From 6da4f8999b5357efcf38e319ff12e42ecca6fb6b Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Mon, 8 Sep 2025 23:21:04 +0530 Subject: [PATCH 01/26] Pipline Helm Charts Signed-off-by: kunal-511 --- experimental/helm/charts/pipeline/Chart.yaml | 37 + experimental/helm/charts/pipeline/README.md | 3 + .../pipeline/ci/values-aws-enhanced.yaml | 207 ++++ .../helm/charts/pipeline/ci/values-aws.yaml | 88 ++ .../pipeline/ci/values-azure-enhanced.yaml | 223 ++++ .../helm/charts/pipeline/ci/values-azure.yaml | 90 ++ .../pipeline/ci/values-dev-enhanced.yaml | 243 ++++ .../helm/charts/pipeline/ci/values-dev.yaml | 190 +++ .../pipeline/ci/values-external-db.yaml | 59 + .../pipeline/ci/values-gcp-enhanced.yaml | 220 ++++ .../helm/charts/pipeline/ci/values-gcp.yaml | 95 ++ .../charts/pipeline/ci/values-kubeflow.yaml | 133 +++ .../ci/values-multi-user-enhanced.yaml | 266 +++++ .../charts/pipeline/ci/values-multi-user.yaml | 231 ++++ .../ci/values-platform-agnostic-enhanced.yaml | 176 +++ .../charts/pipeline/ci/values-postgresql.yaml | 95 ++ .../charts/pipeline/ci/values-production.yaml | 104 ++ .../charts/pipeline/ci/values-standalone.yaml | 56 + .../helm/charts/pipeline/files/sync.py | 395 +++++++ .../charts/pipeline/templates/_helpers.tpl | 450 ++++++++ .../templates/api-server/deployment.yaml | 118 ++ .../pipeline/templates/api-server/role.yaml | 72 ++ .../templates/api-server/rolebinding.yaml | 21 + .../templates/api-server/service.yaml | 31 + .../templates/api-server/serviceaccount.yaml | 16 + .../templates/application/application.yaml | 53 + .../templates/cache-deployer/deployment.yaml | 63 + .../cache-deployer/serviceaccount.yaml | 16 + .../pipeline/templates/cache/deployment.yaml | 105 ++ .../pipeline/templates/cache/service.yaml | 24 + .../templates/cache/serviceaccount.yaml | 16 + .../config/kfp-launcher-configmap.yaml | 12 + .../config/pipeline-install-config.yaml | 43 + .../templates/config/ui-configmap.yaml | 20 + .../templates/crds/application-crd.yaml | 330 ++++++ .../templates/crds/pipelines-crd.yaml | 54 + .../templates/crds/pipelineversions-crd.yaml | 66 ++ .../templates/crds/scheduledworkflow-crd.yaml | 45 + .../pipeline/templates/crds/viewer-crd.yaml | 40 + .../templates/database/mysql-secret.yaml | 16 + .../pipeline/templates/env/aws-patches.yaml | 60 + .../pipeline/templates/env/azure-patches.yaml | 163 +++ .../pipeline/templates/env/gcp-patches.yaml | 134 +++ .../templates/istio/authorization-config.yaml | 154 +++ .../templates/istio/virtual-service.yaml | 29 + .../templates/metadata-writer/deployment.yaml | 65 ++ .../templates/metadata-writer/role.yaml | 40 + .../metadata-writer/rolebinding.yaml | 20 + .../metadata-writer/serviceaccount.yaml | 16 + .../templates/metadata/envoy-deployment.yaml | 59 + .../templates/metadata/envoy-service.yaml | 22 + .../templates/metadata/grpc-configmap.yaml | 15 + .../templates/metadata/grpc-deployment.yaml | 89 ++ .../templates/metadata/grpc-service.yaml | 22 + .../metadata/grpc-serviceaccount.yaml | 15 + .../persistence-agent/deployment.yaml | 81 ++ .../templates/persistence-agent/role.yaml | 44 + .../persistence-agent/rolebinding.yaml | 21 + .../persistence-agent/serviceaccount.yaml | 16 + .../profile-controller/configmap.yaml | 32 + .../decorator-controller.yaml | 50 + .../profile-controller/deployment.yaml | 83 ++ .../templates/profile-controller/service.yaml | 22 + .../rbac/additional-serviceaccounts.yaml | 51 + .../templates/rbac/pipeline-runner-role.yaml | 88 ++ .../rbac/pipeline-runner-rolebinding.yaml | 20 + .../rbac/view-edit-cluster-roles.yaml | 160 +++ .../scheduled-workflow/deployment.yaml | 79 ++ .../templates/scheduled-workflow/role.yaml | 52 + .../scheduled-workflow/rolebinding.yaml | 21 + .../scheduled-workflow/serviceaccount.yaml | 16 + .../pipeline/templates/ui/deployment.yaml | 115 ++ .../charts/pipeline/templates/ui/service.yaml | 24 + .../pipeline/templates/ui/serviceaccount.yaml | 16 + .../templates/viewer-crd/deployment.yaml | 68 ++ .../templates/viewer-crd/serviceaccount.yaml | 16 + .../templates/visualization/deployment.yaml | 90 ++ .../templates/visualization/service.yaml | 24 + .../visualization/serviceaccount.yaml | 16 + .../webhook/mutating-webhook-config.yaml | 34 + .../webhook/validating-webhook-config.yaml | 34 + .../webhook/webhook-server-tls-secret.yaml | 18 + experimental/helm/charts/pipeline/values.yaml | 1026 +++++++++++++++++ tests/helm_kustomize_compare.py | 8 +- tests/helm_kustomize_compare.sh | 72 +- tests/helm_kustomize_compare_all.sh | 6 +- 86 files changed, 7820 insertions(+), 8 deletions(-) create mode 100644 experimental/helm/charts/pipeline/Chart.yaml create mode 100644 experimental/helm/charts/pipeline/README.md create mode 100644 experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-aws.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-azure.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-dev.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-external-db.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-gcp.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-kubeflow.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-multi-user.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-postgresql.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-production.yaml create mode 100644 experimental/helm/charts/pipeline/ci/values-standalone.yaml create mode 100644 experimental/helm/charts/pipeline/files/sync.py create mode 100644 experimental/helm/charts/pipeline/templates/_helpers.tpl create mode 100644 experimental/helm/charts/pipeline/templates/api-server/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/api-server/role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml create mode 100644 experimental/helm/charts/pipeline/templates/api-server/service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/application/application.yaml create mode 100644 experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/cache/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/cache/service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml create mode 100644 experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml create mode 100644 experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml create mode 100644 experimental/helm/charts/pipeline/templates/crds/application-crd.yaml create mode 100644 experimental/helm/charts/pipeline/templates/crds/pipelines-crd.yaml create mode 100644 experimental/helm/charts/pipeline/templates/crds/pipelineversions-crd.yaml create mode 100644 experimental/helm/charts/pipeline/templates/crds/scheduledworkflow-crd.yaml create mode 100644 experimental/helm/charts/pipeline/templates/crds/viewer-crd.yaml create mode 100644 experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml create mode 100644 experimental/helm/charts/pipeline/templates/env/aws-patches.yaml create mode 100644 experimental/helm/charts/pipeline/templates/env/azure-patches.yaml create mode 100644 experimental/helm/charts/pipeline/templates/env/gcp-patches.yaml create mode 100644 experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml create mode 100644 experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml create mode 100644 experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/profile-controller/configmap.yaml create mode 100644 experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml create mode 100644 experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/profile-controller/service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml create mode 100644 experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml create mode 100644 experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml create mode 100644 experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml create mode 100644 experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/ui/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/ui/service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/visualization/deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/visualization/service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/webhook/mutating-webhook-config.yaml create mode 100644 experimental/helm/charts/pipeline/templates/webhook/validating-webhook-config.yaml create mode 100644 experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml create mode 100644 experimental/helm/charts/pipeline/values.yaml diff --git a/experimental/helm/charts/pipeline/Chart.yaml b/experimental/helm/charts/pipeline/Chart.yaml new file mode 100644 index 0000000000..d7f5412d03 --- /dev/null +++ b/experimental/helm/charts/pipeline/Chart.yaml @@ -0,0 +1,37 @@ +apiVersion: v2 + +name: kubeflow-pipelines + +description: A Helm chart for Kubeflow Pipelines - ML Workflows on Kubernetes + +version: 2.5.0 + +appVersion: 2.5.0 + +home: https://github.com/kubeflow/pipelines + +icon: https://github.com/kubeflow/website/blob/master/assets/icons/logo.svg + +sources: +- https://github.com/kubeflow/pipelines +- https://github.com/kubeflow/kubeflow +dependencies: + - name: argo-workflows + version: "~0.40.0" + condition: argo.enabled + repository: "https://argoproj.github.io/argo-helm" + - name: minio + version: "~5.0.0" + condition: minio.enabled + repository: "https://charts.min.io/" + - name: mysql + version: "~9.0.0" + condition: mysql.enabled + repository: "https://charts.bitnami.com/bitnami" + - name: postgresql + version: "~12.0.0" + condition: postgresql.enabled + repository: "https://charts.bitnami.com/bitnami" +annotations: + category: Machine Learning + licenses: Apache-2.0 diff --git a/experimental/helm/charts/pipeline/README.md b/experimental/helm/charts/pipeline/README.md new file mode 100644 index 0000000000..4a0580d1fd --- /dev/null +++ b/experimental/helm/charts/pipeline/README.md @@ -0,0 +1,3 @@ +# Kubeflow Pipelines Helm Chart + +This Helm chart deploys Kubeflow Pipelines on a Kubernetes cluster using the Helm package manager. diff --git a/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml new file mode 100644 index 0000000000..f4e81f57f8 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml @@ -0,0 +1,207 @@ +# AWS-Specific Configuration - Enhanced +# This configuration provides AWS integration with S3 and RDS + +global: + namespace: kubeflow + imageRegistry: ghcr.io/kubeflow + imageTag: "2.5.0" + imagePullPolicy: IfNotPresent + +# Installation mode +installMode: + type: generic + multiUser: + enabled: false + +# Database Configuration +mysql: + enabled: false + +postgresql: + enabled: false + +externalDatabase: + type: mysql + host: "YOUR_RDS_ENDPOINT" + port: 3306 + database: mlpipeline + username: "YOUR_RDS_USERNAME" + password: "YOUR_RDS_PASSWORD" + existingSecret: "mysql-secret" + +# Object Storage Configuration +minio: + enabled: false + +objectStore: + provider: s3 + secure: true + bucketName: "YOUR_S3_BUCKET_NAME" + s3: + region: "YOUR_AWS_REGION" + endpoint: "s3.amazonaws.com" + accessKey: "" + secretKey: "" + +# Database names +database: + pipelinedb: mlpipeline + metadb: metadb + cachedb: cachedb + +# Pipeline Configuration +pipeline: + cache: + image: "registry.k8s.io/busybox" + nodeRestrictions: false + maximumStaleness: "" + defaultStaleness: "" + +# Argo Workflows Configuration +argo: + enabled: true + +env: + platform: aws + aws: + region: "YOUR_AWS_REGION" + rds: + enabled: true + endpoint: "YOUR_RDS_ENDPOINT" + s3: + enabled: true + bucket: "YOUR_S3_BUCKET_NAME" + region: "YOUR_AWS_REGION" + +# AWS API Server configuration +apiServer: + env: + # Object store configuration for S3 + objectStoreSecure: "true" + objectStoreBucketName: "YOUR_S3_BUCKET_NAME" + objectStoreHost: "s3.amazonaws.com" + objectStoreRegion: "YOUR_AWS_REGION" + objectStorePort: "" + + # Service account annotations for IAM roles + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam::ACCOUNT_ID:role/KubeflowPipelinesRole" + + resources: + requests: + cpu: 250m + memory: 500Mi + limits: + cpu: 1 + memory: 2Gi + +# AWS UI configuration +ui: + env: + # MinIO service configuration for S3 + minioServiceHost: "s3.amazonaws.com" + minioServiceRegion: "YOUR_AWS_REGION" + + resources: + requests: + cpu: 10m + memory: 70Mi + limits: + cpu: 100m + memory: 200Mi + +# Enhanced resources for AWS deployment +persistenceAgent: + resources: + requests: + cpu: 120m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +scheduledWorkflow: + resources: + requests: + cpu: 120m + memory: 100Mi + limits: + cpu: 250m + memory: 500Mi + +metadata: + grpc: + resources: + requests: + cpu: 100m + memory: 300Mi + limits: + cpu: 500m + memory: 1Gi + envoy: + resources: + requests: + cpu: 20m + memory: 20Mi + limits: + cpu: 100m + memory: 100Mi + +cache: + server: + resources: + requests: + cpu: 120m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +visualization: + resources: + requests: + cpu: 50m + memory: 200Mi + limits: + cpu: 200m + memory: 500Mi + +viewerCrd: + resources: + requests: + cpu: 120m + memory: 100Mi + limits: + cpu: 250m + memory: 200Mi + +# AWS Load Balancer Controller integration + + service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" + +# Monitoring can be enabled with CloudWatch +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + cloudwatch: + enabled: true + +# Service mesh integration +istio: + enabled: false + +# Cert-manager with AWS integration +certManager: + enabled: true + issuer: "letsencrypt-prod" + issuerKind: "ClusterIssuer" + aws: + region: "YOUR_AWS_REGION" + hostedZoneID: "YOUR_ROUTE53_ZONE_ID" diff --git a/experimental/helm/charts/pipeline/ci/values-aws.yaml b/experimental/helm/charts/pipeline/ci/values-aws.yaml new file mode 100644 index 0000000000..c0e823efe4 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-aws.yaml @@ -0,0 +1,88 @@ +# AWS-specific values for Kubeflow Pipelines +# This configuration uses AWS S3 and RDS + +# Environment configuration +env: + platform: aws + aws: + region: us-west-2 + rds: + enabled: true + s3: + enabled: true + +# Database configuration +mysql: + enabled: false + +postgresql: + enabled: false + +externalDatabase: + type: mysql + host: "YOUR_RDS_ENDPOINT" + port: 3306 + database: "mlpipeline" + username: "root" + password: "" + existingSecret: "mysql-secret" + +# Additional databases +database: + metadb: "metadb" + cachedb: "cachedb" + +# Object storage configuration +objectStore: + provider: s3 + s3: + region: "YOUR_AWS_REGION" + bucket: "YOUR_S3_BUCKET_NAME" + existingSecret: "mlpipeline-minio-artifact" + +minio: + enabled: false + +# Argo Workflows configuration +argo: + enabled: true + +# API Server AWS-specific environment variables +apiServer: + env: + publishLogs: true + pipelineLogLevel: "1" + autoUpdatePipelineDefaultVersion: true + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + additionalEnv: + - name: AWS_REGION + value: "YOUR_AWS_REGION" + - name: OBJECTSTORECONFIG_HOST + value: "s3.amazonaws.com" + - name: OBJECTSTORECONFIG_REGION + value: "YOUR_AWS_REGION" + +# UI AWS-specific configuration +ui: + additionalEnv: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: AWS_REGION + value: "YOUR_AWS_REGION" + +# Monitoring configuration +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + +# Certificate management +certManager: + enabled: false + +# Networking +networking: + istio: + enabled: false diff --git a/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml new file mode 100644 index 0000000000..081c44abea --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml @@ -0,0 +1,223 @@ +# Azure-Specific Configuration - Enhanced +# This configuration provides Azure integration with Blob Storage and Azure SQL + +global: + namespace: kubeflow + imageRegistry: ghcr.io/kubeflow + imageTag: "2.5.0" + imagePullPolicy: IfNotPresent + +# Installation mode +installMode: + type: generic + multiUser: + enabled: false + +# Database Configuration +mysql: + enabled: false + +postgresql: + enabled: false + +externalDatabase: + type: mysql + host: "[SQLSERVERNAME].mysql.database.azure.com" + port: 3306 + database: mlpipeline + username: "YOUR_AZURE_SQL_USERNAME@[SQLSERVERNAME]" + password: "YOUR_AZURE_SQL_PASSWORD" + existingSecret: "mysql-secret" + +# Object Storage Configuration +minio: + enabled: false + +objectStore: + provider: azure + secure: true + bucketName: "mlpipeline" + azure: + storageAccount: "yourstorageaccount" + container: "mlpipeline" + accessKey: "YOUR_STORAGE_ACCOUNT_KEY" + +# Database names +database: + pipelinedb: mlpipeline + metadb: metadb + cachedb: cachedb + +# Pipeline Configuration +pipeline: + cache: + image: "registry.k8s.io/busybox" + nodeRestrictions: false + maximumStaleness: "" + defaultStaleness: "" + +# Argo Workflows Configuration +argo: + enabled: true + # Azure-specific Argo configuration + +# Environment Configuration +env: + platform: azure + azure: + resourceGroup: "yourResourceGroup" + storageAccount: "yourstorageaccount" + sql: + enabled: true + serverName: "[SQLSERVERNAME]" + database: "mlpipeline" + blob: + enabled: true + storageAccount: "yourstorageaccount" + container: "mlpipeline" + +# Azure API Server configuration +apiServer: + env: + # Object store configuration for Azure Blob Storage + objectStoreSecure: "true" + objectStoreBucketName: "mlpipeline" + objectStoreHost: "yourstorageaccount.blob.core.windows.net" + objectStoreRegion: "" + objectStorePort: "" + + # Service account annotations for Azure Managed Identity + serviceAccount: + annotations: + azure.workload.identity/client-id: "YOUR_MANAGED_IDENTITY_CLIENT_ID" + + resources: + requests: + cpu: 250m + memory: 500Mi + limits: + cpu: 1 + memory: 2Gi + +# Azure UI configuration +ui: + env: + # MinIO service configuration for Azure Blob Storage + minioServiceHost: "yourstorageaccount.blob.core.windows.net" + minioServiceRegion: "" + + resources: + requests: + cpu: 10m + memory: 70Mi + limits: + cpu: 100m + memory: 200Mi + +# Enhanced resources for Azure deployment +persistenceAgent: + resources: + requests: + cpu: 120m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +scheduledWorkflow: + resources: + requests: + cpu: 120m + memory: 100Mi + limits: + cpu: 250m + memory: 500Mi + +metadata: + grpc: + resources: + requests: + cpu: 100m + memory: 300Mi + limits: + cpu: 500m + memory: 1Gi + envoy: + resources: + requests: + cpu: 20m + memory: 20Mi + limits: + cpu: 100m + memory: 100Mi + +cache: + server: + resources: + requests: + cpu: 120m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +visualization: + resources: + requests: + cpu: 50m + memory: 200Mi + limits: + cpu: 200m + memory: 500Mi + +viewerCrd: + resources: + requests: + cpu: 120m + memory: 100Mi + limits: + cpu: 250m + memory: 200Mi + +# Azure Load Balancer integration + service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "false" + service.beta.kubernetes.io/azure-dns-label-name: "kubeflow-pipelines" + +# Monitoring with Azure Monitor +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + azureMonitor: + enabled: true + workspaceId: "YOUR_LOG_ANALYTICS_WORKSPACE_ID" + +# Service mesh integration +istio: + enabled: false + +# Cert-manager with Azure DNS +certManager: + enabled: true + issuer: "letsencrypt-prod" + issuerKind: "ClusterIssuer" + azure: + subscriptionId: "YOUR_AZURE_SUBSCRIPTION_ID" + resourceGroup: "yourResourceGroup" + hostedZoneName: "yourdomain.com" + managedIdentityClientId: "YOUR_MANAGED_IDENTITY_CLIENT_ID" + +# Azure-specific MinIO gateway for Blob Storage compatibility +minioAzureGateway: + enabled: true + image: "minio/minio:latest" + storageAccount: "yourstorageaccount" + accessKey: "YOUR_STORAGE_ACCOUNT_KEY" + secretKey: "YOUR_STORAGE_ACCOUNT_SECRET" + service: + type: ClusterIP + port: 9000 diff --git a/experimental/helm/charts/pipeline/ci/values-azure.yaml b/experimental/helm/charts/pipeline/ci/values-azure.yaml new file mode 100644 index 0000000000..66a39cafc4 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-azure.yaml @@ -0,0 +1,90 @@ +# Azure-specific values for Kubeflow Pipelines +# This configuration uses Azure Blob Storage and Azure Database for MySQL + +# Environment configuration +env: + platform: azure + azure: + resourceGroup: "YOUR_RESOURCE_GROUP" + storageAccount: "YOUR_STORAGE_ACCOUNT" + sql: + enabled: true + blob: + enabled: true + +# Database configuration - use external Azure Database for MySQL +mysql: + enabled: false + +postgresql: + enabled: false + +externalDatabase: + type: mysql + host: "[SQLSERVERNAME].mysql.database.azure.com" + port: 3306 + database: "mlpipeline" + username: "root" + password: "" + existingSecret: "mysql-secret" + +database: + metadb: "metadb" + cachedb: "cachedb" + +# Object storage configuration - use Azure Blob Storage +objectStore: + provider: azure + azure: + storageAccount: "YOUR_STORAGE_ACCOUNT" + container: "mlpipeline" + existingSecret: "mlpipeline-minio-artifact" + +minio: + enabled: false + +# Argo Workflows configuration +argo: + enabled: true + +# API Server Azure environment variables +apiServer: + env: + publishLogs: true + pipelineLogLevel: "1" + autoUpdatePipelineDefaultVersion: true + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + additionalEnv: + - name: AZURE_STORAGE_ACCOUNT + value: "YOUR_STORAGE_ACCOUNT" + - name: OBJECTSTORECONFIG_HOST + value: "YOUR_STORAGE_ACCOUNT.blob.core.windows.net" + +# UI Azure configuration +ui: + additionalEnv: + - name: AZURE_STORAGE_ACCOUNT + value: "YOUR_STORAGE_ACCOUNT" + +# Azure AD configuration +azureAD: + enabled: true + tenantId: "YOUR_TENANT_ID" + clientId: "YOUR_CLIENT_ID" + +# Monitoring configuration +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + +# Certificate management +certManager: + enabled: false + +# Networking +networking: + istio: + enabled: false diff --git a/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml new file mode 100644 index 0000000000..46dbcb41f8 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml @@ -0,0 +1,243 @@ +# Development Environment Configuration - Enhanced +# This configuration is optimized for local development and testing + +global: + namespace: kubeflow + imageRegistry: ghcr.io/kubeflow + imageTag: "2.5.0" + imagePullPolicy: IfNotPresent + +# Installation mode +installMode: + type: generic + multiUser: + enabled: false + +# Database Configuration +mysql: + enabled: true + auth: + rootPassword: "dev123" + username: "dev" + password: "dev123" + database: "mlpipeline" + primary: + persistence: + enabled: false + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +postgresql: + enabled: false + +# Object Storage Configuration - Lightweight MinIO +minio: + enabled: true + auth: + rootUser: "dev" + rootPassword: "dev123" + defaultBuckets: "mlpipeline" + persistence: + enabled: false + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + +objectStore: + provider: minio + secure: false + bucketName: mlpipeline + +# Database names +database: + pipelinedb: mlpipeline + metadb: metadb + cachedb: cachedb + +# Pipeline Configuration - Development settings +pipeline: + cache: + image: "registry.k8s.io/busybox" + nodeRestrictions: false + maximumStaleness: "" + defaultStaleness: "" + +# Argo Workflows Configuration - Minimal for dev +argo: + enabled: true + +# Environment Configuration +env: + platform: platform-agnostic + +# Reduced resources for development +apiServer: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 512Mi + env: + logLevel: "2" + +persistenceAgent: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 512Mi + +scheduledWorkflow: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 256Mi + +ui: + replicas: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 50m + memory: 128Mi + +metadata: + grpc: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 512Mi + envoy: + replicas: 1 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 64Mi + +cache: + server: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 512Mi + +cacheDeployer: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 256Mi + +visualization: + replicas: 1 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 256Mi + +viewerCrd: + replicas: 1 + env: + maxNumViewers: "10" + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + +dev: + # Enable debug mode + debug: true + + # Faster probe intervals for development + probes: + readiness: + initialDelaySeconds: 1 + periodSeconds: 2 + liveness: + initialDelaySeconds: 5 + periodSeconds: 5 + + # Development logging + logging: + level: debug + format: text + + # Local port forwarding helpers + portForward: + ui: 8080 + api: 8888 + minio: 9000 + +# Monitoring disabled for development +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + +# Service mesh disabled for development +istio: + enabled: false + +# Cert-manager disabled for development +certManager: + enabled: false + +# Development-specific service types +services: + ui: + type: NodePort + nodePort: 30080 + apiServer: + type: NodePort + nodePort: 30888 + +# Node selectors for development +nodeSelector: {} + +# Tolerations for development +tolerations: [] + +# Affinity for development +affinity: {} diff --git a/experimental/helm/charts/pipeline/ci/values-dev.yaml b/experimental/helm/charts/pipeline/ci/values-dev.yaml new file mode 100644 index 0000000000..8f7689c312 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-dev.yaml @@ -0,0 +1,190 @@ +# Development environment values for Kubeflow Pipelines +# This configuration is optimized for local development and testing + +# Environment configuration +env: + platform: dev + +# Reduced resource requirements for development +apiServer: + replicas: 1 + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 500Mi + +persistenceAgent: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 100Mi + limits: + cpu: 200m + memory: 200Mi + +scheduledWorkflow: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi + +ui: + replicas: 1 + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + cpu: 100m + memory: 100Mi + +viewerCrd: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi + +visualization: + replicas: 1 + resources: + requests: + cpu: 25m + memory: 100Mi + limits: + cpu: 200m + memory: 200Mi + +cache: + server: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi + +cacheDeployer: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi + +metadata: + grpc: + replicas: 1 + resources: + requests: + cpu: 50m + memory: 100Mi + limits: + cpu: 200m + memory: 200Mi + + envoy: + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 100m + memory: 50Mi + +# Database configuration - lightweight MySQL +mysql: + enabled: true + auth: + rootPassword: "dev123" + database: "mlpipeline" + username: "root" + primary: + persistence: + enabled: true + size: 5Gi + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +postgresql: + enabled: false + +# Object storage configuration - lightweight MinIO +minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" + persistence: + enabled: true + size: 5Gi + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + +# Argo Workflows with reduced resources +argo: + enabled: true + controller: + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + +# Development-friendly settings +pipeline: + cache: + # Use faster cache settings for development + maximumStaleness: "P1D" + defaultStaleness: "P1H" + +# Disable production features +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + +certManager: + enabled: false + +networking: + istio: + enabled: false + +# Enable debug logging +global: + logLevel: "debug" + +# Disable security restrictions for easier development +security: + podSecurityStandards: + enabled: false diff --git a/experimental/helm/charts/pipeline/ci/values-external-db.yaml b/experimental/helm/charts/pipeline/ci/values-external-db.yaml new file mode 100644 index 0000000000..025310d008 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-external-db.yaml @@ -0,0 +1,59 @@ +# External database values for kubeflow-pipelines. +# This configuration uses external database instead of embedded MySQL/PostgreSQL + +# Disable embedded databases +mysql: + enabled: false + +postgresql: + enabled: false + +# Configure external database +externalDatabase: + type: mysql + host: "external-db-host.example.com" + port: 3306 + database: "mlpipeline" + username: "mlpipeline" + existingSecret: "external-db-secret" + +# Additional databases for metadata and cache +database: + metadb: "metadb" + cachedb: "cachedb" + connectionMaxLifetime: "300s" + +# Use MinIO for object storage (can be changed to external) +objectStore: + provider: minio + +minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + persistence: + enabled: true + size: 50Gi + +# Standard configuration for other components +apiServer: + replicas: 1 + resources: + requests: + cpu: 250m + memory: 500Mi + +persistenceAgent: + replicas: 1 + resources: + requests: + cpu: 120m + memory: 500Mi + +# Disable monitoring for simplicity +monitoring: + prometheus: + enabled: false + grafana: + enabled: false diff --git a/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml new file mode 100644 index 0000000000..4d20a3a2bd --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml @@ -0,0 +1,220 @@ +# GCP-Specific Configuration - Enhanced +# This configuration provides GCP integration with GCS and Cloud SQL + +global: + namespace: kubeflow + imageRegistry: ghcr.io/kubeflow + imageTag: "2.5.0" + imagePullPolicy: IfNotPresent + +# Installation mode +installMode: + type: generic + multiUser: + enabled: false + +# Database Configuration - External Cloud SQL +mysql: + enabled: false + +postgresql: + enabled: false + +externalDatabase: + type: mysql + host: "127.0.0.1" + port: 3306 + database: pipelinedb + username: "YOUR_CLOUDSQL_USERNAME" + password: "YOUR_CLOUDSQL_PASSWORD" + existingSecret: "mysql-secret" + +# Object Storage Configuration - GCS +minio: + enabled: false + +objectStore: + provider: gcs + secure: true + bucketName: "yourGcsBucketName" + gcs: + projectId: "yourGcsProjectId" + serviceAccount: "" + +# Database names +database: + pipelinedb: pipelinedb + metadb: metadb + cachedb: cachedb + +# Pipeline Configuration +pipeline: + cache: + image: "registry.k8s.io/busybox" + nodeRestrictions: false + maximumStaleness: "" + defaultStaleness: "" + +# Argo Workflows Configuration +argo: + enabled: true + # GCP-specific Argo configuration + +# Environment Configuration +env: + platform: gcp + gcp: + project: "yourGcsProjectId" + cloudsql: + enabled: true + instanceName: "yourCloudSqlInstanceName" + region: "YOUR_GCP_REGION" + gcs: + enabled: true + bucket: "yourGcsBucketName" + projectId: "yourGcsProjectId" + +# GCP-specific API Server configuration +apiServer: + env: + # Object store configuration for GCS + objectStoreSecure: "true" + objectStoreBucketName: "yourGcsBucketName" + objectStoreHost: "storage.googleapis.com" + objectStoreRegion: "" + objectStorePort: "" + + # Service account annotations for Workload Identity + serviceAccount: + annotations: + iam.gke.io/gcp-service-account: "kubeflow-pipelines@yourGcsProjectId.iam.gserviceaccount.com" + + resources: + requests: + cpu: 250m + memory: 500Mi + limits: + cpu: 1 + memory: 2Gi + +# GCP-specific UI configuration +ui: + env: + # MinIO service configuration for GCS + minioServiceHost: "storage.googleapis.com" + minioServiceRegion: "" + + resources: + requests: + cpu: 10m + memory: 70Mi + limits: + cpu: 100m + memory: 200Mi + +# Enhanced resources for GCP deployment +persistenceAgent: + resources: + requests: + cpu: 120m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +scheduledWorkflow: + resources: + requests: + cpu: 120m + memory: 100Mi + limits: + cpu: 250m + memory: 500Mi + +metadata: + grpc: + resources: + requests: + cpu: 100m + memory: 300Mi + limits: + cpu: 500m + memory: 1Gi + envoy: + resources: + requests: + cpu: 20m + memory: 20Mi + limits: + cpu: 100m + memory: 100Mi + +cache: + server: + resources: + requests: + cpu: 120m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +visualization: + resources: + requests: + cpu: 50m + memory: 200Mi + limits: + cpu: 200m + memory: 500Mi + +viewerCrd: + resources: + requests: + cpu: 120m + memory: 100Mi + limits: + cpu: 250m + memory: 200Mi + +# Cloud SQL Proxy sidecar configuration +cloudsqlProxy: + enabled: true + image: "gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.0.0" + instanceConnectionName: "yourGcsProjectId:YOUR_GCP_REGION:yourCloudSqlInstanceName" + serviceAccount: + annotations: + iam.gke.io/gcp-service-account: "cloudsql-proxy@yourGcsProjectId.iam.gserviceaccount.com" + +# GCP Load Balancer integration + service: + type: LoadBalancer + annotations: + cloud.google.com/load-balancer-type: "External" + networking.gke.io/load-balancer-type: "External" + +# Monitoring with Google Cloud Monitoring +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + stackdriver: + enabled: true + projectId: "yourGcsProjectId" + +# Service mesh integration with Istio on GKE +istio: + enabled: true + gateway: "kubeflow-gateway" + gke: + enabled: true + +# Cert-manager with Google Cloud DNS +certManager: + enabled: true + issuer: "letsencrypt-prod" + issuerKind: "ClusterIssuer" + gcp: + projectId: "yourGcsProjectId" + serviceAccountKey: "" # Use Workload Identity instead diff --git a/experimental/helm/charts/pipeline/ci/values-gcp.yaml b/experimental/helm/charts/pipeline/ci/values-gcp.yaml new file mode 100644 index 0000000000..0e71baec00 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-gcp.yaml @@ -0,0 +1,95 @@ +# GCP-specific values for Kubeflow Pipelines +# This configuration uses Google Cloud Storage and Cloud SQL + +# Environment configuration +env: + platform: gcp + gcp: + project: "yourGcsProjectId" + cloudsql: + enabled: true + instanceName: "yourCloudSqlInstanceName" + gcs: + enabled: true + +# Database configuration - use external Cloud SQL +mysql: + enabled: false + +postgresql: + enabled: false + +externalDatabase: + type: mysql + host: "127.0.0.1" + port: 3306 + database: "pipelinedb" + username: "root" + password: "" + existingSecret: "mysql-secret" + +# Additional databases +database: + metadb: "metadb" + cachedb: "cachedb" + +# Object storage configuration - use GCS +objectStore: + provider: gcs + gcs: + bucket: "yourGcsBucketName" + existingSecret: "mlpipeline-minio-artifact" + +minio: + enabled: false + +# Argo Workflows configuration +argo: + enabled: true + +# API Server GCP-specific environment variables +apiServer: + env: + publishLogs: true + pipelineLogLevel: "1" + autoUpdatePipelineDefaultVersion: true + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + additionalEnv: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/gcp-service-account/application_default_credentials.json" + - name: OBJECTSTORECONFIG_HOST + value: "storage.googleapis.com" + +# UI GCP-specific configuration +ui: + additionalEnv: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/gcp-service-account/application_default_credentials.json" + +# Cloud SQL Proxy sidecar configuration +cloudsqlProxy: + enabled: true + image: "gcr.io/cloudsql-docker/gce-proxy:1.33.2" + instanceConnectionName: "yourGcsProjectId:us-central1:yourCloudSqlInstanceName" + +# Workload Identity configuration +workloadIdentity: + enabled: true + serviceAccount: "kf-user@yourGcsProjectId.iam.gserviceaccount.com" + +# Monitoring configuration +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + +# Certificate management +certManager: + enabled: false + +# Networking +networking: + istio: + enabled: false diff --git a/experimental/helm/charts/pipeline/ci/values-kubeflow.yaml b/experimental/helm/charts/pipeline/ci/values-kubeflow.yaml new file mode 100644 index 0000000000..004d9e00c7 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-kubeflow.yaml @@ -0,0 +1,133 @@ +# Kubeflow integration values for kubeflow-pipelines. +# This configuration is optimized for integration with full Kubeflow deployment + + +global: + namespace: kubeflow + logLevel: "info" + +# Installation mode +installMode: + type: multi-user + multiUser: + enabled: true + profileController: + enabled: true + +# Kubeflow-specific RBAC +rbac: + create: true + multiUser: + enabled: true + clusterRoles: + - name: "kubeflow-pipelines-edit" + create: true + - name: "kubeflow-pipelines-view" + create: true + +# Service accounts for Kubeflow integration +serviceAccounts: + pipelineRunner: + create: true + name: "pipeline-runner" + viewer: + create: true + name: "viewer" + containerBuilder: + create: true + name: "container-builder" + metadataWriter: + create: true + name: "metadata-writer" + +# API Server with Kubeflow integration +apiServer: + replicas: 1 + additionalEnv: + - name: MULTI_USER + value: "true" + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + value: "" + +# UI with Kubeflow integration +ui: + replicas: 1 + additionalEnv: + - name: MULTI_USER + value: "true" + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + value: "" + +# Database configuration +mysql: + enabled: true + auth: + rootPassword: "" + database: "mlpipeline" + username: "root" + primary: + persistence: + enabled: true + size: 20Gi + +# Object storage +objectStore: + provider: minio + +minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" + persistence: + enabled: true + size: 20Gi + +# Istio integration (required for Kubeflow) +networking: + istio: + enabled: true + gateway: "kubeflow-gateway" + host: "*" + virtualService: + enabled: true + prefix: "/pipeline" + +# Authorization policies for Kubeflow +authorizationPolicies: + enabled: true + policies: + - name: "ml-pipeline" + action: "ALLOW" + rules: + - from: + - source: + principals: ["cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account"] + - to: + - operation: + methods: ["GET", "POST"] + +# Webhooks for multi-user +webhook: + enabled: true + mutating: + enabled: true + multiUser: true + validating: + enabled: true + multiUser: true + +# Security configuration +security: + podSecurityStandards: + enabled: true + enforce: "restricted" + networkPolicies: + enabled: true + rbacEnforcement: + enabled: true diff --git a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml new file mode 100644 index 0000000000..7f07c86c67 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml @@ -0,0 +1,266 @@ +# Multi-User Configuration - Enhanced +# This configuration provides multi-user deployment with enhanced RBAC and isolation + +global: + namespace: kubeflow + imageRegistry: ghcr.io/kubeflow + imageTag: "2.5.0" + imagePullPolicy: IfNotPresent + +# Installation mode - Multi-user enabled +installMode: + type: multi-user + multiUser: + enabled: true + profileController: + enabled: true + +# Database Configuration - Enhanced for multi-user +mysql: + enabled: true + auth: + rootPassword: "root123" + username: "mlpipeline" + password: "mlpipeline123" + database: "mlpipeline" + primary: + persistence: + enabled: true + size: 50Gi # Larger storage for multi-user + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1 + memory: 2Gi + +postgresql: + enabled: false + +# Object Storage Configuration - Enhanced for multi-user +minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" + persistence: + enabled: true + size: 100Gi # Larger storage for multi-user + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +objectStore: + provider: minio + secure: false + bucketName: mlpipeline + +# Database names +database: + pipelinedb: mlpipeline + metadb: metadb + cachedb: cachedb + +# Pipeline Configuration +pipeline: + cache: + image: "registry.k8s.io/busybox" + nodeRestrictions: true + maximumStaleness: "P0D" + defaultStaleness: "P0D" + +# Argo Workflows Configuration - Multi-user +argo: + enabled: true + # Multi-user specific Argo configuration + +# Environment Configuration +env: + platform: platform-agnostic + +# Enhanced RBAC Configuration +rbac: + create: true + multiUser: + enabled: true + # Additional RBAC rules for multi-user scenarios + +# Multi-user API Server configuration +apiServer: + env: + # Multi-user specific environment variables + multiUser: "true" + kubeflowUserId: "user@example.com" + + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2 + memory: 4Gi + +# Multi-user UI configuration +ui: + env: + # Multi-user UI configuration + multiUser: "true" + + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 1Gi + +# Enhanced resources for multi-user deployment +persistenceAgent: + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: 1 + memory: 2Gi + +scheduledWorkflow: + resources: + requests: + cpu: 250m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +metadata: + grpc: + resources: + requests: + cpu: 250m + memory: 500Mi + limits: + cpu: 1 + memory: 2Gi + envoy: + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 200m + memory: 200Mi + +cache: + # Cache deployer disabled in multi-user mode + enabled: true + server: + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: 1 + memory: 2Gi + +cacheDeployer: + # Typically disabled in multi-user mode + enabled: false + +visualization: + resources: + requests: + cpu: 100m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +viewerCrd: + env: + maxNumViewers: "200" + resources: + requests: + cpu: 250m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +# Profile Controller Configuration (for multi-user) +profileController: + enabled: true + image: + repository: kfp-profile-controller + tag: "2.5.0" + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 1Gi + +# Namespace Isolation Configuration +namespaceIsolation: + enabled: true + # Additional network policies and RBAC for isolation + +# Security Configuration - Enhanced for multi-user +security: + podSecurityStandards: + enabled: true + level: "restricted" + networkPolicies: + enabled: true + +# Monitoring - Enhanced for multi-user +monitoring: + prometheus: + enabled: true + multiUser: true + grafana: + enabled: true + multiUser: true + +# Service mesh - Recommended for multi-user +istio: + enabled: true + gateway: kubeflow-gateway + multiUser: true + authorizationPolicies: + enabled: true + +# Cert-manager for TLS +certManager: + enabled: true + issuer: "kubeflow-self-signing-issuer" + issuerKind: "ClusterIssuer" + +# Additional multi-user specific configurations +multiUserConfig: + # User namespace template + userNamespaceTemplate: "kubeflow-user-{userid}" + + # Default resource quotas per user namespace + resourceQuotas: + requests.cpu: "2" + requests.memory: "4Gi" + limits.cpu: "4" + limits.memory: "8Gi" + persistentvolumeclaims: "10" + + # Network policies for user isolation + networkPolicies: + enabled: true + allowCrossNamespace: false + + # RBAC templates for user permissions + rbacTemplates: + enabled: true diff --git a/experimental/helm/charts/pipeline/ci/values-multi-user.yaml b/experimental/helm/charts/pipeline/ci/values-multi-user.yaml new file mode 100644 index 0000000000..a53894598e --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-multi-user.yaml @@ -0,0 +1,231 @@ +# Multi-user values for Kubeflow Pipelines +# This configuration enables multi-user mode with proper isolation + +# Installation mode configuration +installMode: + type: multi-user + multiUser: + enabled: true + profileController: + enabled: true + +# Environment configuration +env: + platform: platform-agnostic + +# Multi-user specific components +profileController: + enabled: true + name: profiles-controller + + image: + repository: kfp-profile-controller + tag: "2.5.0" + + replicas: 1 + + # Service Account + serviceAccount: + create: true + name: "profiles-controller-service-account" + annotations: {} + + # Resources + resources: + requests: + cpu: 100m + memory: 200Mi + limits: {} + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +# Enhanced API Server configuration for multi-user +apiServer: + env: + publishLogs: true + pipelineLogLevel: "1" + autoUpdatePipelineDefaultVersion: true + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + # Multi-user specific environment variables + multiUser: true + + # Additional environment variables for multi-user mode + additionalEnv: + - name: MULTI_USER + value: "true" + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + value: "" + +# Enhanced UI configuration for multi-user +ui: + additionalEnv: + - name: MULTI_USER + value: "true" + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + value: "" + +# Enhanced RBAC for multi-user +rbac: + create: true + # Multi-user specific RBAC + multiUser: + enabled: true + # Cluster roles for multi-user + clusterRoles: + - name: "kubeflow-pipelines-edit" + create: true + - name: "kubeflow-pipelines-view" + create: true + +# Service accounts for multi-user +serviceAccounts: + pipelineRunner: + create: true + name: "pipeline-runner" + + viewer: + create: true + name: "viewer" + + containerBuilder: + create: true + name: "container-builder" + + metadataWriter: + create: true + name: "metadata-writer" + + # Multi-user specific service accounts + profilesController: + create: true + name: "profiles-controller-service-account" + +# Database configuration +mysql: + enabled: true + auth: + rootPassword: "" + database: "mlpipeline" + username: "root" + primary: + persistence: + enabled: true + size: 20Gi + +postgresql: + enabled: false + +# Object storage configuration +objectStore: + provider: minio + +minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" + persistence: + enabled: true + size: 20Gi + +# Argo Workflows configuration +argo: + enabled: true + +# Istio configuration for multi-user +networking: + istio: + enabled: true + gateway: "kubeflow-gateway" + host: "*" + + # Virtual Service configuration + virtualService: + enabled: true + prefix: "/pipeline" + +# Authorization policies for multi-user +authorizationPolicies: + enabled: true + policies: + - name: "ml-pipeline" + action: "ALLOW" + rules: + - from: + - source: + principals: ["cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account"] + - to: + - operation: + methods: ["GET", "POST"] + + - name: "ml-pipeline-ui" + action: "ALLOW" + rules: + - from: + - source: + principals: ["cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account"] + - to: + - operation: + methods: ["GET", "POST"] + +# Webhook configuration for multi-user +webhook: + enabled: true + + # Mutating webhook + mutating: + enabled: true + name: "cache-webhook-kubeflow" + # Multi-user webhook configuration + multiUser: true + + # Validating webhook + validating: + enabled: true + name: "cache-webhook-kubeflow" + # Multi-user webhook configuration + multiUser: true + +# Security configuration for multi-user +security: + # Pod Security Standards + podSecurityStandards: + enabled: true + enforce: "restricted" + + # Security Context Profile + seccompProfile: + type: "RuntimeDefault" + + # Multi-user security policies + networkPolicies: + enabled: true + + # RBAC enforcement + rbacEnforcement: + enabled: true + +# Monitoring configuration +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + +# Certificate management +certManager: + enabled: false diff --git a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml new file mode 100644 index 0000000000..34e42fe627 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml @@ -0,0 +1,176 @@ +# Platform-Agnostic Configuration - Enhanced +# This configuration provides a complete platform-agnostic deployment +# using MinIO for object storage and MySQL for database + +global: + namespace: kubeflow + imageRegistry: ghcr.io/kubeflow + imageTag: "2.5.0" + imagePullPolicy: IfNotPresent + +# Installation mode +installMode: + type: generic + multiUser: + enabled: false + +# Database Configuration - Using MySQL subchart +mysql: + enabled: true + auth: + rootPassword: "root123" + username: "mlpipeline" + password: "mlpipeline123" + database: "mlpipeline" + primary: + persistence: + enabled: true + size: 20Gi + +postgresql: + enabled: false + +externalDatabase: + type: mysql + host: "" + port: 3306 + database: mlpipeline + username: mlpipeline + password: "" + +# Object Storage Configuration - Using MinIO subchart +minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" + persistence: + enabled: true + size: 20Gi + +objectStore: + provider: minio + secure: false + bucketName: mlpipeline + +# Database names +database: + pipelinedb: mlpipeline + metadb: metadb + cachedb: cachedb + +# Pipeline Configuration +pipeline: + cache: + image: "registry.k8s.io/busybox" + nodeRestrictions: false + maximumStaleness: "" + defaultStaleness: "" + +# Argo Workflows Configuration +argo: + enabled: true + # Basic Argo configuration for platform-agnostic deployment + +# Environment Configuration +env: + platform: platform-agnostic + +# Resource Configuration +apiServer: + resources: + requests: + cpu: 250m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + +persistenceAgent: + resources: + requests: + cpu: 120m + memory: 500Mi + limits: + cpu: 250m + memory: 1Gi + +scheduledWorkflow: + resources: + requests: + cpu: 120m + memory: 100Mi + limits: + cpu: 250m + memory: 500Mi + +ui: + resources: + requests: + cpu: 10m + memory: 70Mi + limits: + cpu: 100m + memory: 200Mi + +metadata: + grpc: + resources: + requests: + cpu: 100m + memory: 300Mi + limits: + cpu: 200m + memory: 600Mi + envoy: + resources: + requests: + cpu: 20m + memory: 20Mi + limits: + cpu: 100m + memory: 100Mi + +cache: + server: + resources: + requests: + cpu: 120m + memory: 500Mi + limits: + cpu: 250m + memory: 1Gi + +visualization: + resources: + requests: + cpu: 50m + memory: 200Mi + limits: + cpu: 100m + memory: 400Mi + +viewerCrd: + resources: + requests: + cpu: 120m + memory: 100Mi + limits: + cpu: 250m + memory: 200Mi + +# Monitoring disabled for platform-agnostic +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + +# Service mesh disabled for platform-agnostic +istio: + enabled: false + +# Cert-manager disabled for platform-agnostic +certManager: + enabled: false diff --git a/experimental/helm/charts/pipeline/ci/values-postgresql.yaml b/experimental/helm/charts/pipeline/ci/values-postgresql.yaml new file mode 100644 index 0000000000..532ad6c882 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-postgresql.yaml @@ -0,0 +1,95 @@ +# PostgreSQL values for kubeflow-pipelines. +# This configuration uses PostgreSQL instead of MySQL + +# Disable MySQL +mysql: + enabled: false + +# Enable PostgreSQL +postgresql: + enabled: true + auth: + postgresPassword: "postgres123" + database: "mlpipeline" + username: "postgres" + primary: + persistence: + enabled: true + size: 20Gi + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 1000m + memory: 1Gi + +# External database configuration pointing to PostgreSQL +externalDatabase: + type: postgresql + host: "" + port: 5432 + database: "mlpipeline" + username: "postgres" + +# Additional databases for PostgreSQL +database: + metadb: "metadb" + cachedb: "cachedb" + connectionMaxLifetime: "300s" + +# Standard MinIO configuration +objectStore: + provider: minio + +minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" + persistence: + enabled: true + size: 20Gi + +# Standard component configuration +apiServer: + replicas: 1 + resources: + requests: + cpu: 250m + memory: 500Mi + +persistenceAgent: + replicas: 1 + resources: + requests: + cpu: 120m + memory: 500Mi + +scheduledWorkflow: + replicas: 1 + resources: + requests: + cpu: 120m + memory: 100Mi + +ui: + replicas: 1 + resources: + requests: + cpu: 10m + memory: 70Mi + +# Disable monitoring for simplicity +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + +# Standard security +security: + podSecurityStandards: + enabled: true + enforce: "restricted" diff --git a/experimental/helm/charts/pipeline/ci/values-production.yaml b/experimental/helm/charts/pipeline/ci/values-production.yaml new file mode 100644 index 0000000000..222695667a --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-production.yaml @@ -0,0 +1,104 @@ +# Production values for kubeflow-pipelines. +# This configuration is optimized for production deployments + +# Global configuration +global: + namespace: kubeflow + logLevel: "info" + +# Production resource settings +apiServer: + replicas: 3 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 2Gi + +persistenceAgent: + replicas: 2 + resources: + requests: + cpu: 250m + memory: 500Mi + limits: + cpu: 1000m + memory: 1Gi + +scheduledWorkflow: + replicas: 2 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 1000m + memory: 512Mi + +ui: + replicas: 2 + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 500Mi + +# High availability database +mysql: + enabled: false + +postgresql: + enabled: false + +# Use external database in production +externalDatabase: + type: mysql + host: "production-db-host" + port: 3306 + database: "mlpipeline" + username: "mlpipeline" + existingSecret: "mysql-secret" + +# Use external object storage +objectStore: + provider: s3 + s3: + region: "us-west-2" + bucket: "production-kfp-artifacts" + existingSecret: "s3-credentials" + +minio: + enabled: false + +# Enable monitoring +monitoring: + prometheus: + enabled: true + serviceMonitor: + enabled: true + grafana: + enabled: true + +# Enable security features +security: + podSecurityStandards: + enabled: true + enforce: "restricted" + +# Enable Istio for production +networking: + istio: + enabled: true + gateway: "kubeflow-gateway" + +# Enable webhooks +webhook: + enabled: true + mutating: + enabled: true + validating: + enabled: true diff --git a/experimental/helm/charts/pipeline/ci/values-standalone.yaml b/experimental/helm/charts/pipeline/ci/values-standalone.yaml new file mode 100644 index 0000000000..15956333f3 --- /dev/null +++ b/experimental/helm/charts/pipeline/ci/values-standalone.yaml @@ -0,0 +1,56 @@ +# Platform-agnostic values for Kubeflow Pipelines +# This configuration uses local storage (MinIO) and MySQL + +# Environment configuration +env: + platform: platform-agnostic + +# Database configuration - use MySQL subchart +mysql: + enabled: true + auth: + rootPassword: "" + database: "mlpipeline" + username: "root" + primary: + persistence: + enabled: true + size: 20Gi + +postgresql: + enabled: false + +externalDatabase: + type: mysql + +# Object storage configuration - use MinIO subchart +objectStore: + provider: minio + +minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" + persistence: + enabled: true + size: 20Gi + +# Argo Workflows configuration +argo: + enabled: true + +# Disable cloud-specific features +monitoring: + prometheus: + enabled: false + grafana: + enabled: false + +certManager: + enabled: false + +networking: + istio: + enabled: false diff --git a/experimental/helm/charts/pipeline/files/sync.py b/experimental/helm/charts/pipeline/files/sync.py new file mode 100644 index 0000000000..3d39e3187f --- /dev/null +++ b/experimental/helm/charts/pipeline/files/sync.py @@ -0,0 +1,395 @@ +# Copyright 2020-2021 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import os +import base64 + + +def main(): + settings = get_settings_from_env() + server = server_factory(**settings) + server.serve_forever() + + +def get_settings_from_env(controller_port=None, + visualization_server_image=None, frontend_image=None, + visualization_server_tag=None, frontend_tag=None, disable_istio_sidecar=None, + minio_access_key=None, minio_secret_key=None, kfp_default_pipeline_root=None): + """ + Returns a dict of settings from environment variables relevant to the controller + + Environment settings can be overridden by passing them here as arguments. + + Settings are pulled from the all-caps version of the setting name. The + following defaults are used if those environment variables are not set + to enable backwards compatibility with previous versions of this script: + visualization_server_image: ghcr.io/kubeflow/kfp-visualization-server + visualization_server_tag: value of KFP_VERSION environment variable + frontend_image: ghcr.io/kubeflow/kfp-frontend + frontend_tag: value of KFP_VERSION environment variable + disable_istio_sidecar: Required (no default) + minio_access_key: Required (no default) + minio_secret_key: Required (no default) + """ + settings = dict() + settings["controller_port"] = \ + controller_port or \ + os.environ.get("CONTROLLER_PORT", "8080") + + settings["visualization_server_image"] = \ + visualization_server_image or \ + os.environ.get("VISUALIZATION_SERVER_IMAGE", "ghcr.io/kubeflow/kfp-visualization-server") + + settings["frontend_image"] = \ + frontend_image or \ + os.environ.get("FRONTEND_IMAGE", "ghcr.io/kubeflow/kfp-frontend") + + # Look for specific tags for each image first, falling back to + # previously used KFP_VERSION environment variable for backwards + # compatibility + settings["visualization_server_tag"] = \ + visualization_server_tag or \ + os.environ.get("VISUALIZATION_SERVER_TAG") or \ + os.environ["KFP_VERSION"] + + settings["frontend_tag"] = \ + frontend_tag or \ + os.environ.get("FRONTEND_TAG") or \ + os.environ["KFP_VERSION"] + + settings["disable_istio_sidecar"] = \ + disable_istio_sidecar if disable_istio_sidecar is not None \ + else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" + + settings["minio_access_key"] = \ + minio_access_key or \ + base64.b64encode(bytes(os.environ.get("MINIO_ACCESS_KEY"), 'utf-8')).decode('utf-8') + + settings["minio_secret_key"] = \ + minio_secret_key or \ + base64.b64encode(bytes(os.environ.get("MINIO_SECRET_KEY"), 'utf-8')).decode('utf-8') + + # KFP_DEFAULT_PIPELINE_ROOT is optional + settings["kfp_default_pipeline_root"] = \ + kfp_default_pipeline_root or \ + os.environ.get("KFP_DEFAULT_PIPELINE_ROOT") + + return settings + + +def server_factory(visualization_server_image, + visualization_server_tag, frontend_image, frontend_tag, + disable_istio_sidecar, minio_access_key, + minio_secret_key, kfp_default_pipeline_root=None, + url="", controller_port=8080): + """ + Returns an HTTPServer populated with Handler with customized settings + """ + class Controller(BaseHTTPRequestHandler): + def sync(self, parent, attachments): + # parent is a namespace + namespace = parent.get("metadata", {}).get("name") + pipeline_enabled = parent.get("metadata", {}).get( + "labels", {}).get("pipelines.kubeflow.org/enabled") + + if pipeline_enabled != "true": + return {"status": {}, "attachments": []} + + desired_configmap_count = 1 + desired_resources = [] + if kfp_default_pipeline_root: + desired_configmap_count = 2 + desired_resources += [{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "kfp-launcher", + "namespace": namespace, + }, + "data": { + "defaultPipelineRoot": kfp_default_pipeline_root, + }, + }] + + + # Compute status based on observed state. + desired_status = { + "kubeflow-pipelines-ready": + len(attachments["Secret.v1"]) == 1 and + len(attachments["ConfigMap.v1"]) == desired_configmap_count and + len(attachments["Deployment.apps/v1"]) == 2 and + len(attachments["Service.v1"]) == 2 and + len(attachments["DestinationRule.networking.istio.io/v1alpha3"]) == 1 and + len(attachments["AuthorizationPolicy.security.istio.io/v1beta1"]) == 1 and + "True" or "False" + } + + # Generate the desired attachment object(s). + desired_resources += [ + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "metadata-grpc-configmap", + "namespace": namespace, + }, + "data": { + "METADATA_GRPC_SERVICE_HOST": + "metadata-grpc-service.kubeflow", + "METADATA_GRPC_SERVICE_PORT": "8080", + }, + }, + # Visualization server related manifests below + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "labels": { + "app": "ml-pipeline-visualizationserver" + }, + "name": "ml-pipeline-visualizationserver", + "namespace": namespace, + }, + "spec": { + "selector": { + "matchLabels": { + "app": "ml-pipeline-visualizationserver" + }, + }, + "template": { + "metadata": { + "labels": { + "app": "ml-pipeline-visualizationserver" + }, + "annotations": disable_istio_sidecar and { + "sidecar.istio.io/inject": "false" + } or {}, + }, + "spec": { + "containers": [{ + "image": f"{visualization_server_image}:{visualization_server_tag}", + "imagePullPolicy": + "IfNotPresent", + "name": + "ml-pipeline-visualizationserver", + "ports": [{ + "containerPort": 8888 + }], + "resources": { + "requests": { + "cpu": "50m", + "memory": "200Mi" + }, + "limits": { + "cpu": "500m", + "memory": "1Gi" + }, + } + }], + "serviceAccountName": + "default-editor", + }, + }, + }, + }, + { + "apiVersion": "networking.istio.io/v1alpha3", + "kind": "DestinationRule", + "metadata": { + "name": "ml-pipeline-visualizationserver", + "namespace": namespace, + }, + "spec": { + "host": "ml-pipeline-visualizationserver", + "trafficPolicy": { + "tls": { + "mode": "ISTIO_MUTUAL" + } + } + } + }, + { + "apiVersion": "security.istio.io/v1beta1", + "kind": "AuthorizationPolicy", + "metadata": { + "name": "ml-pipeline-visualizationserver", + "namespace": namespace, + }, + "spec": { + "selector": { + "matchLabels": { + "app": "ml-pipeline-visualizationserver" + } + }, + "rules": [{ + "from": [{ + "source": { + "principals": ["cluster.local/ns/kubeflow/sa/ml-pipeline"] + } + }] + }] + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "ml-pipeline-visualizationserver", + "namespace": namespace, + }, + "spec": { + "ports": [{ + "name": "http", + "port": 8888, + "protocol": "TCP", + "targetPort": 8888, + }], + "selector": { + "app": "ml-pipeline-visualizationserver", + }, + }, + }, + # Artifact fetcher related resources below. + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "labels": { + "app": "ml-pipeline-ui-artifact" + }, + "name": "ml-pipeline-ui-artifact", + "namespace": namespace, + }, + "spec": { + "selector": { + "matchLabels": { + "app": "ml-pipeline-ui-artifact" + } + }, + "template": { + "metadata": { + "labels": { + "app": "ml-pipeline-ui-artifact" + }, + "annotations": disable_istio_sidecar and { + "sidecar.istio.io/inject": "false" + } or {}, + }, + "spec": { + "containers": [{ + "name": + "ml-pipeline-ui-artifact", + "image": f"{frontend_image}:{frontend_tag}", + "imagePullPolicy": + "IfNotPresent", + "ports": [{ + "containerPort": 3000 + }], + "env": [ + { + "name": "MINIO_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "accesskey", + "name": "mlpipeline-minio-artifact" + } + } + }, + { + "name": "MINIO_SECRET_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "secretkey", + "name": "mlpipeline-minio-artifact" + } + } + } + ], + "resources": { + "requests": { + "cpu": "10m", + "memory": "70Mi" + }, + "limits": { + "cpu": "100m", + "memory": "500Mi" + }, + } + }], + "serviceAccountName": + "default-editor" + } + } + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "ml-pipeline-ui-artifact", + "namespace": namespace, + "labels": { + "app": "ml-pipeline-ui-artifact" + } + }, + "spec": { + "ports": [{ + "name": + "http", # name is required to let istio understand request protocol + "port": 80, + "protocol": "TCP", + "targetPort": 3000 + }], + "selector": { + "app": "ml-pipeline-ui-artifact" + } + } + }, + ] + print('Received request:\n', json.dumps(parent, sort_keys=True)) + print('Desired resources except secrets:\n', json.dumps(desired_resources, sort_keys=True)) + # Moved after the print argument because this is sensitive data. + desired_resources.append({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "mlpipeline-minio-artifact", + "namespace": namespace, + }, + "data": { + "accesskey": minio_access_key, + "secretkey": minio_secret_key, + }, + }) + + return {"status": desired_status, "attachments": desired_resources} + + def do_POST(self): + # Serve the sync() function as a JSON webhook. + observed = json.loads( + self.rfile.read(int(self.headers.get("content-length")))) + desired = self.sync(observed["object"], observed["attachments"]) + + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + self.wfile.write(bytes(json.dumps(desired), 'utf-8')) + + return HTTPServer((url, int(controller_port)), Controller) + + +if __name__ == "__main__": + main() diff --git a/experimental/helm/charts/pipeline/templates/_helpers.tpl b/experimental/helm/charts/pipeline/templates/_helpers.tpl new file mode 100644 index 0000000000..c73fb53c84 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/_helpers.tpl @@ -0,0 +1,450 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "kubeflow-pipelines.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kubeflow-pipelines.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kubeflow-pipelines.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "kubeflow-pipelines.labels" -}} +helm.sh/chart: {{ include "kubeflow-pipelines.chart" . }} +{{ include "kubeflow-pipelines.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: ml-pipeline +application-crd-id: kubeflow-pipelines +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kubeflow-pipelines.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kubeflow-pipelines.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +ML Pipeline specific labels - matching original manifests +*/}} +{{- define "kubeflow-pipelines.mlPipelineLabels" -}} +{{ include "kubeflow-pipelines.labels" . }} +app: ml-pipeline +{{- end }} + +{{/* +ML Pipeline selector labels +*/}} +{{- define "kubeflow-pipelines.mlPipelineSelectorLabels" -}} +app: ml-pipeline +{{- end }} + +{{/* +Create the name of the service account to use for API Server +*/}} +{{- define "kubeflow-pipelines.apiServer.serviceAccountName" -}} +{{- if .Values.apiServer.serviceAccount.create }} +{{- default "ml-pipeline" .Values.apiServer.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.apiServer.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use for Persistence Agent +*/}} +{{- define "kubeflow-pipelines.persistenceAgent.serviceAccountName" -}} +{{- if .Values.persistenceAgent.serviceAccount.create }} +{{- default "ml-pipeline-persistenceagent" .Values.persistenceAgent.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.persistenceAgent.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use for Scheduled Workflow +*/}} +{{- define "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" -}} +{{- if .Values.scheduledWorkflow.serviceAccount.create }} +{{- default "ml-pipeline-scheduledworkflow" .Values.scheduledWorkflow.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.scheduledWorkflow.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use for UI +*/}} +{{- define "kubeflow-pipelines.ui.serviceAccountName" -}} +{{- if .Values.ui.serviceAccount.create }} +{{- default "ml-pipeline-ui" .Values.ui.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.ui.serviceAccount.name }} +{{- end }} +{{- end }} + + + +{{/* +Create the name of the service account to use for Cache Server +*/}} +{{- define "kubeflow-pipelines.cache.serviceAccountName" -}} +{{- if .Values.cache.serviceAccount.create }} +{{- default "cache-server" .Values.cache.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.cache.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use for Cache Deployer +*/}} +{{- define "kubeflow-pipelines.cacheDeployer.serviceAccountName" -}} +{{- if .Values.cacheDeployer.serviceAccount.create }} +{{- default "cache-deployer-sa" .Values.cacheDeployer.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.cacheDeployer.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use for Metadata +*/}} +{{- define "kubeflow-pipelines.metadata.serviceAccountName" -}} +{{- if .Values.metadata.grpc.serviceAccount.create }} +{{- default "metadata-grpc-server" .Values.metadata.grpc.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.metadata.grpc.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use for Viewer CRD Controller +*/}} +{{- define "kubeflow-pipelines.viewerCrd.serviceAccountName" -}} +{{- if .Values.viewerCrd.serviceAccount.create }} +{{- default "ml-pipeline-viewer-crd-service-account" .Values.viewerCrd.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.viewerCrd.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use for Visualization Server +*/}} +{{- define "kubeflow-pipelines.visualization.serviceAccountName" -}} +{{- if .Values.visualization.serviceAccount.create }} +{{- default "ml-pipeline-visualizationserver" .Values.visualization.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.visualization.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use for Metadata Writer +*/}} +{{- define "kubeflow-pipelines.metadataWriter.serviceAccountName" -}} +{{- if .Values.metadataWriter.serviceAccount.create }} +{{- default "kubeflow-pipelines-metadata-writer" .Values.metadataWriter.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.metadataWriter.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Database configuration helpers +*/}} +{{- define "kubeflow-pipelines.database.host" -}} +{{- if .Values.mysql.enabled -}} +mysql +{{- else if .Values.postgresql.enabled -}} +postgresql +{{- else -}} +{{- .Values.externalDatabase.host | default "mysql" }} +{{- end -}} +{{- end }} + +{{- define "kubeflow-pipelines.database.port" -}} +{{- if .Values.mysql.enabled -}} +3306 +{{- else if .Values.postgresql.enabled -}} +5432 +{{- else -}} +{{- .Values.externalDatabase.port }} +{{- end -}} +{{- end }} + +{{- define "kubeflow-pipelines.database.name" -}} +{{- if .Values.mysql.enabled -}} +{{- .Values.mysql.auth.database | default "mlpipeline" }} +{{- else if .Values.postgresql.enabled -}} +{{- .Values.postgresql.auth.database | default "mlpipeline" }} +{{- else -}} +{{- .Values.externalDatabase.database }} +{{- end -}} +{{- end }} + +{{- define "kubeflow-pipelines.database.username" -}} +{{- if .Values.mysql.enabled -}} +{{- .Values.mysql.auth.username | default "mlpipeline" }} +{{- else if .Values.postgresql.enabled -}} +{{- .Values.postgresql.auth.username | default "mlpipeline" }} +{{- else -}} +{{- .Values.externalDatabase.username }} +{{- end -}} +{{- end }} + +{{- define "kubeflow-pipelines.database.secretName" -}} +{{- if .Values.externalDatabase.existingSecret -}} +{{- .Values.externalDatabase.existingSecret }} +{{- else if .Values.mysql.enabled -}} +mysql-secret +{{- else if .Values.postgresql.enabled -}} +postgresql-secret +{{- else -}} +{{ include "kubeflow-pipelines.fullname" . }}-mysql-secret +{{- end -}} +{{- end }} + +{{- define "kubeflow-pipelines.database.secretKey" -}} +{{- if .Values.externalDatabase.existingSecret -}} +{{- .Values.externalDatabase.existingSecretPasswordKey | default "password" }} +{{- else if .Values.mysql.enabled -}} +mysql-password +{{- else if .Values.postgresql.enabled -}} +password +{{- else -}} +password +{{- end -}} +{{- end }} + +{{/* +Object storage configuration helpers +*/}} +{{- define "kubeflow-pipelines.objectStore.endpoint" -}} +{{- if eq .Values.objectStore.provider "minio" -}} +{{- if .Values.minio.enabled -}} +{{- include "minio.fullname" .Subcharts.minio }}:9000 +{{- else -}} +{{- .Values.objectStore.minio.endpoint }} +{{- end -}} +{{- else if eq .Values.objectStore.provider "s3" -}} +s3.amazonaws.com +{{- else if eq .Values.objectStore.provider "gcs" -}} +storage.googleapis.com +{{- else if eq .Values.objectStore.provider "azure" -}} +{{ .Values.objectStore.azure.storageAccount }}.blob.core.windows.net +{{- else -}} +{{ .Values.objectStore.custom.endpoint }} +{{- end -}} +{{- end }} + +{{- define "kubeflow-pipelines.objectStore.bucket" -}} +{{- if eq .Values.objectStore.provider "minio" -}} +{{- .Values.objectStore.minio.bucket | default "mlpipeline" }} +{{- else if eq .Values.objectStore.provider "s3" -}} +{{- .Values.objectStore.s3.bucket }} +{{- else if eq .Values.objectStore.provider "gcs" -}} +{{- .Values.objectStore.gcs.bucket }} +{{- else if eq .Values.objectStore.provider "azure" -}} +{{- .Values.objectStore.azure.container }} +{{- else -}} +{{- .Values.objectStore.custom.bucket }} +{{- end -}} +{{- end }} + +{{- define "kubeflow-pipelines.objectStore.secure" -}} +{{- if eq .Values.objectStore.provider "minio" -}} +{{- .Values.objectStore.minio.secure | default "false" }} +{{- else if eq .Values.objectStore.provider "s3" -}} +"true" +{{- else if eq .Values.objectStore.provider "gcs" -}} +"true" +{{- else if eq .Values.objectStore.provider "azure" -}} +"true" +{{- else -}} +{{- .Values.objectStore.custom.secure | default "true" }} +{{- end -}} +{{- end }} + +{{- define "kubeflow-pipelines.objectStore.secretName" -}} +{{- if eq .Values.objectStore.provider "minio" -}} +{{- if .Values.objectStore.minio.existingSecret -}} +{{- .Values.objectStore.minio.existingSecret }} +{{- else -}} +mlpipeline-minio-artifact +{{- end -}} +{{- else if eq .Values.objectStore.provider "s3" -}} +{{- if .Values.objectStore.s3.existingSecret -}} +{{- .Values.objectStore.s3.existingSecret }} +{{- else -}} +mlpipeline-minio-artifact +{{- end -}} +{{- else if eq .Values.objectStore.provider "gcs" -}} +{{- if .Values.objectStore.gcs.existingSecret -}} +{{- .Values.objectStore.gcs.existingSecret }} +{{- else -}} +mlpipeline-minio-artifact +{{- end -}} +{{- else if eq .Values.objectStore.provider "azure" -}} +{{- if .Values.objectStore.azure.existingSecret -}} +{{- .Values.objectStore.azure.existingSecret }} +{{- else -}} +mlpipeline-minio-artifact +{{- end -}} +{{- else -}} +mlpipeline-minio-artifact +{{- end -}} +{{- end }} + +{{/* +Image helpers +*/}} +{{- define "kubeflow-pipelines.image" -}} +{{- $registry := .context.Values.global.imageRegistry -}} +{{- $repository := .repository -}} +{{- $tag := .tag | default .context.Values.global.imageTag | default .context.Chart.AppVersion -}} +{{- printf "%s/%s:%s" $registry $repository $tag -}} +{{- end }} + +{{/* +Image pull policy +*/}} +{{- define "kubeflow-pipelines.imagePullPolicy" -}} +{{- .pullPolicy | default .context.Values.global.imagePullPolicy -}} +{{- end }} + +{{/* +Image pull secrets +*/}} +{{- define "kubeflow-pipelines.imagePullSecrets" -}} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Environment detection helpers +*/}} +{{- define "kubeflow-pipelines.isMultiUser" -}} +{{- eq .Values.installMode.type "multi-user" -}} +{{- end }} + +{{- define "kubeflow-pipelines.isGeneric" -}} +{{- eq .Values.installMode.type "generic" -}} +{{- end }} + +{{/* +Namespace helper +*/}} +{{- define "kubeflow-pipelines.namespace" -}} +{{- .Values.global.namespace | default .Release.Namespace -}} +{{- end }} + +{{/* +Common annotations +*/}} +{{- define "kubeflow-pipelines.annotations" -}} +{{- with .Values.commonAnnotations }} +{{ toYaml . }} +{{- end }} +{{- with .Values.global.annotations }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Common environment variables for all components +*/}} +{{- define "kubeflow-pipelines.commonEnv" -}} +- name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- if .Values.global.logLevel }} +- name: LOG_LEVEL + value: {{ .Values.global.logLevel | quote }} +{{- end }} +{{- end }} + +{{/* +Database environment variables +*/}} +{{- define "kubeflow-pipelines.databaseEnv" -}} +- name: DBCONFIG_HOST + value: {{ include "kubeflow-pipelines.database.host" . | quote }} +- name: DBCONFIG_PORT + value: {{ include "kubeflow-pipelines.database.port" . | quote }} +- name: DBCONFIG_DBNAME + value: {{ include "kubeflow-pipelines.database.name" . | quote }} +- name: DBCONFIG_USER + valueFrom: + secretKeyRef: + name: {{ include "kubeflow-pipelines.database.secretName" . }} + key: username +- name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "kubeflow-pipelines.database.secretName" . }} + key: {{ include "kubeflow-pipelines.database.secretKey" . }} +- name: DB_DRIVER_NAME + value: {{ .Values.externalDatabase.type | default "mysql" | quote }} +{{- if .Values.database.connectionMaxLifetime }} +- name: DBCONFIG_CONMAXLIFETIME + value: {{ .Values.database.connectionMaxLifetime | quote }} +{{- end }} +{{- end }} + +{{/* +Object store environment variables +*/}} +{{- define "kubeflow-pipelines.objectStoreEnv" -}} +- name: OBJECTSTORECONFIG_SECURE + value: {{ include "kubeflow-pipelines.objectStore.secure" . | quote }} +- name: OBJECTSTORECONFIG_BUCKETNAME + value: {{ include "kubeflow-pipelines.objectStore.bucket" . | quote }} +- name: OBJECTSTORECONFIG_ACCESSKEY + valueFrom: + secretKeyRef: + name: {{ include "kubeflow-pipelines.objectStore.secretName" . }} + key: accesskey +- name: OBJECTSTORECONFIG_SECRETACCESSKEY + valueFrom: + secretKeyRef: + name: {{ include "kubeflow-pipelines.objectStore.secretName" . }} + key: secretkey +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml new file mode 100644 index 0000000000..be0097977e --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml @@ -0,0 +1,118 @@ +{{- if .Values.apiServer.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} + component: api-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.apiServer.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + {{- include "kubeflow-pipelines.mlPipelineSelectorLabels" . | nindent 6 }} + component: api-server + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + {{- include "kubeflow-pipelines.mlPipelineSelectorLabels" . | nindent 8 }} + component: api-server + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + securityContext: + {{- if .Values.security.seccompProfile }} + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} + {{- end }} + containers: + - name: ml-pipeline-api-server + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.apiServer.image.repository "tag" .Values.apiServer.image.tag "context" .) }} + imagePullPolicy: {{ .Values.apiServer.image.pullPolicy | default .Values.global.imagePullPolicy }} + ports: + - name: http + containerPort: 8888 + protocol: TCP + - name: grpc + containerPort: 8887 + protocol: TCP + env: + {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + {{- include "kubeflow-pipelines.databaseEnv" . | nindent 8 }} + {{- include "kubeflow-pipelines.objectStoreEnv" . | nindent 8 }} + # API Server specific environment variables + - name: PUBLISH_LOGS + value: {{ .Values.apiServer.env.publishLogs | quote }} + - name: PIPELINE_LOG_LEVEL + value: {{ .Values.apiServer.env.pipelineLogLevel | quote }} + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + value: {{ .Values.apiServer.env.autoUpdatePipelineDefaultVersion | quote }} + - name: V2_DRIVER_IMAGE + value: {{ .Values.apiServer.env.v2DriverImage | quote }} + - name: V2_LAUNCHER_IMAGE + value: {{ .Values.apiServer.env.v2LauncherImage | quote }} + {{- with .Values.apiServer.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + readinessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - "-" + - http://localhost:8888/apis/v1beta1/healthz + initialDelaySeconds: {{ .Values.apiServer.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.apiServer.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.apiServer.probes.readiness.timeoutSeconds }} + livenessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - "-" + - http://localhost:8888/apis/v1beta1/healthz + initialDelaySeconds: {{ .Values.apiServer.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.apiServer.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.apiServer.probes.liveness.timeoutSeconds }} + startupProbe: + exec: + command: + - wget + - -q + - -S + - -O + - "-" + - http://localhost:8888/apis/v1beta1/healthz + failureThreshold: {{ .Values.apiServer.probes.startup.failureThreshold }} + periodSeconds: {{ .Values.apiServer.probes.startup.periodSeconds }} + timeoutSeconds: {{ .Values.apiServer.probes.startup.timeoutSeconds }} + securityContext: + {{- toYaml .Values.apiServer.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.apiServer.resources | nindent 10 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/api-server/role.yaml b/experimental/helm/charts/pipeline/templates/api-server/role.yaml new file mode 100644 index 0000000000..4b3ffc1c17 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/api-server/role.yaml @@ -0,0 +1,72 @@ +{{- if and .Values.apiServer.enabled .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} + component: api-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + - pods/log + verbs: + - get + - list + - delete +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + verbs: + - create + - get + - list + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows/finalizers + verbs: + - update +- apiGroups: + - pipelines.kubeflow.org + resources: + - pipelines + verbs: + - get + - list + - watch +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml new file mode 100644 index 0000000000..4061568fc2 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.apiServer.enabled .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} + component: api-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "kubeflow-pipelines.fullname" . }}-api-server +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/api-server/service.yaml b/experimental/helm/charts/pipeline/templates/api-server/service.yaml new file mode 100644 index 0000000000..f1582aa3cc --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/api-server/service.yaml @@ -0,0 +1,31 @@ +{{- if .Values.apiServer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} + component: api-server + annotations: + prometheus.io/port: "8888" + prometheus.io/scheme: http + prometheus.io/scrape: "true" + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + type: {{ .Values.apiServer.service.type }} + ports: + - name: http + port: {{ .Values.apiServer.service.port }} + protocol: TCP + targetPort: 8888 + - name: grpc + port: {{ .Values.apiServer.service.grpcPort }} + protocol: TCP + targetPort: 8887 + selector: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + {{- include "kubeflow-pipelines.mlPipelineSelectorLabels" . | nindent 4 }} + component: api-server +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml new file mode 100644 index 0000000000..022868f32f --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.apiServer.enabled .Values.apiServer.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} + component: api-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.apiServer.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/application/application.yaml b/experimental/helm/charts/pipeline/templates/application/application.yaml new file mode 100644 index 0000000000..d0670b24ca --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/application/application.yaml @@ -0,0 +1,53 @@ +{{- if .Values.application.enabled }} +apiVersion: app.k8s.io/v1beta1 +kind: Application +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: application + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + kubernetes-engine.cloud.google.com/icon: >- + data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADMAAAAyCAYAAADx/eOPAAALuUlEQVRogd2afWxd9XnHP99bK4pS3yhiGUIRiiJUVVVqbq8ppdR20ibqpuIMtDRkUYERp29J57gMZVuxsrZiK7oZXVv5re1AiOuoG+N1DMkuytprsGPEVMouxqQZJVHEWIdQlGVxZlmZdb/747zcc869jpMO+seOdPz7nd/L83tev89zzjX8Bq795Rq9o17zXp+Tey+Ijkyboela29DRWkhffyT733pH/Z3este9F2cC6N0kNjxtjD+FdRD8UF9X7u97y7UbQFPAivC0BdllS381slun3s3z3xVhhqeds90tqR/oMB7u68z19ZZra0E/l1if3WOziPx3skrDPTr+bvDxfxImEIJbgX6gGBJ7EfHJX/ySReDHwO9KYAenyWCMFKw21GSeslwa2Z17+TcuzPBRr7B8m6Df5oOJqdPAR/u6cm/2lmv3At+IT3GiXZqbcaxSLsfRoTsvn7XL2jE87ZXGnwf+VGiDY86ETM1wU1+XjvSW/RlgTJADQ2QaCZKWcX1/aDIcjE8i3SdzZLjn0lm8pJXD02417BM+gLmq2Rqjr/d16Vu95dp6wc8Ra5O8NrPIcoZCvIR1H+KZkd2qLcfnRYUZOuorJO+3uQt0RerolGYZR7r5+C9ZATwPviGyQprd6Liszy3bnwVKwGMjPbnFyxJmeNpX2T4gaR/QmmSpyYZTho/2depMb9k/kNh3KawuJ1bWauHzUcyXRpZAv5Zmg7aHBLcmNN9ECAFeAO3s69KZ3nLtDuF9dnBs0IT9JO24rbPb0JfP2syCZpFfE5q1mRWcvlgMNcwMTRq9z/+OWXdx4AGjvX1deqC37DbwPwOrMgsufol5mWMWs1ivEbjTrOCtLNNb+udygqsNbUBtopR/NkuuwTJ6Hxsw67KSuvH5MPDA/nJttfGTdUFCMUlp/ALwOtIs9muBxpnFnBzuSQf21oP/BbXclVvumWuTaDN8WNBm2GizJkxPM0CDMA2WGZ72bbb/Njue2TRj9Il/PcG87SeBz4ZTNaSTsmctHcO8SqDp14d7dCFLZ2v/3OpQ023Ah4n65kohvETUCdcsfmuilD+bpNdgGZvOODuHqYGIVGCec9g7+7o031v2jaBTiD0ysxbHRnZrPktzyz1zK7f0z10nh5pWwLRhvZro1KqznVJhNB8UyDeSsU4zAOiIXV1OuEqQ2AR79nflXgcY6dGLwIvR8q39cy1b+uc2Emo6dI824BpMSxz8iVhy4m/2WiYHdV5UmOHp2mpwm52ESCdwRn+9v0tPAWzpn9sAFAQbMdc60PaHsFZEWd9uxk4z8G3seykECfObTEd2KmuZG4CWyLXkYLMwtiYt+hMsTUdAEZQzjs9apv66SHJRk73ZjBQ+iRu29s+1VEr5OImmXs4MHUahVoLWgK23wbv6OrU4OulcuHYehWsVHhpXwpE2FNRayTszX2cwDpQEzTB+QvrJHCXUaigk+c++aXZiE98YmUVgV19X7u3ypH/fgfUA5h2usY2jNjmWoGVn50nvC9T2NviA5OPBGPW91OlG+0Xa1WJhhqadk3WjpKCilQIQFP19XZocnfIHgIeFWyNh6goXyX6gdNWfU8aJ5tNjEheAHZVS/ruGj0s8k6VPhh6ms6kwgoLl1aGuCEuSpwXfHZ2qrTJ+HHkNCpOjmbdFcEcGUIhUSj/H65rPO6j+766U8i/QXV0z8cqJc4btwF8AtWgtMb1wj+j41Df/s1EYQwdEDiqM3hDes9quGY3IKoYOvCrU7HlCoZtEWapPkzEpsU8uq8b36a6uBqaBv5l45URLpZT/pmGH8LnkvlAdAOt1oeXqRsuYTjlEMJiXvWN/Z+5szfqioKcOKo7qr/nAEesKiOyv2A/q88rOx8+8bPhK5dUTAA8jbUT6MuKnbKteNVHKP23xCeD1LC0F2TWOmzoAKEiWxmC+sr8rN1OerF2HGaqXFcZhDWaYj11S4ZxcXxVqyKqPZOeNTwM7Jkr5BeDPQJ8NFQaoC/gZ26rXT5TyxxAfRx6P94d0gU0pYYama+tsbwix/AHM4fKUrwAeB68kRJ5AZsWWieGTjLipsVCgrKCwKHF7pZQ/RXf104j76i4ZMmquxkzRXb2zUsqfxdxsfCiA70hRjZbpCDHmJcRdeZPDHkVck0Ul5PeHZ81DgHxKtglXaHCxVN9fr5TyR9hW3QA8Amqp5526SyKtBEbZVv1eZeZkbqKU7xfsFJwPqRW29s+11oUxnUhnkHf2dWoB+R5Jv5dNaGHh1wog8d/ZAI+0GgVpFPTp4AfJT2Hup7u6EvMk0tpkboutEz0HMPzHyD+mu3pFpZR/Aug0Pgm0RLkvFzLWYfjDvs7cqfKUt2LuXTLhue5mdWhVDJdEzxDDcRKawceN9lRePVkDfgBcR/LKVqNpz/s08DO6q4VKKT8j8zHgJ1HyzA1P11YZjfV1arw85auBR4RalDB5lEjDKi0CgPPphKZ0QiNRwUQeg88B2ydKreew9yH1NCxe/r4GaZpt1Vsrh/JnDDcBLwPkbLVgf6s86RXYj4KvtJKJM8KsGLkSlsmUL6mSg1RJY1xD7KmU8sfprnYgBqJsGVsiEfupsca7FfMo26p/OfHKiVqllB8HyPV16VxfV66G/G1QBwY5xvCgTT7X3/MTaBbFVr0fJvqw2ASZ+yul/FN0V68CHsesiDl3UopM3CwhDZDD/Dnwj3S/sjoYAMqTtc1YX02jVqYOiuuqsAKIkqZCfFIz/IrfFY8gDrKt2gI8irSuwQezyTeNaOl+6qYb+fpYGKEXJE9GSTObK5ItrheaLHE5/XRKcHul+kYN8x2kzWlLNNuVtUqibzKW5CBjxUoszO7NWrS1E/xWvMeJjck2WQHEKJeMD+qH4gWCSvg00m3AVxv5TMRKsp9Cs0Q/Ka/1BOZQNBSXMz2b9Q5oO9JCKgkqg2aKofl8uvTPeE1w3t5KKf8y26pFxINhLRa5R9JV6huT/aZuFu7Ds+A9jBdj+VIvZz2b9BL2Xi5yJQEgUFqinI9SZBDx358o5Q/HiRGtquOEmxJu6DcbC/afQWxnvHg+Odrwm2bP5txh5OEYjOM3vaiu8qqHJw1mPmK/Xs7HJf0LRncDMF5cAL6NWUxDrX/duwbczljxjSzvTX+gtXU3MBlrRCltrsxBTgorACKrRGf5bczOiVLrhUL74B2F9oHVjBd/iLwTWEhr+CIWaLYumDjIWLHha+aSwvRs1iJmJ9Kb9ZJERTS3ACsMC8i1ZNwgXZDYWTmU/1WhfeAW8Cjo+UL7wDrGik8jfid0kYz/Z2ODepv+GPIY+FAznpcUJhAo9w5mh81CFtEsWieCTzwXkogmfKBSyh8ttA98EDPqoPouYqYLxYEPMVY8itmEeTM+KEaqZhVAkiPPIL6QDPhLFiYQSC9J7M3mGlF/24zWSvwIM1xoH2gF/sFiTcSPxQakqUJxsIPx4jGCr0AzCUYTbROJ7DPAdsbSAX9ZwgDs3qTDiMGUOxF/1DgfekLVsPf0sw8DPARsDNwy8iYBXov4p0L7wC2MF99CfBJ4rqmbJbO/qYE+x1jx5HK8Xtp/aFgHDM/FX+RM9FFjHjjj4NV3HvlPsP4g+SqQgm6zCuvJQnHgi4wVz2JuAj8RnLGEVaCf8Y8cuRQ2L0mYEBB2Gb8ZHKD4NQBx+0Qpf7LQPrAVVGqiiWTpCcEn4QcLxcF7C7+aXMDahT1YX5IS5DHE/ZfC4yULEwr0DtIOWwuuvwZ8rVLKP1soDqzHPGJoyRao9b4SXiQQ30A8eO1/PJ8D7gK+BtQSJcQM8AXGlg747LUkmi91lad8J3CuZ5OeBii0D64ET2FdH1N0omWJvgLPkvwM8LmZf7lrnm3VO4CHsM4DH2P8I8vGSfK67P9q8v9wWPAcQLH4PbBHbK6Pq+3M9+Ml+6FL2dyC+WmhOLiWseKPMDeDd12uIPBrWCZ5Xds++AHsAwGlBKnoB5747c2J+aSJEuvRL8CDv/2Zz+cqh/LL/gPD//vrfwFjcI5oX6jDBwAAAABJRU5ErkJggg== + marketplace.cloud.google.com/deploy-info: '{"partner_id": "google-cloud-ai-platform", "product_id": "kubeflow-pipelines", "partner_name": "Google Cloud AI Platform"}' +spec: + addOwnerRef: true + selector: + matchLabels: + application-crd-id: kubeflow-pipelines + descriptor: + version: {{ .Values.global.imageTag | quote }} + type: Kubeflow Pipelines + description: |- + Reusable end-to-end ML workflow + maintainers: + - name: Google Cloud AI Platform + url: https://cloud.google.com/ai-platform/ + - name: Kubeflow Pipelines + url: https://github.com/kubeflow/pipelines + links: + - description: 'Kubeflow Pipelines Documentation' + url: https://www.kubeflow.org/docs/pipelines/ + notes: |- + Please go to [Hosted Kubeflow Pipelines Console](https://console.cloud.google.com/ai-platform/pipelines/clusters). + info: + - name: Console + value: 'https://console.cloud.google.com/ai-platform/pipelines/clusters' + componentKinds: + - group: v1 + kind: ServiceAccount + - group: rbac.authorization.k8s.io/v1 + kind: Role + - group: rbac.authorization.k8s.io/v1 + kind: RoleBinding + - group: v1 + kind: Service + - group: v1 + kind: ConfigMap + - group: v1 + kind: Secret + - group: apps/v1 + kind: Deployment +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml b/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml new file mode 100644 index 0000000000..d3515e7088 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml @@ -0,0 +1,63 @@ +{{- if .Values.cacheDeployer.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-cache-deployer + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: cache-deployer + component: cache-deployer + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.cacheDeployer.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + app: cache-deployer + component: cache-deployer + strategy: + type: Recreate + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + app: cache-deployer + component: cache-deployer + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.cacheDeployer.serviceAccountName" . }} + containers: + - name: main + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.cacheDeployer.image.repository "tag" .Values.cacheDeployer.image.tag "context" .) }} + imagePullPolicy: {{ .Values.cacheDeployer.image.pullPolicy | default .Values.global.imagePullPolicy }} + env: + {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.cacheDeployer.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.cacheDeployer.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.cacheDeployer.resources | nindent 10 }} + restartPolicy: Always + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml new file mode 100644 index 0000000000..55bc597403 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.cacheDeployer.enabled .Values.cacheDeployer.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.cacheDeployer.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: cache-deployer + component: cache-deployer + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.cacheDeployer.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache/deployment.yaml b/experimental/helm/charts/pipeline/templates/cache/deployment.yaml new file mode 100644 index 0000000000..86d1d70fd7 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/cache/deployment.yaml @@ -0,0 +1,105 @@ +{{- if .Values.cache.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-cache-server + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: cache-server + component: cache-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.cache.server.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + app: cache-server + component: cache-server + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + app: cache-server + component: cache-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + securityContext: + {{- if .Values.security.seccompProfile }} + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} + {{- end }} + containers: + - name: server + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.cache.server.image.repository "tag" .Values.cache.server.image.tag "context" .) }} + imagePullPolicy: {{ .Values.cache.server.image.pullPolicy | default .Values.global.imagePullPolicy }} + env: + {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + {{- include "kubeflow-pipelines.databaseEnv" . | nindent 8 }} + - name: DEFAULT_CACHE_STALENESS + value: {{ .Values.pipeline.cache.defaultStaleness | quote }} + - name: MAXIMUM_CACHE_STALENESS + value: {{ .Values.pipeline.cache.maximumStaleness | quote }} + - name: CACHE_IMAGE + value: {{ .Values.pipeline.cache.image | quote }} + - name: CACHE_NODE_RESTRICTIONS + value: {{ .Values.pipeline.cache.nodeRestrictions | quote }} + - name: DBCONFIG_DRIVER + value: {{ .Values.externalDatabase.type | default "mysql" }} + - name: DBCONFIG_DB_NAME + value: {{ .Values.database.cachedb }} + - name: DBCONFIG_HOST_NAME + value: {{ include "kubeflow-pipelines.database.host" . }} + - name: DBCONFIG_PORT + value: {{ include "kubeflow-pipelines.database.port" . | quote }} + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WEBHOOK_PORT + value: "8443" + {{- with .Values.cache.server.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + args: + - "--db_driver=$(DBCONFIG_DRIVER)" + - "--db_host=$(DBCONFIG_HOST_NAME)" + - "--db_port=$(DBCONFIG_PORT)" + - "--db_name=$(DBCONFIG_DB_NAME)" + - "--db_user=$(DBCONFIG_USER)" + - "--db_password=$(DBCONFIG_PASSWORD)" + - "--namespace_to_watch=$(NAMESPACE_TO_WATCH)" + - "--listen_port=$(WEBHOOK_PORT)" + ports: + - name: webhook-api + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-tls-certs + mountPath: /etc/webhook/certs + readOnly: true + securityContext: + {{- toYaml .Values.cache.server.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.cache.server.resources | nindent 10 }} + volumes: + - name: webhook-tls-certs + secret: + secretName: {{ include "kubeflow-pipelines.fullname" . }}-webhook-server-tls + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache/service.yaml b/experimental/helm/charts/pipeline/templates/cache/service.yaml new file mode 100644 index 0000000000..1b9aa6275b --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/cache/service.yaml @@ -0,0 +1,24 @@ +{{- if .Values.cache.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-cache-server + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: cache-server + component: cache-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + type: {{ .Values.cache.server.service.type }} + ports: + - name: webhook-api + protocol: TCP + port: {{ .Values.cache.server.service.port }} + targetPort: 8443 + selector: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + app: cache-server + component: cache-server +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml new file mode 100644 index 0000000000..82a7294f31 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.cache.enabled .Values.cache.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: cache-server + component: cache-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.cache.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml b/experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml new file mode 100644 index 0000000000..8c2c48f191 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-launcher + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: launcher + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +data: + defaultPipelineRoot: {{ .Values.pipeline.defaultPipelineRoot | quote }} diff --git a/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml b/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml new file mode 100644 index 0000000000..0c56c24f7a --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-install-config + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: config + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +data: + warning: | + 1. Do not use kubectl to edit this configmap, because some values are used + during kustomize build. Instead, change the configmap and apply the entire + kustomize manifests again. + 2. After updating the configmap, some deployments may need to be restarted + until the changes take effect. A quick way to restart all deployments in a + namespace: `kubectl rollout restart deployment -n `. + appName: {{ .Values.pipeline.app.name }} + appVersion: {{ .Values.pipeline.app.version }} + # Database configuration + dbHost: {{ include "kubeflow-pipelines.database.host" . }} + dbPort: {{ include "kubeflow-pipelines.database.port" . | quote }} + dbType: {{ .Values.externalDatabase.type | default "mysql" }} + mysqlHost: {{ include "kubeflow-pipelines.database.host" . }} + mysqlPort: {{ include "kubeflow-pipelines.database.port" . | quote }} + mlmdDb: {{ .Values.database.metadb }} + cacheDb: {{ .Values.database.cachedb }} + pipelineDb: {{ include "kubeflow-pipelines.database.name" . }} + # Object storage configuration + bucketName: {{ include "kubeflow-pipelines.objectStore.bucket" . }} + # Pipeline configuration + defaultPipelineRoot: {{ .Values.pipeline.defaultPipelineRoot | quote }} + autoUpdatePipelineDefaultVersion: {{ .Values.pipeline.autoUpdatePipelineDefaultVersion | quote }} + cronScheduleTimezone: {{ .Values.pipeline.cronScheduleTimezone | quote }} + # Cache configuration + cacheImage: {{ .Values.pipeline.cache.image | quote }} + cacheNodeRestrictions: {{ .Values.pipeline.cache.nodeRestrictions | quote }} + MAXIMUM_CACHE_STALENESS: {{ .Values.pipeline.cache.maximumStaleness | quote }} + DEFAULT_CACHE_STALENESS: {{ .Values.pipeline.cache.defaultStaleness | quote }} + # Connection configuration + ConMaxLifeTime: {{ .Values.database.connectionMaxLifetime | quote }} + LOG_LEVEL: {{ .Values.global.logLevel | quote }} diff --git a/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml b/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml new file mode 100644 index 0000000000..f227b33000 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.ui.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-ui-config + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-ui + component: ui-config + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +data: + viewer-pod-template.json: |- + { + "spec": { + "serviceAccountName": "{{ include "kubeflow-pipelines.fullname" . }}-viewer" + } + } +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/crds/application-crd.yaml b/experimental/helm/charts/pipeline/templates/crds/application-crd.yaml new file mode 100644 index 0000000000..5b4977396f --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/crds/application-crd.yaml @@ -0,0 +1,330 @@ +{{- if .Values.crds.install }} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: applications.app.k8s.io + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + controller-tools.k8s.io: "1.0" + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/application/pull/2 + controller-gen.kubebuilder.io/version: v0.4.0 +spec: + group: app.k8s.io + names: + categories: + - all + kind: Application + listKind: ApplicationList + plural: applications + shortNames: + - app + singular: application + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the application + jsonPath: .spec.descriptor.type + name: Type + type: string + - description: The creation date + jsonPath: .spec.descriptor.version + name: Version + type: string + - description: The application object owns the matched resources + jsonPath: .spec.addOwnerRef + name: Owner + type: boolean + - description: Numbers of components ready + jsonPath: .status.componentsReady + name: Ready + type: string + - description: The creation date + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Application is the Schema for the applications API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object.' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents.' + type: string + metadata: + type: object + spec: + description: ApplicationSpec defines the specification for an Application. + properties: + addOwnerRef: + description: AddOwnerRef objects - flag to indicate if we need to add OwnerRefs to matching objects + type: boolean + assemblyPhase: + description: 'AssemblyPhase represents the current phase of the application''s assembly such as "Pending", "Succeeded", or "Failed".' + type: string + componentKinds: + description: ComponentGroupKinds is a list of Kinds for Application's components + items: + description: GroupKind specifies a Group and a Kind, but does not force a version. + properties: + group: + type: string + kind: + type: string + required: + - group + - kind + type: object + type: array + descriptor: + description: Descriptor regroups information and metadata about an application. + properties: + description: + description: Description is a brief string description of the Application. + type: string + icons: + description: Icons is an optional list of icons for an application. + items: + description: ImageSpec contains information about an image used as an icon. + properties: + size: + description: (optional) The size of the image in pixels (e.g., 25x25). + type: string + src: + description: The source for image represented as either an absolute URL to the image or a Data URL containing the image. + type: string + type: + description: (optional) The mine type of the image (e.g., "image/png"). + type: string + required: + - src + type: object + type: array + keywords: + description: Keywords is an optional list of key words associated with the application + items: + type: string + type: array + links: + description: Links are a list of descriptive URLs intended to be used to surface additional documentation, dashboards, etc. + items: + description: Link contains information about an URL to surface documentation, dashboards, etc. + properties: + description: + description: Description is human readable content explaining the purpose of the link. + type: string + url: + description: Url is typically an HTTP or HTTPS URL. + type: string + type: object + type: array + maintainers: + description: Maintainers is an optional list of maintainers of the application. + items: + description: ContactData contains information about an individual or organization. + properties: + email: + description: Email is the email address of the contact person/organization. + type: string + name: + description: Name is the descriptive name of the contact person/organization + type: string + url: + description: Url could typically be a website address of the contact person/organization. + type: string + type: object + type: array + notes: + description: Notes contain a human readable snippets intended as a quick start for the users of the Application. + type: string + owners: + description: Owners is an optional list of the owners of the installed application. + items: + description: ContactData contains information about an individual or organization. + properties: + email: + description: Email is the email address of the contact person/organization. + type: string + name: + description: Name is the descriptive name of the contact person/organization + type: string + url: + description: Url could typically be a website address of the contact person/organization. + type: string + type: object + type: array + type: + description: Type is the type of the application (e.g. WordPress, MySQL, Cassandra). + type: string + version: + description: Version is an optional version indicator for the Application. + type: string + type: object + info: + description: Info contains human readable key-value pairs for the Application. + items: + description: InfoItem is a human readable key-value pair containing important information about how to access the Application. + properties: + name: + description: Name is a human readable title for this piece of information. + type: string + type: + description: Type of the value for this InfoItem. + type: string + value: + description: Value is human readable content. + type: string + valueFrom: + description: ValueFrom defines a reference to derive the value from another source. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent.' + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + ingressRef: + description: Selects an Ingress. + properties: + host: + description: The host to select. + type: string + name: + description: 'Name of the referent.' + type: string + path: + description: The path to select. + type: string + required: + - name + type: object + secretKeyRef: + description: Selects a key of a secret. + properties: + key: + description: The key of the secret to select from. + type: string + name: + description: 'Name of the referent.' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + serviceRef: + description: Selects a Service. + properties: + name: + description: 'Name of the referent.' + type: string + path: + description: The optional path to select. + type: string + port: + description: The port to select. + format: int32 + type: integer + required: + - name + type: object + type: + description: Type of the value for this InfoItem. + type: string + type: object + required: + - name + type: object + type: array + selector: + description: Selector is a label query over kinds that created by the application. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + type: object + status: + description: ApplicationStatus defines controller's the observed state of Application + properties: + componentsReady: + description: ComponentsReady is a human readable message indicating details about the status of the application. + type: string + conditions: + description: Conditions represents the latest state of the object + items: + description: ApplicationCondition contains details for the current condition of this application. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + lastUpdateTime: + description: Last time the condition was probed + format: date-time + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of application condition. + type: string + required: + - status + - type + type: object + type: array + observedGeneration: + description: ObservedGeneration is the most recent generation observed. It corresponds to the Object's generation, which is updated on mutation by the API Server. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/crds/pipelines-crd.yaml b/experimental/helm/charts/pipeline/templates/crds/pipelines-crd.yaml new file mode 100644 index 0000000000..9d1dcace22 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/crds/pipelines-crd.yaml @@ -0,0 +1,54 @@ +{{- if .Values.crds.install }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + name: pipelines.pipelines.kubeflow.org + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + group: pipelines.kubeflow.org + names: + kind: Pipeline + listKind: PipelineList + plural: pipelines + singular: pipeline + scope: Namespaced + versions: + - name: v2beta1 + schema: + openAPIV3Schema: + description: Pipeline is the Schema for the pipelines API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: PipelineSpec defines the desired state of Pipeline. + properties: + description: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/crds/pipelineversions-crd.yaml b/experimental/helm/charts/pipeline/templates/crds/pipelineversions-crd.yaml new file mode 100644 index 0000000000..bff5908256 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/crds/pipelineversions-crd.yaml @@ -0,0 +1,66 @@ +{{- if .Values.crds.install }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + name: pipelineversions.pipelines.kubeflow.org + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + group: pipelines.kubeflow.org + names: + kind: PipelineVersion + listKind: PipelineVersionList + plural: pipelineversions + singular: pipelineversion + scope: Namespaced + versions: + - name: v2beta1 + schema: + openAPIV3Schema: + description: PipelineVersion is the Schema for the pipelineversions API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: PipelineVersionSpec defines the desired state of PipelineVersion. + properties: + codeSourceURL: + type: string + description: + type: string + pipelineName: + type: string + pipelineSpec: + x-kubernetes-preserve-unknown-fields: true + required: + - pipelineName + - pipelineSpec + type: object + status: + description: PipelineVersionStatus defines the observed state of PipelineVersion. + type: object + type: object + served: true + storage: true + subresources: + status: {} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/crds/scheduledworkflow-crd.yaml b/experimental/helm/charts/pipeline/templates/crds/scheduledworkflow-crd.yaml new file mode 100644 index 0000000000..81069fbc62 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/crds/scheduledworkflow-crd.yaml @@ -0,0 +1,45 @@ +{{- if .Values.crds.install }} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: scheduledworkflows.kubeflow.org + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + group: kubeflow.org + names: + kind: ScheduledWorkflow + listKind: ScheduledWorkflowList + plural: scheduledworkflows + singular: scheduledworkflow + shortNames: + - swf + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + x-kubernetes-map-type: atomic + status: + type: object + x-kubernetes-preserve-unknown-fields: true + x-kubernetes-map-type: atomic + required: + - spec + - status + type: object + served: true + storage: true +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/crds/viewer-crd.yaml b/experimental/helm/charts/pipeline/templates/crds/viewer-crd.yaml new file mode 100644 index 0000000000..ceb3239979 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/crds/viewer-crd.yaml @@ -0,0 +1,40 @@ +{{- if .Values.crds.install }} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: viewers.kubeflow.org + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + group: kubeflow.org + names: + kind: Viewer + listKind: ViewerList + plural: viewers + singular: viewer + shortNames: + - vi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + x-kubernetes-map-type: atomic + required: + - spec + type: object + served: true + storage: true +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml b/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml new file mode 100644 index 0000000000..30ecfdb845 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml @@ -0,0 +1,16 @@ +{{- if and (not .Values.externalDatabase.existingSecret) (not .Values.mysql.enabled) (not .Values.postgresql.enabled) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-mysql-secret + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: database + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +type: Opaque +stringData: + username: {{ .Values.externalDatabase.username | quote }} + password: {{ .Values.externalDatabase.password | quote }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/env/aws-patches.yaml b/experimental/helm/charts/pipeline/templates/env/aws-patches.yaml new file mode 100644 index 0000000000..f1b9351bb2 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/env/aws-patches.yaml @@ -0,0 +1,60 @@ +{{- if eq .Values.env.platform "aws" }} +{{- if .Values.env.aws.s3.enabled }} +--- +# AWS S3 Configuration Patch for API Server +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-aws-config + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: aws-config + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +data: + # AWS S3 specific configuration + OBJECTSTORECONFIG_SECURE: "true" + OBJECTSTORECONFIG_HOST: "s3.amazonaws.com" + OBJECTSTORECONFIG_REGION: {{ .Values.env.aws.s3.region | quote }} + OBJECTSTORECONFIG_BUCKETNAME: {{ .Values.env.aws.s3.bucket | quote }} + OBJECTSTORECONFIG_PORT: "" +{{- end }} + +{{- if .Values.env.aws.iam.enabled }} +--- +# AWS IAM Role Annotation for API Server Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline + component: api-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + eks.amazonaws.com/role-arn: {{ .Values.env.aws.iam.apiServerRoleArn | quote }} +{{- end }} + +{{- if .Values.env.aws.rds.enabled }} +--- +# AWS RDS Configuration Secret +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-aws-rds-secret + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: aws-rds + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +type: Opaque +data: + host: {{ .Values.env.aws.rds.endpoint | b64enc }} + port: {{ "3306" | b64enc }} + database: {{ .Values.database.pipelinedb | b64enc }} +{{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/env/azure-patches.yaml b/experimental/helm/charts/pipeline/templates/env/azure-patches.yaml new file mode 100644 index 0000000000..a0c3629ae2 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/env/azure-patches.yaml @@ -0,0 +1,163 @@ +{{- if eq .Values.env.platform "azure" }} +{{- if .Values.env.azure.blob.enabled }} +--- +# Azure Blob Storage Configuration Patch for API Server +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-azure-config + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: azure-config + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +data: + # Azure Blob Storage specific configuration + OBJECTSTORECONFIG_SECURE: "true" + OBJECTSTORECONFIG_HOST: "{{ .Values.env.azure.blob.storageAccount }}.blob.core.windows.net" + OBJECTSTORECONFIG_REGION: "" + OBJECTSTORECONFIG_BUCKETNAME: {{ .Values.env.azure.blob.container | quote }} + OBJECTSTORECONFIG_PORT: "" + AZURE_STORAGE_ACCOUNT: {{ .Values.env.azure.blob.storageAccount | quote }} +{{- end }} + +{{- if .Values.env.azure.managedIdentity.enabled }} +--- +# Azure Managed Identity Annotation for API Server Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline + component: api-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + azure.workload.identity/client-id: {{ .Values.env.azure.managedIdentity.clientId | quote }} +{{- end }} + +{{- if .Values.env.azure.sql.enabled }} +--- +# Azure SQL Configuration Secret +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-azure-sql-secret + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: azure-sql + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +type: Opaque +data: + host: {{ printf "%s.mysql.database.azure.com" .Values.env.azure.sql.serverName | b64enc }} + port: {{ "3306" | b64enc }} + database: {{ .Values.env.azure.sql.database | b64enc }} +{{- end }} + +{{- if .Values.env.azure.blob.enabled }} +--- +# Azure MinIO Gateway for Blob Storage Compatibility +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-minio-azure-gateway + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: minio-azure-gateway + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + component: minio-azure-gateway + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + component: minio-azure-gateway + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + containers: + - name: minio + image: minio/minio:latest + args: + - gateway + - azure + env: + - name: MINIO_ACCESS_KEY + value: "minio" + - name: MINIO_SECRET_KEY + value: "minio123" + - name: AZURE_STORAGE_ACCOUNT + value: {{ .Values.env.azure.blob.storageAccount | quote }} + - name: AZURE_STORAGE_KEY + valueFrom: + secretKeyRef: + name: {{ include "kubeflow-pipelines.fullname" . }}-azure-storage-secret + key: key + ports: + - name: minio + containerPort: 9000 + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +--- +# Azure MinIO Gateway Service +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-minio-azure-gateway + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: minio-azure-gateway + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: minio + port: 9000 + targetPort: 9000 + selector: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + component: minio-azure-gateway + +--- +# Azure Storage Secret +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-azure-storage-secret + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: azure-storage + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +type: Opaque +data: + key: {{ .Values.objectStore.azure.accessKey | default "YOUR_STORAGE_ACCOUNT_KEY" | b64enc }} +{{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/env/gcp-patches.yaml b/experimental/helm/charts/pipeline/templates/env/gcp-patches.yaml new file mode 100644 index 0000000000..d2cdfa7b58 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/env/gcp-patches.yaml @@ -0,0 +1,134 @@ +{{- if eq .Values.env.platform "gcp" }} +{{- if .Values.env.gcp.gcs.enabled }} +--- +# GCP GCS Configuration Patch for API Server +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-gcp-config + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: gcp-config + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +data: + # GCP GCS specific configuration + OBJECTSTORECONFIG_SECURE: "true" + OBJECTSTORECONFIG_HOST: "storage.googleapis.com" + OBJECTSTORECONFIG_REGION: "" + OBJECTSTORECONFIG_BUCKETNAME: {{ .Values.env.gcp.gcs.bucket | quote }} + OBJECTSTORECONFIG_PORT: "" + GCS_PROJECT_ID: {{ .Values.env.gcp.gcs.projectId | quote }} +{{- end }} + +{{- if .Values.env.gcp.workloadIdentity.enabled }} +--- +# GCP Workload Identity Annotation for API Server Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline + component: api-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + iam.gke.io/gcp-service-account: {{ .Values.env.gcp.workloadIdentity.serviceAccount | quote }} +{{- end }} + +{{- if .Values.env.gcp.cloudsql.enabled }} +--- +# GCP Cloud SQL Proxy Deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-cloudsql-proxy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: cloudsql-proxy + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + component: cloudsql-proxy + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + component: cloudsql-proxy + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + serviceAccountName: {{ include "kubeflow-pipelines.fullname" . }}-cloudsql-proxy + containers: + - name: cloudsql-proxy + image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.0.0 + args: + - "--structured-logs" + - "--port=3306" + - "{{ .Values.env.gcp.project }}:{{ .Values.env.gcp.cloudsql.region }}:{{ .Values.env.gcp.cloudsql.instanceName }}" + ports: + - name: mysql + containerPort: 3306 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + +--- +# GCP Cloud SQL Proxy Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-cloudsql-proxy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: cloudsql-proxy + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- if .Values.env.gcp.workloadIdentity.enabled }} + iam.gke.io/gcp-service-account: cloudsql-proxy@{{ .Values.env.gcp.project }}.iam.gserviceaccount.com + {{- end }} + +--- +# GCP Cloud SQL Proxy Service +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-cloudsql-proxy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: cloudsql-proxy + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: mysql + port: 3306 + targetPort: 3306 + selector: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + component: cloudsql-proxy +{{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml b/experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml new file mode 100644 index 0000000000..a960f8df3a --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml @@ -0,0 +1,154 @@ +{{- if and .Values.networking.istio.enabled .Values.installMode.multiUser.enabled }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-ui-policy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: istio-authorization + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ml-pipeline-ui + rules: + - from: + - source: + principals: + - cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account + + + - when: + - key: request.headers[authorization] + values: + - "*" + - key: request.headers[kubeflow-userid] + notValues: + - "*" + +--- +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-api-policy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: istio-authorization + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ml-pipeline + rules: + - from: + - source: + principals: + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.ui.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + - when: + - key: request.headers[kubeflow-userid] + notValues: ['*'] + +--- +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-visualization-policy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: istio-authorization + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ml-pipeline-visualizationserver + rules: + - from: + - source: + principals: + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.ui.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + +--- +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-cache-server-policy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: istio-authorization + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + selector: + matchLabels: + app: cache-server + rules: + - {} + +--- +apiVersion: "networking.istio.io/v1alpha3" +kind: DestinationRule +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-ui-destination + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: istio-destination-rule + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + host: {{ include "kubeflow-pipelines.fullname" . }}-ui.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL + +--- +apiVersion: "networking.istio.io/v1alpha3" +kind: DestinationRule +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-api-destination + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: istio-destination-rule + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + host: {{ include "kubeflow-pipelines.fullname" . }}-api-server.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL + +--- +apiVersion: "networking.istio.io/v1alpha3" +kind: DestinationRule +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-visualization-destination + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: istio-destination-rule + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + host: {{ include "kubeflow-pipelines.fullname" . }}-visualization.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml b/experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml new file mode 100644 index 0000000000..d3fe09453d --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.networking.istio.enabled .Values.installMode.multiUser.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-ui + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: istio-virtual-service + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + gateways: + - {{ .Values.networking.istio.gateway }} + hosts: + - '*' + http: + - match: + - uri: + prefix: /pipeline + rewrite: + uri: /pipeline + route: + - destination: + host: {{ include "kubeflow-pipelines.fullname" . }}-ui.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + port: + number: {{ .Values.ui.service.port }} + timeout: 300s +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml new file mode 100644 index 0000000000..0eeb7738b3 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml @@ -0,0 +1,65 @@ +{{- if .Values.metadataWriter.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-writer + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: metadata-writer + component: metadata-writer + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.metadataWriter.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + app: metadata-writer + component: metadata-writer + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + app: metadata-writer + component: metadata-writer + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} + securityContext: + {{- if .Values.security.seccompProfile }} + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} + {{- end }} + containers: + - name: main + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.metadataWriter.image.repository "tag" .Values.metadataWriter.image.tag "context" .) }} + imagePullPolicy: {{ .Values.metadataWriter.image.pullPolicy | default .Values.global.imagePullPolicy }} + env: + {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.metadataWriter.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.metadataWriter.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.metadataWriter.resources | nindent 10 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml new file mode 100644 index 0000000000..41a532ccf3 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml @@ -0,0 +1,40 @@ +{{- if .Values.metadataWriter.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-writer-role + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: kubeflow-pipelines-metadata-writer-role + component: metadata-writer + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch + - update + - patch +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml new file mode 100644 index 0000000000..63223f601e --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.metadataWriter.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-writer-binding + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: metadata-writer + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-writer-role +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml new file mode 100644 index 0000000000..05f2410024 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.metadataWriter.enabled .Values.metadataWriter.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: metadata-writer + component: metadata-writer + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.metadataWriter.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml new file mode 100644 index 0000000000..d25a61e55d --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml @@ -0,0 +1,59 @@ +{{- if .Values.metadata.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-envoy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: metadata-envoy + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.metadata.envoy.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + component: metadata-envoy + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + component: metadata-envoy + sidecar.istio.io/inject: "false" + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + containers: + - name: container + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.metadata.envoy.image.repository "tag" .Values.metadata.envoy.image.tag "context" .) }} + imagePullPolicy: {{ .Values.metadata.envoy.image.pullPolicy | default .Values.global.imagePullPolicy }} + ports: + - name: md-envoy + containerPort: 9090 + protocol: TCP + - name: envoy-admin + containerPort: 9901 + protocol: TCP + securityContext: + {{- toYaml .Values.metadata.envoy.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.metadata.envoy.resources | nindent 10 }} + {{- with .Values.metadata.envoy.additionalEnv }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml b/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml new file mode 100644 index 0000000000..78004b7bd1 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.metadata.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-envoy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: metadata-envoy + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + type: {{ .Values.metadata.envoy.service.type }} + ports: + - name: md-envoy + protocol: TCP + port: {{ .Values.metadata.envoy.service.port }} + targetPort: 9090 + selector: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + component: metadata-envoy +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml new file mode 100644 index 0000000000..66fc954e45 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml @@ -0,0 +1,15 @@ +{{- if .Values.metadata.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-grpc-config + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: metadata-grpc-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +data: + METADATA_GRPC_SERVICE_HOST: "{{ include "kubeflow-pipelines.fullname" . }}-metadata-grpc" + METADATA_GRPC_SERVICE_PORT: "{{ .Values.metadata.grpc.service.port }}" +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml new file mode 100644 index 0000000000..ad8ced3210 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml @@ -0,0 +1,89 @@ +{{- if .Values.metadata.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-grpc + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: metadata-grpc-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.metadata.grpc.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + component: metadata-grpc-server + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + component: metadata-grpc-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.metadata.serviceAccountName" . }} + securityContext: + {{- if .Values.security.seccompProfile }} + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} + {{- end }} + containers: + - name: container + image: {{ .Values.metadata.grpc.image.repository }}:{{ .Values.metadata.grpc.image.tag }} + imagePullPolicy: {{ .Values.metadata.grpc.image.pullPolicy | default .Values.global.imagePullPolicy }} + env: + {{- include "kubeflow-pipelines.databaseEnv" . | nindent 8 }} + - name: MYSQL_DATABASE + value: {{ .Values.database.metadb }} + - name: MYSQL_HOST + value: {{ include "kubeflow-pipelines.database.host" . }} + - name: MYSQL_PORT + value: {{ include "kubeflow-pipelines.database.port" . | quote }} + {{- with .Values.metadata.grpc.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + command: ["/bin/metadata_store_server"] + args: + - "--grpc_port=8080" + - "--mysql_config_database=$(MYSQL_DATABASE)" + - "--mysql_config_host=$(MYSQL_HOST)" + - "--mysql_config_port=$(MYSQL_PORT)" + - "--mysql_config_user=$(DBCONFIG_USER)" + - "--mysql_config_password=$(DBCONFIG_PASSWORD)" + - "--enable_database_upgrade=true" + ports: + - name: grpc-api + containerPort: 8080 + protocol: TCP + livenessProbe: + tcpSocket: + port: grpc-api + initialDelaySeconds: {{ .Values.metadata.grpc.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.metadata.grpc.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.metadata.grpc.probes.liveness.timeoutSeconds }} + readinessProbe: + tcpSocket: + port: grpc-api + initialDelaySeconds: {{ .Values.metadata.grpc.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.metadata.grpc.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.metadata.grpc.probes.readiness.timeoutSeconds }} + securityContext: + {{- toYaml .Values.metadata.grpc.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.metadata.grpc.resources | nindent 10 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml new file mode 100644 index 0000000000..2907501695 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.metadata.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-grpc + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: metadata-grpc-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + type: {{ .Values.metadata.grpc.service.type }} + ports: + - name: grpc-api + protocol: TCP + port: {{ .Values.metadata.grpc.service.port }} + targetPort: 8080 + selector: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + component: metadata-grpc-server +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml new file mode 100644 index 0000000000..05f2f926b5 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.metadata.enabled .Values.metadata.grpc.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.metadata.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: metadata-grpc-server + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.metadata.grpc.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml new file mode 100644 index 0000000000..ef0f648d35 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml @@ -0,0 +1,81 @@ +{{- if .Values.persistenceAgent.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-persistence-agent + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-persistenceagent + component: persistence-agent + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.persistenceAgent.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + app: ml-pipeline-persistenceagent + component: persistence-agent + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + app: ml-pipeline-persistenceagent + component: persistence-agent + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} + securityContext: + {{- if .Values.security.seccompProfile }} + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} + {{- end }} + containers: + - name: ml-pipeline-persistenceagent + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.persistenceAgent.image.repository "tag" .Values.persistenceAgent.image.tag "context" .) }} + imagePullPolicy: {{ .Values.persistenceAgent.image.pullPolicy | default .Values.global.imagePullPolicy }} + env: + {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: TTL_SECONDS_AFTER_WORKFLOW_FINISH + value: {{ .Values.persistenceAgent.env.ttlSecondsAfterWorkflowFinish | quote }} + - name: NUM_WORKERS + value: {{ .Values.persistenceAgent.env.numWorkers | quote }} + {{- with .Values.persistenceAgent.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.persistenceAgent.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.persistenceAgent.resources | nindent 10 }} + volumeMounts: + - mountPath: /var/run/secrets/kubeflow/tokens + name: persistenceagent-sa-token + volumes: + - name: persistenceagent-sa-token + projected: + sources: + - serviceAccountToken: + path: persistenceagent-sa-token + expirationSeconds: 3600 + audience: pipelines.kubeflow.org + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml new file mode 100644 index 0000000000..2db8d1c340 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml @@ -0,0 +1,44 @@ +{{- if and .Values.persistenceAgent.enabled .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-persistence-agent + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-persistenceagent + component: persistence-agent + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + verbs: + - get + - list + - watch +- apiGroups: + - pipelines.kubeflow.org + resources: + - scheduledworkflows + - workflows + verbs: + - report +- apiGroups: + - pipelines.kubeflow.org + resources: + - runs + verbs: + - reportMetrics + - readArtifact +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml new file mode 100644 index 0000000000..50518e134c --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.persistenceAgent.enabled .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-persistence-agent + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-persistenceagent + component: persistence-agent + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "kubeflow-pipelines.fullname" . }}-persistence-agent +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml new file mode 100644 index 0000000000..a72727e865 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.persistenceAgent.enabled .Values.persistenceAgent.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-persistenceagent + component: persistence-agent + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.persistenceAgent.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/configmap.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/configmap.yaml new file mode 100644 index 0000000000..e0c41098ea --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/profile-controller/configmap.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.installMode.multiUser.enabled .Values.installMode.multiUser.profileController.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller-env + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: profile-controller + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +data: + CONTROLLER_PORT: "8080" + VISUALIZATION_SERVER_IMAGE: "{{ .Values.global.imageRegistry }}/{{ .Values.visualization.image.repository }}" + FRONTEND_IMAGE: "{{ .Values.global.imageRegistry }}/{{ .Values.ui.image.repository }}" + DISABLE_ISTIO_SIDECAR: {{ .Values.profileController.disableIstioSidecar | quote }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller-code + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: profile-controller + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +data: + sync.py: | +{{ .Files.Get "files/sync.py" | indent 4 }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml new file mode 100644 index 0000000000..eac96a1188 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml @@ -0,0 +1,50 @@ +{{- if and .Values.installMode.multiUser.enabled .Values.installMode.multiUser.profileController.enabled }} +apiVersion: metacontroller.k8s.io/v1alpha1 +kind: DecoratorController +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: profile-controller + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + resyncPeriodSeconds: 3600 + resources: + - apiVersion: v1 + resource: namespaces + labelSelector: + matchLabels: + pipelines.kubeflow.org/enabled: "true" + attachments: + - apiVersion: v1 + resource: secrets + updateStrategy: + method: OnDelete + - apiVersion: v1 + resource: configmaps + updateStrategy: + method: OnDelete + - apiVersion: apps/v1 + resource: deployments + updateStrategy: + method: InPlace + - apiVersion: v1 + resource: services + updateStrategy: + method: InPlace + {{- if .Values.networking.istio.enabled }} + - apiVersion: networking.istio.io/v1alpha3 + resource: destinationrules + updateStrategy: + method: InPlace + - apiVersion: security.istio.io/v1beta1 + resource: authorizationpolicies + updateStrategy: + method: InPlace + {{- end }} + hooks: + sync: + webhook: + url: http://{{ include "kubeflow-pipelines.fullname" . }}-profile-controller/sync +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml new file mode 100644 index 0000000000..86414de3eb --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml @@ -0,0 +1,83 @@ +{{- if and .Values.installMode.multiUser.enabled .Values.installMode.multiUser.profileController.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: profile-controller + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.profileController.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + component: profile-controller + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + component: profile-controller + sidecar.istio.io/inject: "false" + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + containers: + - name: profile-controller + image: {{ .Values.profileController.image.repository }}:{{ .Values.profileController.image.tag }} + imagePullPolicy: {{ .Values.profileController.image.pullPolicy | default .Values.global.imagePullPolicy }} + command: ["python", "/hooks/sync.py"] + envFrom: + - configMapRef: + name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller-env + env: + - name: KFP_VERSION + value: {{ .Values.global.imageTag | quote }} + - name: KFP_DEFAULT_PIPELINE_ROOT + value: {{ .Values.pipeline.defaultPipelineRoot | quote }} + {{- if .Values.minio.enabled }} + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "kubeflow-pipelines.fullname" . }}-minio-artifact + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "kubeflow-pipelines.fullname" . }}-minio-artifact + key: secretkey + {{- end }} + {{- with .Values.profileController.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumeMounts: + - name: hooks + mountPath: /hooks + ports: + - name: http + containerPort: 8080 + protocol: TCP + securityContext: + {{- toYaml .Values.profileController.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.profileController.resources | nindent 10 }} + volumes: + - name: hooks + configMap: + name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller-code + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml new file mode 100644 index 0000000000..b671bf8e37 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.installMode.multiUser.enabled .Values.installMode.multiUser.profileController.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: profile-controller + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + type: {{ .Values.profileController.service.type }} + ports: + - name: http + protocol: TCP + port: {{ .Values.profileController.service.port }} + targetPort: 8080 + selector: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + component: profile-controller +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml b/experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml new file mode 100644 index 0000000000..470eb35883 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml @@ -0,0 +1,51 @@ + +{{- if .Values.additionalServiceAccounts.containerBuilder.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.additionalServiceAccounts.containerBuilder.name }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: container-builder + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.additionalServiceAccounts.containerBuilder.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- if .Values.additionalServiceAccounts.pipelineRunner.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.additionalServiceAccounts.pipelineRunner.name }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: pipeline-runner + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.additionalServiceAccounts.pipelineRunner.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- if .Values.additionalServiceAccounts.viewer.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.additionalServiceAccounts.viewer.name }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: viewer + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.additionalServiceAccounts.viewer.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml new file mode 100644 index 0000000000..d051574ed7 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml @@ -0,0 +1,88 @@ +{{- if .Values.additionalServiceAccounts.pipelineRunner.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-pipeline-runner + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: pipeline-runner + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - persistentvolumes + - persistentvolumeclaims + verbs: + - '*' +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - create + - delete + - get +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - pods + - pods/exec + - pods/log + - services + verbs: + - '*' +- apiGroups: + - "" + - apps + - extensions + resources: + - deployments + - replicasets + verbs: + - '*' +- apiGroups: + - kubeflow.org + resources: + - '*' + verbs: + - '*' +- apiGroups: + - batch + resources: + - jobs + verbs: + - '*' +- apiGroups: + - machinelearning.seldon.io + resources: + - seldondeployments + verbs: + - '*' +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml new file mode 100644 index 0000000000..58e6de2491 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.additionalServiceAccounts.pipelineRunner.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-pipeline-runner-binding + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: pipeline-runner + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "kubeflow-pipelines.fullname" . }}-pipeline-runner +subjects: +- kind: ServiceAccount + name: {{ .Values.additionalServiceAccounts.pipelineRunner.name }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml b/experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml new file mode 100644 index 0000000000..99052744f7 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml @@ -0,0 +1,160 @@ +{{- if .Values.installMode.multiUser.enabled }} +# NOTE: IMPORTANT +# We need to separate out actual rules from aggregation rules due to +# https://github.com/kubernetes/kubernetes/issues/65171 +# TL;DR: We can't have both aggregation and rules in a [Cluster]Role. When that +# is the case, the rules get ignored. +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-edit + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true" + component: multi-user-rbac + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pipelines-edit: "true" +rules: [] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-view + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pipelines-edit: "true" + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true" + component: multi-user-rbac + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pipelines-view: "true" +rules: [] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-aggregate-edit + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pipelines-edit: "true" + component: multi-user-rbac + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +rules: +- apiGroups: + - pipelines.kubeflow.org + resources: + - pipelines + - pipelines/versions + verbs: + - create + - delete + - update +- apiGroups: + - pipelines.kubeflow.org + resources: + - experiments + verbs: + - archive + - create + - delete + - unarchive +- apiGroups: + - pipelines.kubeflow.org + resources: + - runs + verbs: + - archive + - create + - delete + - retry + - terminate + - unarchive + - reportMetrics + - readArtifact +- apiGroups: + - pipelines.kubeflow.org + resources: + - jobs + verbs: + - create + - delete + - disable + - enable +- apiGroups: + - kubeflow.org + verbs: + - '*' + resources: + - scheduledworkflows +- apiGroups: + - argoproj.io + verbs: + - '*' + resources: + - cronworkflows + - cronworkflows/finalizers + - workflows + - workflows/finalizers + - workfloweventbindings + - workflowtemplates + - workflowtaskresults + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-aggregate-view + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pipelines-view: "true" + component: multi-user-rbac + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +rules: +- apiGroups: + - pipelines.kubeflow.org + resources: + - pipelines + - pipelines/versions + - experiments + - jobs + verbs: + - get + - list +- apiGroups: + - pipelines.kubeflow.org + resources: + - runs + verbs: + - get + - list + - readArtifact +- apiGroups: + - kubeflow.org + resources: + - viewers + verbs: + - create + - get + - delete +- apiGroups: + - pipelines.kubeflow.org + resources: + - visualizations + verbs: + - create +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml new file mode 100644 index 0000000000..9445225b3e --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml @@ -0,0 +1,79 @@ +{{- if .Values.scheduledWorkflow.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-scheduled-workflow + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-scheduledworkflow + component: scheduled-workflow + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.scheduledWorkflow.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + app: ml-pipeline-scheduledworkflow + component: scheduled-workflow + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + app: ml-pipeline-scheduledworkflow + component: scheduled-workflow + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} + securityContext: + {{- if .Values.security.seccompProfile }} + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} + {{- end }} + containers: + - name: ml-pipeline-scheduledworkflow + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.scheduledWorkflow.image.repository "tag" .Values.scheduledWorkflow.image.tag "context" .) }} + imagePullPolicy: {{ .Values.scheduledWorkflow.image.pullPolicy | default .Values.global.imagePullPolicy }} + env: + {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CRON_SCHEDULE_TIMEZONE + value: {{ .Values.pipeline.cronScheduleTimezone | quote }} + {{- with .Values.scheduledWorkflow.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.scheduledWorkflow.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.scheduledWorkflow.resources | nindent 10 }} + volumeMounts: + - mountPath: /var/run/secrets/kubeflow/tokens + name: scheduledworkflow-sa-token + volumes: + - name: scheduledworkflow-sa-token + projected: + sources: + - serviceAccountToken: + path: scheduledworkflow-sa-token + expirationSeconds: 3600 + audience: pipelines.kubeflow.org + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml new file mode 100644 index 0000000000..09cc0a52d9 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.scheduledWorkflow.enabled .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-scheduled-workflow + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-scheduledworkflow + component: scheduled-workflow + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + - scheduledworkflows/finalizers + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - pipelines.kubeflow.org + resources: + - runs + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml new file mode 100644 index 0000000000..4012fcf380 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduledWorkflow.enabled .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-scheduled-workflow + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-scheduledworkflow + component: scheduled-workflow + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "kubeflow-pipelines.fullname" . }}-scheduled-workflow +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml new file mode 100644 index 0000000000..9575968046 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.scheduledWorkflow.enabled .Values.scheduledWorkflow.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-scheduledworkflow + component: scheduled-workflow + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.scheduledWorkflow.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/ui/deployment.yaml b/experimental/helm/charts/pipeline/templates/ui/deployment.yaml new file mode 100644 index 0000000000..3314188fb3 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/ui/deployment.yaml @@ -0,0 +1,115 @@ +{{- if .Values.ui.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-ui + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-ui + component: ui + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.ui.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + app: ml-pipeline-ui + component: ui + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + app: ml-pipeline-ui + component: ui + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.ui.serviceAccountName" . }} + securityContext: + {{- if .Values.security.seccompProfile }} + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} + {{- end }} + volumes: + - name: config-volume + configMap: + name: {{ include "kubeflow-pipelines.fullname" . }}-ui-config + containers: + - name: ml-pipeline-ui + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.ui.image.repository "tag" .Values.ui.image.tag "context" .) }} + imagePullPolicy: {{ .Values.ui.image.pullPolicy | default .Values.global.imagePullPolicy }} + ports: + - name: http + containerPort: 3000 + protocol: TCP + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + env: + {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + {{- include "kubeflow-pipelines.objectStoreEnv" . | nindent 8 }} + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: /etc/config/viewer-pod-template.json + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: {{ .Values.ui.env.allowCustomVisualizations | quote }} + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: {{ .Values.ui.env.argoArchiveLogs | quote }} + - name: DISABLE_GKE_METADATA + value: {{ .Values.ui.env.disableGkeMetadata | quote }} + {{- with .Values.ui.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + readinessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - "-" + - http://localhost:3000/apis/v1beta1/healthz + initialDelaySeconds: {{ .Values.ui.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.ui.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.ui.probes.readiness.timeoutSeconds }} + livenessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - "-" + - http://localhost:3000/apis/v1beta1/healthz + initialDelaySeconds: {{ .Values.ui.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.ui.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.ui.probes.liveness.timeoutSeconds }} + securityContext: + {{- toYaml .Values.ui.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.ui.resources | nindent 10 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/ui/service.yaml b/experimental/helm/charts/pipeline/templates/ui/service.yaml new file mode 100644 index 0000000000..3859dd67fe --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/ui/service.yaml @@ -0,0 +1,24 @@ +{{- if .Values.ui.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-ui + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-ui + component: ui + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + type: {{ .Values.ui.service.type }} + ports: + - name: http + protocol: TCP + port: {{ .Values.ui.service.port }} + targetPort: 3000 + selector: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + app: ml-pipeline-ui + component: ui +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml new file mode 100644 index 0000000000..1cb2502773 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.ui.enabled .Values.ui.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.ui.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-ui + component: ui + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.ui.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml b/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml new file mode 100644 index 0000000000..8eba842920 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml @@ -0,0 +1,68 @@ +{{- if .Values.viewerCrd.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-viewer-crd + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-viewer-crd + component: viewer-crd + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.viewerCrd.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + app: ml-pipeline-viewer-crd + component: viewer-crd + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + app: ml-pipeline-viewer-crd + component: viewer-crd + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} + securityContext: + {{- if .Values.security.seccompProfile }} + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} + {{- end }} + containers: + - name: ml-pipeline-viewer-crd + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.viewerCrd.image.repository "tag" .Values.viewerCrd.image.tag "context" .) }} + imagePullPolicy: {{ .Values.viewerCrd.image.pullPolicy | default .Values.global.imagePullPolicy }} + env: + {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + - name: MAX_NUM_VIEWERS + value: {{ .Values.viewerCrd.env.maxNumViewers | quote }} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.viewerCrd.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.viewerCrd.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.viewerCrd.resources | nindent 10 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml new file mode 100644 index 0000000000..fe1aa0caac --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.viewerCrd.enabled .Values.viewerCrd.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-viewer-crd + component: viewer-crd + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.viewerCrd.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml b/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml new file mode 100644 index 0000000000..73ad93313c --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml @@ -0,0 +1,90 @@ +{{- if .Values.visualization.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-visualization + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-visualizationserver + component: visualization + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + replicas: {{ .Values.visualization.replicas }} + selector: + matchLabels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} + app: ml-pipeline-visualizationserver + component: visualization + template: + metadata: + labels: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} + app: ml-pipeline-visualizationserver + component: visualization + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + spec: + {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "kubeflow-pipelines.visualization.serviceAccountName" . }} + securityContext: + {{- if .Values.security.seccompProfile }} + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} + {{- end }} + containers: + - name: ml-pipeline-visualizationserver + image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.visualization.image.repository "tag" .Values.visualization.image.tag "context" .) }} + imagePullPolicy: {{ .Values.visualization.image.pullPolicy | default .Values.global.imagePullPolicy }} + ports: + - name: http + containerPort: 8888 + protocol: TCP + env: + {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + {{- with .Values.visualization.additionalEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + readinessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - "-" + - http://localhost:8888/ + initialDelaySeconds: {{ .Values.visualization.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.visualization.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.visualization.probes.readiness.timeoutSeconds }} + livenessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - "-" + - http://localhost:8888/ + initialDelaySeconds: {{ .Values.visualization.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.visualization.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.visualization.probes.liveness.timeoutSeconds }} + securityContext: + {{- toYaml .Values.visualization.securityContext | nindent 10 }} + resources: + {{- toYaml .Values.visualization.resources | nindent 10 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/visualization/service.yaml b/experimental/helm/charts/pipeline/templates/visualization/service.yaml new file mode 100644 index 0000000000..e77e749a48 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/visualization/service.yaml @@ -0,0 +1,24 @@ +{{- if .Values.visualization.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-visualization + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-visualizationserver + component: visualization + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +spec: + type: {{ .Values.visualization.service.type }} + ports: + - name: http + protocol: TCP + port: {{ .Values.visualization.service.port }} + targetPort: 8888 + selector: + {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + app: ml-pipeline-visualizationserver + component: visualization +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml new file mode 100644 index 0000000000..6096d38ae8 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.visualization.enabled .Values.visualization.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubeflow-pipelines.visualization.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-visualizationserver + component: visualization + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- with .Values.visualization.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/webhook/mutating-webhook-config.yaml b/experimental/helm/charts/pipeline/templates/webhook/mutating-webhook-config.yaml new file mode 100644 index 0000000000..bd2285d3a0 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/webhook/mutating-webhook-config.yaml @@ -0,0 +1,34 @@ +{{- if .Values.webhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-pipelineversions-mutating + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: webhook + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +webhooks: +- name: pipelineversions.pipelines.kubeflow.org + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - pipelines.kubeflow.org + apiVersions: + - v2beta1 + resources: + - pipelineversions + scope: Namespaced + admissionReviewVersions: + - v1 + sideEffects: None + failurePolicy: {{ .Values.webhooks.failurePolicy }} + clientConfig: + service: + name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + namespace: {{ include "kubeflow-pipelines.namespace" . }} + path: /webhooks/mutate-pipelineversion + port: 8443 +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/webhook/validating-webhook-config.yaml b/experimental/helm/charts/pipeline/templates/webhook/validating-webhook-config.yaml new file mode 100644 index 0000000000..58ea502ced --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/webhook/validating-webhook-config.yaml @@ -0,0 +1,34 @@ +{{- if .Values.webhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-pipelineversions-validating + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: webhook + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +webhooks: +- name: pipelineversions.pipelines.kubeflow.org + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - pipelines.kubeflow.org + apiVersions: + - v2beta1 + resources: + - pipelineversions + scope: Namespaced + admissionReviewVersions: + - v1 + sideEffects: None + failurePolicy: {{ .Values.webhooks.failurePolicy }} + clientConfig: + service: + name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + namespace: {{ include "kubeflow-pipelines.namespace" . }} + path: /webhooks/validate-pipelineversion + port: 8443 +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml b/experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml new file mode 100644 index 0000000000..b3edf81f1c --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.cache.enabled (not .Values.webhooks.tls.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kubeflow-pipelines.fullname" . }}-webhook-server-tls + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: webhook + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" +type: kubernetes.io/tls +data: + tls.crt: {{ .Values.webhooks.tls.cert | b64enc }} + tls.key: {{ .Values.webhooks.tls.key | b64enc }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/values.yaml b/experimental/helm/charts/pipeline/values.yaml new file mode 100644 index 0000000000..392379652b --- /dev/null +++ b/experimental/helm/charts/pipeline/values.yaml @@ -0,0 +1,1026 @@ +# Default values for kubeflow-pipelines. + +# -- String to partially override release name. +nameOverride: "" + +# -- String to fully override release name. +fullnameOverride: "" + +# -- Common labels to add to the resources. +commonLabels: {} + +# -- Common annotations to add to the resources. +commonAnnotations: {} + +# Global settings +global: + # -- Namespace to install Kubeflow Pipelines + namespace: kubeflow + # -- Image registry for Kubeflow Pipelines images + imageRegistry: ghcr.io/kubeflow + # -- Global image tag for all Kubeflow Pipelines components + imageTag: 2.5.0 + # -- Global image pull policy + imagePullPolicy: IfNotPresent + # -- Global image pull secrets + imagePullSecrets: [] + # -- Global annotations to add to the resources + annotations: {} + # -- Global logging level + logLevel: "info" + +# Installation mode configuration +installMode: + # -- Installation type (generic or multi-user) + type: generic + + # Multi-user specific configuration + multiUser: + # -- Enable multi-user mode + enabled: false + profileController: + # -- Enable profile controller for multi-user mode + enabled: false + +# Custom Resource Definitions +crds: + # -- Install CRDs + install: true + +# API Server configuration +apiServer: + # -- Enable the Kubeflow Pipelines API Server + enabled: true + # -- Number of API Server replicas + replicas: 1 + + image: + # -- API Server image repository + repository: kfp-api-server + # -- API Server image tag (overrides global.imageTag if set) + tag: "" + # -- API Server image pull policy (overrides global.imagePullPolicy if set) + pullPolicy: "" + + # -- API Server resource requests and limits + resources: + requests: + cpu: 250m + memory: 500Mi + limits: {} + + # Service Account + serviceAccount: + # -- Create service account for API Server + create: true + # -- Service account name (uses default if not set) + name: "ml-pipeline" + # -- Service account annotations + annotations: {} + + # Service configuration + service: + # -- Service type + type: ClusterIP + # -- HTTP service port + port: 8888 + # -- gRPC service port + grpcPort: 8887 + + # Environment variables + env: + # -- Whether to publish component logs to object store + publishLogs: true + # -- Driver/launcher log level during pipeline execution + pipelineLogLevel: "1" + # -- Auto update pipeline default version + autoUpdatePipelineDefaultVersion: true + # -- V2 driver image + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" + # -- V2 launcher image + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + + # Additional environment variables + additionalEnv: [] + + # Probes configuration + probes: + readiness: + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + liveness: + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + startup: + failureThreshold: 12 + periodSeconds: 5 + timeoutSeconds: 2 + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +persistenceAgent: + # -- Enable Persistence Agent + enabled: true + # -- Number of Persistence Agent replicas + replicas: 1 + + image: + # -- Persistence Agent image repository + repository: kfp-persistence-agent + # -- Persistence Agent image tag (overrides global.imageTag if set) + tag: "" + # -- Persistence Agent image pull policy (overrides global.imagePullPolicy if set) + pullPolicy: "" + + # -- Persistence Agent resource requests and limits + resources: + requests: + cpu: 120m + memory: 500Mi + limits: {} + + # Service Account + serviceAccount: + # -- Create service account for Persistence Agent + create: true + # -- Service account name (uses default if not set) + name: "ml-pipeline-persistenceagent" + # -- Service account annotations + annotations: {} + + # Environment variables + env: + # -- TTL seconds after workflow finish + ttlSecondsAfterWorkflowFinish: "86400" + # -- Number of workers + numWorkers: "2" + + # Additional environment variables + additionalEnv: [] + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +scheduledWorkflow: + enabled: true + name: ml-pipeline-scheduledworkflow + + image: + repository: kfp-scheduled-workflow-controller + tag: "2.5.0" + + replicas: 1 + + # Service Account + serviceAccount: + create: true + name: "ml-pipeline-scheduledworkflow" + annotations: {} + + # Resources + resources: + requests: + cpu: 120m + memory: 100Mi + limits: {} + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +ui: + # -- Enable UI + enabled: true + # -- Number of UI replicas + replicas: 1 + + image: + # -- UI image repository + repository: kfp-frontend + # -- UI image tag (overrides global.imageTag if set) + tag: "" + # -- UI image pull policy (overrides global.imagePullPolicy if set) + pullPolicy: "" + + # -- UI resource requests and limits + resources: + requests: + cpu: 10m + memory: 70Mi + limits: {} + + # Service Account + serviceAccount: + # -- Create service account for UI + create: true + # -- Service account name (uses default if not set) + name: "ml-pipeline-ui" + # -- Service account annotations + annotations: {} + + # Service configuration + service: + # -- Service type + type: ClusterIP + # -- Service port + port: 80 + + # Environment variables + env: + # -- Allow custom visualizations + allowCustomVisualizations: true + # -- Archive Argo logs + argoArchiveLogs: true + # -- Disable GKE metadata + disableGkeMetadata: true + + # Additional environment variables + additionalEnv: [] + + # Probes configuration + probes: + readiness: + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + liveness: + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +viewerCrd: + # -- Enable Viewer CRD Controller + enabled: true + # -- Number of Viewer CRD Controller replicas + replicas: 1 + + image: + # -- Viewer CRD Controller image repository + repository: kfp-viewer-crd-controller + # -- Viewer CRD Controller image tag (overrides global.imageTag if set) + tag: "" + # -- Viewer CRD Controller image pull policy (overrides global.imagePullPolicy if set) + pullPolicy: "" + + # -- Viewer CRD Controller resource requests and limits + resources: + requests: + cpu: 120m + memory: 100Mi + limits: {} + + # Service Account + serviceAccount: + # -- Create service account for Viewer CRD Controller + create: true + # -- Service account name (uses default if not set) + name: "ml-pipeline-viewer-crd-service-account" + # -- Service account annotations + annotations: {} + + # Environment variables + env: + # -- Maximum number of viewers + maxNumViewers: "50" + + # Additional environment variables + additionalEnv: [] + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +visualization: + # -- Enable Visualization Server + enabled: true + # -- Number of Visualization Server replicas + replicas: 1 + + image: + # -- Visualization Server image repository + repository: kfp-visualization-server + # -- Visualization Server image tag (overrides global.imageTag if set) + tag: "" + # -- Visualization Server image pull policy (overrides global.imagePullPolicy if set) + pullPolicy: "" + + # -- Visualization Server resource requests and limits + resources: + requests: + cpu: 50m + memory: 200Mi + limits: {} + + # Service Account + serviceAccount: + # -- Create service account for Visualization Server + create: true + # -- Service account name (uses default if not set) + name: "ml-pipeline-visualizationserver" + # -- Service account annotations + annotations: {} + + # Service configuration + service: + # -- Service type + type: ClusterIP + # -- Service port + port: 8888 + + # Additional environment variables + additionalEnv: [] + + # Probes configuration + probes: + readiness: + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + liveness: + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +# Cache System Configuration +cache: + # -- Enable Cache System + enabled: true + + # Service Account + serviceAccount: + # -- Create service account for Cache Server + create: true + # -- Service account name (uses default if not set) + name: "kubeflow-pipelines-cache" + # -- Service account annotations + annotations: {} + + server: + # -- Number of Cache Server replicas + replicas: 1 + + image: + # -- Cache Server image repository + repository: kfp-cache-server + # -- Cache Server image tag (overrides global.imageTag if set) + tag: "" + # -- Cache Server image pull policy (overrides global.imagePullPolicy if set) + pullPolicy: "" + + # -- Cache Server resource requests and limits + resources: + requests: + cpu: 120m + memory: 500Mi + limits: {} + + # Service configuration + service: + # -- Service type + type: ClusterIP + # -- Service port + port: 8443 + + # Additional environment variables + additionalEnv: [] + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +cacheDeployer: + # -- Enable Cache Deployer + enabled: true + # -- Number of Cache Deployer replicas + replicas: 1 + + image: + # -- Cache Deployer image repository + repository: kfp-cache-deployer + # -- Cache Deployer image tag (overrides global.imageTag if set) + tag: "" + # -- Cache Deployer image pull policy (overrides global.imagePullPolicy if set) + pullPolicy: "" + + # -- Cache Deployer resource requests and limits + resources: + requests: + cpu: 120m + memory: 100Mi + limits: {} + + # Service Account + serviceAccount: + # -- Create service account for Cache Deployer + create: true + # -- Service account name (uses default if not set) + name: "kubeflow-pipelines-cache-deployer-sa" + # -- Service account annotations + annotations: {} + + # Additional environment variables + additionalEnv: [] + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +# ML Metadata Configuration +metadata: + # -- Enable ML Metadata + enabled: true + + grpc: + # -- Number of metadata gRPC server replicas + replicas: 1 + + image: + # -- Metadata gRPC server image repository (external image) + repository: gcr.io/tfx-oss-public/ml_metadata_store_server + # -- Metadata gRPC server image tag + tag: "1.14.0" + # -- Metadata gRPC server image pull policy + pullPolicy: "IfNotPresent" + + # -- Metadata gRPC server resource requests and limits + resources: + requests: + cpu: 100m + memory: 300Mi + limits: {} + + # Service Account + serviceAccount: + # -- Create service account for Metadata gRPC server + create: true + # -- Service account name (uses default if not set) + name: "metadata-grpc-server" + # -- Service account annotations + annotations: {} + + # Service configuration + service: + # -- Service type + type: ClusterIP + # -- Service port + port: 8080 + + # Probes configuration + probes: + readiness: + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + liveness: + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + + # Additional environment variables + additionalEnv: [] + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + + envoy: + # -- Number of metadata Envoy replicas + replicas: 1 + + image: + # -- Metadata Envoy image repository + repository: kfp-metadata-envoy + # -- Metadata Envoy image tag (overrides global.imageTag if set) + tag: "" + # -- Metadata Envoy image pull policy (overrides global.imagePullPolicy if set) + pullPolicy: "" + + # -- Metadata Envoy resource requests and limits + resources: + requests: + cpu: 20m + memory: 20Mi + limits: {} + + # Service configuration + service: + # -- Service type + type: ClusterIP + # -- Service port + port: 9090 + + # Additional environment variables + additionalEnv: [] + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Third-party Dependencies Configuration +argo: + enabled: true + # Argo Workflows subchart values will be merged here + +# Database Configuration +mysql: + enabled: true + auth: + rootPassword: "" + database: "mlpipeline" + username: "root" + primary: + persistence: + enabled: true + size: 20Gi + +postgresql: + enabled: false + auth: + postgresPassword: "" + database: "mlpipeline" + username: "postgres" + primary: + persistence: + enabled: true + size: 20Gi + +# External database configuration (when not using subcharts) +externalDatabase: + # Database type: mysql or postgresql + type: mysql + host: "" + port: 3306 + database: "mlpipeline" + username: "root" + password: "" + # Existing secret containing database credentials + existingSecret: "" + existingSecretPasswordKey: "password" + +# Database connection configuration +database: + # Connection max lifetime (important for external databases) + connectionMaxLifetime: "120s" + + # Additional databases + metadb: "metadb" + cachedb: "cachedb" + +# Object Storage Configuration +objectStore: + # Provider: minio, s3, gcs, azure, custom + provider: minio + + minio: + enabled: true + bucket: "mlpipeline" + secure: false + endpoint: "" + existingSecret: "" + + s3: + region: "us-west-2" + bucket: "" + existingSecret: "" + + gcs: + bucket: "" + existingSecret: "" + + azure: + storageAccount: "" + container: "" + existingSecret: "" + + custom: + endpoint: "" + bucket: "" + secure: true + existingSecret: "" + +# MinIO subchart configuration +minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" + persistence: + enabled: true + size: 20Gi + +# Pipeline Configuration +pipeline: + # Application configuration + app: + name: "pipeline" + version: "2.5.0" + + # Pipeline root configuration + defaultPipelineRoot: "minio://mlpipeline/v2/artifacts" + + # Auto update pipeline default version + autoUpdatePipelineDefaultVersion: true + + # Cron schedule timezone + cronScheduleTimezone: "UTC" + + # Cache configuration + cache: + image: "registry.k8s.io/busybox" + nodeRestrictions: false + maximumStaleness: "" + defaultStaleness: "" + +# Environment-specific Configuration +env: + # -- Platform type: platform-agnostic, aws, gcp, azure + platform: platform-agnostic + + # AWS-specific configuration + aws: + # -- AWS region + region: us-west-2 + # RDS configuration + rds: + # -- Enable RDS integration + enabled: false + # -- RDS endpoint + endpoint: "" + # S3 configuration + s3: + # -- Enable S3 integration + enabled: false + # -- S3 bucket name + bucket: "" + # -- S3 region + region: us-west-2 + # IAM configuration + iam: + # -- Enable IAM roles for service accounts (IRSA) + enabled: false + # -- IAM role ARN for API server + apiServerRoleArn: "" + + # GCP-specific configuration + gcp: + # -- GCP project ID + project: "" + # -- GCP region + region: us-central1 + # Cloud SQL configuration + cloudsql: + # -- Enable Cloud SQL integration + enabled: false + # -- Cloud SQL instance name + instanceName: "" + # -- Cloud SQL region + region: us-central1 + # GCS configuration + gcs: + # -- Enable GCS integration + enabled: false + # -- GCS bucket name + bucket: "" + # -- GCS project ID + projectId: "" + # Workload Identity configuration + workloadIdentity: + # -- Enable Workload Identity + enabled: false + # -- GCP service account email + serviceAccount: "" + + # Azure-specific configuration + azure: + # -- Azure subscription ID + subscriptionId: "" + # -- Azure resource group + resourceGroup: "" + # -- Azure region + region: eastus + # Azure SQL configuration + sql: + # -- Enable Azure SQL integration + enabled: false + # -- Azure SQL server name + serverName: "" + # -- Azure SQL database name + database: "" + # Azure Blob Storage configuration + blob: + # -- Enable Azure Blob Storage integration + enabled: false + # -- Storage account name + storageAccount: "" + # -- Container name + container: "" + # Managed Identity configuration + managedIdentity: + # -- Enable Azure Managed Identity + enabled: false + # -- Managed Identity client ID + clientId: "" + +# RBAC Configuration +rbac: + create: true + +# Service Account Configuration +serviceAccounts: + # Additional service accounts + pipelineRunner: + create: true + name: "pipeline-runner" + + viewer: + create: true + name: "viewer" + + containerBuilder: + create: true + name: "container-builder" + + metadataWriter: + create: true + name: "metadata-writer" + +# Networking Configuration +networking: + istio: + enabled: false + gateway: "kubeflow-gateway" + host: "*" + + # Virtual Service configuration + virtualService: + enabled: false + prefix: "/pipeline" + +# Security Configuration +security: + # Pod Security Standards + podSecurityStandards: + enabled: true + enforce: "restricted" + + # Security Context Profile + seccompProfile: + type: "RuntimeDefault" + +# Monitoring Configuration +monitoring: + prometheus: + enabled: false + serviceMonitor: + enabled: false + + grafana: + enabled: false + +# Webhook Configuration +webhooks: + # -- Enable admission webhooks + enabled: true + # -- Webhook failure policy + failurePolicy: "Fail" + + # TLS configuration for webhooks + tls: + # -- Use existing secret for TLS certificates + existingSecret: "" + # -- TLS certificate (base64 encoded) + cert: "" + # -- TLS private key (base64 encoded) + key: "" + +# Metadata Writer Configuration +metadataWriter: + # -- Enable Metadata Writer + enabled: true + # -- Number of Metadata Writer replicas + replicas: 1 + + image: + # -- Metadata Writer image repository + repository: kfp-metadata-writer + # -- Metadata Writer image tag (overrides global.imageTag if set) + tag: "" + # -- Metadata Writer image pull policy (overrides global.imagePullPolicy if set) + pullPolicy: "" + + # -- Metadata Writer resource requests and limits + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 1Gi + + # Service Account + serviceAccount: + # -- Create service account for Metadata Writer + create: true + # -- Service account name (uses default if not set) + name: "kubeflow-pipelines-metadata-writer" + # -- Service account annotations + annotations: {} + + # Additional environment variables + additionalEnv: [] + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + +# Additional Service Accounts Configuration +additionalServiceAccounts: + # Container Builder Service Account + containerBuilder: + # -- Create container builder service account + create: true + # -- Container builder service account name + name: "kubeflow-pipelines-container-builder" + # -- Container builder service account annotations + annotations: {} + + # Pipeline Runner Service Account + pipelineRunner: + # -- Create pipeline runner service account + create: true + # -- Pipeline runner service account name + name: "pipeline-runner" + # -- Pipeline runner service account annotations + annotations: {} + + # Viewer Service Account + viewer: + # -- Create viewer service account + create: true + # -- Viewer service account name + name: "kubeflow-pipelines-viewer" + # -- Viewer service account annotations + annotations: {} + +# Profile Controller Configuration (Multi-User) +profileController: + # -- Enable Profile Controller (only for multi-user mode) + enabled: false + # -- Number of Profile Controller replicas + replicas: 1 + + image: + # -- Profile Controller image repository + repository: public.ecr.aws/docker/library/python + # -- Profile Controller image tag + tag: "3.12" + # -- Profile Controller image pull policy + pullPolicy: IfNotPresent + + # -- Profile Controller resource requests and limits + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 1Gi + + # Service configuration + service: + # -- Service type + type: ClusterIP + # -- Service port + port: 80 + + # -- Disable Istio sidecar injection + disableIstioSidecar: "true" + + # Additional environment variables + additionalEnv: [] + + # Security context + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Application Configuration +application: + # -- Enable Application resource (for Google Cloud Console integration) + enabled: false + +# Certificate Management +certManager: + # -- Enable cert-manager integration + enabled: false + + # Mutating webhook + mutating: + enabled: true + name: "cache-webhook-kubeflow" + + # Validating webhook + validating: + enabled: true + name: "cache-webhook-kubeflow" + + +# Node selector, tolerations, and affinity +nodeSelector: {} +tolerations: [] +affinity: {} + +# Pod annotations +podAnnotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" diff --git a/tests/helm_kustomize_compare.py b/tests/helm_kustomize_compare.py index 11fbabbbe5..a7798b9164 100755 --- a/tests/helm_kustomize_compare.py +++ b/tests/helm_kustomize_compare.py @@ -216,6 +216,8 @@ def get_expected_helm_extras(component: str, scenario: str) -> set: if scenario == "base": return {"AuthorizationPolicy/kserve-models-web-app"} return set() + elif component == "kubeflow-pipelines": + return set() # No extra resources in Helm for Kubeflow Pipelines else: return set() @@ -281,7 +283,7 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena if __name__ == "__main__": if len(sys.argv) < 5: print("Usage: python compare.py [namespace] [--verbose]") - print("Components: katib, model-registry, kserve-models-web-app") + print("Components: katib, model-registry, kserve-models-web-app, kubeflow-pipelines") sys.exit(1) kustomize_file = sys.argv[1] @@ -290,9 +292,9 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena scenario = sys.argv[4] namespace = sys.argv[5] if len(sys.argv) > 5 and not sys.argv[5].startswith('--') else "" - if component not in ["katib", "model-registry", "kserve-models-web-app", "notebook-controller"]: + if component not in ["katib", "model-registry", "kserve-models-web-app", "notebook-controller", "kubeflow-pipelines"]: print(f"ERROR: Unknown component: {component}") - print("Supported components: katib, model-registry, kserve-models-web-app, notebook-controller") + print("Supported components: katib, model-registry, kserve-models-web-app, notebook-controller","kubeflow-pipelines") sys.exit(1) success = compare_manifests(kustomize_file, helm_file, component, scenario, namespace) diff --git a/tests/helm_kustomize_compare.sh b/tests/helm_kustomize_compare.sh index 7cca489db6..718df3946b 100755 --- a/tests/helm_kustomize_compare.sh +++ b/tests/helm_kustomize_compare.sh @@ -11,7 +11,7 @@ ROOT_DIR="$(dirname "$SCRIPT_DIR")" if [[ -z "$COMPONENT" ]]; then echo "ERROR: Component is required" echo "Usage: $0 " - echo "Components: katib, model-registry, kserve-models-web-app, notebook-controller" + echo "Components: katib, model-registry, kserve-models-web-app, notebook-controller, kubeflow-pipelines" exit 1 fi @@ -158,10 +158,65 @@ case "$COMPONENT" in ["production"]="kubeflow" ) ;; + "kubeflow-pipelines") + CHART_DIR="$ROOT_DIR/experimental/helm/charts/kubeflow-pipelines" + MANIFESTS_DIR="$ROOT_DIR/applications/pipeline/upstream" + + declare -A KUSTOMIZE_PATHS=( + ["platform-agnostic"]="$MANIFESTS_DIR/env/platform-agnostic" + ["aws"]="$MANIFESTS_DIR/env/aws" + ["gcp"]="$MANIFESTS_DIR/env/gcp" + ["azure"]="$MANIFESTS_DIR/env/azure" + ["multi-user"]="$MANIFESTS_DIR/base/installs/multi-user" + ["generic"]="$MANIFESTS_DIR/base/installs/generic" + ["dev"]="$MANIFESTS_DIR/env/dev" + ["plain"]="$MANIFESTS_DIR/env/plain" + ["plain-multi-user"]="$MANIFESTS_DIR/env/plain-multi-user" + ["platform-agnostic-emissary"]="$MANIFESTS_DIR/env/platform-agnostic-emissary" + ["platform-agnostic-multi-user"]="$MANIFESTS_DIR/env/platform-agnostic-multi-user" + ["platform-agnostic-multi-user-emissary"]="$MANIFESTS_DIR/env/platform-agnostic-multi-user-emissary" + ["platform-agnostic-multi-user-legacy"]="$MANIFESTS_DIR/env/platform-agnostic-multi-user-legacy" + ["platform-agnostic-postgresql"]="$MANIFESTS_DIR/env/platform-agnostic-postgresql" + ) + + declare -A HELM_VALUES=( + ["platform-agnostic"]="$CHART_DIR/ci/values-platform-agnostic-enhanced.yaml" + ["aws"]="$CHART_DIR/ci/values-aws-enhanced.yaml" + ["gcp"]="$CHART_DIR/ci/values-gcp-enhanced.yaml" + ["azure"]="$CHART_DIR/ci/values-azure-enhanced.yaml" + ["multi-user"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" + ["generic"]="" + ["dev"]="$CHART_DIR/ci/values-dev-enhanced.yaml" + ["plain"]="$CHART_DIR/ci/values-standalone.yaml" + ["plain-multi-user"]="$CHART_DIR/ci/values-multi-user.yaml" + ["platform-agnostic-emissary"]="$CHART_DIR/ci/values-platform-agnostic-enhanced.yaml" + ["platform-agnostic-multi-user"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" + ["platform-agnostic-multi-user-emissary"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" + ["platform-agnostic-multi-user-legacy"]="$CHART_DIR/ci/values-multi-user.yaml" + ["platform-agnostic-postgresql"]="$CHART_DIR/ci/values-postgresql.yaml" + ) + + declare -A NAMESPACES=( + ["platform-agnostic"]="kubeflow" + ["aws"]="kubeflow" + ["gcp"]="kubeflow" + ["azure"]="kubeflow" + ["multi-user"]="kubeflow" + ["generic"]="kubeflow" + ["dev"]="kubeflow" + ["plain"]="kubeflow" + ["plain-multi-user"]="kubeflow" + ["platform-agnostic-emissary"]="kubeflow" + ["platform-agnostic-multi-user"]="kubeflow" + ["platform-agnostic-multi-user-emissary"]="kubeflow" + ["platform-agnostic-multi-user-legacy"]="kubeflow" + ["platform-agnostic-postgresql"]="kubeflow" + ) + ;; *) echo "ERROR: Unknown component: $COMPONENT" - echo "Supported components: katib, model-registry, kserve-models-web-app, notebook-controller" + echo "Supported components: katib, model-registry, kserve-models-web-app, notebook-controller, kubeflow-pipelines"" exit 1 ;; esac @@ -191,7 +246,7 @@ if [ ! -d "$CHART_DIR" ]; then exit 1 fi -if [[ "$COMPONENT" != "kserve-models-web-app" ]] && [ ! -f "$HELM_VALUES_ARG" ]; then +if [[ "$COMPONENT" != "kserve-models-web-app" ]] && [[ "$COMPONENT" != "kubeflow-pipelines" || "$SCENARIO" != "generic" ]] && [ ! -f "$HELM_VALUES_ARG" ]; then echo "ERROR: Helm values file does not exist: $HELM_VALUES_ARG" exit 1 fi @@ -223,6 +278,17 @@ else --namespace "$NAMESPACE" \ --include-crds \ --values "$HELM_VALUES_ARG" > "$HELM_OUTPUT" + elif [[ "$COMPONENT" == "kubeflow-pipelines" ]]; then + if [[ "$SCENARIO" == "generic" ]]; then + helm template kubeflow-pipelines . \ + --namespace "$NAMESPACE" \ + --include-crds > "$HELM_OUTPUT" + else + helm template kubeflow-pipelines . \ + --namespace "$NAMESPACE" \ + --include-crds \ + --values "$HELM_VALUES_ARG" > "$HELM_OUTPUT" + fi else helm template model-registry . \ --namespace "$NAMESPACE" \ diff --git a/tests/helm_kustomize_compare_all.sh b/tests/helm_kustomize_compare_all.sh index 43e3da0b3b..63f4e65785 100755 --- a/tests/helm_kustomize_compare_all.sh +++ b/tests/helm_kustomize_compare_all.sh @@ -13,6 +13,7 @@ declare -A COMPONENT_SCENARIOS=( ["model-registry"]="base overlay-postgres overlay-db controller-manager controller-rbac controller-default controller-prometheus controller-network-policy ui-base ui-standalone ui-integrated ui-istio istio csi" ["kserve-models-web-app"]="base kubeflow" ["notebook-controller"]="base kubeflow standalone" + ["kubeflow-pipelines"]="platform-agnostic aws gcp azure multi-user generic dev plain plain-multi-user platform-agnostic-emissary platform-agnostic-multi-user platform-agnostic-multi-user-emissary platform-agnostic-multi-user-legacy platform-agnostic-postgresql" ) test_component() { @@ -50,7 +51,7 @@ if [[ "$COMPONENT" == "all" ]]; then declare -a passed_components=() declare -a failed_components=() - for comp in katib model-registry kserve-models-web-app notebook-controller; do + for comp in katib model-registry kserve-models-web-app notebook-controller kubeflow-pipelines; do if test_component "$comp"; then passed_components+=("$comp") else @@ -79,6 +80,7 @@ elif [[ "$COMPONENT" == "help" ]] || [[ "$COMPONENT" == "--help" ]] || [[ "$COMP echo " model-registry Test Model Registry scenarios" echo " kserve-models-web-app Test KServe Models Web App scenarios" echo " notebook-controller Test Notebook Controller scenarios" + echo " kubeflow-pipelines Test Kubeflow Pipelines scenarios" echo "" echo "Examples:" echo " $0 # Test all components" @@ -99,7 +101,7 @@ elif [[ "${COMPONENT_SCENARIOS[$COMPONENT]:-}" ]]; then else echo "ERROR: Unknown component: $COMPONENT" - echo "Supported components: katib, model-registry, kserve-models-web-app, notebook-controller, all" + echo "Supported components: katib, model-registry, kserve-models-web-app, notebook-controller,kubeflow-pipelines, all" echo "Use '$0 help' for more information." exit 1 fi \ No newline at end of file From 4cc10b63410b26a3fd406549e9cc67b256c4fe87 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Mon, 22 Sep 2025 17:52:55 +0530 Subject: [PATCH 02/26] Update helm charts Signed-off-by: kunal-511 --- experimental/helm/charts/pipeline/Chart.yaml | 4 +- .../ci/values-platform-agnostic-enhanced.yaml | 65 ++++--- .../charts/pipeline/ci/values-standalone.yaml | 29 ++- .../charts/pipeline/templates/_helpers.tpl | 53 ++++-- .../templates/api-server/deployment.yaml | 168 ++++++++++++------ .../pipeline/templates/api-server/role.yaml | 4 +- .../templates/api-server/rolebinding.yaml | 6 +- .../templates/api-server/service.yaml | 23 +-- .../templates/api-server/serviceaccount.yaml | 9 +- .../templates/cache-deployer/deployment.yaml | 38 +--- .../templates/cache-deployer/role.yaml | 20 +++ .../templates/cache-deployer/rolebinding.yaml | 16 ++ .../cache-deployer/serviceaccount.yaml | 2 +- .../pipeline/templates/cache/deployment.yaml | 123 +++++++------ .../charts/pipeline/templates/cache/role.yaml | 36 ++++ .../pipeline/templates/cache/rolebinding.yaml | 17 ++ .../pipeline/templates/cache/service.yaml | 18 +- .../templates/cache/serviceaccount.yaml | 11 +- .../config/kfp-launcher-configmap.yaml | 7 +- .../config/pipeline-install-config.yaml | 5 +- .../templates/config/ui-configmap.yaml | 8 +- .../templates/database/mysql-secret.yaml | 4 +- .../templates/metadata-writer/deployment.yaml | 49 ++--- .../templates/metadata-writer/role.yaml | 5 +- .../metadata-writer/rolebinding.yaml | 5 +- .../metadata-writer/serviceaccount.yaml | 9 +- .../templates/metadata/envoy-deployment.yaml | 44 ++--- .../templates/metadata/envoy-service.yaml | 17 +- .../templates/metadata/grpc-configmap.yaml | 8 +- .../templates/metadata/grpc-deployment.yaml | 86 ++++----- .../templates/metadata/grpc-service.yaml | 17 +- .../metadata/grpc-serviceaccount.yaml | 8 +- .../persistence-agent/deployment.yaml | 58 ++---- .../templates/persistence-agent/role.yaml | 4 +- .../persistence-agent/rolebinding.yaml | 6 +- .../persistence-agent/serviceaccount.yaml | 9 +- .../rbac/additional-serviceaccounts.yaml | 18 +- .../templates/rbac/pipeline-runner-role.yaml | 5 +- .../rbac/pipeline-runner-rolebinding.yaml | 7 +- .../scheduled-workflow/deployment.yaml | 63 +++---- .../templates/scheduled-workflow/role.yaml | 5 +- .../scheduled-workflow/rolebinding.yaml | 6 +- .../scheduled-workflow/serviceaccount.yaml | 9 +- .../templates/third-party/argo-role.yaml | 124 +++++++++++++ .../third-party/argo-rolebinding.yaml | 19 ++ .../third-party/minio-deployment.yaml | 69 +++++++ .../templates/third-party/minio-pvc.yaml | 15 ++ .../templates/third-party/minio-secret.yaml | 12 ++ .../templates/third-party/minio-service.yaml | 18 ++ .../third-party/mysql-deployment.yaml | 64 +++++++ .../templates/third-party/mysql-pvc.yaml | 15 ++ .../templates/third-party/mysql-service.yaml | 17 ++ .../third-party/mysql-serviceaccount.yaml | 9 + .../workflow-controller-configmap.yaml | 39 ++++ .../workflow-controller-deployment.yaml | 70 ++++++++ .../workflow-controller-priorityclass.yaml | 11 ++ .../workflow-controller-serviceaccount.yaml | 11 ++ .../pipeline/templates/ui/deployment.yaml | 85 ++++----- .../charts/pipeline/templates/ui/role.yaml | 48 +++++ .../pipeline/templates/ui/rolebinding.yaml | 17 ++ .../charts/pipeline/templates/ui/service.yaml | 14 +- .../pipeline/templates/ui/serviceaccount.yaml | 9 +- .../templates/viewer-crd/deployment.yaml | 50 ++---- .../pipeline/templates/viewer-crd/role.yaml | 36 ++++ .../templates/viewer-crd/rolebinding.yaml | 17 ++ .../templates/viewer-crd/serviceaccount.yaml | 9 +- .../templates/visualization/deployment.yaml | 70 +++----- .../templates/visualization/service.yaml | 12 +- .../visualization/serviceaccount.yaml | 9 +- .../webhook/webhook-server-tls-secret.yaml | 2 +- experimental/helm/charts/pipeline/values.yaml | 28 ++- tests/helm_kustomize_compare.py | 25 ++- tests/helm_kustomize_compare.sh | 6 +- 73 files changed, 1299 insertions(+), 735 deletions(-) create mode 100644 experimental/helm/charts/pipeline/templates/cache-deployer/role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/cache-deployer/rolebinding.yaml create mode 100644 experimental/helm/charts/pipeline/templates/cache/role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/argo-role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/argo-rolebinding.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/minio-pvc.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/mysql-pvc.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/mysql-serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/workflow-controller-configmap.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/workflow-controller-priorityclass.yaml create mode 100644 experimental/helm/charts/pipeline/templates/third-party/workflow-controller-serviceaccount.yaml create mode 100644 experimental/helm/charts/pipeline/templates/ui/role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/ui/rolebinding.yaml create mode 100644 experimental/helm/charts/pipeline/templates/viewer-crd/role.yaml create mode 100644 experimental/helm/charts/pipeline/templates/viewer-crd/rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/Chart.yaml b/experimental/helm/charts/pipeline/Chart.yaml index d7f5412d03..a3c321f901 100644 --- a/experimental/helm/charts/pipeline/Chart.yaml +++ b/experimental/helm/charts/pipeline/Chart.yaml @@ -25,11 +25,11 @@ dependencies: condition: minio.enabled repository: "https://charts.min.io/" - name: mysql - version: "~9.0.0" + version: "~14.0.0" condition: mysql.enabled repository: "https://charts.bitnami.com/bitnami" - name: postgresql - version: "~12.0.0" + version: "~16.0.0" condition: postgresql.enabled repository: "https://charts.bitnami.com/bitnami" annotations: diff --git a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml index 34e42fe627..4b62cd5c40 100644 --- a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml @@ -14,9 +14,9 @@ installMode: multiUser: enabled: false -# Database Configuration - Using MySQL subchart +# Database Configuration - Using built-in MySQL mysql: - enabled: true + enabled: false auth: rootPassword: "root123" username: "mlpipeline" @@ -35,12 +35,12 @@ externalDatabase: host: "" port: 3306 database: mlpipeline - username: mlpipeline + username: root password: "" -# Object Storage Configuration - Using MinIO subchart +# Object Storage Configuration - Using built-in MinIO minio: - enabled: true + enabled: false auth: rootUser: "minio" rootPassword: "minio123" @@ -62,57 +62,61 @@ database: # Pipeline Configuration pipeline: + # Platform-agnostic specific: Empty defaultPipelineRoot to match kustomize + defaultPipelineRoot: "" cache: image: "registry.k8s.io/busybox" nodeRestrictions: false maximumStaleness: "" defaultStaleness: "" -# Argo Workflows Configuration +# Argo Workflows Configuration - use built-in argo: - enabled: true + enabled: false # Basic Argo configuration for platform-agnostic deployment +# Built-in third-party components +thirdParty: + mysql: + enabled: true + minio: + enabled: true + argo: + enabled: true + # Environment Configuration env: platform: platform-agnostic -# Resource Configuration +# Resource Configuration apiServer: resources: requests: cpu: 250m memory: 500Mi - limits: - cpu: 500m - memory: 1Gi persistenceAgent: resources: requests: cpu: 120m memory: 500Mi - limits: - cpu: 250m - memory: 1Gi scheduledWorkflow: resources: - requests: - cpu: 120m - memory: 100Mi - limits: - cpu: 250m - memory: 500Mi ui: resources: requests: cpu: 10m memory: 70Mi - limits: - cpu: 100m - memory: 200Mi + +# Disable CRDs for platform-agnostic scenario +crds: + install: false + +# Disable webhooks for platform-agnostic scenario +webhooks: + enabled: false metadata: grpc: @@ -145,20 +149,13 @@ cache: visualization: resources: requests: - cpu: 50m - memory: 200Mi - limits: - cpu: 100m - memory: 400Mi + cpu: 30m + memory: 500Mi viewerCrd: + image: + pullPolicy: Always resources: - requests: - cpu: 120m - memory: 100Mi - limits: - cpu: 250m - memory: 200Mi # Monitoring disabled for platform-agnostic monitoring: diff --git a/experimental/helm/charts/pipeline/ci/values-standalone.yaml b/experimental/helm/charts/pipeline/ci/values-standalone.yaml index 15956333f3..97fc89165b 100644 --- a/experimental/helm/charts/pipeline/ci/values-standalone.yaml +++ b/experimental/helm/charts/pipeline/ci/values-standalone.yaml @@ -5,9 +5,9 @@ env: platform: platform-agnostic -# Database configuration - use MySQL subchart +# Database configuration - use built-in MySQL mysql: - enabled: true + enabled: false auth: rootPassword: "" database: "mlpipeline" @@ -23,12 +23,12 @@ postgresql: externalDatabase: type: mysql -# Object storage configuration - use MinIO subchart +# Object storage configuration - use built-in MinIO objectStore: provider: minio minio: - enabled: true + enabled: false auth: rootUser: "minio" rootPassword: "minio123" @@ -37,9 +37,18 @@ minio: enabled: true size: 20Gi -# Argo Workflows configuration +# Argo Workflows configuration - use built-in argo: - enabled: true + enabled: false + +# Built-in third-party components +thirdParty: + mysql: + enabled: true + minio: + enabled: true + argo: + enabled: true # Disable cloud-specific features monitoring: @@ -54,3 +63,11 @@ certManager: networking: istio: enabled: false + +# Disable CRDs for platform-agnostic scenario +crds: + install: false + +# Disable webhooks for platform-agnostic scenario +webhooks: + enabled: false diff --git a/experimental/helm/charts/pipeline/templates/_helpers.tpl b/experimental/helm/charts/pipeline/templates/_helpers.tpl index c73fb53c84..cb306fdce0 100644 --- a/experimental/helm/charts/pipeline/templates/_helpers.tpl +++ b/experimental/helm/charts/pipeline/templates/_helpers.tpl @@ -31,16 +31,9 @@ Create chart name and version as used by the chart label. {{- end }} {{/* -Common labels +Common labels - simplified to match kustomize output */}} {{- define "kubeflow-pipelines.labels" -}} -helm.sh/chart: {{ include "kubeflow-pipelines.chart" . }} -{{ include "kubeflow-pipelines.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -app.kubernetes.io/component: ml-pipeline application-crd-id: kubeflow-pipelines {{- with .Values.commonLabels }} {{ toYaml . }} @@ -55,12 +48,48 @@ app.kubernetes.io/name: {{ include "kubeflow-pipelines.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{/* +Cache server labels +*/}} +{{- define "kubeflow-pipelines.cacheLabels" -}} +app: cache-server +application-crd-id: kubeflow-pipelines +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Cache deployer labels +*/}} +{{- define "kubeflow-pipelines.cacheDeployerLabels" -}} +app: cache-deployer +application-crd-id: kubeflow-pipelines +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + {{/* ML Pipeline specific labels - matching original manifests */}} {{- define "kubeflow-pipelines.mlPipelineLabels" -}} -{{ include "kubeflow-pipelines.labels" . }} app: ml-pipeline +application-crd-id: kubeflow-pipelines +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +ML Pipeline UI labels +*/}} +{{- define "kubeflow-pipelines.uiLabels" -}} +app: ml-pipeline-ui +application-crd-id: kubeflow-pipelines +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} {{- end }} {{/* @@ -121,7 +150,7 @@ Create the name of the service account to use for Cache Server */}} {{- define "kubeflow-pipelines.cache.serviceAccountName" -}} {{- if .Values.cache.serviceAccount.create }} -{{- default "cache-server" .Values.cache.serviceAccount.name }} +{{- default "kubeflow-pipelines-cache" .Values.cache.serviceAccount.name }} {{- else }} {{- default "default" .Values.cache.serviceAccount.name }} {{- end }} @@ -132,7 +161,7 @@ Create the name of the service account to use for Cache Deployer */}} {{- define "kubeflow-pipelines.cacheDeployer.serviceAccountName" -}} {{- if .Values.cacheDeployer.serviceAccount.create }} -{{- default "cache-deployer-sa" .Values.cacheDeployer.serviceAccount.name }} +{{- default "kubeflow-pipelines-cache-deployer-sa" .Values.cacheDeployer.serviceAccount.name }} {{- else }} {{- default "default" .Values.cacheDeployer.serviceAccount.name }} {{- end }} @@ -233,7 +262,7 @@ mysql-secret {{- else if .Values.postgresql.enabled -}} postgresql-secret {{- else -}} -{{ include "kubeflow-pipelines.fullname" . }}-mysql-secret +mysql-secret {{- end -}} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml index be0097977e..87060d75f4 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml @@ -2,68 +2,133 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} - component: api-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app: ml-pipeline + application-crd-id: kubeflow-pipelines spec: - replicas: {{ .Values.apiServer.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} - {{- include "kubeflow-pipelines.mlPipelineSelectorLabels" . | nindent 6 }} - component: api-server + app: ml-pipeline + application-crd-id: kubeflow-pipelines template: metadata: - labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - {{- include "kubeflow-pipelines.mlPipelineSelectorLabels" . | nindent 8 }} - component: api-server annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + labels: + app: ml-pipeline + application-crd-id: kubeflow-pipelines spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} - securityContext: - {{- if .Values.security.seccompProfile }} - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} - {{- end }} + serviceAccountName: ml-pipeline containers: - name: ml-pipeline-api-server image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.apiServer.image.repository "tag" .Values.apiServer.image.tag "context" .) }} - imagePullPolicy: {{ .Values.apiServer.image.pullPolicy | default .Values.global.imagePullPolicy }} + imagePullPolicy: IfNotPresent ports: - name: http containerPort: 8888 - protocol: TCP - name: grpc containerPort: 8887 - protocol: TCP env: - {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} - {{- include "kubeflow-pipelines.databaseEnv" . | nindent 8 }} - {{- include "kubeflow-pipelines.objectStoreEnv" . | nindent 8 }} # API Server specific environment variables - name: PUBLISH_LOGS value: {{ .Values.apiServer.env.publishLogs | quote }} + - name: LOG_LEVEL + value: {{ .Values.global.logLevel | quote }} - name: PIPELINE_LOG_LEVEL value: {{ .Values.apiServer.env.pipelineLogLevel | quote }} - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - value: {{ .Values.apiServer.env.autoUpdatePipelineDefaultVersion | quote }} + valueFrom: + configMapKeyRef: + key: autoUpdatePipelineDefaultVersion + name: pipeline-install-config + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + valueFrom: + configMapKeyRef: + key: bucketName + name: pipeline-install-config + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + key: username + name: mysql-secret + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: mysql-secret + - name: DBCONFIG_DBNAME + valueFrom: + configMapKeyRef: + key: pipelineDb + name: pipeline-install-config + - name: DBCONFIG_HOST + valueFrom: + configMapKeyRef: + key: dbHost + name: pipeline-install-config + - name: DBCONFIG_PORT + valueFrom: + configMapKeyRef: + key: dbPort + name: pipeline-install-config + - name: DBCONFIG_CONMAXLIFETIME + valueFrom: + configMapKeyRef: + key: ConMaxLifeTime + name: pipeline-install-config + - name: DB_DRIVER_NAME + valueFrom: + configMapKeyRef: + key: dbType + name: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_USER + valueFrom: + secretKeyRef: + key: username + name: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_DBNAME + valueFrom: + configMapKeyRef: + key: pipelineDb + name: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_HOST + valueFrom: + configMapKeyRef: + key: mysqlHost + name: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_PORT + valueFrom: + configMapKeyRef: + key: mysqlPort + name: pipeline-install-config + - name: OBJECTSTORECONFIG_ACCESSKEY + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact - name: V2_DRIVER_IMAGE value: {{ .Values.apiServer.env.v2DriverImage | quote }} - name: V2_LAUNCHER_IMAGE value: {{ .Values.apiServer.env.v2LauncherImage | quote }} - {{- with .Values.apiServer.additionalEnv }} - {{- toYaml . | nindent 8 }} - {{- end }} - readinessProbe: + livenessProbe: exec: command: - wget @@ -72,10 +137,10 @@ spec: - -O - "-" - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: {{ .Values.apiServer.probes.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.apiServer.probes.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.apiServer.probes.readiness.timeoutSeconds }} - livenessProbe: + initialDelaySeconds: {{ .Values.apiServer.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.apiServer.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.apiServer.probes.liveness.timeoutSeconds }} + readinessProbe: exec: command: - wget @@ -84,9 +149,13 @@ spec: - -O - "-" - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: {{ .Values.apiServer.probes.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.apiServer.probes.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.apiServer.probes.liveness.timeoutSeconds }} + initialDelaySeconds: {{ .Values.apiServer.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.apiServer.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.apiServer.probes.readiness.timeoutSeconds }} + resources: + {{- toYaml .Values.apiServer.resources | nindent 10 }} + securityContext: + {{- toYaml .Values.apiServer.securityContext | nindent 10 }} startupProbe: exec: command: @@ -99,20 +168,7 @@ spec: failureThreshold: {{ .Values.apiServer.probes.startup.failureThreshold }} periodSeconds: {{ .Values.apiServer.probes.startup.periodSeconds }} timeoutSeconds: {{ .Values.apiServer.probes.startup.timeoutSeconds }} - securityContext: - {{- toYaml .Values.apiServer.securityContext | nindent 10 }} - resources: - {{- toYaml .Values.apiServer.resources | nindent 10 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + securityContext: + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/api-server/role.yaml b/experimental/helm/charts/pipeline/templates/api-server/role.yaml index 4b3ffc1c17..0c40125791 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/role.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/role.yaml @@ -2,12 +2,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} - component: api-server annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} rules: diff --git a/experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml index 4061568fc2..0c0d58531f 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml @@ -2,18 +2,16 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} - component: api-server annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + name: ml-pipeline subjects: - kind: ServiceAccount name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} diff --git a/experimental/helm/charts/pipeline/templates/api-server/service.yaml b/experimental/helm/charts/pipeline/templates/api-server/service.yaml index f1582aa3cc..26e5b5ea3e 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/service.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/service.yaml @@ -2,30 +2,25 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-api-server + name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} - component: api-server annotations: - prometheus.io/port: "8888" - prometheus.io/scheme: http + prometheus.io/port: {{ .Values.apiServer.service.port | quote }} + prometheus.io/scheme: {{ .Values.apiServer.service.scheme | default "http" }} prometheus.io/scrape: "true" - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} spec: - type: {{ .Values.apiServer.service.type }} ports: - name: http port: {{ .Values.apiServer.service.port }} protocol: TCP - targetPort: 8888 + targetPort: {{ .Values.apiServer.service.port }} - name: grpc port: {{ .Values.apiServer.service.grpcPort }} protocol: TCP - targetPort: 8887 + targetPort: {{ .Values.apiServer.service.grpcPort }} selector: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} - {{- include "kubeflow-pipelines.mlPipelineSelectorLabels" . | nindent 4 }} - component: api-server + app: ml-pipeline + application-crd-id: kubeflow-pipelines {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml index 022868f32f..f37810d5b7 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml @@ -2,15 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} - component: api-server + {{- with .Values.apiServer.serviceAccount.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.apiServer.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml b/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml index d3515e7088..59e6ae288e 100644 --- a/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml @@ -2,62 +2,34 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-cache-deployer + name: cache-deployer-deployment namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: cache-deployer - component: cache-deployer - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: replicas: {{ .Values.cacheDeployer.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} app: cache-deployer - component: cache-deployer + application-crd-id: kubeflow-pipelines strategy: type: Recreate template: metadata: labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} app: cache-deployer - component: cache-deployer - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + application-crd-id: kubeflow-pipelines spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.cacheDeployer.serviceAccountName" . }} + serviceAccountName: kubeflow-pipelines-cache-deployer-sa containers: - name: main image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.cacheDeployer.image.repository "tag" .Values.cacheDeployer.image.tag "context" .) }} - imagePullPolicy: {{ .Values.cacheDeployer.image.pullPolicy | default .Values.global.imagePullPolicy }} + imagePullPolicy: Always env: - {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} - name: NAMESPACE_TO_WATCH valueFrom: fieldRef: fieldPath: metadata.namespace - {{- with .Values.cacheDeployer.additionalEnv }} - {{- toYaml . | nindent 8 }} - {{- end }} - securityContext: - {{- toYaml .Values.cacheDeployer.securityContext | nindent 10 }} - resources: - {{- toYaml .Values.cacheDeployer.resources | nindent 10 }} restartPolicy: Always - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/role.yaml b/experimental/helm/charts/pipeline/templates/cache-deployer/role.yaml new file mode 100644 index 0000000000..56bcc795b9 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/cache-deployer/role.yaml @@ -0,0 +1,20 @@ +{{- if .Values.cacheDeployer.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "kubeflow-pipelines.cacheDeployerLabels" . | nindent 4 }} + name: kubeflow-pipelines-cache-deployer-role + namespace: {{ include "kubeflow-pipelines.namespace" . }} +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - patch + - list +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/cache-deployer/rolebinding.yaml new file mode 100644 index 0000000000..d3a56142c9 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/cache-deployer/rolebinding.yaml @@ -0,0 +1,16 @@ +{{- if .Values.cacheDeployer.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kubeflow-pipelines.cacheDeployerLabels" . | nindent 4 }} + name: kubeflow-pipelines-cache-deployer-rolebinding + namespace: {{ include "kubeflow-pipelines.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeflow-pipelines-cache-deployer-role +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.cacheDeployer.serviceAccountName" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml index 55bc597403..2997df22b7 100644 --- a/experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.cacheDeployer.enabled .Values.cacheDeployer.serviceAccount.create }} +{{- if false }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/experimental/helm/charts/pipeline/templates/cache/deployment.yaml b/experimental/helm/charts/pipeline/templates/cache/deployment.yaml index 86d1d70fd7..c48f634278 100644 --- a/experimental/helm/charts/pipeline/templates/cache/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/cache/deployment.yaml @@ -2,104 +2,109 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-cache-server + name: cache-server namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: cache-server - component: cache-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- include "kubeflow-pipelines.cacheLabels" . | nindent 4 }} spec: replicas: {{ .Values.cache.server.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} app: cache-server - component: cache-server + application-crd-id: kubeflow-pipelines template: metadata: labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - app: cache-server - component: cache-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + {{- include "kubeflow-pipelines.cacheLabels" . | nindent 8 }} spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + serviceAccountName: kubeflow-pipelines-cache securityContext: - {{- if .Values.security.seccompProfile }} seccompProfile: - type: {{ .Values.security.seccompProfile.type }} - {{- end }} + type: RuntimeDefault containers: - name: server image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.cache.server.image.repository "tag" .Values.cache.server.image.tag "context" .) }} - imagePullPolicy: {{ .Values.cache.server.image.pullPolicy | default .Values.global.imagePullPolicy }} + imagePullPolicy: Always + args: + - "--db_driver=$(DBCONFIG_DRIVER)" + - "--db_host=$(DBCONFIG_HOST_NAME)" + - "--db_port=$(DBCONFIG_PORT)" + - "--db_name=$(DBCONFIG_DB_NAME)" + - "--db_user=$(DBCONFIG_USER)" + - "--db_password=$(DBCONFIG_PASSWORD)" + - "--namespace_to_watch=$(NAMESPACE_TO_WATCH)" + - "--listen_port=$(WEBHOOK_PORT)" env: - {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} - {{- include "kubeflow-pipelines.databaseEnv" . | nindent 8 }} - name: DEFAULT_CACHE_STALENESS - value: {{ .Values.pipeline.cache.defaultStaleness | quote }} + valueFrom: + configMapKeyRef: + key: DEFAULT_CACHE_STALENESS + name: pipeline-install-config - name: MAXIMUM_CACHE_STALENESS - value: {{ .Values.pipeline.cache.maximumStaleness | quote }} + valueFrom: + configMapKeyRef: + key: MAXIMUM_CACHE_STALENESS + name: pipeline-install-config - name: CACHE_IMAGE - value: {{ .Values.pipeline.cache.image | quote }} + valueFrom: + configMapKeyRef: + key: cacheImage + name: pipeline-install-config - name: CACHE_NODE_RESTRICTIONS - value: {{ .Values.pipeline.cache.nodeRestrictions | quote }} + valueFrom: + configMapKeyRef: + key: cacheNodeRestrictions + name: pipeline-install-config - name: DBCONFIG_DRIVER - value: {{ .Values.externalDatabase.type | default "mysql" }} + value: mysql - name: DBCONFIG_DB_NAME - value: {{ .Values.database.cachedb }} + valueFrom: + configMapKeyRef: + key: cacheDb + name: pipeline-install-config - name: DBCONFIG_HOST_NAME - value: {{ include "kubeflow-pipelines.database.host" . }} + valueFrom: + configMapKeyRef: + key: dbHost + name: pipeline-install-config - name: DBCONFIG_PORT - value: {{ include "kubeflow-pipelines.database.port" . | quote }} + valueFrom: + configMapKeyRef: + key: dbPort + name: pipeline-install-config + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + key: username + name: mysql-secret + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: mysql-secret - name: NAMESPACE_TO_WATCH valueFrom: fieldRef: fieldPath: metadata.namespace - name: WEBHOOK_PORT value: "8443" - {{- with .Values.cache.server.additionalEnv }} - {{- toYaml . | nindent 8 }} - {{- end }} - args: - - "--db_driver=$(DBCONFIG_DRIVER)" - - "--db_host=$(DBCONFIG_HOST_NAME)" - - "--db_port=$(DBCONFIG_PORT)" - - "--db_name=$(DBCONFIG_DB_NAME)" - - "--db_user=$(DBCONFIG_USER)" - - "--db_password=$(DBCONFIG_PASSWORD)" - - "--namespace_to_watch=$(NAMESPACE_TO_WATCH)" - - "--listen_port=$(WEBHOOK_PORT)" ports: - name: webhook-api containerPort: 8443 - protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 0 + runAsNonRoot: true + runAsUser: 1000 volumeMounts: - name: webhook-tls-certs mountPath: /etc/webhook/certs readOnly: true - securityContext: - {{- toYaml .Values.cache.server.securityContext | nindent 10 }} - resources: - {{- toYaml .Values.cache.server.resources | nindent 10 }} volumes: - name: webhook-tls-certs secret: - secretName: {{ include "kubeflow-pipelines.fullname" . }}-webhook-server-tls - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + secretName: webhook-server-tls {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache/role.yaml b/experimental/helm/charts/pipeline/templates/cache/role.yaml new file mode 100644 index 0000000000..7a6b89b3ba --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/cache/role.yaml @@ -0,0 +1,36 @@ +{{- if .Values.cache.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "kubeflow-pipelines.cacheLabels" . | nindent 4 }} + name: kubeflow-pipelines-cache-role + namespace: {{ include "kubeflow-pipelines.namespace" . }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch + - update + - patch +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml new file mode 100644 index 0000000000..3ecbddb453 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.cache.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kubeflow-pipelines.cacheLabels" . | nindent 4 }} + name: kubeflow-pipelines-cache-binding + namespace: {{ include "kubeflow-pipelines.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeflow-pipelines-cache-role +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache/service.yaml b/experimental/helm/charts/pipeline/templates/cache/service.yaml index 1b9aa6275b..62453e9889 100644 --- a/experimental/helm/charts/pipeline/templates/cache/service.yaml +++ b/experimental/helm/charts/pipeline/templates/cache/service.yaml @@ -2,23 +2,15 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-cache-server + name: cache-server namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: cache-server - component: cache-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- include "kubeflow-pipelines.cacheLabels" . | nindent 4 }} spec: - type: {{ .Values.cache.server.service.type }} ports: - - name: webhook-api - protocol: TCP - port: {{ .Values.cache.server.service.port }} - targetPort: 8443 + - port: {{ .Values.cache.server.service.port }} + targetPort: webhook-api selector: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} app: cache-server - component: cache-server + application-crd-id: kubeflow-pipelines {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml index 82a7294f31..ccd96afafd 100644 --- a/experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml @@ -2,15 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + name: kubeflow-pipelines-cache namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: cache-server - component: cache-server + {{- include "kubeflow-pipelines.cacheLabels" . | nindent 4 }} + {{- with .Values.cache.serviceAccount.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.cache.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml b/experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml index 8c2c48f191..443a0eee94 100644 --- a/experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml +++ b/experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml @@ -1,12 +1,9 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-launcher + name: kfp-launcher namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: launcher - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} data: - defaultPipelineRoot: {{ .Values.pipeline.defaultPipelineRoot | quote }} + defaultPipelineRoot: "" diff --git a/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml b/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml index 0c56c24f7a..e01b571b09 100644 --- a/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml +++ b/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml @@ -1,13 +1,10 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-install-config + name: pipeline-install-config namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: config - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} data: warning: | 1. Do not use kubectl to edit this configmap, because some values are used diff --git a/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml b/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml index f227b33000..bbb6795e38 100644 --- a/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml +++ b/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml @@ -2,19 +2,15 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-ui-config + name: ml-pipeline-ui-configmap namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-ui - component: ui-config - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} data: viewer-pod-template.json: |- { "spec": { - "serviceAccountName": "{{ include "kubeflow-pipelines.fullname" . }}-viewer" + "serviceAccountName": "kubeflow-pipelines-viewer" } } {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml b/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml index 30ecfdb845..44af598c3f 100644 --- a/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml +++ b/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml @@ -2,14 +2,12 @@ apiVersion: v1 kind: Secret metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-mysql-secret + name: mysql-secret namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: database annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -type: Opaque stringData: username: {{ .Values.externalDatabase.username | quote }} password: {{ .Values.externalDatabase.password | quote }} diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml index 0eeb7738b3..66b002c6bd 100644 --- a/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml @@ -2,64 +2,41 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-writer + name: metadata-writer namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: metadata-writer - component: metadata-writer - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + application-crd-id: kubeflow-pipelines spec: replicas: {{ .Values.metadataWriter.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} app: metadata-writer - component: metadata-writer + application-crd-id: kubeflow-pipelines template: metadata: labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} app: metadata-writer - component: metadata-writer - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + application-crd-id: kubeflow-pipelines spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} securityContext: - {{- if .Values.security.seccompProfile }} seccompProfile: - type: {{ .Values.security.seccompProfile.type }} - {{- end }} + type: RuntimeDefault + serviceAccountName: kubeflow-pipelines-metadata-writer containers: - name: main image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.metadataWriter.image.repository "tag" .Values.metadataWriter.image.tag "context" .) }} - imagePullPolicy: {{ .Values.metadataWriter.image.pullPolicy | default .Values.global.imagePullPolicy }} env: - {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} - name: NAMESPACE_TO_WATCH valueFrom: fieldRef: fieldPath: metadata.namespace - {{- with .Values.metadataWriter.additionalEnv }} - {{- toYaml . | nindent 8 }} - {{- end }} securityContext: - {{- toYaml .Values.metadataWriter.securityContext | nindent 10 }} - resources: - {{- toYaml .Values.metadataWriter.resources | nindent 10 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 0 + runAsNonRoot: true + runAsUser: 1000 {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml index 41a532ccf3..c07d9aac97 100644 --- a/experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml @@ -2,12 +2,11 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-writer-role + name: kubeflow-pipelines-metadata-writer-role namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: kubeflow-pipelines-metadata-writer-role - component: metadata-writer + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} rules: diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml index 63223f601e..5e3689b3b5 100644 --- a/experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml @@ -2,17 +2,16 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-writer-binding + name: kubeflow-pipelines-metadata-writer-binding namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: metadata-writer annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-writer-role + name: kubeflow-pipelines-metadata-writer-role subjects: - kind: ServiceAccount name: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml index 05f2410024..fe60fd19e7 100644 --- a/experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml @@ -2,15 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} + name: kubeflow-pipelines-metadata-writer namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: metadata-writer - component: metadata-writer + {{- with .Values.metadataWriter.serviceAccount.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.metadataWriter.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml index d25a61e55d..542cd2a6fe 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml @@ -2,58 +2,40 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-envoy + name: metadata-envoy-deployment namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} component: metadata-envoy - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + application-crd-id: kubeflow-pipelines spec: replicas: {{ .Values.metadata.envoy.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} component: metadata-envoy + application-crd-id: kubeflow-pipelines template: metadata: labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} component: metadata-envoy + application-crd-id: kubeflow-pipelines sidecar.istio.io/inject: "false" - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} containers: - name: container image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.metadata.envoy.image.repository "tag" .Values.metadata.envoy.image.tag "context" .) }} - imagePullPolicy: {{ .Values.metadata.envoy.image.pullPolicy | default .Values.global.imagePullPolicy }} ports: - name: md-envoy containerPort: 9090 - protocol: TCP - name: envoy-admin containerPort: 9901 - protocol: TCP securityContext: - {{- toYaml .Values.metadata.envoy.securityContext | nindent 10 }} - resources: - {{- toYaml .Values.metadata.envoy.resources | nindent 10 }} - {{- with .Values.metadata.envoy.additionalEnv }} - env: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 0 + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml b/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml index 78004b7bd1..9b4e78923f 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml @@ -2,21 +2,18 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-envoy + name: metadata-envoy-service namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: metadata-envoy - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app: metadata-envoy + application-crd-id: kubeflow-pipelines spec: - type: {{ .Values.metadata.envoy.service.type }} ports: - - name: md-envoy - protocol: TCP + - name: {{ .Values.metadata.envoy.service.portName | default "md-envoy" }} port: {{ .Values.metadata.envoy.service.port }} - targetPort: 9090 + protocol: TCP selector: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + application-crd-id: kubeflow-pipelines component: metadata-envoy + type: {{ .Values.metadata.envoy.service.type }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml index 66fc954e45..79e5330bf2 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml @@ -2,14 +2,12 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-grpc-config + name: metadata-grpc-configmap namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} component: metadata-grpc-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} data: - METADATA_GRPC_SERVICE_HOST: "{{ include "kubeflow-pipelines.fullname" . }}-metadata-grpc" - METADATA_GRPC_SERVICE_PORT: "{{ .Values.metadata.grpc.service.port }}" + METADATA_GRPC_SERVICE_HOST: "metadata-grpc-service" + METADATA_GRPC_SERVICE_PORT: "8080" {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml index ad8ced3210..b7a82bb579 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml @@ -2,49 +2,56 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-grpc + name: metadata-grpc-deployment namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + application-crd-id: kubeflow-pipelines component: metadata-grpc-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: replicas: {{ .Values.metadata.grpc.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} component: metadata-grpc-server + application-crd-id: kubeflow-pipelines template: metadata: labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} component: metadata-grpc-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + application-crd-id: kubeflow-pipelines spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.metadata.serviceAccountName" . }} securityContext: - {{- if .Values.security.seccompProfile }} seccompProfile: - type: {{ .Values.security.seccompProfile.type }} - {{- end }} + type: RuntimeDefault + serviceAccountName: metadata-grpc-server containers: - name: container image: {{ .Values.metadata.grpc.image.repository }}:{{ .Values.metadata.grpc.image.tag }} - imagePullPolicy: {{ .Values.metadata.grpc.image.pullPolicy | default .Values.global.imagePullPolicy }} env: - {{- include "kubeflow-pipelines.databaseEnv" . | nindent 8 }} + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + key: username + name: mysql-secret + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: mysql-secret - name: MYSQL_DATABASE - value: {{ .Values.database.metadb }} + valueFrom: + configMapKeyRef: + key: mlmdDb + name: pipeline-install-config - name: MYSQL_HOST - value: {{ include "kubeflow-pipelines.database.host" . }} + valueFrom: + configMapKeyRef: + key: dbHost + name: pipeline-install-config - name: MYSQL_PORT - value: {{ include "kubeflow-pipelines.database.port" . | quote }} - {{- with .Values.metadata.grpc.additionalEnv }} - {{- toYaml . | nindent 8 }} - {{- end }} + valueFrom: + configMapKeyRef: + key: dbPort + name: pipeline-install-config command: ["/bin/metadata_store_server"] args: - "--grpc_port=8080" @@ -57,33 +64,26 @@ spec: ports: - name: grpc-api containerPort: 8080 - protocol: TCP livenessProbe: tcpSocket: port: grpc-api - initialDelaySeconds: {{ .Values.metadata.grpc.probes.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.metadata.grpc.probes.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.metadata.grpc.probes.liveness.timeoutSeconds }} + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 readinessProbe: tcpSocket: port: grpc-api - initialDelaySeconds: {{ .Values.metadata.grpc.probes.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.metadata.grpc.probes.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.metadata.grpc.probes.readiness.timeoutSeconds }} + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 securityContext: - {{- toYaml .Values.metadata.grpc.securityContext | nindent 10 }} - resources: - {{- toYaml .Values.metadata.grpc.resources | nindent 10 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 0 + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml index 2907501695..a4f6752be6 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml @@ -2,21 +2,18 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-metadata-grpc + name: metadata-grpc-service namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: metadata-grpc-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app: metadata + application-crd-id: kubeflow-pipelines spec: - type: {{ .Values.metadata.grpc.service.type }} ports: - - name: grpc-api - protocol: TCP + - name: {{ .Values.metadata.grpc.service.portName | default "grpc-api" }} port: {{ .Values.metadata.grpc.service.port }} - targetPort: 8080 + protocol: TCP selector: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} + application-crd-id: kubeflow-pipelines component: metadata-grpc-server + type: {{ .Values.metadata.grpc.service.type }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml index 05f2f926b5..fc10505bd3 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml @@ -2,14 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.metadata.serviceAccountName" . }} + name: metadata-grpc-server namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: metadata-grpc-server + {{- with .Values.metadata.grpc.serviceAccount.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.metadata.grpc.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml index ef0f648d35..ac74e91ecf 100644 --- a/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml @@ -2,44 +2,30 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-persistence-agent + name: ml-pipeline-persistenceagent namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-persistenceagent - component: persistence-agent - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + application-crd-id: kubeflow-pipelines spec: - replicas: {{ .Values.persistenceAgent.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} app: ml-pipeline-persistenceagent - component: persistence-agent + application-crd-id: kubeflow-pipelines template: metadata: - labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - app: ml-pipeline-persistenceagent - component: persistence-agent annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + labels: + app: ml-pipeline-persistenceagent + application-crd-id: kubeflow-pipelines spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} - securityContext: - {{- if .Values.security.seccompProfile }} - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} - {{- end }} + serviceAccountName: ml-pipeline-persistenceagent containers: - name: ml-pipeline-persistenceagent image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.persistenceAgent.image.repository "tag" .Values.persistenceAgent.image.tag "context" .) }} - imagePullPolicy: {{ .Values.persistenceAgent.image.pullPolicy | default .Values.global.imagePullPolicy }} + imagePullPolicy: IfNotPresent env: - {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} - name: NAMESPACE valueFrom: fieldRef: @@ -48,34 +34,24 @@ spec: value: {{ .Values.persistenceAgent.env.ttlSecondsAfterWorkflowFinish | quote }} - name: NUM_WORKERS value: {{ .Values.persistenceAgent.env.numWorkers | quote }} - {{- with .Values.persistenceAgent.additionalEnv }} - {{- toYaml . | nindent 8 }} - {{- end }} - securityContext: - {{- toYaml .Values.persistenceAgent.securityContext | nindent 10 }} + - name: LOG_LEVEL + value: {{ .Values.global.logLevel | quote }} resources: {{- toYaml .Values.persistenceAgent.resources | nindent 10 }} + securityContext: + {{- toYaml .Values.persistenceAgent.securityContext | nindent 10 }} volumeMounts: - mountPath: /var/run/secrets/kubeflow/tokens name: persistenceagent-sa-token + securityContext: + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} volumes: - name: persistenceagent-sa-token projected: sources: - serviceAccountToken: - path: persistenceagent-sa-token - expirationSeconds: 3600 audience: pipelines.kubeflow.org - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + expirationSeconds: 3600 + path: persistenceagent-sa-token {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml index 2db8d1c340..7d8f127ca1 100644 --- a/experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml @@ -2,12 +2,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-persistence-agent + name: ml-pipeline-persistenceagent-role namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-persistenceagent - component: persistence-agent annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} rules: diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml index 50518e134c..c884606168 100644 --- a/experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml @@ -2,18 +2,16 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-persistence-agent + name: ml-pipeline-persistenceagent-binding namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-persistenceagent - component: persistence-agent annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ include "kubeflow-pipelines.fullname" . }}-persistence-agent + name: ml-pipeline-persistenceagent-role subjects: - kind: ServiceAccount name: {{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml index a72727e865..38b006ff91 100644 --- a/experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml @@ -2,15 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} + name: ml-pipeline-persistenceagent namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-persistenceagent - component: persistence-agent + {{- with .Values.persistenceAgent.serviceAccount.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.persistenceAgent.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml b/experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml index 470eb35883..4b348811fa 100644 --- a/experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml +++ b/experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml @@ -8,12 +8,10 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: container-builder + {{- with .Values.additionalServiceAccounts.containerBuilder.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.additionalServiceAccounts.containerBuilder.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} {{- if .Values.additionalServiceAccounts.pipelineRunner.create }} @@ -25,12 +23,10 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: pipeline-runner + {{- with .Values.additionalServiceAccounts.pipelineRunner.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.additionalServiceAccounts.pipelineRunner.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} {{- if .Values.additionalServiceAccounts.viewer.create }} @@ -42,10 +38,8 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: viewer + {{- with .Values.additionalServiceAccounts.viewer.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.additionalServiceAccounts.viewer.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml index d051574ed7..f017db64a9 100644 --- a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml +++ b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml @@ -2,11 +2,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-pipeline-runner + name: pipeline-runner namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: pipeline-runner + application-crd-id: kubeflow-pipelines annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} rules: diff --git a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml index 58e6de2491..82c6f61ff6 100644 --- a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml +++ b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml @@ -2,17 +2,16 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-pipeline-runner-binding + name: pipeline-runner-binding namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: pipeline-runner + application-crd-id: kubeflow-pipelines annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ include "kubeflow-pipelines.fullname" . }}-pipeline-runner + name: pipeline-runner subjects: - kind: ServiceAccount name: {{ .Values.additionalServiceAccounts.pipelineRunner.name }} diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml index 9445225b3e..ad7f0e4fd4 100644 --- a/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml @@ -2,78 +2,59 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-scheduled-workflow + name: ml-pipeline-scheduledworkflow namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-scheduledworkflow - component: scheduled-workflow - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + application-crd-id: kubeflow-pipelines spec: - replicas: {{ .Values.scheduledWorkflow.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} app: ml-pipeline-scheduledworkflow - component: scheduled-workflow + application-crd-id: kubeflow-pipelines template: metadata: - labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - app: ml-pipeline-scheduledworkflow - component: scheduled-workflow annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + labels: + app: ml-pipeline-scheduledworkflow + application-crd-id: kubeflow-pipelines spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} - securityContext: - {{- if .Values.security.seccompProfile }} - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} - {{- end }} + serviceAccountName: ml-pipeline-scheduledworkflow containers: - name: ml-pipeline-scheduledworkflow image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.scheduledWorkflow.image.repository "tag" .Values.scheduledWorkflow.image.tag "context" .) }} - imagePullPolicy: {{ .Values.scheduledWorkflow.image.pullPolicy | default .Values.global.imagePullPolicy }} + imagePullPolicy: IfNotPresent env: - {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} + - name: LOG_LEVEL + value: {{ .Values.global.logLevel | quote }} - name: NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: CRON_SCHEDULE_TIMEZONE - value: {{ .Values.pipeline.cronScheduleTimezone | quote }} - {{- with .Values.scheduledWorkflow.additionalEnv }} - {{- toYaml . | nindent 8 }} + valueFrom: + configMapKeyRef: + key: cronScheduleTimezone + name: pipeline-install-config + {{- if and .Values.scheduledWorkflow.resources (or .Values.scheduledWorkflow.resources.requests .Values.scheduledWorkflow.resources.limits) }} + resources: + {{- toYaml .Values.scheduledWorkflow.resources | nindent 10 }} {{- end }} securityContext: {{- toYaml .Values.scheduledWorkflow.securityContext | nindent 10 }} - resources: - {{- toYaml .Values.scheduledWorkflow.resources | nindent 10 }} volumeMounts: - mountPath: /var/run/secrets/kubeflow/tokens name: scheduledworkflow-sa-token + securityContext: + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} volumes: - name: scheduledworkflow-sa-token projected: sources: - serviceAccountToken: - path: scheduledworkflow-sa-token - expirationSeconds: 3600 audience: pipelines.kubeflow.org - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + expirationSeconds: 3600 + path: scheduledworkflow-sa-token {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml index 09cc0a52d9..ac6e8f588d 100644 --- a/experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml @@ -2,12 +2,11 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-scheduled-workflow + name: ml-pipeline-scheduledworkflow-role namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + app: ml-pipeline-scheduledworkflow-role {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-scheduledworkflow - component: scheduled-workflow annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} rules: diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml index 4012fcf380..7a3af29d4b 100644 --- a/experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml @@ -2,18 +2,16 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-scheduled-workflow + name: ml-pipeline-scheduledworkflow-binding namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-scheduledworkflow - component: scheduled-workflow annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ include "kubeflow-pipelines.fullname" . }}-scheduled-workflow + name: ml-pipeline-scheduledworkflow-role subjects: - kind: ServiceAccount name: {{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml index 9575968046..8f6cc6f420 100644 --- a/experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml @@ -2,15 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} + name: ml-pipeline-scheduledworkflow namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-scheduledworkflow - component: scheduled-workflow + {{- with .Values.scheduledWorkflow.serviceAccount.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.scheduledWorkflow.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/argo-role.yaml b/experimental/helm/charts/pipeline/templates/third-party/argo-role.yaml new file mode 100644 index 0000000000..54ee92c4a3 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/argo-role.yaml @@ -0,0 +1,124 @@ +{{- if .Values.thirdParty.argo.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + internal.kpt.dev/upstream-identifier: rbac.authorization.k8s.io|Role|default|argo-role + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + name: argo-role + namespace: {{ include "kubeflow-pipelines.namespace" . }} +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - pods + - pods/exec + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - persistentvolumeclaims/finalizers + verbs: + - create + - update + - delete + - get +- apiGroups: + - argoproj.io + resources: + - workflows + - workflows/finalizers + - workflowtasksets + - workflowtasksets/finalizers + - workflowartifactgctasks + verbs: + - get + - list + - watch + - update + - patch + - delete + - create +- apiGroups: + - argoproj.io + resources: + - workflowtemplates + - workflowtemplates/finalizers + verbs: + - get + - list + - watch +- apiGroups: + - argoproj.io + resources: + - workflowtaskresults + verbs: + - list + - watch + - deletecollection +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - list +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +- apiGroups: + - argoproj.io + resources: + - cronworkflows + - cronworkflows/finalizers + verbs: + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - get + - delete +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/argo-rolebinding.yaml b/experimental/helm/charts/pipeline/templates/third-party/argo-rolebinding.yaml new file mode 100644 index 0000000000..b7dd8f4457 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/argo-rolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.thirdParty.argo.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + internal.kpt.dev/upstream-identifier: rbac.authorization.k8s.io|RoleBinding|default|argo-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + name: argo-binding + namespace: {{ include "kubeflow-pipelines.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: argo-role +subjects: +- kind: ServiceAccount + name: argo + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml b/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml new file mode 100644 index 0000000000..c08ba64fd2 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml @@ -0,0 +1,69 @@ +{{- if .Values.thirdParty.minio.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + app: minio + application-crd-id: kubeflow-pipelines +spec: + selector: + matchLabels: + app: minio + application-crd-id: kubeflow-pipelines + strategy: + type: Recreate + template: + metadata: + labels: + app: minio + application-crd-id: kubeflow-pipelines + spec: + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" + seccompProfile: + type: RuntimeDefault + containers: + - args: + - server + - /data + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey + image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance + name: minio + ports: + - containerPort: 9000 + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /data + name: data + subPath: minio + resources: + requests: + cpu: 20m + memory: 100Mi + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-pvc +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-pvc.yaml b/experimental/helm/charts/pipeline/templates/third-party/minio-pvc.yaml new file mode 100644 index 0000000000..6fc93106ac --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/minio-pvc.yaml @@ -0,0 +1,15 @@ +{{- if .Values.thirdParty.minio.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-pvc + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml b/experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml new file mode 100644 index 0000000000..7823542094 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.thirdParty.minio.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: mlpipeline-minio-artifact + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +stringData: + accesskey: minio + secretkey: minio123 +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml b/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml new file mode 100644 index 0000000000..0787ebcc53 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.thirdParty.minio.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: minio-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + ports: + - name: http + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app: minio + application-crd-id: kubeflow-pipelines +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml b/experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml new file mode 100644 index 0000000000..1a7a0c7b2a --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml @@ -0,0 +1,64 @@ +{{- if .Values.thirdParty.mysql.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + app: mysql + application-crd-id: kubeflow-pipelines +spec: + selector: + matchLabels: + app: mysql + application-crd-id: kubeflow-pipelines + strategy: + type: Recreate + template: + metadata: + labels: + app: mysql + application-crd-id: kubeflow-pipelines + spec: + securityContext: + fsGroup: 999 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + serviceAccountName: mysql + containers: + - name: mysql + image: gcr.io/ml-pipeline/mysql:8.0.26 + args: + - --datadir + - /var/lib/mysql + - --default-authentication-plugin=mysql_native_password + # Disable binlog as the logs grow fast and eat up all disk spaces eventually. And KFP doesn't currently utilize binlog. + # https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#option_mysqld_log-bin + - --disable-log-bin + env: + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "true" + ports: + - containerPort: 3306 + name: mysql + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /var/lib/mysql + name: mysql-persistent-storage + resources: + requests: + cpu: 100m + memory: 800Mi + volumes: + - name: mysql-persistent-storage + persistentVolumeClaim: + claimName: mysql-pv-claim +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-pvc.yaml b/experimental/helm/charts/pipeline/templates/third-party/mysql-pvc.yaml new file mode 100644 index 0000000000..828edf1e1f --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/mysql-pvc.yaml @@ -0,0 +1,15 @@ +{{- if .Values.thirdParty.mysql.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-pv-claim + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml b/experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml new file mode 100644 index 0000000000..6ac8d91c1a --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.thirdParty.mysql.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + ports: + - port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: mysql + application-crd-id: kubeflow-pipelines +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/third-party/mysql-serviceaccount.yaml new file mode 100644 index 0000000000..d618ce709e --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/mysql-serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if .Values.thirdParty.mysql.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mysql + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-configmap.yaml b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-configmap.yaml new file mode 100644 index 0000000000..d27afab205 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-configmap.yaml @@ -0,0 +1,39 @@ +{{- if .Values.thirdParty.argo.enabled }} +apiVersion: v1 +data: + artifactRepository: | + archiveLogs: true + s3: + endpoint: "minio-service.kubeflow:9000" + bucket: "mlpipeline" + # keyFormat is a format pattern to define how artifacts will be organized in a bucket. + # It can reference workflow metadata variables such as workflow.namespace, workflow.name, + # pod.name. Can also use strftime formating of workflow.creationTimestamp so that workflow + # artifacts can be organized by date. If omitted, will use `{{`{{workflow.name}}`}}/{{`{{pod.name}}`}}`, + # which has potential for have collisions, because names do not guarantee they are unique + # over the lifetime of the cluster. + # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/names/. + # + # The following format looks like: + # artifacts/my-workflow-abc123/2018/08/23/my-workflow-abc123-1234567890 + # Adding date into the path greatly reduces the chance of {{`{{pod.name}}`}} collision. + keyFormat: "artifacts/{{`{{workflow.name}}`}}/{{`{{workflow.creationTimestamp.Y}}`}}/{{`{{workflow.creationTimestamp.m}}`}}/{{`{{workflow.creationTimestamp.d}}`}}/{{`{{pod.name}}`}}" + # insecure will disable TLS. Primarily used for minio installs not configured with TLS + insecure: true + accessKeySecret: + name: mlpipeline-minio-artifact + key: accesskey + secretKeySecret: + name: mlpipeline-minio-artifact + key: secretkey + executor: | + imagePullPolicy: IfNotPresent +kind: ConfigMap +metadata: + annotations: + internal.kpt.dev/upstream-identifier: '|ConfigMap|default|workflow-controller-configmap' + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + name: workflow-controller-configmap + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml new file mode 100644 index 0000000000..e2e939a8ab --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml @@ -0,0 +1,70 @@ +{{- if .Values.thirdParty.argo.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + internal.kpt.dev/upstream-identifier: apps|Deployment|default|workflow-controller + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + name: workflow-controller + namespace: {{ include "kubeflow-pipelines.namespace" . }} +spec: + selector: + matchLabels: + app: workflow-controller + application-crd-id: kubeflow-pipelines + template: + metadata: + labels: + app: workflow-controller + application-crd-id: kubeflow-pipelines + spec: + containers: + - args: + - --configmap + - workflow-controller-configmap + - --executor-image + - quay.io/argoproj/argoexec:v3.6.7 + - --namespaced + command: + - workflow-controller + env: + - name: LEADER_ELECTION_IDENTITY + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: quay.io/argoproj/workflow-controller:v3.6.7 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 6060 + initialDelaySeconds: 90 + periodSeconds: 60 + timeoutSeconds: 30 + name: workflow-controller + ports: + - containerPort: 9090 + name: metrics + - containerPort: 6060 + resources: + requests: + cpu: 100m + memory: 500Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: workflow-controller + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + serviceAccountName: argo +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-priorityclass.yaml b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-priorityclass.yaml new file mode 100644 index 0000000000..dbf0156379 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-priorityclass.yaml @@ -0,0 +1,11 @@ +{{- if .Values.thirdParty.argo.enabled }} +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + annotations: + internal.kpt.dev/upstream-identifier: scheduling.k8s.io|PriorityClass|default|workflow-controller + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + name: workflow-controller +value: 1000000 +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-serviceaccount.yaml new file mode 100644 index 0000000000..a5d24f7579 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.thirdParty.argo.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + internal.kpt.dev/upstream-identifier: '|ServiceAccount|default|argo' + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + name: argo + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/ui/deployment.yaml b/experimental/helm/charts/pipeline/templates/ui/deployment.yaml index 3314188fb3..1096c9e032 100644 --- a/experimental/helm/charts/pipeline/templates/ui/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/ui/deployment.yaml @@ -2,63 +2,56 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-ui + name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-ui - component: ui - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + application-crd-id: kubeflow-pipelines spec: - replicas: {{ .Values.ui.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} app: ml-pipeline-ui - component: ui + application-crd-id: kubeflow-pipelines template: metadata: - labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - app: ml-pipeline-ui - component: ui annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + labels: + app: ml-pipeline-ui + application-crd-id: kubeflow-pipelines spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.ui.serviceAccountName" . }} - securityContext: - {{- if .Values.security.seccompProfile }} - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} - {{- end }} + serviceAccountName: ml-pipeline-ui volumes: - name: config-volume configMap: - name: {{ include "kubeflow-pipelines.fullname" . }}-ui-config + name: ml-pipeline-ui-configmap containers: - name: ml-pipeline-ui image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.ui.image.repository "tag" .Values.ui.image.tag "context" .) }} - imagePullPolicy: {{ .Values.ui.image.pullPolicy | default .Values.global.imagePullPolicy }} + imagePullPolicy: IfNotPresent ports: - - name: http - containerPort: 3000 - protocol: TCP + - containerPort: 3000 volumeMounts: - name: config-volume mountPath: /etc/config readOnly: true env: - {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} - {{- include "kubeflow-pipelines.objectStoreEnv" . | nindent 8 }} - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH value: /etc/config/viewer-pod-template.json - name: MINIO_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact - name: ALLOW_CUSTOM_VISUALIZATIONS value: {{ .Values.ui.env.allowCustomVisualizations | quote }} - name: FRONTEND_SERVER_NAMESPACE @@ -69,10 +62,7 @@ spec: value: {{ .Values.ui.env.argoArchiveLogs | quote }} - name: DISABLE_GKE_METADATA value: {{ .Values.ui.env.disableGkeMetadata | quote }} - {{- with .Values.ui.additionalEnv }} - {{- toYaml . | nindent 8 }} - {{- end }} - readinessProbe: + livenessProbe: exec: command: - wget @@ -81,10 +71,10 @@ spec: - -O - "-" - http://localhost:3000/apis/v1beta1/healthz - initialDelaySeconds: {{ .Values.ui.probes.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.ui.probes.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.ui.probes.readiness.timeoutSeconds }} - livenessProbe: + initialDelaySeconds: {{ .Values.ui.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.ui.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.ui.probes.liveness.timeoutSeconds }} + readinessProbe: exec: command: - wget @@ -93,23 +83,14 @@ spec: - -O - "-" - http://localhost:3000/apis/v1beta1/healthz - initialDelaySeconds: {{ .Values.ui.probes.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.ui.probes.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.ui.probes.liveness.timeoutSeconds }} - securityContext: - {{- toYaml .Values.ui.securityContext | nindent 10 }} + initialDelaySeconds: {{ .Values.ui.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.ui.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.ui.probes.readiness.timeoutSeconds }} resources: {{- toYaml .Values.ui.resources | nindent 10 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + securityContext: + {{- toYaml .Values.ui.securityContext | nindent 10 }} + securityContext: + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/ui/role.yaml b/experimental/helm/charts/pipeline/templates/ui/role.yaml new file mode 100644 index 0000000000..a4b75afbca --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/ui/role.yaml @@ -0,0 +1,48 @@ +{{- if .Values.ui.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "kubeflow-pipelines.uiLabels" . | nindent 4 }} + name: ml-pipeline-ui + namespace: {{ include "kubeflow-pipelines.namespace" . }} +rules: +- apiGroups: + - "" + resources: + - pods + - pods/log + verbs: + - get +- apiGroups: + - "" + resources: + - events + verbs: + - list +- apiGroups: + - "" + resources: + - secrets + - configmaps + verbs: + - get + - list +- apiGroups: + - kubeflow.org + resources: + - viewers + verbs: + - create + - get + - list + - watch + - delete +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/ui/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/ui/rolebinding.yaml new file mode 100644 index 0000000000..a53196a948 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/ui/rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.ui.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kubeflow-pipelines.uiLabels" . | nindent 4 }} + name: ml-pipeline-ui + namespace: {{ include "kubeflow-pipelines.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ml-pipeline-ui +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.ui.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/ui/service.yaml b/experimental/helm/charts/pipeline/templates/ui/service.yaml index 3859dd67fe..dd34875eba 100644 --- a/experimental/helm/charts/pipeline/templates/ui/service.yaml +++ b/experimental/helm/charts/pipeline/templates/ui/service.yaml @@ -2,23 +2,17 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-ui + name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-ui - component: ui - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + {{- include "kubeflow-pipelines.uiLabels" . | nindent 4 }} spec: - type: {{ .Values.ui.service.type }} ports: - name: http + port: 80 protocol: TCP - port: {{ .Values.ui.service.port }} targetPort: 3000 selector: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} app: ml-pipeline-ui - component: ui + application-crd-id: kubeflow-pipelines {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml index 1cb2502773..1f5d25d711 100644 --- a/experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml @@ -2,15 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.ui.serviceAccountName" . }} + name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-ui - component: ui + {{- with .Values.ui.serviceAccount.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.ui.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml b/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml index 8eba842920..140dfd1355 100644 --- a/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml @@ -2,67 +2,43 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-viewer-crd + name: ml-pipeline-viewer-crd namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-viewer-crd - component: viewer-crd - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + application-crd-id: kubeflow-pipelines spec: - replicas: {{ .Values.viewerCrd.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} app: ml-pipeline-viewer-crd - component: viewer-crd + application-crd-id: kubeflow-pipelines template: metadata: - labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - app: ml-pipeline-viewer-crd - component: viewer-crd annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + labels: + app: ml-pipeline-viewer-crd + application-crd-id: kubeflow-pipelines spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} - securityContext: - {{- if .Values.security.seccompProfile }} - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} - {{- end }} + serviceAccountName: ml-pipeline-viewer-crd-service-account containers: - name: ml-pipeline-viewer-crd image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.viewerCrd.image.repository "tag" .Values.viewerCrd.image.tag "context" .) }} imagePullPolicy: {{ .Values.viewerCrd.image.pullPolicy | default .Values.global.imagePullPolicy }} env: - {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} - name: MAX_NUM_VIEWERS value: {{ .Values.viewerCrd.env.maxNumViewers | quote }} - name: NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - {{- with .Values.viewerCrd.additionalEnv }} - {{- toYaml . | nindent 8 }} + {{- if and .Values.viewerCrd.resources (or .Values.viewerCrd.resources.requests .Values.viewerCrd.resources.limits) }} + resources: + {{- toYaml .Values.viewerCrd.resources | nindent 10 }} {{- end }} securityContext: {{- toYaml .Values.viewerCrd.securityContext | nindent 10 }} - resources: - {{- toYaml .Values.viewerCrd.resources | nindent 10 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + securityContext: + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/role.yaml b/experimental/helm/charts/pipeline/templates/viewer-crd/role.yaml new file mode 100644 index 0000000000..6d58f027a9 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/viewer-crd/role.yaml @@ -0,0 +1,36 @@ +{{- if .Values.viewerCrd.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + name: ml-pipeline-viewer-controller-role + namespace: {{ include "kubeflow-pipelines.namespace" . }} +rules: +- apiGroups: + - '*' + resources: + - deployments + - services + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - viewers + - viewers/finalizers + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/viewer-crd/rolebinding.yaml new file mode 100644 index 0000000000..f48104909f --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/viewer-crd/rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.viewerCrd.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + name: ml-pipeline-viewer-crd-binding + namespace: {{ include "kubeflow-pipelines.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ml-pipeline-viewer-controller-role +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml index fe1aa0caac..c2ce952844 100644 --- a/experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml @@ -2,15 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} + name: ml-pipeline-viewer-crd-service-account namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-viewer-crd - component: viewer-crd + {{- with .Values.viewerCrd.serviceAccount.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.viewerCrd.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml b/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml index 73ad93313c..0bb13bada1 100644 --- a/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml @@ -2,52 +2,33 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-visualization + name: ml-pipeline-visualizationserver namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-visualizationserver - component: visualization - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + application-crd-id: kubeflow-pipelines spec: - replicas: {{ .Values.visualization.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} app: ml-pipeline-visualizationserver - component: visualization + application-crd-id: kubeflow-pipelines template: metadata: - labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - app: ml-pipeline-visualizationserver - component: visualization annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + labels: + app: ml-pipeline-visualizationserver + application-crd-id: kubeflow-pipelines spec: - {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "kubeflow-pipelines.visualization.serviceAccountName" . }} - securityContext: - {{- if .Values.security.seccompProfile }} - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} - {{- end }} + serviceAccountName: ml-pipeline-visualizationserver containers: - name: ml-pipeline-visualizationserver image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.visualization.image.repository "tag" .Values.visualization.image.tag "context" .) }} - imagePullPolicy: {{ .Values.visualization.image.pullPolicy | default .Values.global.imagePullPolicy }} + imagePullPolicy: IfNotPresent ports: - name: http containerPort: 8888 - protocol: TCP - env: - {{- include "kubeflow-pipelines.commonEnv" . | nindent 8 }} - {{- with .Values.visualization.additionalEnv }} - {{- toYaml . | nindent 8 }} - {{- end }} - readinessProbe: + livenessProbe: exec: command: - wget @@ -56,10 +37,10 @@ spec: - -O - "-" - http://localhost:8888/ - initialDelaySeconds: {{ .Values.visualization.probes.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.visualization.probes.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.visualization.probes.readiness.timeoutSeconds }} - livenessProbe: + initialDelaySeconds: {{ .Values.visualization.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.visualization.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.visualization.probes.liveness.timeoutSeconds }} + readinessProbe: exec: command: - wget @@ -68,23 +49,14 @@ spec: - -O - "-" - http://localhost:8888/ - initialDelaySeconds: {{ .Values.visualization.probes.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.visualization.probes.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.visualization.probes.liveness.timeoutSeconds }} - securityContext: - {{- toYaml .Values.visualization.securityContext | nindent 10 }} + initialDelaySeconds: {{ .Values.visualization.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.visualization.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.visualization.probes.readiness.timeoutSeconds }} resources: {{- toYaml .Values.visualization.resources | nindent 10 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + securityContext: + {{- toYaml .Values.visualization.securityContext | nindent 10 }} + securityContext: + seccompProfile: + type: {{ .Values.security.seccompProfile.type }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/visualization/service.yaml b/experimental/helm/charts/pipeline/templates/visualization/service.yaml index e77e749a48..6d075a32a3 100644 --- a/experimental/helm/charts/pipeline/templates/visualization/service.yaml +++ b/experimental/helm/charts/pipeline/templates/visualization/service.yaml @@ -2,23 +2,17 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-visualization + name: ml-pipeline-visualizationserver namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-visualizationserver - component: visualization - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: - type: {{ .Values.visualization.service.type }} ports: - name: http + port: 8888 protocol: TCP - port: {{ .Values.visualization.service.port }} targetPort: 8888 selector: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} app: ml-pipeline-visualizationserver - component: visualization + application-crd-id: kubeflow-pipelines {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml index 6096d38ae8..6442563f1d 100644 --- a/experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml @@ -2,15 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.visualization.serviceAccountName" . }} + name: ml-pipeline-visualizationserver namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-visualizationserver - component: visualization + {{- with .Values.visualization.serviceAccount.annotations }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.visualization.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml b/experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml index b3edf81f1c..e15584f081 100644 --- a/experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml +++ b/experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.cache.enabled (not .Values.webhooks.tls.existingSecret) }} +{{- if false }} apiVersion: v1 kind: Secret metadata: diff --git a/experimental/helm/charts/pipeline/values.yaml b/experimental/helm/charts/pipeline/values.yaml index 392379652b..5131ec395f 100644 --- a/experimental/helm/charts/pipeline/values.yaml +++ b/experimental/helm/charts/pipeline/values.yaml @@ -428,7 +428,7 @@ cache: # -- Service type type: ClusterIP # -- Service port - port: 8443 + port: 443 # Additional environment variables additionalEnv: [] @@ -595,12 +595,29 @@ metadata: # Third-party Dependencies Configuration argo: - enabled: true + enabled: false # Argo Workflows subchart values will be merged here +# Built-in third-party components +thirdParty: + # Use built-in minimal Argo instead of external subchart + argo: + enabled: false + + # Use built-in minimal MySQL instead of external subchart + mysql: + enabled: true + + # Use built-in minimal MinIO instead of external subchart + minio: + enabled: true + # Database Configuration mysql: - enabled: true + enabled: false + global: + security: + allowInsecureImages: true auth: rootPassword: "" database: "mlpipeline" @@ -612,6 +629,9 @@ mysql: postgresql: enabled: false + global: + security: + allowInsecureImages: true auth: postgresPassword: "" database: "mlpipeline" @@ -677,7 +697,7 @@ objectStore: # MinIO subchart configuration minio: - enabled: true + enabled: false auth: rootUser: "minio" rootPassword: "minio123" diff --git a/tests/helm_kustomize_compare.py b/tests/helm_kustomize_compare.py index a7798b9164..8f76cce2c2 100755 --- a/tests/helm_kustomize_compare.py +++ b/tests/helm_kustomize_compare.py @@ -46,6 +46,16 @@ def clean_helm_metadata(obj: Any, component: str = "katib") -> Any: # More restrictive filtering for KServe if not label_key.startswith(('helm.sh/', 'app.kubernetes.io/managed-by')): cleaned_labels[label_key] = label_value + elif component == "pipeline": + helm_specific_labels = [ + 'app.kubernetes.io/managed-by', + 'app.kubernetes.io/version', + 'app.kubernetes.io/name', + 'app.kubernetes.io/instance', + 'app.kubernetes.io/component' + ] + if not label_key.startswith('helm.sh/') and label_key not in helm_specific_labels: + cleaned_labels[label_key] = label_value else: # Standard filtering for Katib and Model Registry helm_labels = [ @@ -216,12 +226,12 @@ def get_expected_helm_extras(component: str, scenario: str) -> set: if scenario == "base": return {"AuthorizationPolicy/kserve-models-web-app"} return set() - elif component == "kubeflow-pipelines": + elif component == "pipeline": return set() # No extra resources in Helm for Kubeflow Pipelines else: return set() -def compare_manifests(kustomize_file: str, helm_file: str, component: str, scenario: str, namespace: str = "") -> bool: +def compare_manifests(kustomize_file: str, helm_file: str, component: str, scenario: str, namespace: str = "", verbose: bool = False) -> bool: """Compare Kustomize and Helm manifests.""" kustomize_manifests = load_manifests(kustomize_file) helm_manifests = load_manifests(helm_file) @@ -271,6 +281,11 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena if differences: print(f"Differences in {key}: {len(differences)} fields") + if verbose: + print(f" Detailed differences for {key}:") + for diff in differences: + print(f" {diff}") + print() differences_found.append(key) success = False @@ -283,7 +298,7 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena if __name__ == "__main__": if len(sys.argv) < 5: print("Usage: python compare.py [namespace] [--verbose]") - print("Components: katib, model-registry, kserve-models-web-app, kubeflow-pipelines") + print("Components: katib, model-registry, kserve-models-web-app, pipeline") sys.exit(1) kustomize_file = sys.argv[1] @@ -296,6 +311,6 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena print(f"ERROR: Unknown component: {component}") print("Supported components: katib, model-registry, kserve-models-web-app, notebook-controller","kubeflow-pipelines") sys.exit(1) - - success = compare_manifests(kustomize_file, helm_file, component, scenario, namespace) + + success = compare_manifests(kustomize_file, helm_file, component, scenario, namespace, verbose) sys.exit(0 if success else 1) \ No newline at end of file diff --git a/tests/helm_kustomize_compare.sh b/tests/helm_kustomize_compare.sh index 718df3946b..0372c11842 100755 --- a/tests/helm_kustomize_compare.sh +++ b/tests/helm_kustomize_compare.sh @@ -246,7 +246,7 @@ if [ ! -d "$CHART_DIR" ]; then exit 1 fi -if [[ "$COMPONENT" != "kserve-models-web-app" ]] && [[ "$COMPONENT" != "kubeflow-pipelines" || "$SCENARIO" != "generic" ]] && [ ! -f "$HELM_VALUES_ARG" ]; then +if [[ "$COMPONENT" != "kserve-models-web-app" ]] && [[ "$COMPONENT" != "pipeline" || "$SCENARIO" != "generic" ]] && [ ! -f "$HELM_VALUES_ARG" ]; then echo "ERROR: Helm values file does not exist: $HELM_VALUES_ARG" exit 1 fi @@ -280,11 +280,11 @@ else --values "$HELM_VALUES_ARG" > "$HELM_OUTPUT" elif [[ "$COMPONENT" == "kubeflow-pipelines" ]]; then if [[ "$SCENARIO" == "generic" ]]; then - helm template kubeflow-pipelines . \ + helm template pipeline . \ --namespace "$NAMESPACE" \ --include-crds > "$HELM_OUTPUT" else - helm template kubeflow-pipelines . \ + helm template pipeline . \ --namespace "$NAMESPACE" \ --include-crds \ --values "$HELM_VALUES_ARG" > "$HELM_OUTPUT" From 1056eda9769a9de37414770f0419d9548f239086 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Wed, 24 Sep 2025 22:06:57 +0530 Subject: [PATCH 03/26] Update helm charts to reduce differences Signed-off-by: kunal-511 --- .../ci/values-multi-user-enhanced.yaml | 59 +- .../charts/pipeline/ci/values-multi-user.yaml | 66 +- .../ci/values-platform-agnostic-enhanced.yaml | 31 + .../charts/pipeline/templates/_helpers.tpl | 132 +++- .../templates/api-server/deployment.yaml | 9 +- .../templates/api-server/service.yaml | 3 +- .../templates/application/application.yaml | 3 +- .../templates/cache-deployer/deployment.yaml | 5 +- .../pipeline/templates/cache/deployment.yaml | 21 +- .../pipeline/templates/cache/service.yaml | 3 +- .../config/pipeline-api-server-config.yaml | 19 + .../config/pipeline-install-config.yaml | 36 +- .../templates/config/ui-configmap.yaml | 43 +- .../templates/istio/authorization-config.yaml | 215 +++++-- .../templates/istio/virtual-service.yaml | 42 +- .../templates/metadata-writer/deployment.yaml | 7 +- .../templates/metadata/envoy-deployment.yaml | 7 +- .../templates/metadata/envoy-service.yaml | 5 +- .../templates/metadata/grpc-deployment.yaml | 78 ++- .../templates/metadata/grpc-service.yaml | 17 +- .../metadata/grpc-serviceaccount.yaml | 10 +- .../persistence-agent/deployment.yaml | 7 +- .../decorator-controller.yaml | 2 +- .../profile-controller/deployment.yaml | 6 +- .../templates/profile-controller/service.yaml | 2 +- .../rbac/argo-workflow-controller-rbac.yaml | 199 ++++++ .../templates/rbac/cluster-role-bindings.yaml | 172 ++++++ .../templates/rbac/cluster-roles.yaml | 579 ++++++++++++++++++ .../templates/rbac/pipeline-runner-role.yaml | 2 +- .../rbac/pipeline-runner-rolebinding.yaml | 2 +- .../rbac/view-edit-cluster-roles.yaml | 28 +- .../scheduled-workflow/deployment.yaml | 7 +- .../third-party/minio-deployment.yaml | 7 +- .../templates/third-party/minio-service.yaml | 3 +- .../third-party/mysql-deployment.yaml | 7 +- .../templates/third-party/mysql-service.yaml | 3 +- .../workflow-controller-deployment.yaml | 5 +- .../pipeline/templates/ui/deployment.yaml | 9 +- .../charts/pipeline/templates/ui/service.yaml | 3 +- .../templates/viewer-crd/deployment.yaml | 7 +- .../templates/visualization/deployment.yaml | 9 +- .../templates/visualization/service.yaml | 3 +- experimental/helm/charts/pipeline/values.yaml | 35 +- 43 files changed, 1639 insertions(+), 269 deletions(-) create mode 100644 experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml create mode 100644 experimental/helm/charts/pipeline/templates/rbac/argo-workflow-controller-rbac.yaml create mode 100644 experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml create mode 100644 experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml index 7f07c86c67..136938bca9 100644 --- a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml @@ -5,7 +5,7 @@ global: namespace: kubeflow imageRegistry: ghcr.io/kubeflow imageTag: "2.5.0" - imagePullPolicy: IfNotPresent + imagePullPolicy: Always # Installation mode - Multi-user enabled installMode: @@ -71,9 +71,9 @@ database: pipeline: cache: image: "registry.k8s.io/busybox" - nodeRestrictions: true - maximumStaleness: "P0D" - defaultStaleness: "P0D" + nodeRestrictions: false + maximumStaleness: "" + defaultStaleness: "" # Argo Workflows Configuration - Multi-user argo: @@ -114,23 +114,24 @@ ui: resources: requests: - cpu: 100m - memory: 200Mi - limits: - cpu: 500m - memory: 1Gi + cpu: 10m + memory: 70Mi + # Enhanced resources for multi-user deployment persistenceAgent: resources: requests: - cpu: 250m - memory: 1Gi - limits: - cpu: 1 - memory: 2Gi + cpu: 120m + memory: 500Mi scheduledWorkflow: + additionalEnv: + - name: NAMESPACE + value: '' + - name: LOG_LEVEL + value: "info" + valueFrom: null resources: requests: cpu: 250m @@ -168,6 +169,24 @@ cache: limits: cpu: 1 memory: 2Gi + additionalEnv: + - name: NAMESPACE_TO_WATCH + - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + - name: DBCONFIG_DRIVER + - name: DBCONFIG_DB_NAME + key: cacheDb + - name: DBCONFIG_HOST_NAME + key: dbHost + - name: DBCONFIG_PORT + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD cacheDeployer: # Typically disabled in multi-user mode @@ -176,15 +195,12 @@ cacheDeployer: visualization: resources: requests: - cpu: 100m + cpu: 30m memory: 500Mi - limits: - cpu: 500m - memory: 1Gi viewerCrd: env: - maxNumViewers: "200" + maxNumViewers: "" resources: requests: cpu: 250m @@ -192,6 +208,11 @@ viewerCrd: limits: cpu: 500m memory: 1Gi + additionalEnv: + - name: NAMESPACE + value: "" + - name: MAX_NUM_VIEWERS + value: 50 # Profile Controller Configuration (for multi-user) profileController: diff --git a/experimental/helm/charts/pipeline/ci/values-multi-user.yaml b/experimental/helm/charts/pipeline/ci/values-multi-user.yaml index a53894598e..1dd6b6e30b 100644 --- a/experimental/helm/charts/pipeline/ci/values-multi-user.yaml +++ b/experimental/helm/charts/pipeline/ci/values-multi-user.yaml @@ -113,38 +113,54 @@ serviceAccounts: create: true name: "profiles-controller-service-account" -# Database configuration -mysql: - enabled: true - auth: - rootPassword: "" - database: "mlpipeline" - username: "root" - primary: +# Third party services configuration +thirdParty: + # Argo Workflows configuration + argo: + enabled: true + + # Database configuration + mysql: + enabled: true + auth: + rootPassword: "" + database: "mlpipeline" + username: "root" + primary: + persistence: + enabled: true + size: 20Gi + + # Object storage configuration + minio: + enabled: true + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" persistence: enabled: true size: 20Gi -postgresql: - enabled: false - -# Object storage configuration -objectStore: - provider: minio - -minio: - enabled: true - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: + # Metacontroller for profile controller + metacontroller: enabled: true - size: 20Gi -# Argo Workflows configuration -argo: +# Metadata service configuration +metadata: enabled: true + grpc: + image: + repository: gcr.io/tfx-oss-public/ml_metadata_store_server + tag: "1.14.0" + pullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 100Mi + +postgresql: + enabled: false # Istio configuration for multi-user networking: diff --git a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml index 4b62cd5c40..ee901e5e61 100644 --- a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml @@ -103,6 +103,13 @@ persistenceAgent: scheduledWorkflow: resources: + additionalEnv: + - name: LOG_LEVEL + value: "info" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace ui: resources: @@ -145,6 +152,25 @@ cache: limits: cpu: 250m memory: 1Gi + additionalEnv: + - name: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + - name: DBCONFIG_DB_NAME + - name: DBCONFIG_HOST_NAME + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_USER + - name: DBCONFIG_PASSWORD + key: password + - name: NAMESPACE_TO_WATCH + visualization: resources: @@ -156,6 +182,11 @@ viewerCrd: image: pullPolicy: Always resources: + additionalEnv: + - name: MAX_NUM_VIEWERS + value: 50 + - name: NAMESPACE + value: "" # Monitoring disabled for platform-agnostic monitoring: diff --git a/experimental/helm/charts/pipeline/templates/_helpers.tpl b/experimental/helm/charts/pipeline/templates/_helpers.tpl index cb306fdce0..405b931d2b 100644 --- a/experimental/helm/charts/pipeline/templates/_helpers.tpl +++ b/experimental/helm/charts/pipeline/templates/_helpers.tpl @@ -31,9 +31,11 @@ Create chart name and version as used by the chart label. {{- end }} {{/* -Common labels - simplified to match kustomize output +Common labels */}} {{- define "kubeflow-pipelines.labels" -}} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines application-crd-id: kubeflow-pipelines {{- with .Values.commonLabels }} {{ toYaml . }} @@ -53,6 +55,8 @@ Cache server labels */}} {{- define "kubeflow-pipelines.cacheLabels" -}} app: cache-server +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines application-crd-id: kubeflow-pipelines {{- with .Values.commonLabels }} {{ toYaml . }} @@ -64,6 +68,8 @@ Cache deployer labels */}} {{- define "kubeflow-pipelines.cacheDeployerLabels" -}} app: cache-deployer +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines application-crd-id: kubeflow-pipelines {{- with .Values.commonLabels }} {{ toYaml . }} @@ -75,6 +81,8 @@ ML Pipeline specific labels - matching original manifests */}} {{- define "kubeflow-pipelines.mlPipelineLabels" -}} app: ml-pipeline +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines application-crd-id: kubeflow-pipelines {{- with .Values.commonLabels }} {{ toYaml . }} @@ -86,6 +94,8 @@ ML Pipeline UI labels */}} {{- define "kubeflow-pipelines.uiLabels" -}} app: ml-pipeline-ui +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines application-crd-id: kubeflow-pipelines {{- with .Values.commonLabels }} {{ toYaml . }} @@ -97,6 +107,124 @@ ML Pipeline selector labels */}} {{- define "kubeflow-pipelines.mlPipelineSelectorLabels" -}} app: ml-pipeline +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Cache server selector labels +*/}} +{{- define "kubeflow-pipelines.cacheSelectorLabels" -}} +app: cache-server +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +UI selector labels +*/}} +{{- define "kubeflow-pipelines.uiSelectorLabels" -}} +app: ml-pipeline-ui +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Viewer CRD selector labels +*/}} +{{- define "kubeflow-pipelines.viewerCrdSelectorLabels" -}} +app: ml-pipeline-viewer-crd +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Visualization server labels +*/}} +{{- define "kubeflow-pipelines.visualizationLabels" -}} +app: ml-pipeline-visualizationserver +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +application-crd-id: kubeflow-pipelines +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Visualization server selector labels +*/}} +{{- define "kubeflow-pipelines.visualizationSelectorLabels" -}} +app: ml-pipeline-visualizationserver +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +MySQL selector labels +*/}} +{{- define "kubeflow-pipelines.mysqlSelectorLabels" -}} +app: mysql +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +MinIO selector labels +*/}} +{{- define "kubeflow-pipelines.minioSelectorLabels" -}} +app: minio +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Metadata GRPC selector labels +*/}} +{{- define "kubeflow-pipelines.metadataGrpcSelectorLabels" -}} +component: metadata-grpc-server +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Metadata Envoy selector labels +*/}} +{{- define "kubeflow-pipelines.metadataEnvoySelectorLabels" -}} +component: metadata-envoy +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Cache deployer selector labels +*/}} +{{- define "kubeflow-pipelines.cacheDeployerSelectorLabels" -}} +app: cache-deployer +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Metadata writer selector labels +*/}} +{{- define "kubeflow-pipelines.metadataWriterSelectorLabels" -}} +app: metadata-writer +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Persistence agent selector labels +*/}} +{{- define "kubeflow-pipelines.persistenceAgentSelectorLabels" -}} +app: ml-pipeline-persistenceagent +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Scheduled workflow selector labels +*/}} +{{- define "kubeflow-pipelines.scheduledWorkflowSelectorLabels" -}} +app: ml-pipeline-scheduledworkflow +application-crd-id: kubeflow-pipelines +{{- end }} + +{{/* +Workflow controller selector labels +*/}} +{{- define "kubeflow-pipelines.workflowControllerSelectorLabels" -}} +app: workflow-controller +application-crd-id: kubeflow-pipelines {{- end }} {{/* @@ -303,7 +431,7 @@ storage.googleapis.com {{- if eq .Values.objectStore.provider "minio" -}} {{- .Values.objectStore.minio.bucket | default "mlpipeline" }} {{- else if eq .Values.objectStore.provider "s3" -}} -{{- .Values.objectStore.s3.bucket }} +{{- .Values.objectStore.bucketName | default .Values.objectStore.s3.bucket }} {{- else if eq .Values.objectStore.provider "gcs" -}} {{- .Values.objectStore.gcs.bucket }} {{- else if eq .Values.objectStore.provider "azure" -}} diff --git a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml index 87060d75f4..b361449d87 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml @@ -5,20 +5,17 @@ metadata: name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 4 }} spec: selector: matchLabels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.mlPipelineSelectorLabels" . | nindent 6 }} template: metadata: annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.mlPipelineLabels" . | nindent 8 }} spec: serviceAccountName: ml-pipeline containers: diff --git a/experimental/helm/charts/pipeline/templates/api-server/service.yaml b/experimental/helm/charts/pipeline/templates/api-server/service.yaml index 26e5b5ea3e..a7dd68959e 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/service.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/service.yaml @@ -21,6 +21,5 @@ spec: protocol: TCP targetPort: {{ .Values.apiServer.service.grpcPort }} selector: - app: ml-pipeline - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.mlPipelineSelectorLabels" . | nindent 4 }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/application/application.yaml b/experimental/helm/charts/pipeline/templates/application/application.yaml index d0670b24ca..66df704ca5 100644 --- a/experimental/helm/charts/pipeline/templates/application/application.yaml +++ b/experimental/helm/charts/pipeline/templates/application/application.yaml @@ -16,7 +16,8 @@ spec: addOwnerRef: true selector: matchLabels: - application-crd-id: kubeflow-pipelines + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines descriptor: version: {{ .Values.global.imageTag | quote }} type: Kubeflow Pipelines diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml b/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml index 59e6ae288e..4a99d1e690 100644 --- a/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml @@ -11,15 +11,14 @@ spec: replicas: {{ .Values.cacheDeployer.replicas }} selector: matchLabels: - app: cache-deployer - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.cacheDeployerSelectorLabels" . | nindent 6 }} strategy: type: Recreate template: metadata: labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: cache-deployer - application-crd-id: kubeflow-pipelines spec: serviceAccountName: kubeflow-pipelines-cache-deployer-sa containers: diff --git a/experimental/helm/charts/pipeline/templates/cache/deployment.yaml b/experimental/helm/charts/pipeline/templates/cache/deployment.yaml index c48f634278..2b1e2af396 100644 --- a/experimental/helm/charts/pipeline/templates/cache/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/cache/deployment.yaml @@ -10,8 +10,7 @@ spec: replicas: {{ .Values.cache.server.replicas }} selector: matchLabels: - app: cache-server - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.cacheSelectorLabels" . | nindent 6 }} template: metadata: labels: @@ -38,50 +37,50 @@ spec: - name: DEFAULT_CACHE_STALENESS valueFrom: configMapKeyRef: - key: DEFAULT_CACHE_STALENESS name: pipeline-install-config + key: DEFAULT_CACHE_STALENESS - name: MAXIMUM_CACHE_STALENESS valueFrom: configMapKeyRef: - key: MAXIMUM_CACHE_STALENESS name: pipeline-install-config + key: MAXIMUM_CACHE_STALENESS - name: CACHE_IMAGE valueFrom: configMapKeyRef: - key: cacheImage name: pipeline-install-config + key: cacheImage - name: CACHE_NODE_RESTRICTIONS valueFrom: configMapKeyRef: - key: cacheNodeRestrictions name: pipeline-install-config + key: cacheNodeRestrictions - name: DBCONFIG_DRIVER value: mysql - name: DBCONFIG_DB_NAME valueFrom: configMapKeyRef: - key: cacheDb name: pipeline-install-config + key: cacheDb - name: DBCONFIG_HOST_NAME valueFrom: configMapKeyRef: - key: dbHost name: pipeline-install-config + key: dbHost - name: DBCONFIG_PORT valueFrom: configMapKeyRef: - key: dbPort name: pipeline-install-config + key: dbPort - name: DBCONFIG_USER valueFrom: secretKeyRef: - key: username name: mysql-secret + key: username - name: DBCONFIG_PASSWORD valueFrom: secretKeyRef: - key: password name: mysql-secret + key: password - name: NAMESPACE_TO_WATCH valueFrom: fieldRef: diff --git a/experimental/helm/charts/pipeline/templates/cache/service.yaml b/experimental/helm/charts/pipeline/templates/cache/service.yaml index 62453e9889..da8ecaa94f 100644 --- a/experimental/helm/charts/pipeline/templates/cache/service.yaml +++ b/experimental/helm/charts/pipeline/templates/cache/service.yaml @@ -11,6 +11,5 @@ spec: - port: {{ .Values.cache.server.service.port }} targetPort: webhook-api selector: - app: cache-server - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.cacheSelectorLabels" . | nindent 4 }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml b/experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml new file mode 100644 index 0000000000..dca1fbf52b --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml @@ -0,0 +1,19 @@ +{{- if .Values.apiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: pipeline-api-server-config + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +data: + {{- if and .Values.installMode.multiUser.enabled (eq .Values.installMode.type "multi-user") }} + MULTIUSER: "true" + DEFAULTPIPELINERUNNERSERVICEACCOUNT: "default-editor" + {{- else }} + MULTIUSER: "false" + DEFAULTPIPELINERUNNERSERVICEACCOUNT: "pipeline-runner" + {{- end }} + VISUALIZATIONSERVICE_NAME: "ml-pipeline-visualizationserver" + VISUALIZATIONSERVICE_PORT: "8888" +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml b/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml index e01b571b09..2e74241e87 100644 --- a/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml +++ b/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml @@ -16,10 +16,16 @@ data: appName: {{ .Values.pipeline.app.name }} appVersion: {{ .Values.pipeline.app.version }} # Database configuration - dbHost: {{ include "kubeflow-pipelines.database.host" . }} +{{- $dbHost := include "kubeflow-pipelines.database.host" . -}} +{{- if and .Values.env .Values.env.aws .Values.env.aws.rds .Values.env.aws.rds.enabled -}} + {{- $dbHost = .Values.env.aws.rds.endpoint -}} +{{- else if and .Values.env .Values.env.azure .Values.env.azure.sql .Values.env.azure.sql.enabled -}} + {{- $dbHost = printf "%s.mysql.database.azure.com" .Values.env.azure.sql.serverName -}} +{{- end }} + dbHost: {{ $dbHost | quote }} dbPort: {{ include "kubeflow-pipelines.database.port" . | quote }} - dbType: {{ .Values.externalDatabase.type | default "mysql" }} - mysqlHost: {{ include "kubeflow-pipelines.database.host" . }} + dbType: {{ .Values.externalDatabase.type | default "mysql" | quote }} + mysqlHost: {{ $dbHost | quote }} mysqlPort: {{ include "kubeflow-pipelines.database.port" . | quote }} mlmdDb: {{ .Values.database.metadb }} cacheDb: {{ .Values.database.cachedb }} @@ -28,13 +34,33 @@ data: bucketName: {{ include "kubeflow-pipelines.objectStore.bucket" . }} # Pipeline configuration defaultPipelineRoot: {{ .Values.pipeline.defaultPipelineRoot | quote }} - autoUpdatePipelineDefaultVersion: {{ .Values.pipeline.autoUpdatePipelineDefaultVersion | quote }} + autoUpdatePipelineDefaultVersion: {{ .Values.pipeline.autoUpdatePipelineDefaultVersion | toString | quote }} cronScheduleTimezone: {{ .Values.pipeline.cronScheduleTimezone | quote }} # Cache configuration cacheImage: {{ .Values.pipeline.cache.image | quote }} - cacheNodeRestrictions: {{ .Values.pipeline.cache.nodeRestrictions | quote }} + cacheNodeRestrictions: {{ .Values.pipeline.cache.nodeRestrictions | toString | quote }} MAXIMUM_CACHE_STALENESS: {{ .Values.pipeline.cache.maximumStaleness | quote }} DEFAULT_CACHE_STALENESS: {{ .Values.pipeline.cache.defaultStaleness | quote }} # Connection configuration ConMaxLifeTime: {{ .Values.database.connectionMaxLifetime | quote }} LOG_LEVEL: {{ .Values.global.logLevel | quote }} +{{- if eq .Values.objectStore.provider "s3" }} + # AWS S3 configuration + minioServiceHost: {{ .Values.objectStore.s3.endpoint | default "s3.amazonaws.com" | quote }} +{{- if .Values.objectStore.s3.region }} + minioServiceRegion: {{ .Values.objectStore.s3.region | quote }} +{{- else if and .Values.env .Values.env.aws .Values.env.aws.region }} + minioServiceRegion: {{ .Values.env.aws.region | quote }} +{{- end }} +{{- end }} +{{- if eq .Values.objectStore.provider "gcs" }} + # GCP GCS configuration +{{- if .Values.objectStore.gcs.projectId }} + gcsProjectId: {{ .Values.objectStore.gcs.projectId | quote }} +{{- else if and .Values.env .Values.env.gcp .Values.env.gcp.project }} + gcsProjectId: {{ .Values.env.gcp.project | quote }} +{{- end }} +{{- if and .Values.env .Values.env.gcp .Values.env.gcp.cloudsql .Values.env.gcp.cloudsql.enabled }} + gcsCloudSqlInstanceName: {{ .Values.env.gcp.cloudsql.instanceName | quote }} +{{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml b/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml index bbb6795e38..2ec1f76abc 100644 --- a/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml +++ b/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml @@ -8,9 +8,50 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} data: viewer-pod-template.json: |- +{{- if eq .Values.objectStore.provider "s3" }} + { + "spec": { + "containers": [ + { + "env": [ + { + "name": "AWS_ACCESS_KEY_ID", + "valueFrom": { + "secretKeyRef": { + "name": "mlpipeline-minio-artifact", + "key": "accesskey" + } + } + }, + { + "name": "AWS_SECRET_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "name": "mlpipeline-minio-artifact", + "key": "secretkey" + } + } + }, + { + "name": "AWS_REGION", + "valueFrom": { + "configMapKeyRef": { + "name": "pipeline-install-config", + "key": "minioServiceRegion" + } + } + } + ] + } + ], + "serviceAccountName": "{{- if and .Values.installMode.multiUser.enabled (eq .Values.installMode.type "multi-user") }}default-editor{{- else }}kubeflow-pipelines-viewer{{- end }}" + } + } +{{- else }} { "spec": { - "serviceAccountName": "kubeflow-pipelines-viewer" + "serviceAccountName": "{{- if and .Values.installMode.multiUser.enabled (eq .Values.installMode.type "multi-user") }}default-editor{{- else }}kubeflow-pipelines-viewer{{- end }}" } } {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml b/experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml index a960f8df3a..2181aea7b3 100644 --- a/experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml +++ b/experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml @@ -2,24 +2,25 @@ apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-ui-policy + name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: istio-authorization - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: selector: matchLabels: app: ml-pipeline-ui rules: + # Allow all requests from the ingress gateway - from: - source: principals: - cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account - + # Allow all requests with an `authorization` header but NOT a `kubeflow-userid` header. + # This is needed to allow Kubernetes JWTs to be passed to the KFP API. - when: - key: request.headers[authorization] values: @@ -32,13 +33,12 @@ spec: apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-api-policy + name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: istio-authorization - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: selector: matchLabels: @@ -47,12 +47,13 @@ spec: - from: - source: principals: - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.ui.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-ui + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-persistenceagent + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-scheduledworkflow + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-viewer-crd-service-account + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/kubeflow-pipelines-cache + # Allow all requests that dont have a `kubeflow-userid` header. - when: - key: request.headers[kubeflow-userid] notValues: ['*'] @@ -61,13 +62,12 @@ spec: apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-visualization-policy + name: ml-pipeline-visualizationserver namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: istio-authorization - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: selector: matchLabels: @@ -76,24 +76,23 @@ spec: - from: - source: principals: - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.ui.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/{{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-ui + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-persistenceagent + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-scheduledworkflow + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-viewer-crd-service-account + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/kubeflow-pipelines-cache --- apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-cache-server-policy + name: service-cache-server namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: istio-authorization - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: selector: matchLabels: @@ -101,19 +100,95 @@ spec: rules: - {} +--- +{{- if .Values.thirdParty.minio.enabled }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: minio-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +spec: + action: ALLOW + selector: + matchLabels: + app: minio + rules: + - from: + - source: + principals: + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline + - from: + - source: + principals: + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-ui + # Allow traffic from User Pipeline Pods, which don't have a sidecar. + - {} +{{- end }} + +--- +{{- if .Values.thirdParty.mysql.enabled }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: mysql + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +spec: + selector: + matchLabels: + app: mysql + rules: + - from: + - source: + principals: + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-ui + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-persistenceagent + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-scheduledworkflow + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-viewer-crd-service-account + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/kubeflow-pipelines-cache + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/metadata-grpc-server +{{- end }} + +--- +{{- if .Values.metadata.enabled }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: metadata-grpc-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +spec: + action: ALLOW + selector: + matchLabels: + component: metadata-grpc-server + rules: + - {} +{{- end }} + --- apiVersion: "networking.istio.io/v1alpha3" kind: DestinationRule metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-ui-destination + name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: istio-destination-rule - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: - host: {{ include "kubeflow-pipelines.fullname" . }}-ui.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + host: ml-pipeline-ui.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local trafficPolicy: tls: mode: ISTIO_MUTUAL @@ -122,15 +197,14 @@ spec: apiVersion: "networking.istio.io/v1alpha3" kind: DestinationRule metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-api-destination + name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: istio-destination-rule - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: - host: {{ include "kubeflow-pipelines.fullname" . }}-api-server.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + host: ml-pipeline.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local trafficPolicy: tls: mode: ISTIO_MUTUAL @@ -139,16 +213,69 @@ spec: apiVersion: "networking.istio.io/v1alpha3" kind: DestinationRule metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-visualization-destination + name: ml-pipeline-visualizationserver namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: istio-destination-rule - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: - host: {{ include "kubeflow-pipelines.fullname" . }}-visualization.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + host: ml-pipeline-visualizationserver.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local trafficPolicy: tls: mode: ISTIO_MUTUAL + +--- +{{- if .Values.thirdParty.minio.enabled }} +apiVersion: "networking.istio.io/v1alpha3" +kind: DestinationRule +metadata: + name: ml-pipeline-minio + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +spec: + host: minio-service.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +{{- end }} + +--- +{{- if .Values.thirdParty.mysql.enabled }} +apiVersion: "networking.istio.io/v1alpha3" +kind: DestinationRule +metadata: + name: ml-pipeline-mysql + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +spec: + host: mysql.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +{{- end }} + +--- +{{- if .Values.metadata.enabled }} +apiVersion: "networking.istio.io/v1alpha3" +kind: DestinationRule +metadata: + name: metadata-grpc-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +spec: + host: metadata-grpc-service.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +{{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml b/experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml index d3fe09453d..7c3af25ab6 100644 --- a/experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml +++ b/experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml @@ -2,16 +2,15 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-ui + name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: istio-virtual-service - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: gateways: - - {{ .Values.networking.istio.gateway }} + - {{ .Values.networking.istio.gateway | default "kubeflow-gateway" }} hosts: - '*' http: @@ -22,8 +21,37 @@ spec: uri: /pipeline route: - destination: - host: {{ include "kubeflow-pipelines.fullname" . }}-ui.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + host: ml-pipeline-ui.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local port: - number: {{ .Values.ui.service.port }} + number: 80 timeout: 300s + +--- +{{- if .Values.metadata.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: metadata-grpc + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +spec: + gateways: + - {{ .Values.networking.istio.gateway | default "kubeflow-gateway" }} + hosts: + - '*' + http: + - match: + - uri: + prefix: /ml_metadata + rewrite: + uri: /ml_metadata + route: + - destination: + host: metadata-envoy-service.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + port: + number: 9090 +{{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml index 66b002c6bd..442485cdc1 100644 --- a/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml @@ -5,19 +5,18 @@ metadata: name: metadata-writer namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: metadata-writer - application-crd-id: kubeflow-pipelines spec: replicas: {{ .Values.metadataWriter.replicas }} selector: matchLabels: - app: metadata-writer - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.metadataWriterSelectorLabels" . | nindent 6 }} template: metadata: labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: metadata-writer - application-crd-id: kubeflow-pipelines spec: securityContext: seccompProfile: diff --git a/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml index 542cd2a6fe..f4e18423d4 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml @@ -6,18 +6,19 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: component: metadata-envoy - application-crd-id: kubeflow-pipelines + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: replicas: {{ .Values.metadata.envoy.replicas }} selector: matchLabels: component: metadata-envoy - application-crd-id: kubeflow-pipelines template: metadata: labels: component: metadata-envoy - application-crd-id: kubeflow-pipelines + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines sidecar.istio.io/inject: "false" spec: containers: diff --git a/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml b/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml index 9b4e78923f..2dc727afd4 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml @@ -5,15 +5,14 @@ metadata: name: metadata-envoy-service namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: metadata-envoy - application-crd-id: kubeflow-pipelines spec: ports: - name: {{ .Values.metadata.envoy.service.portName | default "md-envoy" }} port: {{ .Values.metadata.envoy.service.port }} protocol: TCP selector: - application-crd-id: kubeflow-pipelines - component: metadata-envoy + {{- include "kubeflow-pipelines.metadataEnvoySelectorLabels" . | nindent 4 }} type: {{ .Values.metadata.envoy.service.type }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml index b7a82bb579..85aec707d2 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml @@ -5,62 +5,75 @@ metadata: name: metadata-grpc-deployment namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} component: metadata-grpc-server + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: - replicas: {{ .Values.metadata.grpc.replicas }} + replicas: 1 selector: matchLabels: component: metadata-grpc-server - application-crd-id: kubeflow-pipelines template: metadata: labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} component: metadata-grpc-server - application-crd-id: kubeflow-pipelines + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines spec: securityContext: seccompProfile: type: RuntimeDefault - serviceAccountName: metadata-grpc-server containers: - name: container - image: {{ .Values.metadata.grpc.image.repository }}:{{ .Values.metadata.grpc.image.tag }} + image: {{ .Values.metadata.grpc.image.repository }}:{{ .Values.metadata.grpc.image.tag | default "1.14.0" }} + imagePullPolicy: {{ .Values.metadata.grpc.image.pullPolicy | default "IfNotPresent" }} + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL env: - name: DBCONFIG_USER valueFrom: secretKeyRef: + name: {{ .Values.thirdParty.mysql.secretName | default "mysql-secret" }} key: username - name: mysql-secret - name: DBCONFIG_PASSWORD valueFrom: secretKeyRef: + name: {{ .Values.thirdParty.mysql.secretName | default "mysql-secret" }} key: password - name: mysql-secret - name: MYSQL_DATABASE valueFrom: configMapKeyRef: - key: mlmdDb name: pipeline-install-config + key: mlmdDb - name: MYSQL_HOST valueFrom: configMapKeyRef: - key: dbHost name: pipeline-install-config + key: dbHost - name: MYSQL_PORT valueFrom: configMapKeyRef: - key: dbPort name: pipeline-install-config + key: dbPort command: ["/bin/metadata_store_server"] - args: - - "--grpc_port=8080" - - "--mysql_config_database=$(MYSQL_DATABASE)" - - "--mysql_config_host=$(MYSQL_HOST)" - - "--mysql_config_port=$(MYSQL_PORT)" - - "--mysql_config_user=$(DBCONFIG_USER)" - - "--mysql_config_password=$(DBCONFIG_PASSWORD)" - - "--enable_database_upgrade=true" + args: ["--grpc_port=8080", + "--mysql_config_database=$(MYSQL_DATABASE)", + "--mysql_config_host=$(MYSQL_HOST)", + "--mysql_config_port=$(MYSQL_PORT)", + "--mysql_config_user=$(DBCONFIG_USER)", + "--mysql_config_password=$(DBCONFIG_PASSWORD)", + "--enable_database_upgrade=true" + ] ports: - name: grpc-api containerPort: 8080 @@ -76,14 +89,19 @@ spec: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsGroup: 0 - runAsNonRoot: true - runAsUser: 1000 - seccompProfile: - type: RuntimeDefault -{{- end }} + resources: + {{- toYaml .Values.metadata.grpc.resources | nindent 10 }} + serviceAccountName: metadata-grpc-server + {{- with .Values.metadata.grpc.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.metadata.grpc.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.metadata.grpc.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml index a4f6752be6..d0ee4c8f57 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml @@ -1,19 +1,18 @@ {{- if .Values.metadata.enabled }} -apiVersion: v1 kind: Service +apiVersion: v1 metadata: name: metadata-grpc-service namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: metadata - application-crd-id: kubeflow-pipelines spec: + selector: + {{- include "kubeflow-pipelines.metadataGrpcSelectorLabels" . | nindent 4 }} + type: ClusterIP ports: - - name: {{ .Values.metadata.grpc.service.portName | default "grpc-api" }} - port: {{ .Values.metadata.grpc.service.port }} + - port: 8080 protocol: TCP - selector: - application-crd-id: kubeflow-pipelines - component: metadata-grpc-server - type: {{ .Values.metadata.grpc.service.type }} -{{- end }} + name: grpc-api +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml index fc10505bd3..c73d78f25a 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.metadata.enabled .Values.metadata.grpc.serviceAccount.create }} +{{- if .Values.metadata.enabled }} apiVersion: v1 kind: ServiceAccount metadata: @@ -6,8 +6,6 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - {{- with .Values.metadata.grpc.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml index ac74e91ecf..091d7cce5d 100644 --- a/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml @@ -5,20 +5,19 @@ metadata: name: ml-pipeline-persistenceagent namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-persistenceagent - application-crd-id: kubeflow-pipelines spec: selector: matchLabels: - app: ml-pipeline-persistenceagent - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.persistenceAgentSelectorLabels" . | nindent 6 }} template: metadata: annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: ml-pipeline-persistenceagent - application-crd-id: kubeflow-pipelines spec: serviceAccountName: ml-pipeline-persistenceagent containers: diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml index eac96a1188..f146e7ed89 100644 --- a/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml +++ b/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml @@ -2,7 +2,7 @@ apiVersion: metacontroller.k8s.io/v1alpha1 kind: DecoratorController metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller + name: kubeflow-pipelines-profile-controller labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} component: profile-controller diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml index 86414de3eb..1f47e722cb 100644 --- a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller + name: kubeflow-pipelines-profile-controller namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} @@ -32,7 +32,7 @@ spec: command: ["python", "/hooks/sync.py"] envFrom: - configMapRef: - name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller-env + name: kubeflow-pipelines-profile-controller-env env: - name: KFP_VERSION value: {{ .Values.global.imageTag | quote }} @@ -67,7 +67,7 @@ spec: volumes: - name: hooks configMap: - name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller-code + name: kubeflow-pipelines-profile-controller-code {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml index b671bf8e37..9759c8555e 100644 --- a/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml +++ b/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller + name: kubeflow-pipelines-profile-controller namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/argo-workflow-controller-rbac.yaml b/experimental/helm/charts/pipeline/templates/rbac/argo-workflow-controller-rbac.yaml new file mode 100644 index 0000000000..7d02f84856 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/rbac/argo-workflow-controller-rbac.yaml @@ -0,0 +1,199 @@ +{{- if and .Values.thirdParty.argo.enabled .Values.installMode.multiUser.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-cluster-role + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + - pods/exec + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - persistentvolumeclaims/finalizers + verbs: + - create + - update + - delete + - get +- apiGroups: + - argoproj.io + resources: + - workflows + - workflows/finalizers + - workflowtaskresults + - workflowtaskresults/finalizers + - workflowartifactgctasks + - workflowartifactgctasks/finalizers + verbs: + - get + - list + - watch + - update + - patch + - delete + - create +- apiGroups: + - argoproj.io + resources: + - workflowtemplates + - workflowtemplates/finalizers + - clusterworkflowtemplates + - clusterworkflowtemplates/finalizers + - cronworkflows + - cronworkflows/finalizers + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - list +- apiGroups: + - argoproj.io + resources: + - workfloweventbindings + - workfloweventbindings/finalizers + verbs: + - get + - list + - watch +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - get + - delete + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: argo-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: argo-cluster-role +subjects: +- kind: ServiceAccount + name: argo + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-aggregate-to-view + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + - workflows/finalizers + - workfloweventbindings + - workfloweventbindings/finalizers + - workflowtemplates + - workflowtemplates/finalizers + - cronworkflows + - cronworkflows/finalizers + - clusterworkflowtemplates + - clusterworkflowtemplates/finalizers + - workflowtaskresults + - workflowtaskresults/finalizers + verbs: + - get + - list + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-aggregate-to-edit + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + - workflows/finalizers + - workfloweventbindings + - workfloweventbindings/finalizers + - workflowtemplates + - workflowtemplates/finalizers + - cronworkflows + - cronworkflows/finalizers + - clusterworkflowtemplates + - clusterworkflowtemplates/finalizers + - workflowtaskresults + - workflowtaskresults/finalizers + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-aggregate-to-admin + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + - workflows/finalizers + - workfloweventbindings + - workfloweventbindings/finalizers + - workflowtemplates + - workflowtemplates/finalizers + - cronworkflows + - cronworkflows/finalizers + - clusterworkflowtemplates + - clusterworkflowtemplates/finalizers + - workflowtaskresults + - workflowtaskresults/finalizers + verbs: + - "*" +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml b/experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml new file mode 100644 index 0000000000..125436dd03 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml @@ -0,0 +1,172 @@ +{{- if .Values.installMode.multiUser.enabled }} +# API Server ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +# UI ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline-ui + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline-ui +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.ui.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +# Scheduled Workflow ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline-scheduledworkflow-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline-scheduledworkflow-role +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +# Persistence Agent ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline-persistenceagent-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline-persistenceagent-role +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +# Viewer Controller ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline-viewer-crd-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline-viewer-controller-role +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +# Cache ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeflow-pipelines-cache-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-pipelines-cache-role +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +# Cache Deployer ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeflow-pipelines-cache-deployer-clusterrolebinding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-pipelines-cache-deployer-clusterrole +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.cacheDeployer.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +# Metadata Writer ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeflow-pipelines-metadata-writer-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-pipelines-metadata-writer-role +subjects: +- kind: ServiceAccount + name: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +# Argo ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: argo-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: argo-cluster-role +subjects: +- kind: ServiceAccount + name: argo + namespace: {{ include "kubeflow-pipelines.namespace" . }} + +--- +# Metacontroller ClusterRoleBinding +{{- if .Values.thirdParty.metacontroller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: meta-controller-cluster-role-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-metacontroller +subjects: +- kind: ServiceAccount + name: meta-controller-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml b/experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml new file mode 100644 index 0000000000..f4ef3982e5 --- /dev/null +++ b/experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml @@ -0,0 +1,579 @@ +{{- if .Values.installMode.multiUser.enabled }} +# API Server ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - "" + resources: + - pods + - pods/log + verbs: + - get + - list + - delete +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + verbs: + - create + - get + - list + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows/finalizers + verbs: + - update +- apiGroups: + - pipelines.kubeflow.org + resources: + - pipelines + verbs: + - get + - list + - watch +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + +--- +# UI ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline-ui + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - "" + resources: + - pods + - pods/log + verbs: + - get + - list +- apiGroups: + - "" + resources: + - events + verbs: + - list +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - kubeflow.org + resources: + - viewers + verbs: + - create + - get + - list + - watch + - delete + +--- +# Scheduled Workflow ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline-scheduledworkflow-role + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows/finalizers + verbs: + - update + +--- +# Persistence Agent ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline-persistenceagent-role + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + verbs: + - get + - list + - watch + +--- +# Viewer Controller ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline-viewer-controller-role + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - '*' + resources: + - deployments + - replicasets + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - viewers + verbs: + - create + - get + - list + - watch + - update + - patch + - delete + +--- +# Cache ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-pipelines-cache-role + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch + - update + - patch + +--- +# Cache Deployer ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-pipelines-cache-deployer-clusterrole + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + - certificatesigningrequests/approval + verbs: + - create + - delete + - get + - update +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - kubernetes.io/* + verbs: + - approve + +--- +# Metadata Writer ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-pipelines-metadata-writer-role + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch + - update + - patch + +--- +# Argo Cluster Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-cluster-role + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - "" + resources: + - pods + - pods/exec + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - persistentvolumeclaims/finalizers + verbs: + - create + - update + - delete + - get +- apiGroups: + - argoproj.io + resources: + - workflows + - workflows/finalizers + - workflowtasksets + - workflowtasksets/finalizers + - workflowartifactgctasks + verbs: + - get + - list + - watch + - update + - patch + - delete + - create +- apiGroups: + - argoproj.io + resources: + - workflowtemplates + - workflowtemplates/finalizers + - clusterworkflowtemplates + - clusterworkflowtemplates/finalizers + verbs: + - get + - list + - watch +- apiGroups: + - argoproj.io + resources: + - workflowtaskresults + verbs: + - list + - watch + - deletecollection +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - list +- apiGroups: + - argoproj.io + resources: + - cronworkflows + - cronworkflows/finalizers + verbs: + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "policy" + resources: + - poddisruptionbudgets + verbs: + - create + - get + - delete +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + resourceNames: + - argo-workflows-agent-ca-certificates + +--- +# Argo Aggregate Roles +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-aggregate-to-view + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + - workflows/finalizers + - workfloweventbindings + - workfloweventbindings/finalizers + - workflowtemplates + - workflowtemplates/finalizers + - cronworkflows + - cronworkflows/finalizers + - clusterworkflowtemplates + - clusterworkflowtemplates/finalizers + - workflowtaskresults + - workflowtaskresults/finalizers + verbs: + - get + - list + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-aggregate-to-edit + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + - workflows/finalizers + - workfloweventbindings + - workfloweventbindings/finalizers + - workflowtemplates + - workflowtemplates/finalizers + - cronworkflows + - cronworkflows/finalizers + - clusterworkflowtemplates + - clusterworkflowtemplates/finalizers + - workflowtaskresults + - workflowtaskresults/finalizers + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-aggregate-to-admin + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + - workflows/finalizers + - workfloweventbindings + - workfloweventbindings/finalizers + - workflowtemplates + - workflowtemplates/finalizers + - cronworkflows + - cronworkflows/finalizers + - clusterworkflowtemplates + - clusterworkflowtemplates/finalizers + - workflowtasksets + - workflowtasksets/finalizers + - workflowtaskresults + - workflowtaskresults/finalizers + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + +--- +# Metacontroller ClusterRole +{{- if .Values.thirdParty.metacontroller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-metacontroller + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +{{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml index f017db64a9..6cd5469e5b 100644 --- a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml +++ b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml @@ -5,7 +5,7 @@ metadata: name: pipeline-runner namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} rules: diff --git a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml index 82c6f61ff6..8f7c1a48b1 100644 --- a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml +++ b/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml @@ -5,7 +5,7 @@ metadata: name: pipeline-runner-binding namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} roleRef: diff --git a/experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml b/experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml index 99052744f7..7d7b182ae8 100644 --- a/experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml +++ b/experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml @@ -8,13 +8,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-edit + name: kubeflow-pipelines-edit labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true" - component: multi-user-rbac - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines aggregationRule: clusterRoleSelectors: - matchLabels: @@ -26,14 +25,13 @@ rules: [] apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-view + name: kubeflow-pipelines-view labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pipelines-edit: "true" rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true" - component: multi-user-rbac - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines aggregationRule: clusterRoleSelectors: - matchLabels: @@ -45,13 +43,12 @@ rules: [] apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-aggregate-edit + name: aggregate-to-kubeflow-pipelines-edit labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pipelines-edit: "true" - component: multi-user-rbac - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines rules: - apiGroups: - pipelines.kubeflow.org @@ -117,13 +114,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-aggregate-view + name: aggregate-to-kubeflow-pipelines-view labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pipelines-view: "true" - component: multi-user-rbac - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines rules: - apiGroups: - pipelines.kubeflow.org diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml index ad7f0e4fd4..e8cf33f87b 100644 --- a/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml @@ -5,20 +5,19 @@ metadata: name: ml-pipeline-scheduledworkflow namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-scheduledworkflow - application-crd-id: kubeflow-pipelines spec: selector: matchLabels: - app: ml-pipeline-scheduledworkflow - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.scheduledWorkflowSelectorLabels" . | nindent 6 }} template: metadata: annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: ml-pipeline-scheduledworkflow - application-crd-id: kubeflow-pipelines spec: serviceAccountName: ml-pipeline-scheduledworkflow containers: diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml b/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml index c08ba64fd2..e036736b9f 100644 --- a/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml @@ -5,20 +5,19 @@ metadata: name: minio namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: minio - application-crd-id: kubeflow-pipelines spec: selector: matchLabels: - app: minio - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.minioSelectorLabels" . | nindent 6 }} strategy: type: Recreate template: metadata: labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: minio - application-crd-id: kubeflow-pipelines spec: securityContext: fsGroup: 1000 diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml b/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml index 0787ebcc53..0e0ce28640 100644 --- a/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml +++ b/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml @@ -13,6 +13,5 @@ spec: protocol: TCP targetPort: 9000 selector: - app: minio - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.minioSelectorLabels" . | nindent 4 }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml b/experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml index 1a7a0c7b2a..5b7aa6dfed 100644 --- a/experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml @@ -5,20 +5,19 @@ metadata: name: mysql namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: mysql - application-crd-id: kubeflow-pipelines spec: selector: matchLabels: - app: mysql - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.mysqlSelectorLabels" . | nindent 6 }} strategy: type: Recreate template: metadata: labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: mysql - application-crd-id: kubeflow-pipelines spec: securityContext: fsGroup: 999 diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml b/experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml index 6ac8d91c1a..8652dc97c0 100644 --- a/experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml +++ b/experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml @@ -12,6 +12,5 @@ spec: protocol: TCP targetPort: 3306 selector: - app: mysql - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.mysqlSelectorLabels" . | nindent 4 }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml index e2e939a8ab..3ed4b83f17 100644 --- a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml @@ -11,13 +11,12 @@ metadata: spec: selector: matchLabels: - app: workflow-controller - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.workflowControllerSelectorLabels" . | nindent 6 }} template: metadata: labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: workflow-controller - application-crd-id: kubeflow-pipelines spec: containers: - args: diff --git a/experimental/helm/charts/pipeline/templates/ui/deployment.yaml b/experimental/helm/charts/pipeline/templates/ui/deployment.yaml index 1096c9e032..d1bcef47d1 100644 --- a/experimental/helm/charts/pipeline/templates/ui/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/ui/deployment.yaml @@ -5,20 +5,17 @@ metadata: name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.uiLabels" . | nindent 4 }} spec: selector: matchLabels: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.uiSelectorLabels" . | nindent 6 }} template: metadata: annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" labels: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.uiLabels" . | nindent 8 }} spec: serviceAccountName: ml-pipeline-ui volumes: diff --git a/experimental/helm/charts/pipeline/templates/ui/service.yaml b/experimental/helm/charts/pipeline/templates/ui/service.yaml index dd34875eba..2f5dc5cf0e 100644 --- a/experimental/helm/charts/pipeline/templates/ui/service.yaml +++ b/experimental/helm/charts/pipeline/templates/ui/service.yaml @@ -13,6 +13,5 @@ spec: protocol: TCP targetPort: 3000 selector: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.uiSelectorLabels" . | nindent 4 }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml b/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml index 140dfd1355..f486d9f960 100644 --- a/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml @@ -5,20 +5,19 @@ metadata: name: ml-pipeline-viewer-crd namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-viewer-crd - application-crd-id: kubeflow-pipelines spec: selector: matchLabels: - app: ml-pipeline-viewer-crd - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.viewerCrdSelectorLabels" . | nindent 6 }} template: metadata: annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: ml-pipeline-viewer-crd - application-crd-id: kubeflow-pipelines spec: serviceAccountName: ml-pipeline-viewer-crd-service-account containers: diff --git a/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml b/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml index 0bb13bada1..dd47c19ca9 100644 --- a/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml @@ -5,20 +5,17 @@ metadata: name: ml-pipeline-visualizationserver namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - app: ml-pipeline-visualizationserver - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.visualizationLabels" . | nindent 4 }} spec: selector: matchLabels: - app: ml-pipeline-visualizationserver - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.visualizationSelectorLabels" . | nindent 6 }} template: metadata: annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" labels: - app: ml-pipeline-visualizationserver - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.visualizationLabels" . | nindent 8 }} spec: serviceAccountName: ml-pipeline-visualizationserver containers: diff --git a/experimental/helm/charts/pipeline/templates/visualization/service.yaml b/experimental/helm/charts/pipeline/templates/visualization/service.yaml index 6d075a32a3..437406ad7a 100644 --- a/experimental/helm/charts/pipeline/templates/visualization/service.yaml +++ b/experimental/helm/charts/pipeline/templates/visualization/service.yaml @@ -13,6 +13,5 @@ spec: protocol: TCP targetPort: 8888 selector: - app: ml-pipeline-visualizationserver - application-crd-id: kubeflow-pipelines + {{- include "kubeflow-pipelines.visualizationSelectorLabels" . | nindent 4 }} {{- end }} diff --git a/experimental/helm/charts/pipeline/values.yaml b/experimental/helm/charts/pipeline/values.yaml index 5131ec395f..b26f6d765d 100644 --- a/experimental/helm/charts/pipeline/values.yaml +++ b/experimental/helm/charts/pipeline/values.yaml @@ -40,7 +40,7 @@ installMode: enabled: false profileController: # -- Enable profile controller for multi-user mode - enabled: false + enabled: true # Custom Resource Definitions crds: @@ -67,7 +67,6 @@ apiServer: requests: cpu: 250m memory: 500Mi - limits: {} # Service Account serviceAccount: @@ -195,11 +194,7 @@ scheduledWorkflow: annotations: {} # Resources - resources: - requests: - cpu: 120m - memory: 100Mi - limits: {} + resources: {} # Security context securityContext: @@ -296,11 +291,7 @@ viewerCrd: pullPolicy: "" # -- Viewer CRD Controller resource requests and limits - resources: - requests: - cpu: 120m - memory: 100Mi - limits: {} + resources: {} # Service Account serviceAccount: @@ -489,7 +480,7 @@ cacheDeployer: # ML Metadata Configuration metadata: # -- Enable ML Metadata - enabled: true + enabled: false grpc: # -- Number of metadata gRPC server replicas @@ -602,15 +593,19 @@ argo: thirdParty: # Use built-in minimal Argo instead of external subchart argo: - enabled: false - - # Use built-in minimal MySQL instead of external subchart - mysql: enabled: true + # Use built-in minimal MySQL instead of external subchart + mysql: + enabled: false + # Use built-in minimal MinIO instead of external subchart minio: - enabled: true + enabled: false + + # Metacontroller configuration + metacontroller: + enabled: false # Database Configuration mysql: @@ -714,7 +709,7 @@ pipeline: version: "2.5.0" # Pipeline root configuration - defaultPipelineRoot: "minio://mlpipeline/v2/artifacts" + defaultPipelineRoot: "" # Auto update pipeline default version autoUpdatePipelineDefaultVersion: true @@ -878,7 +873,7 @@ monitoring: # Webhook Configuration webhooks: # -- Enable admission webhooks - enabled: true + enabled: false # -- Webhook failure policy failurePolicy: "Fail" From 27921515408c0f46bcd6f70cb3f088d07c7002c3 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Thu, 25 Sep 2025 15:38:35 +0530 Subject: [PATCH 04/26] modified labels and env Signed-off-by: kunal-511 --- .../ci/values-multi-user-enhanced.yaml | 25 ++----- .../ci/values-platform-agnostic-enhanced.yaml | 7 +- .../charts/pipeline/templates/_helpers.tpl | 57 ++++++++++++++++ .../templates/api-server/deployment.yaml | 11 +++ .../pipeline/templates/cache/deployment.yaml | 67 +++++++------------ .../pipeline/templates/cache/rolebinding.yaml | 2 +- .../templates/metadata-writer/deployment.yaml | 16 +++++ .../decorator-controller.yaml | 10 +-- .../profile-controller/deployment.yaml | 39 ++++++----- .../scheduled-workflow/deployment.yaml | 27 ++++++-- .../templates/viewer-crd/deployment.yaml | 21 ++++-- 11 files changed, 183 insertions(+), 99 deletions(-) diff --git a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml index 136938bca9..9648e4cd01 100644 --- a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml @@ -100,11 +100,8 @@ apiServer: resources: requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 2 - memory: 4Gi + cpu: 250m + memory: 500Mi # Multi-user UI configuration ui: @@ -132,13 +129,6 @@ scheduledWorkflow: - name: LOG_LEVEL value: "info" valueFrom: null - resources: - requests: - cpu: 250m - memory: 500Mi - limits: - cpu: 500m - memory: 1Gi metadata: grpc: @@ -171,6 +161,7 @@ cache: memory: 2Gi additionalEnv: - name: NAMESPACE_TO_WATCH + key: null - name: DEFAULT_CACHE_STALENESS key: DEFAULT_CACHE_STALENESS - name: MAXIMUM_CACHE_STALENESS @@ -178,15 +169,18 @@ cache: - name: CACHE_IMAGE key: cacheImage - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions - name: DBCONFIG_DRIVER - name: DBCONFIG_DB_NAME key: cacheDb - name: DBCONFIG_HOST_NAME key: dbHost - name: DBCONFIG_PORT + key: dbPort - name: DBCONFIG_USER key: username - name: DBCONFIG_PASSWORD + key: password cacheDeployer: # Typically disabled in multi-user mode @@ -201,13 +195,6 @@ visualization: viewerCrd: env: maxNumViewers: "" - resources: - requests: - cpu: 250m - memory: 500Mi - limits: - cpu: 500m - memory: 1Gi additionalEnv: - name: NAMESPACE value: "" diff --git a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml index ee901e5e61..c828a937f7 100644 --- a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml @@ -154,6 +154,7 @@ cache: memory: 1Gi additionalEnv: - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS - name: MAXIMUM_CACHE_STALENESS key: MAXIMUM_CACHE_STALENESS - name: CACHE_IMAGE @@ -162,11 +163,13 @@ cache: key: cacheNodeRestrictions - name: DBCONFIG_DRIVER - name: DBCONFIG_DB_NAME + key: cacheDb - name: DBCONFIG_HOST_NAME key: dbHost - name: DBCONFIG_PORT key: dbPort - name: DBCONFIG_USER + key: username - name: DBCONFIG_PASSWORD key: password - name: NAMESPACE_TO_WATCH @@ -186,7 +189,9 @@ viewerCrd: - name: MAX_NUM_VIEWERS value: 50 - name: NAMESPACE - value: "" + valueFrom: + fieldRef: + fieldPath: metadata.namespace # Monitoring disabled for platform-agnostic monitoring: diff --git a/experimental/helm/charts/pipeline/templates/_helpers.tpl b/experimental/helm/charts/pipeline/templates/_helpers.tpl index 405b931d2b..88f761c4b2 100644 --- a/experimental/helm/charts/pipeline/templates/_helpers.tpl +++ b/experimental/helm/charts/pipeline/templates/_helpers.tpl @@ -36,7 +36,9 @@ Common labels {{- define "kubeflow-pipelines.labels" -}} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if ne .Values.installMode.type "multi-user" }} application-crd-id: kubeflow-pipelines +{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} @@ -57,7 +59,9 @@ Cache server labels app: cache-server app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if ne .Values.installMode.type "multi-user" }} application-crd-id: kubeflow-pipelines +{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} @@ -83,7 +87,9 @@ ML Pipeline specific labels - matching original manifests app: ml-pipeline app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if ne .Values.installMode.type "multi-user" }} application-crd-id: kubeflow-pipelines +{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} @@ -107,16 +113,26 @@ ML Pipeline selector labels */}} {{- define "kubeflow-pipelines.mlPipelineSelectorLabels" -}} app: ml-pipeline +{{- if eq .Values.installMode.type "multi-user" }} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- else }} application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} {{/* Cache server selector labels */}} {{- define "kubeflow-pipelines.cacheSelectorLabels" -}} app: cache-server +{{- if eq .Values.installMode.type "multi-user" }} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- else }} application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} {{/* UI selector labels @@ -126,13 +142,33 @@ app: ml-pipeline-ui application-crd-id: kubeflow-pipelines {{- end }} +{{/* +Viewer CRD labels +*/}} +{{- define "kubeflow-pipelines.viewerCrdLabels" -}} +{{- if eq .Values.installMode.type "multi-user" }} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +app: ml-pipeline-viewer-crd +{{- else }} +{{- include "kubeflow-pipelines.labels" . }} +app: ml-pipeline-viewer-crd +{{- end }} +{{- end }} + {{/* Viewer CRD selector labels */}} {{- define "kubeflow-pipelines.viewerCrdSelectorLabels" -}} +{{- if eq .Values.installMode.type "multi-user" }} +app: ml-pipeline-viewer-crd +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- else }} app: ml-pipeline-viewer-crd application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} {{/* Visualization server labels @@ -151,9 +187,15 @@ application-crd-id: kubeflow-pipelines Visualization server selector labels */}} {{- define "kubeflow-pipelines.visualizationSelectorLabels" -}} +{{- if eq .Values.installMode.type "multi-user" }} +app: ml-pipeline-visualizationserver +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- else }} app: ml-pipeline-visualizationserver application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} {{/* MySQL selector labels @@ -200,24 +242,39 @@ Metadata writer selector labels */}} {{- define "kubeflow-pipelines.metadataWriterSelectorLabels" -}} app: metadata-writer +{{- if eq .Values.installMode.type "multi-user" }} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- else }} application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} {{/* Persistence agent selector labels */}} {{- define "kubeflow-pipelines.persistenceAgentSelectorLabels" -}} app: ml-pipeline-persistenceagent +{{- if eq .Values.installMode.type "multi-user" }} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- else }} application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} {{/* Scheduled workflow selector labels */}} {{- define "kubeflow-pipelines.scheduledWorkflowSelectorLabels" -}} app: ml-pipeline-scheduledworkflow +{{- if eq .Values.installMode.type "multi-user" }} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- else }} application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} {{/* Workflow controller selector labels diff --git a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml index b361449d87..4dbb02e052 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml @@ -29,6 +29,12 @@ spec: containerPort: 8887 env: # API Server specific environment variables + {{- if eq .Values.installMode.type "multi-user" }} + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + value: "" + {{- end }} - name: PUBLISH_LOGS value: {{ .Values.apiServer.env.publishLogs | quote }} - name: LOG_LEVEL @@ -125,6 +131,11 @@ spec: value: {{ .Values.apiServer.env.v2DriverImage | quote }} - name: V2_LAUNCHER_IMAGE value: {{ .Values.apiServer.env.v2LauncherImage | quote }} + {{- if eq .Values.installMode.type "multi-user" }} + envFrom: + - configMapRef: + name: pipeline-api-server-config + {{- end }} livenessProbe: exec: command: diff --git a/experimental/helm/charts/pipeline/templates/cache/deployment.yaml b/experimental/helm/charts/pipeline/templates/cache/deployment.yaml index 2b1e2af396..50f2cd016b 100644 --- a/experimental/helm/charts/pipeline/templates/cache/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/cache/deployment.yaml @@ -34,57 +34,36 @@ spec: - "--namespace_to_watch=$(NAMESPACE_TO_WATCH)" - "--listen_port=$(WEBHOOK_PORT)" env: - - name: DEFAULT_CACHE_STALENESS - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: mysql - - name: DBCONFIG_DB_NAME - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: cacheDb - - name: DBCONFIG_HOST_NAME - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: dbHost - - name: DBCONFIG_PORT - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: dbPort - - name: DBCONFIG_USER + {{- range .Values.cache.server.additionalEnv }} + - name: {{ .name }} + {{- if .value }} + value: {{ .value | quote }} + {{- else if and .key (ne .key "null") }} + {{- if or (eq .name "DBCONFIG_USER") (eq .name "DBCONFIG_PASSWORD") }} valueFrom: secretKeyRef: name: mysql-secret - key: username - - name: DBCONFIG_PASSWORD + key: {{ .key }} + {{- else }} valueFrom: - secretKeyRef: - name: mysql-secret - key: password - - name: NAMESPACE_TO_WATCH + configMapKeyRef: + name: pipeline-install-config + key: {{ .key }} + {{- end }} + {{- else if eq .name "NAMESPACE_TO_WATCH" }} + {{- if eq $.Values.installMode.type "multi-user" }} + value: "" + {{- else }} valueFrom: fieldRef: fieldPath: metadata.namespace + {{- end }} + {{- else if eq .name "DBCONFIG_DRIVER" }} + value: mysql + {{- else }} + value: "" + {{- end }} + {{- end }} - name: WEBHOOK_PORT value: "8443" ports: diff --git a/experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml b/experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml index 3ecbddb453..dff7c890e3 100644 --- a/experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml +++ b/experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml @@ -1,4 +1,4 @@ -{{- if .Values.cache.enabled }} +{{- if and .Values.cache.enabled (ne .Values.installMode.type "multi-user") }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml index 442485cdc1..652f8dc8d4 100644 --- a/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml @@ -5,8 +5,14 @@ metadata: name: metadata-writer namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: +{{- if eq .Values.installMode.type "multi-user" }} + app: metadata-writer + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +{{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: metadata-writer +{{- end }} spec: replicas: {{ .Values.metadataWriter.replicas }} selector: @@ -15,8 +21,14 @@ spec: template: metadata: labels: +{{- if eq .Values.installMode.type "multi-user" }} + app: metadata-writer + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +{{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: metadata-writer +{{- end }} spec: securityContext: seccompProfile: @@ -27,9 +39,13 @@ spec: image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.metadataWriter.image.repository "tag" .Values.metadataWriter.image.tag "context" .) }} env: - name: NAMESPACE_TO_WATCH +{{- if eq .Values.installMode.type "multi-user" }} + value: "" +{{- else }} valueFrom: fieldRef: fieldPath: metadata.namespace +{{- end }} securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml index f146e7ed89..ea04317666 100644 --- a/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml +++ b/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml @@ -3,9 +3,11 @@ apiVersion: metacontroller.k8s.io/v1alpha1 kind: DecoratorController metadata: name: kubeflow-pipelines-profile-controller + namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: profile-controller + app: kubeflow-pipelines-profile-controller + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: @@ -33,7 +35,6 @@ spec: resource: services updateStrategy: method: InPlace - {{- if .Values.networking.istio.enabled }} - apiVersion: networking.istio.io/v1alpha3 resource: destinationrules updateStrategy: @@ -42,9 +43,8 @@ spec: resource: authorizationpolicies updateStrategy: method: InPlace - {{- end }} hooks: sync: webhook: - url: http://{{ include "kubeflow-pipelines.fullname" . }}-profile-controller/sync + url: http://kubeflow-pipelines-profile-controller/sync {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml index 1f47e722cb..3fb3aa2c75 100644 --- a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml @@ -5,21 +5,24 @@ metadata: name: kubeflow-pipelines-profile-controller namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: profile-controller + app: kubeflow-pipelines-profile-controller + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: replicas: {{ .Values.profileController.replicas }} selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} - component: profile-controller + app: kubeflow-pipelines-profile-controller + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines template: metadata: labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - component: profile-controller + app: kubeflow-pipelines-profile-controller + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines sidecar.istio.io/inject: "false" annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} @@ -27,27 +30,33 @@ spec: {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} containers: - name: profile-controller - image: {{ .Values.profileController.image.repository }}:{{ .Values.profileController.image.tag }} - imagePullPolicy: {{ .Values.profileController.image.pullPolicy | default .Values.global.imagePullPolicy }} + image: public.ecr.aws/docker/library/python:3.12 command: ["python", "/hooks/sync.py"] envFrom: - configMapRef: name: kubeflow-pipelines-profile-controller-env env: - name: KFP_VERSION - value: {{ .Values.global.imageTag | quote }} + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: appVersion - name: KFP_DEFAULT_PIPELINE_ROOT - value: {{ .Values.pipeline.defaultPipelineRoot | quote }} + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: defaultPipelineRoot + optional: true {{- if .Values.minio.enabled }} - name: MINIO_ACCESS_KEY valueFrom: secretKeyRef: - name: {{ include "kubeflow-pipelines.fullname" . }}-minio-artifact + name: mlpipeline-minio-artifact key: accesskey - name: MINIO_SECRET_KEY valueFrom: secretKeyRef: - name: {{ include "kubeflow-pipelines.fullname" . }}-minio-artifact + name: mlpipeline-minio-artifact key: secretkey {{- end }} {{- with .Values.profileController.additionalEnv }} @@ -57,13 +66,9 @@ spec: - name: hooks mountPath: /hooks ports: - - name: http - containerPort: 8080 - protocol: TCP + - containerPort: 8080 securityContext: {{- toYaml .Values.profileController.securityContext | nindent 10 }} - resources: - {{- toYaml .Values.profileController.resources | nindent 10 }} volumes: - name: hooks configMap: diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml b/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml index e8cf33f87b..50155cfccd 100644 --- a/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml @@ -5,8 +5,14 @@ metadata: name: ml-pipeline-scheduledworkflow namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: +{{- if eq .Values.installMode.type "multi-user" }} + app: ml-pipeline-scheduledworkflow + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +{{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-scheduledworkflow +{{- end }} spec: selector: matchLabels: @@ -16,8 +22,14 @@ spec: annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" labels: +{{- if eq .Values.installMode.type "multi-user" }} + app: ml-pipeline-scheduledworkflow + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines +{{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: ml-pipeline-scheduledworkflow +{{- end }} spec: serviceAccountName: ml-pipeline-scheduledworkflow containers: @@ -25,12 +37,17 @@ spec: image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.scheduledWorkflow.image.repository "tag" .Values.scheduledWorkflow.image.tag "context" .) }} imagePullPolicy: IfNotPresent env: - - name: LOG_LEVEL - value: {{ .Values.global.logLevel | quote }} - - name: NAMESPACE + {{- range .Values.scheduledWorkflow.additionalEnv }} + - name: {{ .name }} + {{- if hasKey . "value" }} + value: {{ .value | quote }} + {{- else if and (hasKey . "valueFrom") .valueFrom (not (eq .valueFrom nil)) }} valueFrom: - fieldRef: - fieldPath: metadata.namespace + {{- toYaml .valueFrom | nindent 12 }} + {{- else }} + value: "" + {{- end }} + {{- end }} - name: CRON_SCHEDULE_TIMEZONE valueFrom: configMapKeyRef: diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml b/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml index f486d9f960..8338bc0bd7 100644 --- a/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml @@ -5,8 +5,7 @@ metadata: name: ml-pipeline-viewer-crd namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline-viewer-crd + {{- include "kubeflow-pipelines.viewerCrdLabels" . | nindent 4 }} spec: selector: matchLabels: @@ -16,8 +15,7 @@ spec: annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" labels: - {{- include "kubeflow-pipelines.labels" . | nindent 8 }} - app: ml-pipeline-viewer-crd + {{- include "kubeflow-pipelines.viewerCrdLabels" . | nindent 8 }} spec: serviceAccountName: ml-pipeline-viewer-crd-service-account containers: @@ -25,12 +23,21 @@ spec: image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.viewerCrd.image.repository "tag" .Values.viewerCrd.image.tag "context" .) }} imagePullPolicy: {{ .Values.viewerCrd.image.pullPolicy | default .Values.global.imagePullPolicy }} env: - - name: MAX_NUM_VIEWERS - value: {{ .Values.viewerCrd.env.maxNumViewers | quote }} - - name: NAMESPACE + {{- range .Values.viewerCrd.additionalEnv }} + - name: {{ .name }} + {{- if hasKey . "value" }} + value: {{ .value | quote }} + {{- else if and (hasKey . "valueFrom") .valueFrom (not (eq .valueFrom nil)) }} + valueFrom: + {{- toYaml .valueFrom | nindent 12 }} + {{- else if eq .name "NAMESPACE" }} valueFrom: fieldRef: fieldPath: metadata.namespace + {{- else }} + value: "" + {{- end }} + {{- end }} {{- if and .Values.viewerCrd.resources (or .Values.viewerCrd.resources.requests .Values.viewerCrd.resources.limits) }} resources: {{- toYaml .Values.viewerCrd.resources | nindent 10 }} From b23e2506b56b619fd96ebdf8f9e227329d7f91d3 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Thu, 25 Sep 2025 23:25:13 +0530 Subject: [PATCH 05/26] update labels and env Signed-off-by: kunal-511 --- .../pipeline/ci/values-aws-enhanced.yaml | 70 ++++++++++++------- .../pipeline/ci/values-azure-enhanced.yaml | 15 ++-- .../pipeline/ci/values-dev-enhanced.yaml | 38 ++-------- .../pipeline/ci/values-gcp-enhanced.yaml | 19 ++--- .../ci/values-multi-user-enhanced.yaml | 21 +++--- .../ci/values-platform-agnostic-enhanced.yaml | 12 ++-- .../charts/pipeline/ci/values-standalone.yaml | 62 ++++++++++++++++ .../charts/pipeline/templates/_helpers.tpl | 9 +++ .../config/pipeline-api-server-config.yaml | 2 +- .../config/pipeline-install-config.yaml | 2 +- .../templates/config/ui-configmap.yaml | 5 +- .../templates/metadata/envoy-deployment.yaml | 7 +- .../templates/metadata/grpc-deployment.yaml | 6 +- .../persistence-agent/deployment.yaml | 17 ++++- .../profile-controller/deployment.yaml | 3 +- .../templates/profile-controller/service.yaml | 8 +-- .../templates/rbac/cluster-role-bindings.yaml | 2 + .../templates/rbac/cluster-roles.yaml | 43 ++++++++++-- experimental/helm/charts/pipeline/values.yaml | 6 +- 19 files changed, 222 insertions(+), 125 deletions(-) diff --git a/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml index f4e81f57f8..e47dad28b7 100644 --- a/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml @@ -59,7 +59,7 @@ pipeline: # Argo Workflows Configuration argo: - enabled: true + enabled: false env: platform: aws @@ -107,28 +107,29 @@ ui: requests: cpu: 10m memory: 70Mi - limits: - cpu: 100m - memory: 200Mi + # Enhanced resources for AWS deployment persistenceAgent: + additionalEnv: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace resources: requests: cpu: 120m memory: 500Mi - limits: - cpu: 500m - memory: 1Gi scheduledWorkflow: resources: - requests: - cpu: 120m - memory: 100Mi - limits: - cpu: 250m - memory: 500Mi + additionalEnv: + - name: LOG_LEVEL + value: "info" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace metadata: grpc: @@ -150,6 +151,31 @@ metadata: cache: server: + additionalEnv: + - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: "mysql" + - name: DBCONFIG_DB_NAME + key: cacheDb + - name: DBCONFIG_HOST_NAME + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace resources: requests: cpu: 120m @@ -161,20 +187,16 @@ cache: visualization: resources: requests: - cpu: 50m - memory: 200Mi - limits: - cpu: 200m + cpu: 30m memory: 500Mi viewerCrd: - resources: - requests: - cpu: 120m - memory: 100Mi - limits: - cpu: 250m - memory: 200Mi + image: + pullPolicy: Always + additionalEnv: + - name: MAX_NUM_VIEWERS + value: 50 + - name: NAMESPACE # AWS Load Balancer Controller integration diff --git a/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml index 081c44abea..52134c53f2 100644 --- a/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml @@ -58,7 +58,7 @@ pipeline: # Argo Workflows Configuration argo: - enabled: true + enabled: false # Azure-specific Argo configuration # Environment Configuration @@ -110,9 +110,6 @@ ui: requests: cpu: 10m memory: 70Mi - limits: - cpu: 100m - memory: 200Mi # Enhanced resources for Azure deployment persistenceAgent: @@ -120,9 +117,6 @@ persistenceAgent: requests: cpu: 120m memory: 500Mi - limits: - cpu: 500m - memory: 1Gi scheduledWorkflow: resources: @@ -164,10 +158,7 @@ cache: visualization: resources: requests: - cpu: 50m - memory: 200Mi - limits: - cpu: 200m + cpu: 30m memory: 500Mi viewerCrd: @@ -178,6 +169,8 @@ viewerCrd: limits: cpu: 250m memory: 200Mi + image: + pullPolicy: Always # Azure Load Balancer integration service: diff --git a/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml index 46dbcb41f8..a354a5b39b 100644 --- a/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml @@ -96,32 +96,17 @@ persistenceAgent: replicas: 1 resources: requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 512Mi - + cpu: 120m + memory: 500Mi scheduledWorkflow: replicas: 1 - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 100m - memory: 256Mi ui: replicas: 1 resources: requests: cpu: 10m - memory: 32Mi - limits: - cpu: 50m - memory: 128Mi - + memory: 70Mi metadata: grpc: replicas: 1 @@ -167,24 +152,15 @@ visualization: replicas: 1 resources: requests: - cpu: 25m - memory: 64Mi - limits: - cpu: 100m - memory: 256Mi + cpu: 30m + memory: 500Mi viewerCrd: replicas: 1 + image: + pullPolicy: Always env: maxNumViewers: "10" - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 100m - memory: 128Mi - dev: # Enable debug mode debug: true diff --git a/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml index 4d20a3a2bd..81d59bd36c 100644 --- a/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml @@ -57,7 +57,7 @@ pipeline: # Argo Workflows Configuration argo: - enabled: true + enabled: false # GCP-specific Argo configuration # Environment Configuration @@ -108,9 +108,6 @@ ui: requests: cpu: 10m memory: 70Mi - limits: - cpu: 100m - memory: 200Mi # Enhanced resources for GCP deployment persistenceAgent: @@ -162,20 +159,12 @@ cache: visualization: resources: requests: - cpu: 50m - memory: 200Mi - limits: - cpu: 200m + cpu: 30m memory: 500Mi viewerCrd: - resources: - requests: - cpu: 120m - memory: 100Mi - limits: - cpu: 250m - memory: 200Mi + image: + pullPolicy: Always # Cloud SQL Proxy sidecar configuration cloudsqlProxy: diff --git a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml index 9648e4cd01..ca4c1fc933 100644 --- a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml @@ -17,7 +17,7 @@ installMode: # Database Configuration - Enhanced for multi-user mysql: - enabled: true + enabled: false auth: rootPassword: "root123" username: "mlpipeline" @@ -40,7 +40,7 @@ postgresql: # Object Storage Configuration - Enhanced for multi-user minio: - enabled: true + enabled: false auth: rootUser: "minio" rootPassword: "minio123" @@ -77,8 +77,8 @@ pipeline: # Argo Workflows Configuration - Multi-user argo: - enabled: true - # Multi-user specific Argo configuration + enabled: false + # Multi-user uses built-in minimal Argo instead of subchart # Environment Configuration env: @@ -121,6 +121,9 @@ persistenceAgent: requests: cpu: 120m memory: 500Mi + additionalEnv: + - name: NAMESPACE + value: "" scheduledWorkflow: additionalEnv: @@ -238,11 +241,11 @@ monitoring: multiUser: true # Service mesh - Recommended for multi-user -istio: - enabled: true - gateway: kubeflow-gateway - multiUser: true - authorizationPolicies: +networking: + istio: + enabled: true + gateway: kubeflow-gateway + virtualService: enabled: true # Cert-manager for TLS diff --git a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml index c828a937f7..a16e5e1529 100644 --- a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml @@ -100,6 +100,11 @@ persistenceAgent: requests: cpu: 120m memory: 500Mi + additionalEnv: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace scheduledWorkflow: resources: @@ -126,14 +131,9 @@ webhooks: enabled: false metadata: + enabled: true grpc: resources: - requests: - cpu: 100m - memory: 300Mi - limits: - cpu: 200m - memory: 600Mi envoy: resources: requests: diff --git a/experimental/helm/charts/pipeline/ci/values-standalone.yaml b/experimental/helm/charts/pipeline/ci/values-standalone.yaml index 97fc89165b..a36ec32d48 100644 --- a/experimental/helm/charts/pipeline/ci/values-standalone.yaml +++ b/experimental/helm/charts/pipeline/ci/values-standalone.yaml @@ -50,6 +50,17 @@ thirdParty: argo: enabled: true +# Workflow controller configuration +workflowController: + priorityClass: + create: false + +# Enable metadata server for standalone deployment +metadata: + enabled: true + grpc: + resources: + # Disable cloud-specific features monitoring: prometheus: @@ -71,3 +82,54 @@ crds: # Disable webhooks for platform-agnostic scenario webhooks: enabled: false + + +cache: + server: + additionalEnv: + - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: "mysql" + - name: DBCONFIG_DB_NAME + key: cacheDb + - name: DBCONFIG_HOST_NAME + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace +scheduledWorkflow: + additionalEnv: + - name: LOG_LEVEL + value: "info" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +persistenceAgent: + additionalEnv: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +viewerCrd: + additionalEnv: + - name: MAX_NUM_VIEWERS + value: "50" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace \ No newline at end of file diff --git a/experimental/helm/charts/pipeline/templates/_helpers.tpl b/experimental/helm/charts/pipeline/templates/_helpers.tpl index 88f761c4b2..6061067598 100644 --- a/experimental/helm/charts/pipeline/templates/_helpers.tpl +++ b/experimental/helm/charts/pipeline/templates/_helpers.tpl @@ -102,7 +102,9 @@ ML Pipeline UI labels app: ml-pipeline-ui app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if ne .Values.installMode.type "multi-user" }} application-crd-id: kubeflow-pipelines +{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} @@ -139,8 +141,13 @@ UI selector labels */}} {{- define "kubeflow-pipelines.uiSelectorLabels" -}} app: ml-pipeline-ui +{{- if eq .Values.installMode.type "multi-user" }} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- else }} application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} {{/* Viewer CRD labels @@ -177,7 +184,9 @@ Visualization server labels app: ml-pipeline-visualizationserver app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if ne .Values.installMode.type "multi-user" }} application-crd-id: kubeflow-pipelines +{{- end}} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml b/experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml index dca1fbf52b..eabf0841e0 100644 --- a/experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml +++ b/experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml @@ -1,4 +1,4 @@ -{{- if .Values.apiServer.enabled }} +{{- if and .Values.apiServer.enabled .Values.installMode.multiUser.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml b/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml index 2e74241e87..d60bfd942c 100644 --- a/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml +++ b/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml @@ -25,7 +25,7 @@ data: dbHost: {{ $dbHost | quote }} dbPort: {{ include "kubeflow-pipelines.database.port" . | quote }} dbType: {{ .Values.externalDatabase.type | default "mysql" | quote }} - mysqlHost: {{ $dbHost | quote }} + mysqlHost: "mysql" mysqlPort: {{ include "kubeflow-pipelines.database.port" . | quote }} mlmdDb: {{ .Values.database.metadb }} cacheDb: {{ .Values.database.cachedb }} diff --git a/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml b/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml index 2ec1f76abc..4acd6987d8 100644 --- a/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml +++ b/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml @@ -43,14 +43,13 @@ data: } ] } - ], - "serviceAccountName": "{{- if and .Values.installMode.multiUser.enabled (eq .Values.installMode.type "multi-user") }}default-editor{{- else }}kubeflow-pipelines-viewer{{- end }}" + ] } } {{- else }} { "spec": { - "serviceAccountName": "{{- if and .Values.installMode.multiUser.enabled (eq .Values.installMode.type "multi-user") }}default-editor{{- else }}kubeflow-pipelines-viewer{{- end }}" + "serviceAccountName": "{{ if and .Values.installMode.multiUser.enabled (eq .Values.installMode.type "multi-user") }}default-editor{{ else }}kubeflow-pipelines-viewer{{ end }}" } } {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml index f4e18423d4..34837c39cc 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml @@ -5,20 +5,19 @@ metadata: name: metadata-envoy-deployment namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} component: metadata-envoy - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines spec: replicas: {{ .Values.metadata.envoy.replicas }} selector: matchLabels: component: metadata-envoy + application-crd-id: kubeflow-pipelines template: metadata: labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} component: metadata-envoy - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines sidecar.istio.io/inject: "false" spec: containers: diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml b/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml index 85aec707d2..5354d08173 100644 --- a/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml @@ -7,20 +7,17 @@ metadata: labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} component: metadata-grpc-server - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines spec: replicas: 1 selector: matchLabels: component: metadata-grpc-server + application-crd-id: kubeflow-pipelines template: metadata: labels: {{- include "kubeflow-pipelines.labels" . | nindent 8 }} component: metadata-grpc-server - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines spec: securityContext: seccompProfile: @@ -28,7 +25,6 @@ spec: containers: - name: container image: {{ .Values.metadata.grpc.image.repository }}:{{ .Values.metadata.grpc.image.tag | default "1.14.0" }} - imagePullPolicy: {{ .Values.metadata.grpc.image.pullPolicy | default "IfNotPresent" }} securityContext: allowPrivilegeEscalation: false seccompProfile: diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml b/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml index 091d7cce5d..6fa53a4b1e 100644 --- a/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml @@ -25,10 +25,25 @@ spec: image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.persistenceAgent.image.repository "tag" .Values.persistenceAgent.image.tag "context" .) }} imagePullPolicy: IfNotPresent env: - - name: NAMESPACE + {{- range .Values.persistenceAgent.additionalEnv }} + - name: {{ .name }} + {{- if hasKey . "value" }} + value: {{ .value | quote }} + {{- else if and (hasKey . "valueFrom") .valueFrom (not (eq .valueFrom nil)) }} + valueFrom: + {{- toYaml .valueFrom | nindent 12 }} + {{- else if eq .name "NAMESPACE" }} + {{- if eq $.Values.installMode.type "multi-user" }} + value: "" + {{- else }} valueFrom: fieldRef: fieldPath: metadata.namespace + {{- end }} + {{- else }} + value: "" + {{- end }} + {{- end }} - name: TTL_SECONDS_AFTER_WORKFLOW_FINISH value: {{ .Values.persistenceAgent.env.ttlSecondsAfterWorkflowFinish | quote }} - name: NUM_WORKERS diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml index 3fb3aa2c75..0622599298 100644 --- a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml @@ -5,9 +5,8 @@ metadata: name: kubeflow-pipelines-profile-controller namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: kubeflow-pipelines-profile-controller - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml index 9759c8555e..f00b79ea80 100644 --- a/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml +++ b/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml @@ -6,17 +6,17 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: profile-controller + app: kubeflow-pipelines-profile-controller annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: - type: {{ .Values.profileController.service.type }} ports: - name: http protocol: TCP port: {{ .Values.profileController.service.port }} targetPort: 8080 selector: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} - component: profile-controller + app: kubeflow-pipelines-profile-controller + app.kubernetes.io/component: ml-pipeline + app.kubernetes.io/name: kubeflow-pipelines {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml b/experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml index 125436dd03..2b43fb8040 100644 --- a/experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml +++ b/experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml @@ -23,6 +23,7 @@ metadata: name: ml-pipeline-ui labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: ml-pipeline-ui roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -91,6 +92,7 @@ metadata: name: kubeflow-pipelines-cache-binding labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: cache-server roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml b/experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml index f4ef3982e5..ea58bc837d 100644 --- a/experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml +++ b/experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml @@ -78,6 +78,7 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines + app: ml-pipeline-ui rules: - apiGroups: - "" @@ -86,7 +87,6 @@ rules: - pods/log verbs: - get - - list - apiGroups: - "" resources: @@ -110,6 +110,14 @@ rules: - list - watch - delete +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + --- # Scheduled Workflow ClusterRole @@ -134,10 +142,17 @@ rules: - update - patch - delete +- apiGroups: + - pipelines.kubeflow.org + resources: + - runs + verbs: + - create - apiGroups: - kubeflow.org resources: - scheduledworkflows + - scheduledworkflows/finalizers verbs: - create - get @@ -147,11 +162,12 @@ rules: - patch - delete - apiGroups: - - kubeflow.org + - "" resources: - - scheduledworkflows/finalizers + - events verbs: - - update + - create + - patch --- # Persistence Agent ClusterRole @@ -180,6 +196,20 @@ rules: - get - list - watch +- apiGroups: + - pipelines.kubeflow.org + resources: + - scheduledworkflows + - workflows + verbs: + - report +- apiGroups: + - pipelines.kubeflow.org + resources: + - runs + verbs: + - reportMetrics + - readArtifact --- # Viewer Controller ClusterRole @@ -196,7 +226,7 @@ rules: - '*' resources: - deployments - - replicasets + - services verbs: - create - get @@ -209,6 +239,7 @@ rules: - kubeflow.org resources: - viewers + - viewers/finalizers verbs: - create - get @@ -228,6 +259,7 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines + app: cache-server rules: - apiGroups: - "" @@ -266,6 +298,7 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines + app: kubeflow-pipelines-cache-deployer-clusterrole rules: - apiGroups: - certificates.k8s.io diff --git a/experimental/helm/charts/pipeline/values.yaml b/experimental/helm/charts/pipeline/values.yaml index b26f6d765d..c8954296b4 100644 --- a/experimental/helm/charts/pipeline/values.yaml +++ b/experimental/helm/charts/pipeline/values.yaml @@ -288,7 +288,7 @@ viewerCrd: # -- Viewer CRD Controller image tag (overrides global.imageTag if set) tag: "" # -- Viewer CRD Controller image pull policy (overrides global.imagePullPolicy if set) - pullPolicy: "" + pullPolicy: "Always" # -- Viewer CRD Controller resource requests and limits resources: {} @@ -337,8 +337,8 @@ visualization: # -- Visualization Server resource requests and limits resources: requests: - cpu: 50m - memory: 200Mi + cpu: 30m + memory: 500Mi limits: {} # Service Account From 123e42c52e14316185ed517825d1b97a3603b92d Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Fri, 26 Sep 2025 19:09:58 +0530 Subject: [PATCH 06/26] Update values Signed-off-by: kunal-511 --- .../pipeline/ci/values-aws-enhanced.yaml | 126 +++++++++++++++-- .../pipeline/ci/values-azure-enhanced.yaml | 103 +++++++++++--- .../pipeline/ci/values-dev-enhanced.yaml | 11 +- .../pipeline/ci/values-gcp-enhanced.yaml | 85 ++++++++++++ .../ci/values-multi-user-enhanced.yaml | 118 +++++++++++++++- .../ci/values-platform-agnostic-enhanced.yaml | 29 +++- .../charts/pipeline/ci/values-standalone.yaml | 30 +++- .../templates/api-server/deployment.yaml | 130 ++++++------------ .../templates/api-server/serviceaccount.yaml | 2 - .../templates/database/mysql-secret.yaml | 7 +- .../profile-controller/deployment.yaml | 38 ++--- .../third-party/minio-deployment.yaml | 29 +--- .../templates/third-party/minio-secret.yaml | 12 +- .../templates/third-party/minio-service.yaml | 3 +- .../pipeline/templates/ui/deployment.yaml | 45 +++--- 15 files changed, 565 insertions(+), 203 deletions(-) diff --git a/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml index e47dad28b7..1d5ede41c4 100644 --- a/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml @@ -92,9 +92,75 @@ apiServer: requests: cpu: 250m memory: 500Mi - limits: - cpu: 1 - memory: 2Gi + additionalEnv: + - name: OBJECTSTORECONFIG_SECURE + value: "true" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + configMapName: pipeline-install-config + - name: OBJECTSTORECONFIG_HOST + key: minioServiceHost + configMapName: pipeline-install-config + - name: OBJECTSTORECONFIG_REGION + key: minioServiceRegion + configMapName: pipeline-install-config + - name: OBJECTSTORECONFIG_PORT + value: "" + - name: PUBLISH_LOGS + value: "true" + - name: LOG_LEVEL + value: "info" + - name: PIPELINE_LOG_LEVEL + value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + configMapName: pipeline-install-config + - name: POD_NAMESPACE + - name: DBCONFIG_USER + key: username + secretName: mysql-secret + - name: DBCONFIG_PASSWORD + key: password + secretName: mysql-secret + - name: DBCONFIG_DBNAME + key: pipelineDb + configMapName: pipeline-install-config + - name: DBCONFIG_HOST + key: dbHost + configMapName: pipeline-install-config + - name: DBCONFIG_PORT + key: dbPort + configMapName: pipeline-install-config + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + configMapName: pipeline-install-config + - name: DB_DRIVER_NAME + key: dbType + secretName: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + secretName: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + secretName: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + configMapName: pipeline-install-config + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + secretName: mlpipeline-minio-artifact + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey + - name: V2_DRIVER_IMAGE + value: "ghcr.io/kubeflow/kfp-driver:2.5.0" + - name: V2_LAUNCHER_IMAGE + value: "ghcr.io/kubeflow/kfp-launcher:2.5.0" # AWS UI configuration ui: @@ -107,6 +173,43 @@ ui: requests: cpu: 10m memory: 70Mi + additionalEnv: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" # Enhanced resources for AWS deployment @@ -132,14 +235,9 @@ scheduledWorkflow: fieldPath: metadata.namespace metadata: + enabled: true grpc: resources: - requests: - cpu: 100m - memory: 300Mi - limits: - cpu: 500m - memory: 1Gi envoy: resources: requests: @@ -227,3 +325,13 @@ certManager: aws: region: "YOUR_AWS_REGION" hostedZoneID: "YOUR_ROUTE53_ZONE_ID" + +profileController: + additionalEnv: + - name: AWS_REGION + value: "YOUR_AWS_REGION" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact \ No newline at end of file diff --git a/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml index 52134c53f2..0d837f903b 100644 --- a/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml @@ -27,7 +27,7 @@ externalDatabase: database: mlpipeline username: "YOUR_AZURE_SQL_USERNAME@[SQLSERVERNAME]" password: "YOUR_AZURE_SQL_PASSWORD" - existingSecret: "mysql-secret" + existingSecret: "" # Object Storage Configuration minio: @@ -75,6 +75,8 @@ env: enabled: true storageAccount: "yourstorageaccount" container: "mlpipeline" + managedIdentity: + enabled: false # Azure API Server configuration apiServer: @@ -95,9 +97,6 @@ apiServer: requests: cpu: 250m memory: 500Mi - limits: - cpu: 1 - memory: 2Gi # Azure UI configuration ui: @@ -110,6 +109,38 @@ ui: requests: cpu: 10m memory: 70Mi + additionalEnv: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" + +# Third-party components configuration +thirdParty: + minio: + enabled: true # Enhanced resources for Azure deployment persistenceAgent: @@ -117,25 +148,27 @@ persistenceAgent: requests: cpu: 120m memory: 500Mi + additionalEnv: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace scheduledWorkflow: resources: - requests: - cpu: 120m - memory: 100Mi - limits: - cpu: 250m - memory: 500Mi + additionalEnv: + - name: LOG_LEVEL + value: "info" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + metadata: + enabled: true grpc: resources: - requests: - cpu: 100m - memory: 300Mi - limits: - cpu: 500m - memory: 1Gi envoy: resources: requests: @@ -154,6 +187,31 @@ cache: limits: cpu: 500m memory: 1Gi + additionalEnv: + - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: "mysql" + - name: DBCONFIG_DB_NAME + key: cacheDb + - name: DBCONFIG_HOST_NAME + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace visualization: resources: @@ -163,14 +221,15 @@ visualization: viewerCrd: resources: - requests: - cpu: 120m - memory: 100Mi - limits: - cpu: 250m - memory: 200Mi image: pullPolicy: Always + additionalEnv: + - name: MAX_NUM_VIEWERS + value: "50" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace # Azure Load Balancer integration service: diff --git a/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml index a354a5b39b..f1a3169c44 100644 --- a/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml @@ -84,13 +84,13 @@ apiServer: replicas: 1 resources: requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 512Mi + cpu: 250m + memory: 500Mi env: logLevel: "2" + additionalEnv: + - name: PIPELINE_LOG_LEVEL + value: "1" persistenceAgent: replicas: 1 @@ -108,6 +108,7 @@ ui: cpu: 10m memory: 70Mi metadata: + enabled: true grpc: replicas: 1 resources: diff --git a/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml index 81d59bd36c..c47a26c4cc 100644 --- a/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml @@ -108,6 +108,33 @@ ui: requests: cpu: 10m memory: 70Mi + additionalEnv: + - name: DISABLE_GKE_METADATA + value: "false" + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" # Enhanced resources for GCP deployment persistenceAgent: @@ -129,6 +156,7 @@ scheduledWorkflow: memory: 500Mi metadata: + enabled: true grpc: resources: requests: @@ -155,6 +183,56 @@ cache: limits: cpu: 500m memory: 1Gi + additionalEnv: + - name: DEFAULT_CACHE_STALENESS + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CACHE_NODE_RESTRICTIONS + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: DBCONFIG_DRIVER + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: DBCONFIG_DB_NAME + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact + - name: DBCONFIG_HOST_NAME + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: DBCONFIG_PORT + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: DBCONFIG_PASSWORD + valueFrom: + + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace + visualization: resources: @@ -163,6 +241,13 @@ visualization: memory: 500Mi viewerCrd: + additionalEnv: + - name: MAX_NUM_VIEWERS + value: "50" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: pullPolicy: Always diff --git a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml index ca4c1fc933..5602314e28 100644 --- a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml @@ -102,18 +102,123 @@ apiServer: requests: cpu: 250m memory: 500Mi + additionalEnv: + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + value: "" + - name: PUBLISH_LOGS + value: "true" + - name: LOG_LEVEL + value: "info" + - name: PIPELINE_LOG_LEVEL + value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + configMapName: pipeline-install-config + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + configMapName: pipeline-install-config + - name: DBCONFIG_USER + key: username + secretName: mysql-secret + - name: DBCONFIG_PASSWORD + key: password + secretName: mysql-secret + - name: DBCONFIG_DBNAME + key: pipelineDb + configMapName: pipeline-install-config + - name: DBCONFIG_HOST + key: dbHost + configMapName: pipeline-install-config + - name: DBCONFIG_PORT + key: dbPort + configMapName: pipeline-install-config + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + configMapName: pipeline-install-config + - name: DB_DRIVER_NAME + key: dbType + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + secretName: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + secretName: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + configMapName: pipeline-install-config + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + secretName: mlpipeline-minio-artifact + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey + secretName: mlpipeline-minio-artifact + - name: V2_DRIVER_IMAGE + value: "ghcr.io/kubeflow/kfp-driver:2.5.0" + - name: V2_LAUNCHER_IMAGE + value: "ghcr.io/kubeflow/kfp-launcher:2.5.0" # Multi-user UI configuration ui: env: # Multi-user UI configuration multiUser: "true" - + resources: requests: cpu: 10m memory: 70Mi + additionalEnv: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: DEPLOYMENT + value: "KUBEFLOW" + - name: ARTIFACTS_SERVICE_PROXY_NAME + value: "ml-pipeline-ui-artifact" + - name: ARTIFACTS_SERVICE_PROXY_PORT + value: "80" + - name: ARTIFACTS_SERVICE_PROXY_ENABLED + value: "true" + - name: ENABLE_AUTHZ + value: "true" + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + key: null + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + key: accesskey + - name: MINIO_SECRET_KEY + key: secretkey + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" + # Enhanced resources for multi-user deployment persistenceAgent: @@ -174,6 +279,7 @@ cache: - name: CACHE_NODE_RESTRICTIONS key: cacheNodeRestrictions - name: DBCONFIG_DRIVER + value: "mysql" - name: DBCONFIG_DB_NAME key: cacheDb - name: DBCONFIG_HOST_NAME @@ -218,6 +324,16 @@ profileController: cpu: 500m memory: 1Gi + additionalEnv: + - name: KFP_VERSION + key: appVersion + - name: KFP_DEFAULT_PIPELINE_ROOT + key: defaultPipelineRoot + - name: MINIO_ACCESS_KEY + key: accesskey + - name: MINIO_SECRET_KEY + key: secretkey + # Namespace Isolation Configuration namespaceIsolation: enabled: true diff --git a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml index a16e5e1529..b491e7fc2c 100644 --- a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml +++ b/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml @@ -62,7 +62,7 @@ database: # Pipeline Configuration pipeline: - # Platform-agnostic specific: Empty defaultPipelineRoot to match kustomize + # Platform-agnostic specific: Empty defaultPipelineRoot defaultPipelineRoot: "" cache: image: "registry.k8s.io/busybox" @@ -122,6 +122,33 @@ ui: cpu: 10m memory: 70Mi + additionalEnv: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" # Disable CRDs for platform-agnostic scenario crds: install: false diff --git a/experimental/helm/charts/pipeline/ci/values-standalone.yaml b/experimental/helm/charts/pipeline/ci/values-standalone.yaml index a36ec32d48..4be61b650b 100644 --- a/experimental/helm/charts/pipeline/ci/values-standalone.yaml +++ b/experimental/helm/charts/pipeline/ci/values-standalone.yaml @@ -132,4 +132,32 @@ viewerCrd: - name: NAMESPACE valueFrom: fieldRef: - fieldPath: metadata.namespace \ No newline at end of file + fieldPath: metadata.namespace +ui: + additionalEnv: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" \ No newline at end of file diff --git a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml index 4dbb02e052..40c961b541 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml @@ -28,109 +28,59 @@ spec: - name: grpc containerPort: 8887 env: - # API Server specific environment variables - {{- if eq .Values.installMode.type "multi-user" }} - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - value: "" - {{- end }} - - name: PUBLISH_LOGS - value: {{ .Values.apiServer.env.publishLogs | quote }} - - name: LOG_LEVEL - value: {{ .Values.global.logLevel | quote }} - - name: PIPELINE_LOG_LEVEL - value: {{ .Values.apiServer.env.pipelineLogLevel | quote }} - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - valueFrom: - configMapKeyRef: - key: autoUpdatePipelineDefaultVersion - name: pipeline-install-config - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - valueFrom: - configMapKeyRef: - key: bucketName - name: pipeline-install-config - - name: DBCONFIG_USER + {{- range .Values.apiServer.additionalEnv }} + - name: {{ .name }} + {{- if .value }} + value: {{ .value | quote }} + {{- else if and .key (ne .key "null") }} + {{- if and .secretName .secretKey }} valueFrom: secretKeyRef: - key: username - name: mysql-secret - - name: DBCONFIG_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: mysql-secret - - name: DBCONFIG_DBNAME + name: {{ .secretName }} + key: {{ .secretKey }} + {{- else if and .configMapName .configMapKey }} valueFrom: configMapKeyRef: - key: pipelineDb - name: pipeline-install-config - - name: DBCONFIG_HOST - valueFrom: - configMapKeyRef: - key: dbHost - name: pipeline-install-config - - name: DBCONFIG_PORT - valueFrom: - configMapKeyRef: - key: dbPort - name: pipeline-install-config - - name: DBCONFIG_CONMAXLIFETIME - valueFrom: - configMapKeyRef: - key: ConMaxLifeTime - name: pipeline-install-config - - name: DB_DRIVER_NAME - valueFrom: - configMapKeyRef: - key: dbType - name: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_USER + name: {{ .configMapName }} + key: {{ .configMapKey }} + {{- else if or (eq .name "DBCONFIG_USER") (eq .name "DBCONFIG_PASSWORD") (eq .name "DBCONFIG_MYSQLCONFIG_USER") (eq .name "DBCONFIG_MYSQLCONFIG_PASSWORD") }} valueFrom: secretKeyRef: - key: username name: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: {{ .key }} + {{- else if or (eq .name "OBJECTSTORECONFIG_ACCESSKEY") (eq .name "OBJECTSTORECONFIG_SECRETACCESSKEY") }} valueFrom: secretKeyRef: - key: password - name: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_DBNAME - valueFrom: - configMapKeyRef: - key: pipelineDb - name: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_HOST - valueFrom: - configMapKeyRef: - key: mysqlHost - name: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_PORT + name: mlpipeline-minio-artifact + key: {{ .key }} + {{- else }} valueFrom: configMapKeyRef: - key: mysqlPort name: pipeline-install-config - - name: OBJECTSTORECONFIG_ACCESSKEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: {{ .key }} + {{- end }} + {{- else if eq .name "POD_NAMESPACE" }} valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: V2_DRIVER_IMAGE - value: {{ .Values.apiServer.env.v2DriverImage | quote }} - - name: V2_LAUNCHER_IMAGE - value: {{ .Values.apiServer.env.v2LauncherImage | quote }} + fieldRef: + fieldPath: metadata.namespace + {{- else if eq .name "KUBEFLOW_USERID_HEADER" }} + {{- if eq $.Values.installMode.type "multi-user" }} + value: "kubeflow-userid" + {{- else }} + value: "" + {{- end }} + {{- else if eq .name "KUBEFLOW_USERID_PREFIX" }} + {{- if eq $.Values.installMode.type "multi-user" }} + value: "" + {{- else }} + value: "" + {{- end }} + {{- else if eq .name "OBJECTSTORECONFIG_SECURE" }} + value: "false" + {{- else }} + value: "" + {{- end }} + {{- end }} {{- if eq .Values.installMode.type "multi-user" }} envFrom: - configMapRef: diff --git a/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml b/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml index f37810d5b7..6f89594c56 100644 --- a/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml +++ b/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml @@ -7,7 +7,5 @@ metadata: labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} {{- with .Values.apiServer.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml b/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml index 44af598c3f..aa09bc6274 100644 --- a/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml +++ b/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml @@ -8,7 +8,8 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -stringData: - username: {{ .Values.externalDatabase.username | quote }} - password: {{ .Values.externalDatabase.password | quote }} +type: Opaque +data: + username: {{ .Values.externalDatabase.username | b64enc | quote }} + password: {{ .Values.externalDatabase.password | b64enc | quote }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml index 0622599298..7d84a9d398 100644 --- a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml @@ -35,31 +35,31 @@ spec: - configMapRef: name: kubeflow-pipelines-profile-controller-env env: - - name: KFP_VERSION + {{- range .Values.profileController.additionalEnv }} + - name: {{ .name }} + {{- if .value }} + value: {{ .value | quote }} + {{- else if and .key (ne .key "null") }} + {{- if or (eq .name "MINIO_ACCESS_KEY") (eq .name "MINIO_SECRET_KEY") (eq .name "AWS_ACCESS_KEY_ID") (eq .name "AWS_SECRET_ACCESS_KEY") }} valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: appVersion - - name: KFP_DEFAULT_PIPELINE_ROOT + secretKeyRef: + name: mlpipeline-minio-artifact + key: {{ .key }} + {{- else }} valueFrom: configMapKeyRef: name: pipeline-install-config - key: defaultPipelineRoot + key: {{ .key }} + {{- if eq .name "KFP_DEFAULT_PIPELINE_ROOT" }} optional: true - {{- if .Values.minio.enabled }} - - name: MINIO_ACCESS_KEY + {{- end }} + {{- end }} + {{- else if .valueFrom }} valueFrom: - secretKeyRef: - name: mlpipeline-minio-artifact - key: accesskey - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: mlpipeline-minio-artifact - key: secretkey - {{- end }} - {{- with .Values.profileController.additionalEnv }} - {{- toYaml . | nindent 8 }} + {{- toYaml .valueFrom | nindent 12 }} + {{- else }} + value: "" + {{- end }} {{- end }} volumeMounts: - name: hooks diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml b/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml index e036736b9f..fdbbfca9ed 100644 --- a/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml @@ -19,15 +19,10 @@ spec: {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: minio spec: - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: "OnRootMismatch" - seccompProfile: - type: RuntimeDefault containers: - args: - - server - - /data + - gateway + - azure env: - name: MINIO_ACCESS_KEY valueFrom: @@ -43,26 +38,8 @@ spec: name: minio ports: - containerPort: 9000 - securityContext: - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - - ALL - volumeMounts: - - mountPath: /data - name: data - subPath: minio resources: requests: cpu: 20m - memory: 100Mi - volumes: - - name: data - persistentVolumeClaim: - claimName: minio-pvc + memory: 25Mi {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml b/experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml index 7823542094..eb11e882af 100644 --- a/experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml +++ b/experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml @@ -6,7 +6,13 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -stringData: - accesskey: minio - secretkey: minio123 +type: Opaque +data: + {{- if eq .Values.env.platform "azure" }} + accesskey: {{ .Values.env.azure.storageAccount | b64enc | quote }} + secretkey: {{ .Values.objectStore.azure.accessKey | b64enc | quote }} + {{- else }} + accesskey: {{ "minio" | b64enc | quote }} + secretkey: {{ "minio123" | b64enc | quote }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml b/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml index 0e0ce28640..f693f1e720 100644 --- a/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml +++ b/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml @@ -8,8 +8,7 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} spec: ports: - - name: http - port: 9000 + - port: 9000 protocol: TCP targetPort: 9000 selector: diff --git a/experimental/helm/charts/pipeline/templates/ui/deployment.yaml b/experimental/helm/charts/pipeline/templates/ui/deployment.yaml index d1bcef47d1..eec9d07bae 100644 --- a/experimental/helm/charts/pipeline/templates/ui/deployment.yaml +++ b/experimental/helm/charts/pipeline/templates/ui/deployment.yaml @@ -33,32 +33,39 @@ spec: mountPath: /etc/config readOnly: true env: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: /etc/config/viewer-pod-template.json - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY + {{- range .Values.ui.additionalEnv }} + - name: {{ .name }} + {{- if .value }} + value: {{ .value | quote }} + {{- else if and .key (ne .key "null") }} + {{- if or (eq .name "MINIO_ACCESS_KEY") (eq .name "MINIO_SECRET_KEY") }} valueFrom: secretKeyRef: - key: accesskey name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY + key: {{ .key }} + {{- else }} valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: {{ .Values.ui.env.allowCustomVisualizations | quote }} - - name: FRONTEND_SERVER_NAMESPACE + configMapKeyRef: + name: pipeline-install-config + key: {{ .key }} + {{- end }} + {{- else if .valueFrom }} + valueFrom: + {{- toYaml .valueFrom | nindent 12 }} + {{- else if or (eq .name "MINIO_NAMESPACE") (eq .name "FRONTEND_SERVER_NAMESPACE") }} valueFrom: fieldRef: fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: {{ .Values.ui.env.argoArchiveLogs | quote }} - - name: DISABLE_GKE_METADATA - value: {{ .Values.ui.env.disableGkeMetadata | quote }} + {{- else if eq .name "ALLOW_CUSTOM_VISUALIZATIONS" }} + value: {{ $.Values.ui.env.allowCustomVisualizations | quote }} + {{- else if eq .name "ARGO_ARCHIVE_LOGS" }} + value: {{ $.Values.ui.env.argoArchiveLogs | quote }} + {{- else if eq .name "DISABLE_GKE_METADATA" }} + value: {{ $.Values.ui.env.disableGkeMetadata | quote }} + {{- else }} + value: "" + {{- end }} + {{- end }} livenessProbe: exec: command: From 3683cd83d1b8002a3f9e59d2d1d09252d9bf9461 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sun, 28 Sep 2025 14:25:39 +0530 Subject: [PATCH 07/26] Fix syntax Signed-off-by: kunal-511 --- tests/helm_kustomize_compare.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/helm_kustomize_compare.sh b/tests/helm_kustomize_compare.sh index 0372c11842..c988d90bf9 100755 --- a/tests/helm_kustomize_compare.sh +++ b/tests/helm_kustomize_compare.sh @@ -213,6 +213,33 @@ case "$COMPONENT" in ["platform-agnostic-postgresql"]="kubeflow" ) ;; + + "notebook-controller") + CHART_DIR="$ROOT_DIR/experimental/helm/charts/notebook-controller" + MANIFESTS_DIR="$ROOT_DIR/applications/jupyter/notebook-controller/upstream" + + declare -A KUSTOMIZE_PATHS=( + ["base"]="$MANIFESTS_DIR/base" + ["kubeflow"]="$MANIFESTS_DIR/overlays/kubeflow" + ["standalone"]="$MANIFESTS_DIR/overlays/standalone" + ) + + declare -A HELM_VALUES=( + ["base"]="$CHART_DIR/ci/base-values.yaml" + ["kubeflow"]="$CHART_DIR/ci/kubeflow-values.yaml" + ["standalone"]="$CHART_DIR/ci/standalone-values.yaml" + ["webhook"]="$CHART_DIR/ci/webhook-values.yaml" + ["production"]="$CHART_DIR/ci/production-values.yaml" + ) + + declare -A NAMESPACES=( + ["base"]="notebook-controller-system" + ["kubeflow"]="kubeflow" + ["standalone"]="notebook-controller-system" + ["webhook"]="kubeflow" + ["production"]="kubeflow" + ) + ;; *) echo "ERROR: Unknown component: $COMPONENT" From 9e2580a0ed3f8494013b8098ee8367c8aa20f2ea Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sun, 28 Sep 2025 15:04:36 +0530 Subject: [PATCH 08/26] fix syntax and naming Signed-off-by: kunal-511 --- tests/helm_kustomize_compare.py | 7 ++++--- tests/helm_kustomize_compare.sh | 10 +++++----- tests/helm_kustomize_compare_all.sh | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/helm_kustomize_compare.py b/tests/helm_kustomize_compare.py index 8f76cce2c2..54bf9e203b 100755 --- a/tests/helm_kustomize_compare.py +++ b/tests/helm_kustomize_compare.py @@ -306,10 +306,11 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena component = sys.argv[3] scenario = sys.argv[4] namespace = sys.argv[5] if len(sys.argv) > 5 and not sys.argv[5].startswith('--') else "" - - if component not in ["katib", "model-registry", "kserve-models-web-app", "notebook-controller", "kubeflow-pipelines"]: + verbose = '--verbose' in sys.argv + + if component not in ["katib", "model-registry", "kserve-models-web-app", "notebook-controller", "pipelines"]: print(f"ERROR: Unknown component: {component}") - print("Supported components: katib, model-registry, kserve-models-web-app, notebook-controller","kubeflow-pipelines") + print("Supported components: katib, model-registry, kserve-models-web-app, notebook-controller, pipelines") sys.exit(1) success = compare_manifests(kustomize_file, helm_file, component, scenario, namespace, verbose) diff --git a/tests/helm_kustomize_compare.sh b/tests/helm_kustomize_compare.sh index c988d90bf9..c91accdea0 100755 --- a/tests/helm_kustomize_compare.sh +++ b/tests/helm_kustomize_compare.sh @@ -11,7 +11,7 @@ ROOT_DIR="$(dirname "$SCRIPT_DIR")" if [[ -z "$COMPONENT" ]]; then echo "ERROR: Component is required" echo "Usage: $0 " - echo "Components: katib, model-registry, kserve-models-web-app, notebook-controller, kubeflow-pipelines" + echo "Components: katib, model-registry, kserve-models-web-app, notebook-controller, pipelines" exit 1 fi @@ -158,8 +158,8 @@ case "$COMPONENT" in ["production"]="kubeflow" ) ;; - "kubeflow-pipelines") - CHART_DIR="$ROOT_DIR/experimental/helm/charts/kubeflow-pipelines" + "pipelines") + CHART_DIR="$ROOT_DIR/experimental/helm/charts/pipelines" MANIFESTS_DIR="$ROOT_DIR/applications/pipeline/upstream" declare -A KUSTOMIZE_PATHS=( @@ -243,7 +243,7 @@ case "$COMPONENT" in *) echo "ERROR: Unknown component: $COMPONENT" - echo "Supported components: katib, model-registry, kserve-models-web-app, notebook-controller, kubeflow-pipelines"" + echo "Supported components: katib, model-registry, kserve-models-web-app, notebook-controller, pipelines" exit 1 ;; esac @@ -305,7 +305,7 @@ else --namespace "$NAMESPACE" \ --include-crds \ --values "$HELM_VALUES_ARG" > "$HELM_OUTPUT" - elif [[ "$COMPONENT" == "kubeflow-pipelines" ]]; then + elif [[ "$COMPONENT" == "pipelines" ]]; then if [[ "$SCENARIO" == "generic" ]]; then helm template pipeline . \ --namespace "$NAMESPACE" \ diff --git a/tests/helm_kustomize_compare_all.sh b/tests/helm_kustomize_compare_all.sh index 63f4e65785..4e9837e530 100755 --- a/tests/helm_kustomize_compare_all.sh +++ b/tests/helm_kustomize_compare_all.sh @@ -13,7 +13,7 @@ declare -A COMPONENT_SCENARIOS=( ["model-registry"]="base overlay-postgres overlay-db controller-manager controller-rbac controller-default controller-prometheus controller-network-policy ui-base ui-standalone ui-integrated ui-istio istio csi" ["kserve-models-web-app"]="base kubeflow" ["notebook-controller"]="base kubeflow standalone" - ["kubeflow-pipelines"]="platform-agnostic aws gcp azure multi-user generic dev plain plain-multi-user platform-agnostic-emissary platform-agnostic-multi-user platform-agnostic-multi-user-emissary platform-agnostic-multi-user-legacy platform-agnostic-postgresql" + ["pipelines"]="platform-agnostic aws gcp azure multi-user generic dev plain plain-multi-user platform-agnostic-emissary platform-agnostic-multi-user platform-agnostic-multi-user-emissary platform-agnostic-multi-user-legacy platform-agnostic-postgresql" ) test_component() { @@ -51,7 +51,7 @@ if [[ "$COMPONENT" == "all" ]]; then declare -a passed_components=() declare -a failed_components=() - for comp in katib model-registry kserve-models-web-app notebook-controller kubeflow-pipelines; do + for comp in katib model-registry kserve-models-web-app notebook-controller pipelines; do if test_component "$comp"; then passed_components+=("$comp") else @@ -80,7 +80,7 @@ elif [[ "$COMPONENT" == "help" ]] || [[ "$COMPONENT" == "--help" ]] || [[ "$COMP echo " model-registry Test Model Registry scenarios" echo " kserve-models-web-app Test KServe Models Web App scenarios" echo " notebook-controller Test Notebook Controller scenarios" - echo " kubeflow-pipelines Test Kubeflow Pipelines scenarios" + echo " pipelines Test Kubeflow Pipelines scenarios" echo "" echo "Examples:" echo " $0 # Test all components" @@ -101,7 +101,7 @@ elif [[ "${COMPONENT_SCENARIOS[$COMPONENT]:-}" ]]; then else echo "ERROR: Unknown component: $COMPONENT" - echo "Supported components: katib, model-registry, kserve-models-web-app, notebook-controller,kubeflow-pipelines, all" + echo "Supported components: katib, model-registry, kserve-models-web-app, notebook-controller,pipelines, all" echo "Use '$0 help' for more information." exit 1 fi \ No newline at end of file From 24be78e25d85bd83b1f893331d4e680f84ccb07a Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sun, 28 Sep 2025 15:11:38 +0530 Subject: [PATCH 09/26] naming change Signed-off-by: kunal-511 --- experimental/helm/charts/{pipeline => pipelines}/Chart.yaml | 0 experimental/helm/charts/{pipeline => pipelines}/README.md | 0 .../charts/{pipeline => pipelines}/ci/values-aws-enhanced.yaml | 0 .../helm/charts/{pipeline => pipelines}/ci/values-aws.yaml | 0 .../charts/{pipeline => pipelines}/ci/values-azure-enhanced.yaml | 0 .../helm/charts/{pipeline => pipelines}/ci/values-azure.yaml | 0 .../charts/{pipeline => pipelines}/ci/values-dev-enhanced.yaml | 0 .../helm/charts/{pipeline => pipelines}/ci/values-dev.yaml | 0 .../charts/{pipeline => pipelines}/ci/values-external-db.yaml | 0 .../charts/{pipeline => pipelines}/ci/values-gcp-enhanced.yaml | 0 .../helm/charts/{pipeline => pipelines}/ci/values-gcp.yaml | 0 .../helm/charts/{pipeline => pipelines}/ci/values-kubeflow.yaml | 0 .../{pipeline => pipelines}/ci/values-multi-user-enhanced.yaml | 0 .../helm/charts/{pipeline => pipelines}/ci/values-multi-user.yaml | 0 .../ci/values-platform-agnostic-enhanced.yaml | 0 .../helm/charts/{pipeline => pipelines}/ci/values-postgresql.yaml | 0 .../helm/charts/{pipeline => pipelines}/ci/values-production.yaml | 0 .../helm/charts/{pipeline => pipelines}/ci/values-standalone.yaml | 0 experimental/helm/charts/{pipeline => pipelines}/files/sync.py | 0 .../helm/charts/{pipeline => pipelines}/templates/_helpers.tpl | 0 .../{pipeline => pipelines}/templates/api-server/deployment.yaml | 0 .../charts/{pipeline => pipelines}/templates/api-server/role.yaml | 0 .../{pipeline => pipelines}/templates/api-server/rolebinding.yaml | 0 .../{pipeline => pipelines}/templates/api-server/service.yaml | 0 .../templates/api-server/serviceaccount.yaml | 0 .../templates/application/application.yaml | 0 .../templates/cache-deployer/deployment.yaml | 0 .../{pipeline => pipelines}/templates/cache-deployer/role.yaml | 0 .../templates/cache-deployer/rolebinding.yaml | 0 .../templates/cache-deployer/serviceaccount.yaml | 0 .../{pipeline => pipelines}/templates/cache/deployment.yaml | 0 .../helm/charts/{pipeline => pipelines}/templates/cache/role.yaml | 0 .../{pipeline => pipelines}/templates/cache/rolebinding.yaml | 0 .../charts/{pipeline => pipelines}/templates/cache/service.yaml | 0 .../{pipeline => pipelines}/templates/cache/serviceaccount.yaml | 0 .../templates/config/kfp-launcher-configmap.yaml | 0 .../templates/config/pipeline-api-server-config.yaml | 0 .../templates/config/pipeline-install-config.yaml | 0 .../{pipeline => pipelines}/templates/config/ui-configmap.yaml | 0 .../{pipeline => pipelines}/templates/crds/application-crd.yaml | 0 .../{pipeline => pipelines}/templates/crds/pipelines-crd.yaml | 0 .../templates/crds/pipelineversions-crd.yaml | 0 .../templates/crds/scheduledworkflow-crd.yaml | 0 .../charts/{pipeline => pipelines}/templates/crds/viewer-crd.yaml | 0 .../{pipeline => pipelines}/templates/database/mysql-secret.yaml | 0 .../charts/{pipeline => pipelines}/templates/env/aws-patches.yaml | 0 .../{pipeline => pipelines}/templates/env/azure-patches.yaml | 0 .../charts/{pipeline => pipelines}/templates/env/gcp-patches.yaml | 0 .../templates/istio/authorization-config.yaml | 0 .../{pipeline => pipelines}/templates/istio/virtual-service.yaml | 0 .../templates/metadata-writer/deployment.yaml | 0 .../{pipeline => pipelines}/templates/metadata-writer/role.yaml | 0 .../templates/metadata-writer/rolebinding.yaml | 0 .../templates/metadata-writer/serviceaccount.yaml | 0 .../templates/metadata/envoy-deployment.yaml | 0 .../{pipeline => pipelines}/templates/metadata/envoy-service.yaml | 0 .../templates/metadata/grpc-configmap.yaml | 0 .../templates/metadata/grpc-deployment.yaml | 0 .../{pipeline => pipelines}/templates/metadata/grpc-service.yaml | 0 .../templates/metadata/grpc-serviceaccount.yaml | 0 .../templates/persistence-agent/deployment.yaml | 0 .../{pipeline => pipelines}/templates/persistence-agent/role.yaml | 0 .../templates/persistence-agent/rolebinding.yaml | 0 .../templates/persistence-agent/serviceaccount.yaml | 0 .../templates/profile-controller/configmap.yaml | 0 .../templates/profile-controller/decorator-controller.yaml | 0 .../templates/profile-controller/deployment.yaml | 0 .../templates/profile-controller/service.yaml | 0 .../templates/rbac/additional-serviceaccounts.yaml | 0 .../templates/rbac/argo-workflow-controller-rbac.yaml | 0 .../templates/rbac/cluster-role-bindings.yaml | 0 .../{pipeline => pipelines}/templates/rbac/cluster-roles.yaml | 0 .../templates/rbac/pipeline-runner-role.yaml | 0 .../templates/rbac/pipeline-runner-rolebinding.yaml | 0 .../templates/rbac/view-edit-cluster-roles.yaml | 0 .../templates/scheduled-workflow/deployment.yaml | 0 .../templates/scheduled-workflow/role.yaml | 0 .../templates/scheduled-workflow/rolebinding.yaml | 0 .../templates/scheduled-workflow/serviceaccount.yaml | 0 .../{pipeline => pipelines}/templates/third-party/argo-role.yaml | 0 .../templates/third-party/argo-rolebinding.yaml | 0 .../templates/third-party/minio-deployment.yaml | 0 .../{pipeline => pipelines}/templates/third-party/minio-pvc.yaml | 0 .../templates/third-party/minio-secret.yaml | 0 .../templates/third-party/minio-service.yaml | 0 .../templates/third-party/mysql-deployment.yaml | 0 .../{pipeline => pipelines}/templates/third-party/mysql-pvc.yaml | 0 .../templates/third-party/mysql-service.yaml | 0 .../templates/third-party/mysql-serviceaccount.yaml | 0 .../templates/third-party/workflow-controller-configmap.yaml | 0 .../templates/third-party/workflow-controller-deployment.yaml | 0 .../templates/third-party/workflow-controller-priorityclass.yaml | 0 .../templates/third-party/workflow-controller-serviceaccount.yaml | 0 .../charts/{pipeline => pipelines}/templates/ui/deployment.yaml | 0 .../helm/charts/{pipeline => pipelines}/templates/ui/role.yaml | 0 .../charts/{pipeline => pipelines}/templates/ui/rolebinding.yaml | 0 .../helm/charts/{pipeline => pipelines}/templates/ui/service.yaml | 0 .../{pipeline => pipelines}/templates/ui/serviceaccount.yaml | 0 .../{pipeline => pipelines}/templates/viewer-crd/deployment.yaml | 0 .../charts/{pipeline => pipelines}/templates/viewer-crd/role.yaml | 0 .../{pipeline => pipelines}/templates/viewer-crd/rolebinding.yaml | 0 .../templates/viewer-crd/serviceaccount.yaml | 0 .../templates/visualization/deployment.yaml | 0 .../{pipeline => pipelines}/templates/visualization/service.yaml | 0 .../templates/visualization/serviceaccount.yaml | 0 .../templates/webhook/mutating-webhook-config.yaml | 0 .../templates/webhook/validating-webhook-config.yaml | 0 .../templates/webhook/webhook-server-tls-secret.yaml | 0 experimental/helm/charts/{pipeline => pipelines}/values.yaml | 0 109 files changed, 0 insertions(+), 0 deletions(-) rename experimental/helm/charts/{pipeline => pipelines}/Chart.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/README.md (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-aws-enhanced.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-aws.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-azure-enhanced.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-azure.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-dev-enhanced.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-dev.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-external-db.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-gcp-enhanced.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-gcp.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-kubeflow.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-multi-user-enhanced.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-multi-user.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-platform-agnostic-enhanced.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-postgresql.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-production.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/ci/values-standalone.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/files/sync.py (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/_helpers.tpl (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/api-server/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/api-server/role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/api-server/rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/api-server/service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/api-server/serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/application/application.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/cache-deployer/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/cache-deployer/role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/cache-deployer/rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/cache-deployer/serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/cache/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/cache/role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/cache/rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/cache/service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/cache/serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/config/kfp-launcher-configmap.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/config/pipeline-api-server-config.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/config/pipeline-install-config.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/config/ui-configmap.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/crds/application-crd.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/crds/pipelines-crd.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/crds/pipelineversions-crd.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/crds/scheduledworkflow-crd.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/crds/viewer-crd.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/database/mysql-secret.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/env/aws-patches.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/env/azure-patches.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/env/gcp-patches.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/istio/authorization-config.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/istio/virtual-service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata-writer/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata-writer/role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata-writer/rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata-writer/serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata/envoy-deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata/envoy-service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata/grpc-configmap.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata/grpc-deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata/grpc-service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/metadata/grpc-serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/persistence-agent/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/persistence-agent/role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/persistence-agent/rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/persistence-agent/serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/profile-controller/configmap.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/profile-controller/decorator-controller.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/profile-controller/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/profile-controller/service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/rbac/additional-serviceaccounts.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/rbac/argo-workflow-controller-rbac.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/rbac/cluster-role-bindings.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/rbac/cluster-roles.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/rbac/pipeline-runner-role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/rbac/pipeline-runner-rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/rbac/view-edit-cluster-roles.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/scheduled-workflow/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/scheduled-workflow/role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/scheduled-workflow/rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/scheduled-workflow/serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/argo-role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/argo-rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/minio-deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/minio-pvc.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/minio-secret.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/minio-service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/mysql-deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/mysql-pvc.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/mysql-service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/mysql-serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/workflow-controller-configmap.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/workflow-controller-deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/workflow-controller-priorityclass.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/third-party/workflow-controller-serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/ui/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/ui/role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/ui/rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/ui/service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/ui/serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/viewer-crd/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/viewer-crd/role.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/viewer-crd/rolebinding.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/viewer-crd/serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/visualization/deployment.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/visualization/service.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/visualization/serviceaccount.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/webhook/mutating-webhook-config.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/webhook/validating-webhook-config.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/templates/webhook/webhook-server-tls-secret.yaml (100%) rename experimental/helm/charts/{pipeline => pipelines}/values.yaml (100%) diff --git a/experimental/helm/charts/pipeline/Chart.yaml b/experimental/helm/charts/pipelines/Chart.yaml similarity index 100% rename from experimental/helm/charts/pipeline/Chart.yaml rename to experimental/helm/charts/pipelines/Chart.yaml diff --git a/experimental/helm/charts/pipeline/README.md b/experimental/helm/charts/pipelines/README.md similarity index 100% rename from experimental/helm/charts/pipeline/README.md rename to experimental/helm/charts/pipelines/README.md diff --git a/experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-aws-enhanced.yaml rename to experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-aws.yaml b/experimental/helm/charts/pipelines/ci/values-aws.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-aws.yaml rename to experimental/helm/charts/pipelines/ci/values-aws.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-azure-enhanced.yaml rename to experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-azure.yaml b/experimental/helm/charts/pipelines/ci/values-azure.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-azure.yaml rename to experimental/helm/charts/pipelines/ci/values-azure.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-dev-enhanced.yaml rename to experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-dev.yaml b/experimental/helm/charts/pipelines/ci/values-dev.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-dev.yaml rename to experimental/helm/charts/pipelines/ci/values-dev.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-external-db.yaml b/experimental/helm/charts/pipelines/ci/values-external-db.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-external-db.yaml rename to experimental/helm/charts/pipelines/ci/values-external-db.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-gcp-enhanced.yaml rename to experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-gcp.yaml b/experimental/helm/charts/pipelines/ci/values-gcp.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-gcp.yaml rename to experimental/helm/charts/pipelines/ci/values-gcp.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-kubeflow.yaml b/experimental/helm/charts/pipelines/ci/values-kubeflow.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-kubeflow.yaml rename to experimental/helm/charts/pipelines/ci/values-kubeflow.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-multi-user-enhanced.yaml rename to experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-multi-user.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-multi-user.yaml rename to experimental/helm/charts/pipelines/ci/values-multi-user.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-platform-agnostic-enhanced.yaml rename to experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-postgresql.yaml b/experimental/helm/charts/pipelines/ci/values-postgresql.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-postgresql.yaml rename to experimental/helm/charts/pipelines/ci/values-postgresql.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-production.yaml b/experimental/helm/charts/pipelines/ci/values-production.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-production.yaml rename to experimental/helm/charts/pipelines/ci/values-production.yaml diff --git a/experimental/helm/charts/pipeline/ci/values-standalone.yaml b/experimental/helm/charts/pipelines/ci/values-standalone.yaml similarity index 100% rename from experimental/helm/charts/pipeline/ci/values-standalone.yaml rename to experimental/helm/charts/pipelines/ci/values-standalone.yaml diff --git a/experimental/helm/charts/pipeline/files/sync.py b/experimental/helm/charts/pipelines/files/sync.py similarity index 100% rename from experimental/helm/charts/pipeline/files/sync.py rename to experimental/helm/charts/pipelines/files/sync.py diff --git a/experimental/helm/charts/pipeline/templates/_helpers.tpl b/experimental/helm/charts/pipelines/templates/_helpers.tpl similarity index 100% rename from experimental/helm/charts/pipeline/templates/_helpers.tpl rename to experimental/helm/charts/pipelines/templates/_helpers.tpl diff --git a/experimental/helm/charts/pipeline/templates/api-server/deployment.yaml b/experimental/helm/charts/pipelines/templates/api-server/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/api-server/deployment.yaml rename to experimental/helm/charts/pipelines/templates/api-server/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/api-server/role.yaml b/experimental/helm/charts/pipelines/templates/api-server/role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/api-server/role.yaml rename to experimental/helm/charts/pipelines/templates/api-server/role.yaml diff --git a/experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/api-server/rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/api-server/rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/api-server/rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/api-server/service.yaml b/experimental/helm/charts/pipelines/templates/api-server/service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/api-server/service.yaml rename to experimental/helm/charts/pipelines/templates/api-server/service.yaml diff --git a/experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/api-server/serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/api-server/serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/api-server/serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/application/application.yaml b/experimental/helm/charts/pipelines/templates/application/application.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/application/application.yaml rename to experimental/helm/charts/pipelines/templates/application/application.yaml diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/cache-deployer/deployment.yaml rename to experimental/helm/charts/pipelines/templates/cache-deployer/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/role.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/cache-deployer/role.yaml rename to experimental/helm/charts/pipelines/templates/cache-deployer/role.yaml diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/cache-deployer/rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/cache-deployer/serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/cache/deployment.yaml b/experimental/helm/charts/pipelines/templates/cache/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/cache/deployment.yaml rename to experimental/helm/charts/pipelines/templates/cache/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/cache/role.yaml b/experimental/helm/charts/pipelines/templates/cache/role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/cache/role.yaml rename to experimental/helm/charts/pipelines/templates/cache/role.yaml diff --git a/experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/cache/rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/cache/service.yaml b/experimental/helm/charts/pipelines/templates/cache/service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/cache/service.yaml rename to experimental/helm/charts/pipelines/templates/cache/service.yaml diff --git a/experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/cache/serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/cache/serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/cache/serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml b/experimental/helm/charts/pipelines/templates/config/kfp-launcher-configmap.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/config/kfp-launcher-configmap.yaml rename to experimental/helm/charts/pipelines/templates/config/kfp-launcher-configmap.yaml diff --git a/experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml b/experimental/helm/charts/pipelines/templates/config/pipeline-api-server-config.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/config/pipeline-api-server-config.yaml rename to experimental/helm/charts/pipelines/templates/config/pipeline-api-server-config.yaml diff --git a/experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml b/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/config/pipeline-install-config.yaml rename to experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml diff --git a/experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml b/experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/config/ui-configmap.yaml rename to experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml diff --git a/experimental/helm/charts/pipeline/templates/crds/application-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/application-crd.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/crds/application-crd.yaml rename to experimental/helm/charts/pipelines/templates/crds/application-crd.yaml diff --git a/experimental/helm/charts/pipeline/templates/crds/pipelines-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/crds/pipelines-crd.yaml rename to experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml diff --git a/experimental/helm/charts/pipeline/templates/crds/pipelineversions-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/crds/pipelineversions-crd.yaml rename to experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml diff --git a/experimental/helm/charts/pipeline/templates/crds/scheduledworkflow-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/scheduledworkflow-crd.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/crds/scheduledworkflow-crd.yaml rename to experimental/helm/charts/pipelines/templates/crds/scheduledworkflow-crd.yaml diff --git a/experimental/helm/charts/pipeline/templates/crds/viewer-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/viewer-crd.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/crds/viewer-crd.yaml rename to experimental/helm/charts/pipelines/templates/crds/viewer-crd.yaml diff --git a/experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml b/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/database/mysql-secret.yaml rename to experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml diff --git a/experimental/helm/charts/pipeline/templates/env/aws-patches.yaml b/experimental/helm/charts/pipelines/templates/env/aws-patches.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/env/aws-patches.yaml rename to experimental/helm/charts/pipelines/templates/env/aws-patches.yaml diff --git a/experimental/helm/charts/pipeline/templates/env/azure-patches.yaml b/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/env/azure-patches.yaml rename to experimental/helm/charts/pipelines/templates/env/azure-patches.yaml diff --git a/experimental/helm/charts/pipeline/templates/env/gcp-patches.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/env/gcp-patches.yaml rename to experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml diff --git a/experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml b/experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/istio/authorization-config.yaml rename to experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml diff --git a/experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml b/experimental/helm/charts/pipelines/templates/istio/virtual-service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/istio/virtual-service.yaml rename to experimental/helm/charts/pipelines/templates/istio/virtual-service.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata-writer/deployment.yaml rename to experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml b/experimental/helm/charts/pipelines/templates/metadata-writer/role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata-writer/role.yaml rename to experimental/helm/charts/pipelines/templates/metadata-writer/role.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/metadata-writer/rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata-writer/rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/metadata-writer/rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/metadata-writer/serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata-writer/serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/metadata-writer/serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/envoy-deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata/envoy-deployment.yaml rename to experimental/helm/charts/pipelines/templates/metadata/envoy-deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml b/experimental/helm/charts/pipelines/templates/metadata/envoy-service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata/envoy-service.yaml rename to experimental/helm/charts/pipelines/templates/metadata/envoy-service.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml b/experimental/helm/charts/pipelines/templates/metadata/grpc-configmap.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata/grpc-configmap.yaml rename to experimental/helm/charts/pipelines/templates/metadata/grpc-configmap.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata/grpc-deployment.yaml rename to experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml b/experimental/helm/charts/pipelines/templates/metadata/grpc-service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata/grpc-service.yaml rename to experimental/helm/charts/pipelines/templates/metadata/grpc-service.yaml diff --git a/experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/metadata/grpc-serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/metadata/grpc-serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/metadata/grpc-serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml b/experimental/helm/charts/pipelines/templates/persistence-agent/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/persistence-agent/deployment.yaml rename to experimental/helm/charts/pipelines/templates/persistence-agent/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml b/experimental/helm/charts/pipelines/templates/persistence-agent/role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/persistence-agent/role.yaml rename to experimental/helm/charts/pipelines/templates/persistence-agent/role.yaml diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/persistence-agent/rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/persistence-agent/rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/persistence-agent/rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/persistence-agent/serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/persistence-agent/serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/persistence-agent/serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/configmap.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/profile-controller/configmap.yaml rename to experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/profile-controller/decorator-controller.yaml rename to experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/profile-controller/deployment.yaml rename to experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/profile-controller/service.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/profile-controller/service.yaml rename to experimental/helm/charts/pipelines/templates/profile-controller/service.yaml diff --git a/experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml b/experimental/helm/charts/pipelines/templates/rbac/additional-serviceaccounts.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/rbac/additional-serviceaccounts.yaml rename to experimental/helm/charts/pipelines/templates/rbac/additional-serviceaccounts.yaml diff --git a/experimental/helm/charts/pipeline/templates/rbac/argo-workflow-controller-rbac.yaml b/experimental/helm/charts/pipelines/templates/rbac/argo-workflow-controller-rbac.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/rbac/argo-workflow-controller-rbac.yaml rename to experimental/helm/charts/pipelines/templates/rbac/argo-workflow-controller-rbac.yaml diff --git a/experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml b/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/rbac/cluster-role-bindings.yaml rename to experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml diff --git a/experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml b/experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/rbac/cluster-roles.yaml rename to experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml diff --git a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml b/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-role.yaml rename to experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-role.yaml diff --git a/experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/rbac/pipeline-runner-rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml b/experimental/helm/charts/pipelines/templates/rbac/view-edit-cluster-roles.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/rbac/view-edit-cluster-roles.yaml rename to experimental/helm/charts/pipelines/templates/rbac/view-edit-cluster-roles.yaml diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml b/experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/scheduled-workflow/deployment.yaml rename to experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml b/experimental/helm/charts/pipelines/templates/scheduled-workflow/role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/scheduled-workflow/role.yaml rename to experimental/helm/charts/pipelines/templates/scheduled-workflow/role.yaml diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/scheduled-workflow/rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/scheduled-workflow/rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/scheduled-workflow/rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/scheduled-workflow/serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/scheduled-workflow/serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/scheduled-workflow/serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/argo-role.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/argo-role.yaml rename to experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/argo-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/argo-rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/minio-deployment.yaml rename to experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-pvc.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/minio-pvc.yaml rename to experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/minio-secret.yaml rename to experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/minio-service.yaml rename to experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/mysql-deployment.yaml rename to experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-pvc.yaml b/experimental/helm/charts/pipelines/templates/third-party/mysql-pvc.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/mysql-pvc.yaml rename to experimental/helm/charts/pipelines/templates/third-party/mysql-pvc.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml b/experimental/helm/charts/pipelines/templates/third-party/mysql-service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/mysql-service.yaml rename to experimental/helm/charts/pipelines/templates/third-party/mysql-service.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/mysql-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/third-party/mysql-serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/mysql-serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/third-party/mysql-serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-configmap.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/workflow-controller-configmap.yaml rename to experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/workflow-controller-deployment.yaml rename to experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-priorityclass.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-priorityclass.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/workflow-controller-priorityclass.yaml rename to experimental/helm/charts/pipelines/templates/third-party/workflow-controller-priorityclass.yaml diff --git a/experimental/helm/charts/pipeline/templates/third-party/workflow-controller-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/third-party/workflow-controller-serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/third-party/workflow-controller-serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/ui/deployment.yaml b/experimental/helm/charts/pipelines/templates/ui/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/ui/deployment.yaml rename to experimental/helm/charts/pipelines/templates/ui/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/ui/role.yaml b/experimental/helm/charts/pipelines/templates/ui/role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/ui/role.yaml rename to experimental/helm/charts/pipelines/templates/ui/role.yaml diff --git a/experimental/helm/charts/pipeline/templates/ui/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/ui/rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/ui/rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/ui/rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/ui/service.yaml b/experimental/helm/charts/pipelines/templates/ui/service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/ui/service.yaml rename to experimental/helm/charts/pipelines/templates/ui/service.yaml diff --git a/experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/ui/serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/ui/serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/ui/serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml b/experimental/helm/charts/pipelines/templates/viewer-crd/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/viewer-crd/deployment.yaml rename to experimental/helm/charts/pipelines/templates/viewer-crd/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/role.yaml b/experimental/helm/charts/pipelines/templates/viewer-crd/role.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/viewer-crd/role.yaml rename to experimental/helm/charts/pipelines/templates/viewer-crd/role.yaml diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/viewer-crd/rolebinding.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/viewer-crd/rolebinding.yaml rename to experimental/helm/charts/pipelines/templates/viewer-crd/rolebinding.yaml diff --git a/experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/viewer-crd/serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/viewer-crd/serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/viewer-crd/serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/visualization/deployment.yaml b/experimental/helm/charts/pipelines/templates/visualization/deployment.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/visualization/deployment.yaml rename to experimental/helm/charts/pipelines/templates/visualization/deployment.yaml diff --git a/experimental/helm/charts/pipeline/templates/visualization/service.yaml b/experimental/helm/charts/pipelines/templates/visualization/service.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/visualization/service.yaml rename to experimental/helm/charts/pipelines/templates/visualization/service.yaml diff --git a/experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/visualization/serviceaccount.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/visualization/serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/visualization/serviceaccount.yaml diff --git a/experimental/helm/charts/pipeline/templates/webhook/mutating-webhook-config.yaml b/experimental/helm/charts/pipelines/templates/webhook/mutating-webhook-config.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/webhook/mutating-webhook-config.yaml rename to experimental/helm/charts/pipelines/templates/webhook/mutating-webhook-config.yaml diff --git a/experimental/helm/charts/pipeline/templates/webhook/validating-webhook-config.yaml b/experimental/helm/charts/pipelines/templates/webhook/validating-webhook-config.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/webhook/validating-webhook-config.yaml rename to experimental/helm/charts/pipelines/templates/webhook/validating-webhook-config.yaml diff --git a/experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml b/experimental/helm/charts/pipelines/templates/webhook/webhook-server-tls-secret.yaml similarity index 100% rename from experimental/helm/charts/pipeline/templates/webhook/webhook-server-tls-secret.yaml rename to experimental/helm/charts/pipelines/templates/webhook/webhook-server-tls-secret.yaml diff --git a/experimental/helm/charts/pipeline/values.yaml b/experimental/helm/charts/pipelines/values.yaml similarity index 100% rename from experimental/helm/charts/pipeline/values.yaml rename to experimental/helm/charts/pipelines/values.yaml From 4c5802416c21781f759170f1aea9b1218b9166d6 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Mon, 29 Sep 2025 19:11:14 +0530 Subject: [PATCH 10/26] Update image tag and add addition env Signed-off-by: kunal-511 --- experimental/helm/charts/pipelines/Chart.yaml | 4 +- .../pipelines/ci/values-aws-enhanced.yaml | 8 +- .../helm/charts/pipelines/ci/values-aws.yaml | 4 +- .../pipelines/ci/values-azure-enhanced.yaml | 50 ++++++- .../charts/pipelines/ci/values-azure.yaml | 4 +- .../pipelines/ci/values-dev-enhanced.yaml | 132 +++++++++++++++-- .../pipelines/ci/values-gcp-enhanced.yaml | 138 ++++++++++-------- .../helm/charts/pipelines/ci/values-gcp.yaml | 4 +- .../ci/values-multi-user-enhanced.yaml | 10 +- .../pipelines/ci/values-multi-user.yaml | 6 +- .../ci/values-platform-agnostic-enhanced.yaml | 51 ++++++- .../templates/api-server/deployment.yaml | 33 ++--- .../templates/metadata/grpc-deployment.yaml | 3 +- .../templates/rbac/pipeline-runner-role.yaml | 7 + .../templates/third-party/argo-role.yaml | 2 - .../third-party/argo-rolebinding.yaml | 2 - .../third-party/minio-deployment.yaml | 42 +++++- .../templates/third-party/minio-service.yaml | 3 +- .../third-party/mysql-deployment.yaml | 5 +- .../workflow-controller-deployment.yaml | 2 - .../workflow-controller-priorityclass.yaml | 2 - .../pipelines/templates/ui/deployment.yaml | 22 +-- .../templates/visualization/deployment.yaml | 22 +-- .../helm/charts/pipelines/values.yaml | 12 +- 24 files changed, 398 insertions(+), 170 deletions(-) diff --git a/experimental/helm/charts/pipelines/Chart.yaml b/experimental/helm/charts/pipelines/Chart.yaml index a3c321f901..6c9c82a338 100644 --- a/experimental/helm/charts/pipelines/Chart.yaml +++ b/experimental/helm/charts/pipelines/Chart.yaml @@ -4,9 +4,9 @@ name: kubeflow-pipelines description: A Helm chart for Kubeflow Pipelines - ML Workflows on Kubernetes -version: 2.5.0 +version: 2.14.3 -appVersion: 2.5.0 +appVersion: 2.14.3 home: https://github.com/kubeflow/pipelines diff --git a/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml index 1d5ede41c4..5b4e659284 100644 --- a/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml @@ -4,7 +4,7 @@ global: namespace: kubeflow imageRegistry: ghcr.io/kubeflow - imageTag: "2.5.0" + imageTag: "2.14.3" imagePullPolicy: IfNotPresent # Installation mode @@ -52,7 +52,7 @@ database: # Pipeline Configuration pipeline: cache: - image: "registry.k8s.io/busybox" + image: "ghcr.io/containerd/busybox" nodeRestrictions: false maximumStaleness: "" defaultStaleness: "" @@ -158,9 +158,9 @@ apiServer: - name: OBJECTSTORECONFIG_SECRETACCESSKEY key: secretkey - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.5.0" + value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" # AWS UI configuration ui: diff --git a/experimental/helm/charts/pipelines/ci/values-aws.yaml b/experimental/helm/charts/pipelines/ci/values-aws.yaml index c0e823efe4..e2675a59ea 100644 --- a/experimental/helm/charts/pipelines/ci/values-aws.yaml +++ b/experimental/helm/charts/pipelines/ci/values-aws.yaml @@ -53,8 +53,8 @@ apiServer: publishLogs: true pipelineLogLevel: "1" autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" additionalEnv: - name: AWS_REGION value: "YOUR_AWS_REGION" diff --git a/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml index 0d837f903b..39ca4cb77a 100644 --- a/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml @@ -4,7 +4,7 @@ global: namespace: kubeflow imageRegistry: ghcr.io/kubeflow - imageTag: "2.5.0" + imageTag: "2.14.3" imagePullPolicy: IfNotPresent # Installation mode @@ -51,7 +51,7 @@ database: # Pipeline Configuration pipeline: cache: - image: "registry.k8s.io/busybox" + image: "ghcr.io/containerd/busybox" nodeRestrictions: false maximumStaleness: "" defaultStaleness: "" @@ -97,6 +97,52 @@ apiServer: requests: cpu: 250m memory: 500Mi + additionalEnv: + - name: PUBLISH_LOGS + value: "true" + - name: LOG_LEVEL + value: "info" + - name: PIPELINE_LOG_LEVEL + value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: DBCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_HOST + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + - name: DB_DRIVER_NAME + key: dbType + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey + - name: V2_DRIVER_IMAGE + value: "ghcr.io/kubeflow/kfp-driver:2.14.3" + - name: V2_LAUNCHER_IMAGE + value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" # Azure UI configuration ui: diff --git a/experimental/helm/charts/pipelines/ci/values-azure.yaml b/experimental/helm/charts/pipelines/ci/values-azure.yaml index 66a39cafc4..a2219b6200 100644 --- a/experimental/helm/charts/pipelines/ci/values-azure.yaml +++ b/experimental/helm/charts/pipelines/ci/values-azure.yaml @@ -53,8 +53,8 @@ apiServer: publishLogs: true pipelineLogLevel: "1" autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" additionalEnv: - name: AZURE_STORAGE_ACCOUNT value: "YOUR_STORAGE_ACCOUNT" diff --git a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml index f1a3169c44..3310eac1de 100644 --- a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml @@ -4,7 +4,7 @@ global: namespace: kubeflow imageRegistry: ghcr.io/kubeflow - imageTag: "2.5.0" + imageTag: "master" imagePullPolicy: IfNotPresent # Installation mode @@ -15,7 +15,7 @@ installMode: # Database Configuration mysql: - enabled: true + enabled: false auth: rootPassword: "dev123" username: "dev" @@ -37,7 +37,7 @@ postgresql: # Object Storage Configuration - Lightweight MinIO minio: - enabled: true + enabled: false auth: rootUser: "dev" rootPassword: "dev123" @@ -66,14 +66,14 @@ database: # Pipeline Configuration - Development settings pipeline: cache: - image: "registry.k8s.io/busybox" + image: "ghcr.io/containerd/busybox" nodeRestrictions: false maximumStaleness: "" defaultStaleness: "" # Argo Workflows Configuration - Minimal for dev argo: - enabled: true + enabled: false # Environment Configuration env: @@ -89,8 +89,54 @@ apiServer: env: logLevel: "2" additionalEnv: + - name: V2_DRIVER_IMAGE + value: "ghcr.io/kubeflow/kfp-driver:master" + - name: V2_LAUNCHER_IMAGE + value: "ghcr.io/kubeflow/kfp-launcher:master" + - name: PUBLISH_LOGS + value: "true" + - name: LOG_LEVEL + value: "info" - name: PIPELINE_LOG_LEVEL value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: DBCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_HOST + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + - name: DB_DRIVER_NAME + key: dbType + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey persistenceAgent: replicas: 1 @@ -98,8 +144,23 @@ persistenceAgent: requests: cpu: 120m memory: 500Mi + additionalEnv: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace scheduledWorkflow: replicas: 1 + image: + repository: kfp-scheduled-workflow-controller + tag: "master" + additionalEnv: + - name: LOG_LEVEL + value: "info" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace ui: replicas: 1 @@ -107,17 +168,33 @@ ui: requests: cpu: 10m memory: 70Mi + additionalEnv: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + key: accesskey + - name: MINIO_SECRET_KEY + key: secretkey + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" + metadata: enabled: true grpc: replicas: 1 resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 512Mi envoy: replicas: 1 resources: @@ -138,6 +215,31 @@ cache: limits: cpu: 200m memory: 512Mi + additionalEnv: + - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: "mysql" + - name: DBCONFIG_DB_NAME + key: cacheDb + - name: DBCONFIG_HOST_NAME + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace cacheDeployer: replicas: 1 @@ -148,7 +250,6 @@ cacheDeployer: limits: cpu: 100m memory: 256Mi - visualization: replicas: 1 resources: @@ -162,6 +263,13 @@ viewerCrd: pullPolicy: Always env: maxNumViewers: "10" + additionalEnv: + - name: MAX_NUM_VIEWERS + value: "50" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace dev: # Enable debug mode debug: true diff --git a/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml index c47a26c4cc..9458dece94 100644 --- a/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml @@ -4,7 +4,7 @@ global: namespace: kubeflow imageRegistry: ghcr.io/kubeflow - imageTag: "2.5.0" + imageTag: "2.14.3" imagePullPolicy: IfNotPresent # Installation mode @@ -22,7 +22,7 @@ postgresql: externalDatabase: type: mysql - host: "127.0.0.1" + host: "mysql" port: 3306 database: pipelinedb username: "YOUR_CLOUDSQL_USERNAME" @@ -36,7 +36,7 @@ minio: objectStore: provider: gcs secure: true - bucketName: "yourGcsBucketName" + bucketName: "mlpipeline" gcs: projectId: "yourGcsProjectId" serviceAccount: "" @@ -50,7 +50,7 @@ database: # Pipeline Configuration pipeline: cache: - image: "registry.k8s.io/busybox" + image: "ghcr.io/containerd/busybox" nodeRestrictions: false maximumStaleness: "" defaultStaleness: "" @@ -84,6 +84,61 @@ apiServer: objectStoreRegion: "" objectStorePort: "" + additionalEnv: + - name: HAS_DEFAULT_BUCKET + value: "true" + - name: BUCKET_NAME + key: bucketName + - name: PROJECT_ID + key: gcsProjectId + - name: PUBLISH_LOGS + value: "true" + - name: LOG_LEVEL + value: "info" + - name: PIPELINE_LOG_LEVEL + value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: DBCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_HOST + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + - name: DB_DRIVER_NAME + key: dbType + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey + - name: V2_DRIVER_IMAGE + value: "ghcr.io/kubeflow/kfp-driver:2.14.3" + - name: V2_LAUNCHER_IMAGE + value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" # Service account annotations for Workload Identity serviceAccount: annotations: @@ -93,9 +148,6 @@ apiServer: requests: cpu: 250m memory: 500Mi - limits: - cpu: 1 - memory: 2Gi # GCP-specific UI configuration ui: @@ -142,29 +194,24 @@ persistenceAgent: requests: cpu: 120m memory: 500Mi - limits: - cpu: 500m - memory: 1Gi - + additionalEnv: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace scheduledWorkflow: resources: - requests: - cpu: 120m - memory: 100Mi - limits: - cpu: 250m - memory: 500Mi - + additionalEnv: + - name: LOG_LEVEL + value: "info" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace metadata: enabled: true grpc: resources: - requests: - cpu: 100m - memory: 300Mi - limits: - cpu: 500m - memory: 1Gi envoy: resources: requests: @@ -185,53 +232,30 @@ cache: memory: 1Gi additionalEnv: - name: DEFAULT_CACHE_STALENESS - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS - name: MAXIMUM_CACHE_STALENESS - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS - name: CACHE_IMAGE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + key: cacheImage - name: CACHE_NODE_RESTRICTIONS - valueFrom: - fieldRef: - fieldPath: metadata.namespace + key: cacheNodeRestrictions - name: DBCONFIG_DRIVER valueFrom: secretKeyRef: key: accesskey name: mlpipeline-minio-artifact - name: DBCONFIG_DB_NAME - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact + key: cacheDb - name: DBCONFIG_HOST_NAME - valueFrom: - fieldRef: - fieldPath: metadata.namespace + key: dbHost - name: DBCONFIG_PORT - valueFrom: - fieldRef: - fieldPath: metadata.namespace + key: dbPort - name: DBCONFIG_USER - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact + key: username - name: DBCONFIG_PASSWORD - valueFrom: - + key: password - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace + key: null visualization: diff --git a/experimental/helm/charts/pipelines/ci/values-gcp.yaml b/experimental/helm/charts/pipelines/ci/values-gcp.yaml index 0e71baec00..3e5c4bd165 100644 --- a/experimental/helm/charts/pipelines/ci/values-gcp.yaml +++ b/experimental/helm/charts/pipelines/ci/values-gcp.yaml @@ -53,8 +53,8 @@ apiServer: publishLogs: true pipelineLogLevel: "1" autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" additionalEnv: - name: GOOGLE_APPLICATION_CREDENTIALS value: "/etc/gcp-service-account/application_default_credentials.json" diff --git a/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml index 5602314e28..0dd9900d7b 100644 --- a/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml @@ -4,7 +4,7 @@ global: namespace: kubeflow imageRegistry: ghcr.io/kubeflow - imageTag: "2.5.0" + imageTag: "2.14.3" imagePullPolicy: Always # Installation mode - Multi-user enabled @@ -70,7 +70,7 @@ database: # Pipeline Configuration pipeline: cache: - image: "registry.k8s.io/busybox" + image: "ghcr.io/containerd/busybox" nodeRestrictions: false maximumStaleness: "" defaultStaleness: "" @@ -168,9 +168,9 @@ apiServer: key: secretkey secretName: mlpipeline-minio-artifact - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.5.0" + value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" # Multi-user UI configuration ui: @@ -315,7 +315,7 @@ profileController: enabled: true image: repository: kfp-profile-controller - tag: "2.5.0" + tag: "2.14.3" resources: requests: cpu: 100m diff --git a/experimental/helm/charts/pipelines/ci/values-multi-user.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user.yaml index 1dd6b6e30b..295a829716 100644 --- a/experimental/helm/charts/pipelines/ci/values-multi-user.yaml +++ b/experimental/helm/charts/pipelines/ci/values-multi-user.yaml @@ -20,7 +20,7 @@ profileController: image: repository: kfp-profile-controller - tag: "2.5.0" + tag: "2.14.3" replicas: 1 @@ -53,8 +53,8 @@ apiServer: publishLogs: true pipelineLogLevel: "1" autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" # Multi-user specific environment variables multiUser: true diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml index b491e7fc2c..8d4e5c7144 100644 --- a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml @@ -5,7 +5,7 @@ global: namespace: kubeflow imageRegistry: ghcr.io/kubeflow - imageTag: "2.5.0" + imageTag: "2.14.3" imagePullPolicy: IfNotPresent # Installation mode @@ -65,7 +65,7 @@ pipeline: # Platform-agnostic specific: Empty defaultPipelineRoot defaultPipelineRoot: "" cache: - image: "registry.k8s.io/busybox" + image: "ghcr.io/containerd/busybox" nodeRestrictions: false maximumStaleness: "" defaultStaleness: "" @@ -81,6 +81,7 @@ thirdParty: enabled: true minio: enabled: true + portName: http argo: enabled: true @@ -94,6 +95,52 @@ apiServer: requests: cpu: 250m memory: 500Mi + additionalEnv: + - name: PUBLISH_LOGS + value: "true" + - name: LOG_LEVEL + value: "info" + - name: PIPELINE_LOG_LEVEL + value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: DBCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_HOST + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + - name: DB_DRIVER_NAME + key: dbType + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey + - name: V2_DRIVER_IMAGE + value: "ghcr.io/kubeflow/kfp-driver:2.14.3" + - name: V2_LAUNCHER_IMAGE + value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" persistenceAgent: resources: diff --git a/experimental/helm/charts/pipelines/templates/api-server/deployment.yaml b/experimental/helm/charts/pipelines/templates/api-server/deployment.yaml index 40c961b541..7cfcae452d 100644 --- a/experimental/helm/charts/pipelines/templates/api-server/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/api-server/deployment.yaml @@ -87,26 +87,16 @@ spec: name: pipeline-api-server-config {{- end }} livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - "-" - - http://localhost:8888/apis/v1beta1/healthz + httpGet: + path: /apis/v1beta1/healthz + port: 8888 initialDelaySeconds: {{ .Values.apiServer.probes.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.apiServer.probes.liveness.periodSeconds }} timeoutSeconds: {{ .Values.apiServer.probes.liveness.timeoutSeconds }} readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - "-" - - http://localhost:8888/apis/v1beta1/healthz + httpGet: + path: /apis/v1beta1/healthz + port: 8888 initialDelaySeconds: {{ .Values.apiServer.probes.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.apiServer.probes.readiness.periodSeconds }} timeoutSeconds: {{ .Values.apiServer.probes.readiness.timeoutSeconds }} @@ -115,14 +105,9 @@ spec: securityContext: {{- toYaml .Values.apiServer.securityContext | nindent 10 }} startupProbe: - exec: - command: - - wget - - -q - - -S - - -O - - "-" - - http://localhost:8888/apis/v1beta1/healthz + httpGet: + path: /apis/v1beta1/healthz + port: 8888 failureThreshold: {{ .Values.apiServer.probes.startup.failureThreshold }} periodSeconds: {{ .Values.apiServer.probes.startup.periodSeconds }} timeoutSeconds: {{ .Values.apiServer.probes.startup.timeoutSeconds }} diff --git a/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml index 5354d08173..f35f4f9d85 100644 --- a/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml @@ -68,7 +68,8 @@ spec: "--mysql_config_port=$(MYSQL_PORT)", "--mysql_config_user=$(DBCONFIG_USER)", "--mysql_config_password=$(DBCONFIG_PASSWORD)", - "--enable_database_upgrade=true" + "--enable_database_upgrade=true", + "--grpc_channel_arguments=grpc.max_metadata_size=16384" ] ports: - name: grpc-api diff --git a/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-role.yaml b/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-role.yaml index 6cd5469e5b..21e6858fd0 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-role.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-role.yaml @@ -84,4 +84,11 @@ rules: - seldondeployments verbs: - '*' +- apiGroups: + - argoproj.io + resources: + - workflowtaskresults + verbs: + - create + - patch {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml index 54ee92c4a3..bf5767f63c 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml @@ -2,8 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - annotations: - internal.kpt.dev/upstream-identifier: rbac.authorization.k8s.io|Role|default|argo-role labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} name: argo-role diff --git a/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml index b7dd8f4457..058879bfcc 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml @@ -2,8 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - annotations: - internal.kpt.dev/upstream-identifier: rbac.authorization.k8s.io|RoleBinding|default|argo-binding labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} name: argo-binding diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml index fdbbfca9ed..8c8dcacbf9 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml @@ -19,10 +19,24 @@ spec: {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: minio spec: + {{- if ne .Values.env.platform "azure" }} + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" + seccompProfile: + type: RuntimeDefault + {{- end }} containers: - - args: + - name: minio + image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance + args: + {{- if eq .Values.env.platform "azure" }} - gateway - azure + {{- else }} + - server + - /data + {{- end }} env: - name: MINIO_ACCESS_KEY valueFrom: @@ -34,12 +48,34 @@ spec: secretKeyRef: name: mlpipeline-minio-artifact key: secretkey - image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance - name: minio ports: - containerPort: 9000 resources: requests: cpu: 20m + {{- if eq .Values.env.platform "azure" }} memory: 25Mi + {{- else }} + memory: 100Mi + {{- end }} + {{- if ne .Values.env.platform "azure" }} + securityContext: + runAsUser: 1000 + runAsGroup: 0 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /data + name: data + subPath: minio + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-pvc + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml index f693f1e720..56c20a515f 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml @@ -8,7 +8,8 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} spec: ports: - - port: 9000 + - name: {{ .Values.thirdParty.minio.portName }} + port: 9000 protocol: TCP targetPort: 9000 selector: diff --git a/experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml index 5b7aa6dfed..7a74b494a0 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml @@ -27,11 +27,12 @@ spec: serviceAccountName: mysql containers: - name: mysql - image: gcr.io/ml-pipeline/mysql:8.0.26 + image: mysql:8.4 args: - --datadir - /var/lib/mysql - - --default-authentication-plugin=mysql_native_password + - --authentication-policy=mysql_native_password + - --mysql-native-password=ON # Disable binlog as the logs grow fast and eat up all disk spaces eventually. And KFP doesn't currently utilize binlog. # https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#option_mysqld_log-bin - --disable-log-bin diff --git a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml index 3ed4b83f17..40db343332 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml @@ -2,8 +2,6 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - internal.kpt.dev/upstream-identifier: apps|Deployment|default|workflow-controller labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} name: workflow-controller diff --git a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-priorityclass.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-priorityclass.yaml index dbf0156379..dab0f03518 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-priorityclass.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-priorityclass.yaml @@ -2,8 +2,6 @@ apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: - annotations: - internal.kpt.dev/upstream-identifier: scheduling.k8s.io|PriorityClass|default|workflow-controller labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} name: workflow-controller diff --git a/experimental/helm/charts/pipelines/templates/ui/deployment.yaml b/experimental/helm/charts/pipelines/templates/ui/deployment.yaml index eec9d07bae..8a89408e3e 100644 --- a/experimental/helm/charts/pipelines/templates/ui/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/ui/deployment.yaml @@ -67,26 +67,16 @@ spec: {{- end }} {{- end }} livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - "-" - - http://localhost:3000/apis/v1beta1/healthz + httpGet: + path: /apis/v1beta1/healthz + port: 3000 initialDelaySeconds: {{ .Values.ui.probes.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.ui.probes.liveness.periodSeconds }} timeoutSeconds: {{ .Values.ui.probes.liveness.timeoutSeconds }} readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - "-" - - http://localhost:3000/apis/v1beta1/healthz + httpGet: + path: /apis/v1beta1/healthz + port: 3000 initialDelaySeconds: {{ .Values.ui.probes.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.ui.probes.readiness.periodSeconds }} timeoutSeconds: {{ .Values.ui.probes.readiness.timeoutSeconds }} diff --git a/experimental/helm/charts/pipelines/templates/visualization/deployment.yaml b/experimental/helm/charts/pipelines/templates/visualization/deployment.yaml index dd47c19ca9..f50653a85f 100644 --- a/experimental/helm/charts/pipelines/templates/visualization/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/visualization/deployment.yaml @@ -26,26 +26,16 @@ spec: - name: http containerPort: 8888 livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - "-" - - http://localhost:8888/ + httpGet: + path: / + port: 8888 initialDelaySeconds: {{ .Values.visualization.probes.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.visualization.probes.liveness.periodSeconds }} timeoutSeconds: {{ .Values.visualization.probes.liveness.timeoutSeconds }} readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - "-" - - http://localhost:8888/ + httpGet: + path: / + port: 8888 initialDelaySeconds: {{ .Values.visualization.probes.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.visualization.probes.readiness.periodSeconds }} timeoutSeconds: {{ .Values.visualization.probes.readiness.timeoutSeconds }} diff --git a/experimental/helm/charts/pipelines/values.yaml b/experimental/helm/charts/pipelines/values.yaml index c8954296b4..869ba02101 100644 --- a/experimental/helm/charts/pipelines/values.yaml +++ b/experimental/helm/charts/pipelines/values.yaml @@ -19,7 +19,7 @@ global: # -- Image registry for Kubeflow Pipelines images imageRegistry: ghcr.io/kubeflow # -- Global image tag for all Kubeflow Pipelines components - imageTag: 2.5.0 + imageTag: 2.14.3 # -- Global image pull policy imagePullPolicy: IfNotPresent # -- Global image pull secrets @@ -95,9 +95,9 @@ apiServer: # -- Auto update pipeline default version autoUpdatePipelineDefaultVersion: true # -- V2 driver image - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.5.0" + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" # -- V2 launcher image - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.5.0" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" # Additional environment variables additionalEnv: [] @@ -183,7 +183,7 @@ scheduledWorkflow: image: repository: kfp-scheduled-workflow-controller - tag: "2.5.0" + tag: "2.14.3" replicas: 1 @@ -706,7 +706,7 @@ pipeline: # Application configuration app: name: "pipeline" - version: "2.5.0" + version: "2.14.3" # Pipeline root configuration defaultPipelineRoot: "" @@ -719,7 +719,7 @@ pipeline: # Cache configuration cache: - image: "registry.k8s.io/busybox" + image: "ghcr.io/containerd/busybox" nodeRestrictions: false maximumStaleness: "" defaultStaleness: "" From 8f70cf197d6d48532c7cdf44114e90bfaac50301 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Thu, 2 Oct 2025 01:25:35 +0530 Subject: [PATCH 11/26] Add missing resources Signed-off-by: kunal-511 --- .../pipelines/ci/values-aws-enhanced.yaml | 10 + .../pipelines/ci/values-dev-enhanced.yaml | 26 + .../pipelines/ci/values-gcp-enhanced.yaml | 22 +- .../ci/values-multi-user-enhanced.yaml | 2 + .../pipelines/ci/values-multi-user.yaml | 2 + ...platform-agnostic-multi-user-enhanced.yaml | 342 +++++++ .../pipelines/ci/values-postgresql.yaml | 190 +++- .../pipelines/ci/values-standalone.yaml | 52 +- .../charts/pipelines/templates/_helpers.tpl | 61 +- .../application/controller-deployment.yaml | 39 + .../application/controller-role.yaml | 27 + .../application/controller-rolebinding.yaml | 18 + .../application/controller-service.yaml | 19 + .../controller-serviceaccount.yaml | 12 + .../templates/cache-deployer/rolebinding.yaml | 3 + .../pipelines/templates/cache/deployment.yaml | 13 + .../templates/crds/metacontroller-crds.yaml | 837 ++++++++++++++++++ .../templates/database/mysql-secret.yaml | 13 +- .../env/gcp-proxy-agent-configmap.yaml | 12 + .../env/gcp-proxy-agent-deployment.yaml | 32 + .../templates/env/gcp-proxy-agent-role.yaml | 18 + .../env/gcp-proxy-agent-rolebinding.yaml | 19 + .../env/gcp-proxy-agent-serviceaccount.yaml | 12 + .../templates/istio/authorization-config.yaml | 28 +- .../templates/metadata-writer/deployment.yaml | 6 + .../templates/metadata/grpc-deployment.yaml | 45 +- .../profile-controller/configmap.yaml | 4 +- .../decorator-controller.yaml | 3 +- .../profile-controller/deployment.yaml | 6 +- .../templates/profile-controller/service.yaml | 3 + .../rbac/metacontroller-clusterrole.yaml | 37 + .../metacontroller-clusterrolebinding.yaml | 17 + .../scheduled-workflow/deployment.yaml | 6 + .../templates/third-party/argo-role.yaml | 2 +- .../third-party/argo-rolebinding.yaml | 2 +- .../metacontroller-serviceaccount.yaml | 10 + .../metacontroller-statefulset.yaml | 61 ++ .../templates/third-party/minio-secret.yaml | 16 +- .../third-party/postgresql-deployment.yaml | 54 ++ .../templates/third-party/postgresql-pvc.yaml | 16 + .../postgresql-secret-extended.yaml | 11 + .../third-party/postgresql-secret.yaml | 10 + .../third-party/postgresql-service.yaml | 18 + .../postgresql-serviceaccount.yaml | 7 + .../workflow-controller-configmap.yaml | 2 - .../workflow-controller-deployment.yaml | 2 + .../workflow-controller-serviceaccount.yaml | 2 - .../helm/charts/pipelines/values.yaml | 43 +- tests/helm_kustomize_compare.sh | 6 +- 49 files changed, 2097 insertions(+), 101 deletions(-) create mode 100644 experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml create mode 100644 experimental/helm/charts/pipelines/templates/application/controller-deployment.yaml create mode 100644 experimental/helm/charts/pipelines/templates/application/controller-role.yaml create mode 100644 experimental/helm/charts/pipelines/templates/application/controller-rolebinding.yaml create mode 100644 experimental/helm/charts/pipelines/templates/application/controller-service.yaml create mode 100644 experimental/helm/charts/pipelines/templates/application/controller-serviceaccount.yaml create mode 100644 experimental/helm/charts/pipelines/templates/crds/metacontroller-crds.yaml create mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-configmap.yaml create mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-deployment.yaml create mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-role.yaml create mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-rolebinding.yaml create mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-serviceaccount.yaml create mode 100644 experimental/helm/charts/pipelines/templates/rbac/metacontroller-clusterrole.yaml create mode 100644 experimental/helm/charts/pipelines/templates/rbac/metacontroller-clusterrolebinding.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/metacontroller-serviceaccount.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/metacontroller-statefulset.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-deployment.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-pvc.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-service.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml diff --git a/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml index 5b4e659284..7436223003 100644 --- a/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml @@ -61,6 +61,16 @@ pipeline: argo: enabled: false +# Built-in third-party components +thirdParty: + argo: + enabled: true + mysql: + enabled: true + minio: + enabled: true + portName: http + env: platform: aws aws: diff --git a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml index 3310eac1de..50de321bab 100644 --- a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml @@ -35,6 +35,21 @@ mysql: postgresql: enabled: false +# Third-party components configuration +thirdParty: + # Use built-in MySQL instead of external subchart + mysql: + enabled: true + + # Use built-in MinIO instead of external subchart + minio: + enabled: true + portName: http + + # Use built-in Argo instead of external subchart + argo: + enabled: true + # Object Storage Configuration - Lightweight MinIO minio: enabled: false @@ -75,9 +90,20 @@ pipeline: argo: enabled: false +# Application Configuration +application: + enabled: true + controller: + enabled: true + # Environment Configuration env: platform: platform-agnostic + gcp: + proxyAgent: + enabled: true + image: + tag: "master" # Reduced resources for development apiServer: diff --git a/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml index 9458dece94..43190a1c92 100644 --- a/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml @@ -60,6 +60,21 @@ argo: enabled: false # GCP-specific Argo configuration +# Built-in third-party components +thirdParty: + argo: + enabled: true + mysql: + enabled: false + minio: + enabled: false + +# Application Configuration +application: + enabled: true + controller: + enabled: true + # Environment Configuration env: platform: gcp @@ -73,6 +88,8 @@ env: enabled: true bucket: "yourGcsBucketName" projectId: "yourGcsProjectId" + proxyAgent: + enabled: true # GCP-specific API Server configuration apiServer: @@ -240,10 +257,7 @@ cache: - name: CACHE_NODE_RESTRICTIONS key: cacheNodeRestrictions - name: DBCONFIG_DRIVER - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact + value: "mysql" - name: DBCONFIG_DB_NAME key: cacheDb - name: DBCONFIG_HOST_NAME diff --git a/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml index 0dd9900d7b..6a0f222548 100644 --- a/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml @@ -14,6 +14,8 @@ installMode: enabled: true profileController: enabled: true + # Legacy mode disabled - excludes application-crd-id label for regular multi-user resources + legacyLabels: false # Database Configuration - Enhanced for multi-user mysql: diff --git a/experimental/helm/charts/pipelines/ci/values-multi-user.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user.yaml index 295a829716..ff229dfaca 100644 --- a/experimental/helm/charts/pipelines/ci/values-multi-user.yaml +++ b/experimental/helm/charts/pipelines/ci/values-multi-user.yaml @@ -8,6 +8,8 @@ installMode: enabled: true profileController: enabled: true + # Legacy mode - includes application-crd-id label on all resources + legacyLabels: true # Environment configuration env: diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml new file mode 100644 index 0000000000..bf6cb12b4e --- /dev/null +++ b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml @@ -0,0 +1,342 @@ +# Platform-Agnostic Multi-User Configuration - Enhanced +# This configuration provides platform-agnostic multi-user deployment + +global: + namespace: kubeflow + imageRegistry: ghcr.io/kubeflow + imageTag: "2.14.3" + imagePullPolicy: Always + +# Installation mode - Multi-user enabled with legacy labels +installMode: + type: multi-user + multiUser: + enabled: true + profileController: + enabled: true + # Legacy mode enabled - includes application-crd-id label + legacyLabels: true + +# Database Configuration - Enhanced for multi-user +mysql: + enabled: false + auth: + rootPassword: "root123" + username: "mlpipeline" + password: "mlpipeline123" + database: "mlpipeline" + primary: + persistence: + enabled: true + size: 50Gi + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1 + memory: 2Gi + +postgresql: + enabled: false + +# Object Storage Configuration - Enhanced for multi-user +minio: + enabled: false + auth: + rootUser: "minio" + rootPassword: "minio123" + defaultBuckets: "mlpipeline" + persistence: + enabled: true + size: 100Gi + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +objectStore: + provider: minio + secure: false + bucketName: mlpipeline + +# Database names +database: + pipelinedb: mlpipeline + metadb: metadb + cachedb: cachedb + +# Pipeline Configuration +pipeline: + cache: + image: "ghcr.io/containerd/busybox" + nodeRestrictions: false + maximumStaleness: "" + defaultStaleness: "" + +# Argo Workflows Configuration +argo: + enabled: false + +# Built-in third-party components +thirdParty: + argo: + enabled: true + mysql: + enabled: true + minio: + enabled: true + portName: http + metacontroller: + enabled: true + +# Application Configuration +application: + enabled: false + controller: + enabled: false + +# Metadata (MLMD) Configuration +metadata: + enabled: true + +# Environment Configuration +env: + platform: platform-agnostic + +# Enhanced RBAC Configuration +rbac: + create: true + multiUser: + enabled: true + +# Multi-user API Server configuration +apiServer: + env: + publishLogs: true + pipelineLogLevel: "1" + autoUpdatePipelineDefaultVersion: true + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" + multiUser: true + + additionalEnv: + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + value: "" + - name: PUBLISH_LOGS + value: "true" + - name: LOG_LEVEL + value: "info" + - name: PIPELINE_LOG_LEVEL + value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + configMapName: pipeline-install-config + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + configMapName: pipeline-install-config + - name: DBCONFIG_USER + key: username + secretName: mysql-secret + - name: DBCONFIG_PASSWORD + key: password + secretName: mysql-secret + - name: DBCONFIG_DBNAME + key: pipelineDb + configMapName: pipeline-install-config + - name: DBCONFIG_HOST + key: dbHost + configMapName: pipeline-install-config + - name: DBCONFIG_PORT + key: dbPort + configMapName: pipeline-install-config + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + configMapName: pipeline-install-config + - name: DB_DRIVER_NAME + key: dbType + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + secretName: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + secretName: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + configMapName: pipeline-install-config + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + secretName: mlpipeline-minio-artifact + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey + secretName: mlpipeline-minio-artifact + - name: V2_DRIVER_IMAGE + value: "ghcr.io/kubeflow/kfp-driver:2.14.3" + - name: V2_LAUNCHER_IMAGE + value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" + +# Enhanced UI configuration +ui: + additionalEnv: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: DEPLOYMENT + value: "KUBEFLOW" + - name: ARTIFACTS_SERVICE_PROXY_NAME + value: "ml-pipeline-ui-artifact" + - name: ARTIFACTS_SERVICE_PROXY_PORT + value: "80" + - name: ARTIFACTS_SERVICE_PROXY_ENABLED + value: "true" + - name: ENABLE_AUTHZ + value: "true" + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + key: null + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + key: accesskey + - name: MINIO_SECRET_KEY + key: secretkey + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" + +persistenceAgent: + resources: + requests: + cpu: 120m + memory: 500Mi + additionalEnv: + - name: NAMESPACE + value: "" + +scheduledWorkflow: + additionalEnv: + - name: NAMESPACE + value: '' + - name: LOG_LEVEL + value: "info" + valueFrom: null + +cache: + server: + additionalEnv: + - name: NAMESPACE_TO_WATCH + key: null + - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: "mysql" + - name: DBCONFIG_DB_NAME + key: cacheDb + - name: DBCONFIG_HOST_NAME + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + +viewerCrd: + env: + maxNumViewers: "" + additionalEnv: + - name: NAMESPACE + value: "" + - name: MAX_NUM_VIEWERS + value: 50 + +# Profile Controller Configuration +profileController: + enabled: true + replicas: 1 + + image: + repository: public.ecr.aws/docker/library/python + tag: "3.12" + pullPolicy: IfNotPresent + + additionalEnv: + - name: KFP_VERSION + key: appVersion + - name: KFP_DEFAULT_PIPELINE_ROOT + key: defaultPipelineRoot + - name: MINIO_ACCESS_KEY + key: accesskey + - name: MINIO_SECRET_KEY + key: secretkey + + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 1Gi + + service: + type: ClusterIP + port: 80 + + disableIstioSidecar: "true" + + + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Istio configuration for multi-user +istio: + enabled: true + +# Networking configuration +networking: + istio: + enabled: true + gateway: "kubeflow-gateway" + host: "*" + diff --git a/experimental/helm/charts/pipelines/ci/values-postgresql.yaml b/experimental/helm/charts/pipelines/ci/values-postgresql.yaml index 532ad6c882..b6f9e2f8c8 100644 --- a/experimental/helm/charts/pipelines/ci/values-postgresql.yaml +++ b/experimental/helm/charts/pipelines/ci/values-postgresql.yaml @@ -1,28 +1,42 @@ # PostgreSQL values for kubeflow-pipelines. # This configuration uses PostgreSQL instead of MySQL -# Disable MySQL +# CRDs are not included in Kustomize platform-agnostic-postgresql +crds: + install: false + +# Third-party services configuration +thirdParty: + argo: + enabled: true + mysql: + enabled: false + minio: + enabled: true + portName: http + metacontroller: + enabled: false + # Use built-in PostgreSQL deployment (not subchart) + postgresql: + enabled: true + image: + repository: postgres + tag: "14.7-alpine3.17" + database: "postgres" + username: "user" + password: "password" + secretName: "postgres-secret" + secretKey: "root_password" + persistence: + size: "20Gi" + +# Disable MySQL subchart mysql: enabled: false -# Enable PostgreSQL +# Disable postgresql subchart postgresql: - enabled: true - auth: - postgresPassword: "postgres123" - database: "mlpipeline" - username: "postgres" - primary: - persistence: - enabled: true - size: 20Gi - resources: - requests: - cpu: 250m - memory: 256Mi - limits: - cpu: 1000m - memory: 1Gi + enabled: false # External database configuration pointing to PostgreSQL externalDatabase: @@ -36,21 +50,25 @@ externalDatabase: database: metadb: "metadb" cachedb: "cachedb" - connectionMaxLifetime: "300s" + connectionMaxLifetime: "120s" # Standard MinIO configuration objectStore: provider: minio +# Disable MinIO subchart minio: + enabled: false + +# Metadata service configuration +metadata: enabled: true - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: - enabled: true - size: 20Gi + grpc: + image: + repository: gcr.io/tfx-oss-public/ml_metadata_store_server + tag: "1.14.0" + pullPolicy: IfNotPresent + resources: # Standard component configuration apiServer: @@ -59,6 +77,40 @@ apiServer: requests: cpu: 250m memory: 500Mi + additionalEnv: + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + configMapName: pipeline-install-config + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + - name: DB_DRIVER_NAME + key: dbType + - name: DBCONFIG_POSTGRESQLCONFIG_USER + key: username + secretName: postgres-secret-extended + secretKey: username + - name: DBCONFIG_POSTGRESQLCONFIG_PASSWORD + key: password + secretName: postgres-secret-extended + secretKey: password + - name: DBCONFIG_POSTGRESQLCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_POSTGRESQLCONFIG_HOST + key: postgresHost + - name: DBCONFIG_POSTGRESQLCONFIG_PORT + key: postgresPort + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey persistenceAgent: replicas: 1 @@ -66,13 +118,62 @@ persistenceAgent: requests: cpu: 120m memory: 500Mi + additionalEnv: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + scheduledWorkflow: replicas: 1 resources: - requests: - cpu: 120m - memory: 100Mi + additionalEnv: + - name: LOG_LEVEL + value: "info" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + +cache: + server: + additionalEnv: + - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: "pgx" + - name: DBCONFIG_DB_NAME + key: cacheDb + - name: DBCONFIG_HOST_NAME + key: postgresHost + - name: DBCONFIG_PORT + key: postgresPort + - name: DBCONFIG_USER + secretName: postgres-secret-extended + secretKey: username + - name: DBCONFIG_PASSWORD + secretName: postgres-secret-extended + secretKey: password + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace + +viewerCrd: + additionalEnv: + - name: MAX_NUM_VIEWERS + value: "50" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace ui: replicas: 1 @@ -80,6 +181,35 @@ ui: requests: cpu: 10m memory: 70Mi + additionalEnv: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" + + # Disable monitoring for simplicity monitoring: @@ -92,4 +222,4 @@ monitoring: security: podSecurityStandards: enabled: true - enforce: "restricted" + enforce: "restricted" \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/ci/values-standalone.yaml b/experimental/helm/charts/pipelines/ci/values-standalone.yaml index 4be61b650b..550b607d4f 100644 --- a/experimental/helm/charts/pipelines/ci/values-standalone.yaml +++ b/experimental/helm/charts/pipelines/ci/values-standalone.yaml @@ -47,6 +47,7 @@ thirdParty: enabled: true minio: enabled: true + portName: http argo: enabled: true @@ -83,7 +84,56 @@ crds: webhooks: enabled: false - +apiServer: + additionalEnv: + - name: PUBLISH_LOGS + value: "true" + - name: LOG_LEVEL + value: "info" + - name: PIPELINE_LOG_LEVEL + value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: DBCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_HOST + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + - name: DB_DRIVER_NAME + key: dbType + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey + - name: V2_DRIVER_IMAGE + value: "ghcr.io/kubeflow/kfp-driver:2.14.3" + - name: V2_LAUNCHER_IMAGE + value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" cache: server: additionalEnv: diff --git a/experimental/helm/charts/pipelines/templates/_helpers.tpl b/experimental/helm/charts/pipelines/templates/_helpers.tpl index 6061067598..d56d670392 100644 --- a/experimental/helm/charts/pipelines/templates/_helpers.tpl +++ b/experimental/helm/charts/pipelines/templates/_helpers.tpl @@ -31,12 +31,26 @@ Create chart name and version as used by the chart label. {{- end }} {{/* -Common labels +Common labels */}} {{- define "kubeflow-pipelines.labels" -}} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if ne .Values.installMode.type "multi-user" }} +{{- if or (ne .Values.installMode.type "multi-user") (.Values.installMode.legacyLabels) }} +application-crd-id: kubeflow-pipelines +{{- end }} +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Authorization Policy labels - includes application-crd-id for legacy multi-user mode +*/}} +{{- define "kubeflow-pipelines.authorizationPolicyLabels" -}} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- if or (ne .Values.installMode.type "multi-user") (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines {{- end }} {{- with .Values.commonLabels }} @@ -59,7 +73,7 @@ Cache server labels app: cache-server app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if ne .Values.installMode.type "multi-user" }} +{{- if or (ne .Values.installMode.type "multi-user") (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines {{- end }} {{- with .Values.commonLabels }} @@ -87,7 +101,7 @@ ML Pipeline specific labels - matching original manifests app: ml-pipeline app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if ne .Values.installMode.type "multi-user" }} +{{- if or (ne .Values.installMode.type "multi-user") (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines {{- end }} {{- with .Values.commonLabels }} @@ -102,7 +116,7 @@ ML Pipeline UI labels app: ml-pipeline-ui app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if ne .Values.installMode.type "multi-user" }} +{{- if or (ne .Values.installMode.type "multi-user") (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines {{- end }} {{- with .Values.commonLabels }} @@ -118,6 +132,9 @@ app: ml-pipeline {{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} application-crd-id: kubeflow-pipelines {{- end }} @@ -131,6 +148,9 @@ app: cache-server {{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} application-crd-id: kubeflow-pipelines {{- end }} @@ -144,6 +164,9 @@ app: ml-pipeline-ui {{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} application-crd-id: kubeflow-pipelines {{- end }} @@ -157,6 +180,9 @@ Viewer CRD labels app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines app: ml-pipeline-viewer-crd +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} {{- include "kubeflow-pipelines.labels" . }} app: ml-pipeline-viewer-crd @@ -171,6 +197,9 @@ Viewer CRD selector labels app: ml-pipeline-viewer-crd app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} app: ml-pipeline-viewer-crd application-crd-id: kubeflow-pipelines @@ -184,7 +213,7 @@ Visualization server labels app: ml-pipeline-visualizationserver app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if ne .Values.installMode.type "multi-user" }} +{{- if or (ne .Values.installMode.type "multi-user") (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines {{- end}} {{- with .Values.commonLabels }} @@ -200,6 +229,9 @@ Visualization server selector labels app: ml-pipeline-visualizationserver app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} app: ml-pipeline-visualizationserver application-crd-id: kubeflow-pipelines @@ -243,8 +275,16 @@ Cache deployer selector labels */}} {{- define "kubeflow-pipelines.cacheDeployerSelectorLabels" -}} app: cache-deployer +{{- if eq .Values.installMode.type "multi-user" }} +app.kubernetes.io/component: ml-pipeline +app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} +{{- else }} application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} {{/* Metadata writer selector labels @@ -254,6 +294,9 @@ app: metadata-writer {{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} application-crd-id: kubeflow-pipelines {{- end }} @@ -267,6 +310,9 @@ app: ml-pipeline-persistenceagent {{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} application-crd-id: kubeflow-pipelines {{- end }} @@ -280,6 +326,9 @@ app: ml-pipeline-scheduledworkflow {{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} +application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} application-crd-id: kubeflow-pipelines {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/application/controller-deployment.yaml b/experimental/helm/charts/pipelines/templates/application/controller-deployment.yaml new file mode 100644 index 0000000000..315dec428b --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/application/controller-deployment.yaml @@ -0,0 +1,39 @@ +{{- if .Values.application.controller.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + control-plane: controller-manager + controller-tools.k8s.io: "1.0" +spec: + selector: + matchLabels: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + application-crd-id: kubeflow-pipelines + template: + metadata: + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + spec: + serviceAccountName: application + containers: + - name: manager + image: {{ .Values.application.controller.image.repository }}:{{ .Values.application.controller.image.tag }} + imagePullPolicy: {{ .Values.application.controller.image.pullPolicy | default .Values.global.imagePullPolicy }} + command: + - /kube-app-manager + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + {{- toYaml .Values.application.controller.resources | nindent 10 }} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/application/controller-role.yaml b/experimental/helm/charts/pipelines/templates/application/controller-role.yaml new file mode 100644 index 0000000000..538c8e107e --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/application/controller-role.yaml @@ -0,0 +1,27 @@ +{{- if .Values.application.controller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: application-manager-role + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - update + - patch + - watch +- apiGroups: + - app.k8s.io + resources: + - '*' + verbs: + - '*' +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/application/controller-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/application/controller-rolebinding.yaml new file mode 100644 index 0000000000..886726f6f7 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/application/controller-rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if .Values.application.controller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: application-manager-rolebinding + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: application-manager-role +subjects: +- kind: ServiceAccount + name: application + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/application/controller-service.yaml b/experimental/helm/charts/pipelines/templates/application/controller-service.yaml new file mode 100644 index 0000000000..f96d6ac642 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/application/controller-service.yaml @@ -0,0 +1,19 @@ +{{- if .Values.application.controller.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: controller-manager-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + control-plane: controller-manager + controller-tools.k8s.io: "1.0" +spec: + selector: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + application-crd-id: kubeflow-pipelines + ports: + - port: 443 +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/application/controller-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/application/controller-serviceaccount.yaml new file mode 100644 index 0000000000..b586cc37a8 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/application/controller-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.application.controller.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: application + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml index d3a56142c9..64d88a09c8 100644 --- a/experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml @@ -13,4 +13,7 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "kubeflow-pipelines.cacheDeployer.serviceAccountName" . }} + {{- if .Values.installMode.multiUser.enabled }} + namespace: {{ include "kubeflow-pipelines.namespace" . }} + {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/cache/deployment.yaml b/experimental/helm/charts/pipelines/templates/cache/deployment.yaml index 50f2cd016b..d8d1d0332d 100644 --- a/experimental/helm/charts/pipelines/templates/cache/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/cache/deployment.yaml @@ -38,6 +38,19 @@ spec: - name: {{ .name }} {{- if .value }} value: {{ .value | quote }} + {{- else if .valueFrom }} + valueFrom: + {{- toYaml .valueFrom | nindent 12 }} + {{- else if and .secretName .secretKey }} + valueFrom: + secretKeyRef: + name: {{ .secretName }} + key: {{ .secretKey }} + {{- else if and .configMapName .configMapKey }} + valueFrom: + configMapKeyRef: + name: {{ .configMapName }} + key: {{ .configMapKey }} {{- else if and .key (ne .key "null") }} {{- if or (eq .name "DBCONFIG_USER") (eq .name "DBCONFIG_PASSWORD") }} valueFrom: diff --git a/experimental/helm/charts/pipelines/templates/crds/metacontroller-crds.yaml b/experimental/helm/charts/pipelines/templates/crds/metacontroller-crds.yaml new file mode 100644 index 0000000000..75b8bb0147 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/crds/metacontroller-crds.yaml @@ -0,0 +1,837 @@ +{{- if .Values.thirdParty.metacontroller.enabled }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved, request not yet submitted + controller-gen.kubebuilder.io/version: v0.13.0 + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + kustomize.component: metacontroller + name: compositecontrollers.metacontroller.k8s.io +spec: + group: metacontroller.k8s.io + names: + kind: CompositeController + listKind: CompositeControllerList + plural: compositecontrollers + shortNames: + - cc + - cctl + singular: compositecontroller + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: CompositeController + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + childResources: + items: + properties: + apiVersion: + type: string + resource: + type: string + updateStrategy: + properties: + method: + enum: + - OnDelete + - Recreate + - InPlace + - RollingRecreate + - RollingInPlace + type: string + statusChecks: + properties: + conditions: + items: + properties: + reason: + type: string + status: + type: string + type: + type: string + required: + - type + type: object + type: array + type: object + type: object + required: + - apiVersion + - resource + type: object + type: array + generateSelector: + type: boolean + hooks: + properties: + customize: + properties: + version: + default: v1 + enum: + - v1 + - v2 + type: string + webhook: + properties: + etag: + properties: + cacheCleanupSeconds: + format: int32 + type: integer + cacheTimeoutSeconds: + format: int32 + type: integer + enabled: + type: boolean + type: object + path: + type: string + responseUnMarshallMode: + description: Sets the json unmarshall mode. One of the + 'loose' or 'strict'. In 'strict' mode additional checks + are performed to detect unknown and duplicated fields. + enum: + - loose + - strict + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + format: duration + type: string + url: + type: string + type: object + type: object + finalize: + properties: + version: + default: v1 + enum: + - v1 + - v2 + type: string + webhook: + properties: + etag: + properties: + cacheCleanupSeconds: + format: int32 + type: integer + cacheTimeoutSeconds: + format: int32 + type: integer + enabled: + type: boolean + type: object + path: + type: string + responseUnMarshallMode: + description: Sets the json unmarshall mode. One of the + 'loose' or 'strict'. In 'strict' mode additional checks + are performed to detect unknown and duplicated fields. + enum: + - loose + - strict + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + format: duration + type: string + url: + type: string + type: object + type: object + postUpdateChild: + properties: + version: + default: v1 + enum: + - v1 + - v2 + type: string + webhook: + properties: + etag: + properties: + cacheCleanupSeconds: + format: int32 + type: integer + cacheTimeoutSeconds: + format: int32 + type: integer + enabled: + type: boolean + type: object + path: + type: string + responseUnMarshallMode: + description: Sets the json unmarshall mode. One of the + 'loose' or 'strict'. In 'strict' mode additional checks + are performed to detect unknown and duplicated fields. + enum: + - loose + - strict + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + format: duration + type: string + url: + type: string + type: object + type: object + preUpdateChild: + properties: + version: + default: v1 + enum: + - v1 + - v2 + type: string + webhook: + properties: + etag: + properties: + cacheCleanupSeconds: + format: int32 + type: integer + cacheTimeoutSeconds: + format: int32 + type: integer + enabled: + type: boolean + type: object + path: + type: string + responseUnMarshallMode: + description: Sets the json unmarshall mode. One of the + 'loose' or 'strict'. In 'strict' mode additional checks + are performed to detect unknown and duplicated fields. + enum: + - loose + - strict + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + format: duration + type: string + url: + type: string + type: object + type: object + sync: + properties: + version: + default: v1 + enum: + - v1 + - v2 + type: string + webhook: + properties: + etag: + properties: + cacheCleanupSeconds: + format: int32 + type: integer + cacheTimeoutSeconds: + format: int32 + type: integer + enabled: + type: boolean + type: object + path: + type: string + responseUnMarshallMode: + description: Sets the json unmarshall mode. One of the + 'loose' or 'strict'. In 'strict' mode additional checks + are performed to detect unknown and duplicated fields. + enum: + - loose + - strict + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + format: duration + type: string + url: + type: string + type: object + type: object + type: object + parentResource: + properties: + apiVersion: + type: string + labelSelector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An + empty label selector matches all objects. A null label selector + matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + resource: + type: string + revisionHistory: + properties: + fieldPaths: + items: + type: string + type: array + type: object + required: + - apiVersion + - resource + type: object + resyncPeriodSeconds: + format: int32 + type: integer + required: + - parentResource + type: object + status: + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved, request not yet submitted + controller-gen.kubebuilder.io/version: v0.13.0 + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + kustomize.component: metacontroller + name: controllerrevisions.metacontroller.k8s.io +spec: + group: metacontroller.k8s.io + names: + kind: ControllerRevision + listKind: ControllerRevisionList + plural: controllerrevisions + singular: controllerrevision + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ControllerRevision + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + children: + items: + properties: + apiGroup: + type: string + kind: + type: string + names: + items: + type: string + type: array + required: + - apiGroup + - kind + - names + type: object + type: array + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + parentPatch: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - parentPatch + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved, request not yet submitted + controller-gen.kubebuilder.io/version: v0.13.0 + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + kustomize.component: metacontroller + name: decoratorcontrollers.metacontroller.k8s.io +spec: + group: metacontroller.k8s.io + names: + kind: DecoratorController + listKind: DecoratorControllerList + plural: decoratorcontrollers + shortNames: + - dec + - decorators + singular: decoratorcontroller + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: DecoratorController + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + attachments: + items: + properties: + apiVersion: + type: string + resource: + type: string + updateStrategy: + properties: + method: + enum: + - OnDelete + - Recreate + - InPlace + - RollingRecreate + - RollingInPlace + type: string + type: object + required: + - apiVersion + - resource + type: object + type: array + hooks: + properties: + customize: + properties: + version: + default: v1 + enum: + - v1 + - v2 + type: string + webhook: + properties: + etag: + properties: + cacheCleanupSeconds: + format: int32 + type: integer + cacheTimeoutSeconds: + format: int32 + type: integer + enabled: + type: boolean + type: object + path: + type: string + responseUnMarshallMode: + description: Sets the json unmarshall mode. One of the + 'loose' or 'strict'. In 'strict' mode additional checks + are performed to detect unknown and duplicated fields. + enum: + - loose + - strict + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + format: duration + type: string + url: + type: string + type: object + type: object + finalize: + properties: + version: + default: v1 + enum: + - v1 + - v2 + type: string + webhook: + properties: + etag: + properties: + cacheCleanupSeconds: + format: int32 + type: integer + cacheTimeoutSeconds: + format: int32 + type: integer + enabled: + type: boolean + type: object + path: + type: string + responseUnMarshallMode: + description: Sets the json unmarshall mode. One of the + 'loose' or 'strict'. In 'strict' mode additional checks + are performed to detect unknown and duplicated fields. + enum: + - loose + - strict + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + format: duration + type: string + url: + type: string + type: object + type: object + sync: + properties: + version: + default: v1 + enum: + - v1 + - v2 + type: string + webhook: + properties: + etag: + properties: + cacheCleanupSeconds: + format: int32 + type: integer + cacheTimeoutSeconds: + format: int32 + type: integer + enabled: + type: boolean + type: object + path: + type: string + responseUnMarshallMode: + description: Sets the json unmarshall mode. One of the + 'loose' or 'strict'. In 'strict' mode additional checks + are performed to detect unknown and duplicated fields. + enum: + - loose + - strict + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + format: duration + type: string + url: + type: string + type: object + type: object + type: object + resources: + items: + properties: + annotationSelector: + properties: + matchAnnotations: + additionalProperties: + type: string + type: object + matchExpressions: + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + apiVersion: + type: string + labelSelector: + description: A label selector is a label query over a set of + resources. The result of matchLabels and matchExpressions + are ANDed. An empty label selector matches all objects. A + null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + resource: + type: string + required: + - apiVersion + - resource + type: object + type: array + resyncPeriodSeconds: + format: int32 + type: integer + required: + - resources + type: object + status: + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {}{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml b/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml index aa09bc6274..fb7b64afd1 100644 --- a/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml +++ b/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml @@ -4,12 +4,15 @@ kind: Secret metadata: name: mysql-secret namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- if not (or (eq .Values.env.platform "gcp") ) }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +{{- end }} +{{- if or (eq .Values.env.platform "aws") (eq .Values.env.platform "gcp") (eq .Values.env.platform "azure") }} type: Opaque -data: - username: {{ .Values.externalDatabase.username | b64enc | quote }} - password: {{ .Values.externalDatabase.password | b64enc | quote }} +{{- else }} +stringData: + username: {{ .Values.externalDatabase.username | quote }} + password: {{ .Values.externalDatabase.password | quote }} +{{- end }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-configmap.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-configmap.yaml new file mode 100644 index 0000000000..30d9322fb9 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-configmap.yaml @@ -0,0 +1,12 @@ +{{- if .Values.env.gcp.proxyAgent.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: inverse-proxy-config + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-deployment.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-deployment.yaml new file mode 100644 index 0000000000..b985b88467 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-deployment.yaml @@ -0,0 +1,32 @@ +{{- if .Values.env.gcp.proxyAgent.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: proxy-agent + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: proxy-agent +spec: + selector: + matchLabels: + app: proxy-agent + application-crd-id: kubeflow-pipelines + template: + metadata: + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} + app: proxy-agent + spec: + hostNetwork: true + serviceAccountName: proxy-agent-runner + containers: + - name: proxy-agent + image: {{ .Values.env.gcp.proxyAgent.image.repository }}:{{ .Values.env.gcp.proxyAgent.image.tag }} + imagePullPolicy: {{ .Values.env.gcp.proxyAgent.image.pullPolicy | default .Values.global.imagePullPolicy }} + {{- if .Values.env.gcp.proxyAgent.resources }} + resources: + {{- toYaml .Values.env.gcp.proxyAgent.resources | nindent 10 }} + {{- end }} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-role.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-role.yaml new file mode 100644 index 0000000000..1bcd20256e --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-role.yaml @@ -0,0 +1,18 @@ +{{- if .Values.env.gcp.proxyAgent.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: proxy-agent-runner + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: proxy-agent-runner +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-rolebinding.yaml new file mode 100644 index 0000000000..fbbf802dc3 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-rolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.env.gcp.proxyAgent.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: proxy-agent-runner + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: proxy-agent-runner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: proxy-agent-runner +subjects: +- kind: ServiceAccount + name: proxy-agent-runner + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-serviceaccount.yaml new file mode 100644 index 0000000000..0874c2e2cb --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.env.gcp.proxyAgent.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxy-agent-runner + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + annotations: + {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml b/experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml index 2181aea7b3..bfda14cc50 100644 --- a/experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml +++ b/experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml @@ -5,9 +5,7 @@ metadata: name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines + {{- include "kubeflow-pipelines.authorizationPolicyLabels" . | nindent 4 }} spec: selector: matchLabels: @@ -36,9 +34,7 @@ metadata: name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines + {{- include "kubeflow-pipelines.authorizationPolicyLabels" . | nindent 4 }} spec: selector: matchLabels: @@ -65,9 +61,7 @@ metadata: name: ml-pipeline-visualizationserver namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines + {{- include "kubeflow-pipelines.authorizationPolicyLabels" . | nindent 4 }} spec: selector: matchLabels: @@ -90,9 +84,7 @@ metadata: name: service-cache-server namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines + {{- include "kubeflow-pipelines.authorizationPolicyLabels" . | nindent 4 }} spec: selector: matchLabels: @@ -108,9 +100,7 @@ metadata: name: minio-service namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines + {{- include "kubeflow-pipelines.authorizationPolicyLabels" . | nindent 4 }} spec: action: ALLOW selector: @@ -137,9 +127,7 @@ metadata: name: mysql namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines + {{- include "kubeflow-pipelines.authorizationPolicyLabels" . | nindent 4 }} spec: selector: matchLabels: @@ -165,9 +153,7 @@ metadata: name: metadata-grpc-service namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines + {{- include "kubeflow-pipelines.authorizationPolicyLabels" . | nindent 4 }} spec: action: ALLOW selector: diff --git a/experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml index 652f8dc8d4..afd37f5b47 100644 --- a/experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml @@ -9,6 +9,9 @@ metadata: app: metadata-writer app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: metadata-writer @@ -25,6 +28,9 @@ spec: app: metadata-writer app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: metadata-writer diff --git a/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml index f35f4f9d85..2c70621f48 100644 --- a/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml @@ -35,6 +35,26 @@ spec: capabilities: drop: - ALL + {{- if .Values.thirdParty.postgresql.enabled }} + env: [] + envFrom: + - configMapRef: + name: metadata-postgres-db-parameters + - secretRef: + name: metadata-postgres-db-secrets + - configMapRef: + name: metadata-grpc-configmap + command: ["/bin/metadata_store_server"] + args: + - --grpc_port=$(METADATA_GRPC_SERVICE_PORT) + - --metadata_source_config_type=postgresql + - --postgres_config_host=metadata-postgres-db + - --postgres_config_port=$(POSTGRES_PORT) + - --postgres_config_dbname=$(POSTGRES_DBNAME) + - --postgres_config_user=$(POSTGRES_USER) + - --postgres_config_password=$(POSTGRES_PASSWORD) + - --enable_database_upgrade=true + {{- else }} env: - name: DBCONFIG_USER valueFrom: @@ -62,15 +82,16 @@ spec: name: pipeline-install-config key: dbPort command: ["/bin/metadata_store_server"] - args: ["--grpc_port=8080", - "--mysql_config_database=$(MYSQL_DATABASE)", - "--mysql_config_host=$(MYSQL_HOST)", - "--mysql_config_port=$(MYSQL_PORT)", - "--mysql_config_user=$(DBCONFIG_USER)", - "--mysql_config_password=$(DBCONFIG_PASSWORD)", - "--enable_database_upgrade=true", - "--grpc_channel_arguments=grpc.max_metadata_size=16384" - ] + args: + - --grpc_port=8080 + - --mysql_config_database=$(MYSQL_DATABASE) + - --mysql_config_host=$(MYSQL_HOST) + - --mysql_config_port=$(MYSQL_PORT) + - --mysql_config_user=$(DBCONFIG_USER) + - --mysql_config_password=$(DBCONFIG_PASSWORD) + - --enable_database_upgrade=true + - --grpc_channel_arguments=grpc.max_metadata_size=16384 + {{- end }} ports: - name: grpc-api containerPort: 8080 @@ -86,8 +107,12 @@ spec: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 + {{- if not .Values.thirdParty.postgresql.enabled }} + {{- with .Values.metadata.grpc.resources }} resources: - {{- toYaml .Values.metadata.grpc.resources | nindent 10 }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} serviceAccountName: metadata-grpc-server {{- with .Values.metadata.grpc.nodeSelector }} nodeSelector: diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml index e0c41098ea..5dd982c31a 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller-env + name: kubeflow-pipelines-profile-controller-env namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} @@ -19,7 +19,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-profile-controller-code + name: kubeflow-pipelines-profile-controller-code namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml index ea04317666..883e6863bc 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml @@ -5,9 +5,8 @@ metadata: name: kubeflow-pipelines-profile-controller namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: kubeflow-pipelines-profile-controller - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml index 7d84a9d398..a3b321844f 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml @@ -16,12 +16,14 @@ spec: app: kubeflow-pipelines-profile-controller app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines +{{- end }} template: metadata: labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: kubeflow-pipelines-profile-controller - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines sidecar.istio.io/inject: "false" annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/service.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/service.yaml index f00b79ea80..56c79723fc 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/service.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/service.yaml @@ -19,4 +19,7 @@ spec: app: kubeflow-pipelines-profile-controller app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines +{{- end }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/rbac/metacontroller-clusterrole.yaml b/experimental/helm/charts/pipelines/templates/rbac/metacontroller-clusterrole.yaml new file mode 100644 index 0000000000..9a43e35dc2 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/rbac/metacontroller-clusterrole.yaml @@ -0,0 +1,37 @@ +{{- if .Values.thirdParty.metacontroller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-metacontroller + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + kustomize.component: metacontroller +rules: +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: [""] + resources: ["namespaces/status"] + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: [""] + resources: ["secrets", "configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["networking.istio.io"] + resources: ["destinationrules"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["security.istio.io"] + resources: ["authorizationpolicies"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["metacontroller.k8s.io"] + resources: ["compositecontrollers", "controllerrevisions", "decoratorcontrollers"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/rbac/metacontroller-clusterrolebinding.yaml b/experimental/helm/charts/pipelines/templates/rbac/metacontroller-clusterrolebinding.yaml new file mode 100644 index 0000000000..bd75342bfc --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/rbac/metacontroller-clusterrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.thirdParty.metacontroller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: meta-controller-cluster-role-binding + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + kustomize.component: metacontroller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-metacontroller +subjects: +- kind: ServiceAccount + name: meta-controller-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml b/experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml index 50155cfccd..9cca3cd1c9 100644 --- a/experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml @@ -9,6 +9,9 @@ metadata: app: ml-pipeline-scheduledworkflow app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-scheduledworkflow @@ -26,6 +29,9 @@ spec: app: ml-pipeline-scheduledworkflow app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines +{{- end }} {{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: ml-pipeline-scheduledworkflow diff --git a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml index bf5767f63c..ed1c60d8cd 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml @@ -1,4 +1,4 @@ -{{- if .Values.thirdParty.argo.enabled }} +{{- if and .Values.thirdParty.argo.enabled (ne .Values.installMode.type "multi-user") }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml index 058879bfcc..cbc0aad61d 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml @@ -1,4 +1,4 @@ -{{- if .Values.thirdParty.argo.enabled }} +{{- if and .Values.thirdParty.argo.enabled (ne .Values.installMode.type "multi-user") }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/experimental/helm/charts/pipelines/templates/third-party/metacontroller-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/third-party/metacontroller-serviceaccount.yaml new file mode 100644 index 0000000000..b86f9dc776 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/metacontroller-serviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.thirdParty.metacontroller.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: meta-controller-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + kustomize.component: metacontroller +{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/metacontroller-statefulset.yaml b/experimental/helm/charts/pipelines/templates/third-party/metacontroller-statefulset.yaml new file mode 100644 index 0000000000..d52c18008a --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/metacontroller-statefulset.yaml @@ -0,0 +1,61 @@ +{{- if .Values.thirdParty.metacontroller.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: metacontroller + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: metacontroller + kustomize.component: metacontroller +spec: + replicas: 1 + selector: + matchLabels: + app: metacontroller + serviceName: "" + template: + metadata: + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} + app: metacontroller + kustomize.component: metacontroller + sidecar.istio.io/inject: "false" + spec: + containers: + - name: metacontroller + image: 'ghcr.io/metacontroller/metacontroller:v4.11.22' + command: ["/usr/bin/metacontroller"] + args: + - --zap-log-level=4 + - --discovery-interval=3600s + resources: + limits: + cpu: '1' + memory: 1Gi + requests: + cpu: 100m + memory: 256Mi + livenessProbe: + httpGet: + port: 8081 + path: /healthz + readinessProbe: + httpGet: + port: 8081 + path: /readyz + securityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + privileged: false + allowPrivilegeEscalation: false + serviceAccountName: meta-controller-service + # Workaround for https://github.com/kubernetes-sigs/kustomize/issues/677 + volumeClaimTemplates: [] +{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml index eb11e882af..abed4614e4 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml @@ -4,15 +4,15 @@ kind: Secret metadata: name: mlpipeline-minio-artifact namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- if not (or (eq .Values.env.platform "gcp") ) }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +{{- end }} +{{- if or (eq .Values.env.platform "aws") (eq .Values.env.platform "gcp") (eq .Values.env.platform "azure") }} type: Opaque -data: - {{- if eq .Values.env.platform "azure" }} - accesskey: {{ .Values.env.azure.storageAccount | b64enc | quote }} - secretkey: {{ .Values.objectStore.azure.accessKey | b64enc | quote }} - {{- else }} - accesskey: {{ "minio" | b64enc | quote }} - secretkey: {{ "minio123" | b64enc | quote }} - {{- end }} +{{- else }} +stringData: + accesskey: "minio" + secretkey: "minio123" +{{- end }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-deployment.yaml new file mode 100644 index 0000000000..fb3dd4d40b --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/postgresql-deployment.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + app: postgres + application-crd-id: kubeflow-pipelines +spec: + selector: + matchLabels: + app: postgres + application-crd-id: kubeflow-pipelines + strategy: + type: Recreate + template: + metadata: + labels: + app: postgres + application-crd-id: kubeflow-pipelines + spec: + serviceAccountName: postgresql + containers: + - image: {{ .Values.thirdParty.postgresql.image.repository }}:{{ .Values.thirdParty.postgresql.image.tag }} + name: postgres + env: + - name: POSTGRES_DB + value: {{ .Values.thirdParty.postgresql.database | default "postgres" }} + - name: POSTGRES_USER + value: {{ .Values.thirdParty.postgresql.username | default "user" }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.thirdParty.postgresql.secretName | default "postgres-secret" }} + key: {{ .Values.thirdParty.postgresql.secretKey | default "root_password" }} + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 + name: postgres + readinessProbe: + exec: + command: ["psql", "-U", "{{ .Values.thirdParty.postgresql.username | default "user" }}", "-d", "{{ .Values.thirdParty.postgresql.database | default "postgres" }}", "-c", "SELECT 1"] + initialDelaySeconds: 15 + timeoutSeconds: 2 + volumeMounts: + - name: postgres-stateful-data + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-stateful-data + persistentVolumeClaim: + claimName: postgres-pvc +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-pvc.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-pvc.yaml new file mode 100644 index 0000000000..857ea60a1f --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/postgresql-pvc.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + app: postgres + application-crd-id: kubeflow-pipelines +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.thirdParty.postgresql.persistence.size | default "20Gi" }} +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml new file mode 100644 index 0000000000..ea6ca75fd1 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret-extended + namespace: {{ include "kubeflow-pipelines.namespace" . }} +type: Opaque +stringData: + username: {{ .Values.thirdParty.postgresql.username | default "user" }} + password: {{ .Values.thirdParty.postgresql.password | default "password" }} +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml new file mode 100644 index 0000000000..95a99aef40 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret + namespace: {{ include "kubeflow-pipelines.namespace" . }} +type: Opaque +data: + root_password: {{ .Values.thirdParty.postgresql.password | b64enc }} +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-service.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-service.yaml new file mode 100644 index 0000000000..7d3de5660c --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/postgresql-service.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: postgres-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + app: postgres + application-crd-id: kubeflow-pipelines +spec: + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 + selector: + app: postgres + application-crd-id: kubeflow-pipelines +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml new file mode 100644 index 0000000000..01d57fd809 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml @@ -0,0 +1,7 @@ +{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: postgresql + namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml index d27afab205..1e49306b11 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml @@ -30,8 +30,6 @@ data: imagePullPolicy: IfNotPresent kind: ConfigMap metadata: - annotations: - internal.kpt.dev/upstream-identifier: '|ConfigMap|default|workflow-controller-configmap' labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} name: workflow-controller-configmap diff --git a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml index 40db343332..84adb61b74 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml @@ -22,7 +22,9 @@ spec: - workflow-controller-configmap - --executor-image - quay.io/argoproj/argoexec:v3.6.7 + {{- if ne .Values.installMode.type "multi-user" }} - --namespaced + {{- end }} command: - workflow-controller env: diff --git a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-serviceaccount.yaml index a5d24f7579..99122d6f69 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-serviceaccount.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-serviceaccount.yaml @@ -2,8 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: - annotations: - internal.kpt.dev/upstream-identifier: '|ServiceAccount|default|argo' labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} name: argo diff --git a/experimental/helm/charts/pipelines/values.yaml b/experimental/helm/charts/pipelines/values.yaml index 869ba02101..dd6fa773dd 100644 --- a/experimental/helm/charts/pipelines/values.yaml +++ b/experimental/helm/charts/pipelines/values.yaml @@ -594,7 +594,7 @@ thirdParty: # Use built-in minimal Argo instead of external subchart argo: enabled: true - + # Use built-in minimal MySQL instead of external subchart mysql: enabled: false @@ -603,6 +603,10 @@ thirdParty: minio: enabled: false + # Use built-in minimal PostgreSQL instead of external subchart + postgresql: + enabled: false + # Metacontroller configuration metacontroller: enabled: false @@ -782,6 +786,21 @@ env: enabled: false # -- GCP service account email serviceAccount: "" + # Inverse Proxy Agent configuration + proxyAgent: + # -- Enable inverse proxy agent for GCP + enabled: false + + image: + # -- Proxy agent image repository + repository: ghcr.io/kubeflow/kfp-inverse-proxy-agent + # -- Proxy agent image tag + tag: "2.14.3" + # -- Proxy agent image pull policy + pullPolicy: IfNotPresent + + # -- Proxy agent resource requests and limits + resources: {} # Azure-specific configuration azure: @@ -1014,6 +1033,28 @@ profileController: application: # -- Enable Application resource (for Google Cloud Console integration) enabled: false + + # Application controller configuration + controller: + # -- Enable Application controller deployment (controller-manager) + enabled: false + + image: + # -- Application controller image repository + repository: gcr.io/ml-pipeline/application-crd-controller + # -- Application controller image tag + tag: "20231101" + # -- Application controller image pull policy + pullPolicy: IfNotPresent + + # -- Application controller resource requests and limits + resources: + requests: + cpu: 100m + memory: 20Mi + limits: + cpu: 100m + memory: 30Mi # Certificate Management certManager: diff --git a/tests/helm_kustomize_compare.sh b/tests/helm_kustomize_compare.sh index c91accdea0..c85dcffc6e 100755 --- a/tests/helm_kustomize_compare.sh +++ b/tests/helm_kustomize_compare.sh @@ -185,13 +185,13 @@ case "$COMPONENT" in ["gcp"]="$CHART_DIR/ci/values-gcp-enhanced.yaml" ["azure"]="$CHART_DIR/ci/values-azure-enhanced.yaml" ["multi-user"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" - ["generic"]="" + ["generic"]="$CHART_DIR/values.yaml" ["dev"]="$CHART_DIR/ci/values-dev-enhanced.yaml" ["plain"]="$CHART_DIR/ci/values-standalone.yaml" ["plain-multi-user"]="$CHART_DIR/ci/values-multi-user.yaml" ["platform-agnostic-emissary"]="$CHART_DIR/ci/values-platform-agnostic-enhanced.yaml" - ["platform-agnostic-multi-user"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" - ["platform-agnostic-multi-user-emissary"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" + ["platform-agnostic-multi-user"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" + ["platform-agnostic-multi-user-emissary"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" ["platform-agnostic-multi-user-legacy"]="$CHART_DIR/ci/values-multi-user.yaml" ["platform-agnostic-postgresql"]="$CHART_DIR/ci/values-postgresql.yaml" ) From 03d5f940ed2eeed4ef5b9414b22ac192fafe712d Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Fri, 3 Oct 2025 23:30:41 +0530 Subject: [PATCH 12/26] remove extra resources Signed-off-by: kunal-511 --- .../pipelines/ci/values-aws-enhanced.yaml | 10 +- .../pipelines/ci/values-azure-enhanced.yaml | 18 +- .../pipelines/ci/values-dev-enhanced.yaml | 10 + .../pipelines/ci/values-gcp-enhanced.yaml | 15 +- .../ci/values-multi-user-enhanced.yaml | 11 +- .../pipelines/ci/values-multi-user.yaml | 175 ++++++++++- ...platform-agnostic-multi-user-enhanced.yaml | 2 + .../pipelines/ci/values-standalone.yaml | 17 ++ .../charts/pipelines/templates/_helpers.tpl | 2 + .../templates/application/application.yaml | 8 +- .../cache-deployer/serviceaccount.yaml | 2 +- .../templates/cache/rolebinding.yaml | 2 +- .../templates/crds/application-crd.yaml | 2 +- .../templates/crds/pipelines-crd.yaml | 2 +- .../templates/crds/pipelineversions-crd.yaml | 2 +- .../templates/env/azure-patches.yaml | 75 ++--- .../pipelines/templates/env/gcp-patches.yaml | 287 +++++++++++++----- .../templates/rbac/cluster-role-bindings.yaml | 2 + .../templates/rbac/cluster-roles.yaml | 2 + .../templates/third-party/argo-role.yaml | 2 +- .../third-party/argo-rolebinding.yaml | 2 +- .../metacontroller-statefulset.yaml | 2 + .../helm/charts/pipelines/values.yaml | 2 + tests/helm_kustomize_compare.sh | 4 +- 24 files changed, 504 insertions(+), 152 deletions(-) diff --git a/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml index 7436223003..9d140211f9 100644 --- a/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml @@ -13,6 +13,14 @@ installMode: multiUser: enabled: false +crds: + install: false + +cacheDeployer: + enabled: true + serviceAccount: + create: false + # Database Configuration mysql: enabled: false @@ -79,7 +87,7 @@ env: enabled: true endpoint: "YOUR_RDS_ENDPOINT" s3: - enabled: true + enabled: false bucket: "YOUR_S3_BUCKET_NAME" region: "YOUR_AWS_REGION" diff --git a/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml index 39ca4cb77a..3b48f99102 100644 --- a/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml @@ -17,6 +17,14 @@ installMode: mysql: enabled: false +crds: + install: false + +cacheDeployer: + enabled: true + serviceAccount: + create: false + postgresql: enabled: false @@ -68,13 +76,15 @@ env: resourceGroup: "yourResourceGroup" storageAccount: "yourstorageaccount" sql: - enabled: true + enabled: false serverName: "[SQLSERVERNAME]" database: "mlpipeline" blob: - enabled: true + enabled: true storageAccount: "yourstorageaccount" container: "mlpipeline" + createMinioGateway: true + createMinioSecret: true managedIdentity: enabled: false @@ -186,7 +196,9 @@ ui: # Third-party components configuration thirdParty: minio: - enabled: true + enabled: false # Azure uses custom minio-azure-gateway, not the built-in MinIO + mysql: + enabled: false # Azure uses external MySQL/Azure SQL # Enhanced resources for Azure deployment persistenceAgent: diff --git a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml index 50de321bab..90408b9adc 100644 --- a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml @@ -13,6 +13,16 @@ installMode: multiUser: enabled: false +crds: + install: true + application: true + +# Cache deployer for generic mode +cacheDeployer: + enabled: true + serviceAccount: + create: true + # Database Configuration mysql: enabled: false diff --git a/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml index 43190a1c92..05859dacda 100644 --- a/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml @@ -12,11 +12,20 @@ installMode: type: generic multiUser: enabled: false + legacyLabels: true # Database Configuration - External Cloud SQL mysql: enabled: false +crds: + install: false + +cacheDeployer: + enabled: true + serviceAccount: + create: false + postgresql: enabled: false @@ -35,7 +44,7 @@ minio: objectStore: provider: gcs - secure: true + secure: false bucketName: "mlpipeline" gcs: projectId: "yourGcsProjectId" @@ -86,10 +95,12 @@ env: region: "YOUR_GCP_REGION" gcs: enabled: true - bucket: "yourGcsBucketName" + bucket: "mlpipeline" projectId: "yourGcsProjectId" proxyAgent: enabled: true + workloadIdentity: + enabled: false # GCP-specific API Server configuration apiServer: diff --git a/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml index 6a0f222548..0573cbd73e 100644 --- a/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml @@ -82,6 +82,13 @@ argo: enabled: false # Multi-user uses built-in minimal Argo instead of subchart +# Third Party Components +thirdParty: + argo: + enabled: false + metacontroller: + enabled: false + # Environment Configuration env: platform: platform-agnostic @@ -294,8 +301,8 @@ cache: key: password cacheDeployer: - # Typically disabled in multi-user mode - enabled: false + # Cache deployer is included in base multi-user install + enabled: true visualization: resources: diff --git a/experimental/helm/charts/pipelines/ci/values-multi-user.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user.yaml index ff229dfaca..c65769c696 100644 --- a/experimental/helm/charts/pipelines/ci/values-multi-user.yaml +++ b/experimental/helm/charts/pipelines/ci/values-multi-user.yaml @@ -31,7 +31,16 @@ profileController: create: true name: "profiles-controller-service-account" annotations: {} - + additionalEnv: + - name: KFP_VERSION + key: appVersion + - name: KFP_DEFAULT_PIPELINE_ROOT + key: defaultPipelineRoot + - name: MINIO_ACCESS_KEY + key: accesskey + - name: MINIO_SECRET_KEY + key: secretkey + # Resources resources: requests: @@ -62,23 +71,131 @@ apiServer: # Additional environment variables for multi-user mode additionalEnv: - - name: MULTI_USER - value: "true" - name: KUBEFLOW_USERID_HEADER value: "kubeflow-userid" - name: KUBEFLOW_USERID_PREFIX value: "" + - name: PUBLISH_LOGS + value: "true" + - name: LOG_LEVEL + value: "info" + - name: PIPELINE_LOG_LEVEL + value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + configMapName: pipeline-install-config + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + configMapName: pipeline-install-config + - name: DBCONFIG_USER + key: username + secretName: mysql-secret + - name: DBCONFIG_PASSWORD + key: password + secretName: mysql-secret + - name: DBCONFIG_DBNAME + key: pipelineDb + configMapName: pipeline-install-config + - name: DBCONFIG_HOST + key: dbHost + configMapName: pipeline-install-config + - name: DBCONFIG_PORT + key: dbPort + configMapName: pipeline-install-config + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + configMapName: pipeline-install-config + - name: DB_DRIVER_NAME + key: dbType + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + secretName: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + secretName: mysql-secret + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + configMapName: pipeline-install-config + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + configMapName: pipeline-install-config + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + secretName: mlpipeline-minio-artifact + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey + secretName: mlpipeline-minio-artifact + - name: V2_DRIVER_IMAGE + value: "ghcr.io/kubeflow/kfp-driver:2.14.3" + - name: V2_LAUNCHER_IMAGE + value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" + # Enhanced UI configuration for multi-user ui: additionalEnv: - - name: MULTI_USER + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: DEPLOYMENT + value: "KUBEFLOW" + - name: ARTIFACTS_SERVICE_PROXY_NAME + value: "ml-pipeline-ui-artifact" + - name: ARTIFACTS_SERVICE_PROXY_PORT + value: "80" + - name: ARTIFACTS_SERVICE_PROXY_ENABLED + value: "true" + - name: ENABLE_AUTHZ value: "true" - name: KUBEFLOW_USERID_HEADER value: "kubeflow-userid" - name: KUBEFLOW_USERID_PREFIX + key: null + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + key: accesskey + - name: MINIO_SECRET_KEY + key: secretkey + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" + +persistenceAgent: + resources: + requests: + cpu: 120m + memory: 500Mi + additionalEnv: + - name: NAMESPACE value: "" +scheduledWorkflow: + additionalEnv: + - name: NAMESPACE + value: '' + - name: LOG_LEVEL + value: "info" + valueFrom: null + # Enhanced RBAC for multi-user rbac: create: true @@ -119,7 +236,7 @@ serviceAccounts: thirdParty: # Argo Workflows configuration argo: - enabled: true + enabled: false # Database configuration mysql: @@ -157,13 +274,55 @@ metadata: tag: "1.14.0" pullPolicy: IfNotPresent resources: - requests: - cpu: 100m - memory: 100Mi postgresql: enabled: false +cache: + # Cache deployer disabled in multi-user mode + enabled: true + server: + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: 1 + memory: 2Gi + additionalEnv: + - name: NAMESPACE_TO_WATCH + key: null + - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: "mysql" + - name: DBCONFIG_DB_NAME + key: cacheDb + - name: DBCONFIG_HOST_NAME + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + +viewerCrd: + env: + maxNumViewers: "" + additionalEnv: + - name: NAMESPACE + value: "" + - name: MAX_NUM_VIEWERS + value: 50 + + # Istio configuration for multi-user networking: istio: diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml index bf6cb12b4e..88e4e7f013 100644 --- a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml @@ -102,6 +102,8 @@ application: # Metadata (MLMD) Configuration metadata: enabled: true + grpc: + resources: # Environment Configuration env: diff --git a/experimental/helm/charts/pipelines/ci/values-standalone.yaml b/experimental/helm/charts/pipelines/ci/values-standalone.yaml index 550b607d4f..8e9e0a838f 100644 --- a/experimental/helm/charts/pipelines/ci/values-standalone.yaml +++ b/experimental/helm/charts/pipelines/ci/values-standalone.yaml @@ -1,6 +1,23 @@ # Platform-agnostic values for Kubeflow Pipelines # This configuration uses local storage (MinIO) and MySQL +# Installation mode +installMode: + type: generic + multiUser: + enabled: false + +# CRDs configuration +crds: + install: false + application: false + +# Cache deployer for generic mode +cacheDeployer: + enabled: true + serviceAccount: + create: false + # Environment configuration env: platform: platform-agnostic diff --git a/experimental/helm/charts/pipelines/templates/_helpers.tpl b/experimental/helm/charts/pipelines/templates/_helpers.tpl index d56d670392..48b9fa4ec8 100644 --- a/experimental/helm/charts/pipelines/templates/_helpers.tpl +++ b/experimental/helm/charts/pipelines/templates/_helpers.tpl @@ -88,7 +88,9 @@ Cache deployer labels app: cache-deployer app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines +{{- if or (ne .Values.installMode.type "multi-user") (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines +{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/application/application.yaml b/experimental/helm/charts/pipelines/templates/application/application.yaml index 66df704ca5..d7af0bc176 100644 --- a/experimental/helm/charts/pipelines/templates/application/application.yaml +++ b/experimental/helm/charts/pipelines/templates/application/application.yaml @@ -2,11 +2,8 @@ apiVersion: app.k8s.io/v1beta1 kind: Application metadata: - name: {{ include "kubeflow-pipelines.fullname" . }} + name: pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: application annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} kubernetes-engine.cloud.google.com/icon: >- @@ -16,8 +13,7 @@ spec: addOwnerRef: true selector: matchLabels: - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines + application-crd-id: kubeflow-pipelines descriptor: version: {{ .Values.global.imageTag | quote }} type: Kubeflow Pipelines diff --git a/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml index 2997df22b7..55bc597403 100644 --- a/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml +++ b/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if false }} +{{- if and .Values.cacheDeployer.enabled .Values.cacheDeployer.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml index dff7c890e3..3ecbddb453 100644 --- a/experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.cache.enabled (ne .Values.installMode.type "multi-user") }} +{{- if .Values.cache.enabled }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/experimental/helm/charts/pipelines/templates/crds/application-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/application-crd.yaml index 5b4977396f..70146a9e0b 100644 --- a/experimental/helm/charts/pipelines/templates/crds/application-crd.yaml +++ b/experimental/helm/charts/pipelines/templates/crds/application-crd.yaml @@ -1,4 +1,4 @@ -{{- if .Values.crds.install }} +{{- if and .Values.crds.install .Values.crds.application }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: diff --git a/experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml index 9d1dcace22..14ed1d5183 100644 --- a/experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml +++ b/experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml @@ -1,4 +1,4 @@ -{{- if .Values.crds.install }} +{{- if and .Values.crds.install .Values.webhooks.enabled }} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml index bff5908256..467539c576 100644 --- a/experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml +++ b/experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml @@ -1,4 +1,4 @@ -{{- if .Values.crds.install }} +{{- if and .Values.crds.install .Values.webhooks.enabled }} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml b/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml index a0c3629ae2..06d2f4b31f 100644 --- a/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml +++ b/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml @@ -1,5 +1,5 @@ {{- if eq .Values.env.platform "azure" }} -{{- if .Values.env.azure.blob.enabled }} +{{- if and .Values.env.azure.blob.enabled (not .Values.env.azure.blob.createMinioGateway) }} --- # Azure Blob Storage Configuration Patch for API Server apiVersion: v1 @@ -59,105 +59,90 @@ data: database: {{ .Values.env.azure.sql.database | b64enc }} {{- end }} -{{- if .Values.env.azure.blob.enabled }} +{{- if and .Values.env.azure.blob.enabled .Values.env.azure.blob.createMinioGateway }} --- # Azure MinIO Gateway for Blob Storage Compatibility apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-minio-azure-gateway + name: minio namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: minio-azure-gateway + app: minio annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: - replicas: 1 selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} - component: minio-azure-gateway + app: minio + application-crd-id: kubeflow-pipelines + strategy: + type: Recreate template: metadata: labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - component: minio-azure-gateway - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + app: minio + application-crd-id: kubeflow-pipelines spec: containers: - name: minio - image: minio/minio:latest + image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance args: - gateway - azure env: - name: MINIO_ACCESS_KEY - value: "minio" + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey - name: MINIO_SECRET_KEY - value: "minio123" - - name: AZURE_STORAGE_ACCOUNT - value: {{ .Values.env.azure.blob.storageAccount | quote }} - - name: AZURE_STORAGE_KEY valueFrom: secretKeyRef: - name: {{ include "kubeflow-pipelines.fullname" . }}-azure-storage-secret - key: key + name: mlpipeline-minio-artifact + key: secretkey ports: - - name: minio - containerPort: 9000 - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL + - containerPort: 9000 resources: requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi + cpu: 20m + memory: 25Mi --- # Azure MinIO Gateway Service apiVersion: v1 kind: Service metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-minio-azure-gateway + name: minio-service namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: minio-azure-gateway annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} spec: - type: ClusterIP ports: - - name: minio - port: 9000 + - port: 9000 targetPort: 9000 + protocol: TCP selector: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} - component: minio-azure-gateway + app: minio + application-crd-id: kubeflow-pipelines --- -# Azure Storage Secret +# MinIO Artifact Secret apiVersion: v1 kind: Secret metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-azure-storage-secret + name: mlpipeline-minio-artifact namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: azure-storage annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} type: Opaque -data: - key: {{ .Values.objectStore.azure.accessKey | default "YOUR_STORAGE_ACCOUNT_KEY" | b64enc }} +stringData: + accesskey: {{ .Values.objectStore.azure.accessKey | default "minio" | quote }} + secretkey: {{ .Values.objectStore.azure.secretKey | default "minio123" | quote }} {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml index d2cdfa7b58..bd8a1f9634 100644 --- a/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml +++ b/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml @@ -1,26 +1,4 @@ {{- if eq .Values.env.platform "gcp" }} -{{- if .Values.env.gcp.gcs.enabled }} ---- -# GCP GCS Configuration Patch for API Server -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-gcp-config - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: gcp-config - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -data: - # GCP GCS specific configuration - OBJECTSTORECONFIG_SECURE: "true" - OBJECTSTORECONFIG_HOST: "storage.googleapis.com" - OBJECTSTORECONFIG_REGION: "" - OBJECTSTORECONFIG_BUCKETNAME: {{ .Values.env.gcp.gcs.bucket | quote }} - OBJECTSTORECONFIG_PORT: "" - GCS_PROJECT_ID: {{ .Values.env.gcp.gcs.projectId | quote }} -{{- end }} {{- if .Values.env.gcp.workloadIdentity.enabled }} --- @@ -40,95 +18,252 @@ metadata: {{- end }} {{- if .Values.env.gcp.cloudsql.enabled }} +--- +# GCP Cloud SQL Proxy ServiceAccount +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeflow-pipelines-cloudsql-proxy + namespace: {{ include "kubeflow-pipelines.namespace" . }} + {{- if .Values.installMode.legacyLabels }} + labels: + application-crd-id: kubeflow-pipelines + {{- end }} + {{- if .Values.env.gcp.workloadIdentity.enabled }} + annotations: + iam.gke.io/gcp-service-account: cloudsql-proxy@{{ .Values.env.gcp.project }}.iam.gserviceaccount.com + {{- end }} + --- # GCP Cloud SQL Proxy Deployment apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-cloudsql-proxy + name: cloudsqlproxy namespace: {{ include "kubeflow-pipelines.namespace" . }} + {{- if .Values.installMode.legacyLabels }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: cloudsql-proxy - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + app: cloudsqlproxy + application-crd-id: kubeflow-pipelines + {{- end }} spec: - replicas: 1 selector: matchLabels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 6 }} - component: cloudsql-proxy + app: cloudsqlproxy + {{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines + {{- end }} + replicas: 1 template: metadata: labels: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 8 }} - component: cloudsql-proxy - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} + app: cloudsqlproxy + {{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines + {{- end }} spec: - serviceAccountName: {{ include "kubeflow-pipelines.fullname" . }}-cloudsql-proxy + serviceAccountName: kubeflow-pipelines-cloudsql-proxy containers: - - name: cloudsql-proxy - image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.0.0 - args: - - "--structured-logs" - - "--port=3306" - - "{{ .Values.env.gcp.project }}:{{ .Values.env.gcp.cloudsql.region }}:{{ .Values.env.gcp.cloudsql.instanceName }}" + - image: gcr.io/cloudsql-docker/gce-proxy:1.25.0 + name: cloudsqlproxy + env: + - name: GCP_CLOUDSQL_INSTANCE_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: gcsCloudSqlInstanceName + command: ["/cloud_sql_proxy", + "-dir=/cloudsql", + "-instances=$(GCP_CLOUDSQL_INSTANCE_NAME)=tcp:0.0.0.0:3306", + "-use_http_health_check", + "term_timeout=10s"] + lifecycle: + preStop: + exec: + command: ['sleep', '10'] + livenessProbe: + httpGet: + path: /liveness + port: 8090 + initialDelaySeconds: 0 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8090 + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 1 + startupProbe: + httpGet: + path: /startup + port: 8090 + periodSeconds: 1 + timeoutSeconds: 5 + failureThreshold: 20 ports: - name: mysql containerPort: 3306 - securityContext: - runAsNonRoot: true - runAsUser: 65532 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi + volumeMounts: + - mountPath: /cloudsql + name: cloudsql + volumes: + - name: cloudsql + emptyDir: + +--- +# MySQL Service for Cloud SQL Proxy +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: {{ include "kubeflow-pipelines.namespace" . }} + {{- if .Values.installMode.legacyLabels }} + labels: + application-crd-id: kubeflow-pipelines + {{- end }} +spec: + ports: + - port: 3306 + selector: + app: cloudsqlproxy + {{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines + {{- end }} + +--- +# MySQL Secret +apiVersion: v1 +kind: Secret +metadata: + name: mysql-secret + namespace: {{ include "kubeflow-pipelines.namespace" . }} + {{- if .Values.installMode.legacyLabels }} + labels: + application-crd-id: kubeflow-pipelines + {{- end }} +stringData: + password: "" + username: root +{{- end }} +{{- if .Values.env.gcp.gcs.enabled }} --- -# GCP Cloud SQL Proxy Service Account +# MinIO GCS Gateway ServiceAccount apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-cloudsql-proxy + name: kubeflow-pipelines-minio-gcs-gateway namespace: {{ include "kubeflow-pipelines.namespace" . }} + {{- if .Values.installMode.legacyLabels }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: cloudsql-proxy + application-crd-id: kubeflow-pipelines + {{- end }} + {{- if .Values.env.gcp.workloadIdentity.enabled }} annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- if .Values.env.gcp.workloadIdentity.enabled }} - iam.gke.io/gcp-service-account: cloudsql-proxy@{{ .Values.env.gcp.project }}.iam.gserviceaccount.com - {{- end }} + iam.gke.io/gcp-service-account: {{ .Values.env.gcp.workloadIdentity.serviceAccount | quote }} + {{- end }} --- -# GCP Cloud SQL Proxy Service +# MinIO GCS Gateway Deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: {{ include "kubeflow-pipelines.namespace" . }} + {{- if .Values.installMode.legacyLabels }} + labels: + app: minio + application-crd-id: kubeflow-pipelines + {{- end }} +spec: + selector: + matchLabels: + app: minio + {{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines + {{- end }} + strategy: + type: Recreate + template: + metadata: + labels: + app: minio + {{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines + {{- end }} + spec: + serviceAccountName: kubeflow-pipelines-minio-gcs-gateway + containers: + - name: minio + image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance + args: + - gateway + - gcs + - $(GCP_PROJECT_ID) + env: + - name: GCP_PROJECT_ID + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: gcsProjectId + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey + resources: + requests: + cpu: 20m + memory: 25Mi + ports: + - containerPort: 9000 + +--- +# MinIO GCS Gateway Service apiVersion: v1 kind: Service metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-cloudsql-proxy + name: minio-service namespace: {{ include "kubeflow-pipelines.namespace" . }} + {{- if .Values.installMode.legacyLabels }} labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: cloudsql-proxy - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} + application-crd-id: kubeflow-pipelines + {{- end }} spec: - type: ClusterIP ports: - - name: mysql - port: 3306 - targetPort: 3306 + - port: 9000 + protocol: TCP + targetPort: 9000 selector: - {{- include "kubeflow-pipelines.selectorLabels" . | nindent 4 }} - component: cloudsql-proxy + app: minio + {{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines + {{- end }} + +--- +# MinIO Artifact Secret +apiVersion: v1 +kind: Secret +metadata: + name: mlpipeline-minio-artifact + namespace: {{ include "kubeflow-pipelines.namespace" . }} + {{- if .Values.installMode.legacyLabels }} + labels: + application-crd-id: kubeflow-pipelines + {{- end }} +type: Opaque +data: + accesskey: {{ "minio" | b64enc | quote }} + secretkey: {{ "minio123" | b64enc | quote }} {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml b/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml index 2b43fb8040..43cdd75680 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml @@ -136,6 +136,7 @@ subjects: name: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- if .Values.thirdParty.argo.enabled }} --- # Argo ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -152,6 +153,7 @@ subjects: - kind: ServiceAccount name: argo namespace: {{ include "kubeflow-pipelines.namespace" . }} +{{- end }} --- # Metacontroller ClusterRoleBinding diff --git a/experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml b/experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml index ea58bc837d..0037f1b2c4 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml @@ -367,6 +367,7 @@ rules: - update - patch +{{- if .Values.thirdParty.argo.enabled }} --- # Argo Cluster Role apiVersion: rbac.authorization.k8s.io/v1 @@ -592,6 +593,7 @@ rules: - patch - update - watch +{{- end }} --- # Metacontroller ClusterRole diff --git a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml index ed1c60d8cd..bf5767f63c 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.thirdParty.argo.enabled (ne .Values.installMode.type "multi-user") }} +{{- if .Values.thirdParty.argo.enabled }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml index cbc0aad61d..058879bfcc 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/argo-rolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.thirdParty.argo.enabled (ne .Values.installMode.type "multi-user") }} +{{- if .Values.thirdParty.argo.enabled }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/experimental/helm/charts/pipelines/templates/third-party/metacontroller-statefulset.yaml b/experimental/helm/charts/pipelines/templates/third-party/metacontroller-statefulset.yaml index d52c18008a..a005ea63bb 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/metacontroller-statefulset.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/metacontroller-statefulset.yaml @@ -13,6 +13,8 @@ spec: selector: matchLabels: app: metacontroller + kustomize.component: metacontroller + application-crd-id: kubeflow-pipelines serviceName: "" template: metadata: diff --git a/experimental/helm/charts/pipelines/values.yaml b/experimental/helm/charts/pipelines/values.yaml index dd6fa773dd..9ea0837cce 100644 --- a/experimental/helm/charts/pipelines/values.yaml +++ b/experimental/helm/charts/pipelines/values.yaml @@ -46,6 +46,8 @@ installMode: crds: # -- Install CRDs install: true + # -- Install Application CRD + application: false # API Server configuration apiServer: diff --git a/tests/helm_kustomize_compare.sh b/tests/helm_kustomize_compare.sh index c85dcffc6e..90ce1b89ae 100755 --- a/tests/helm_kustomize_compare.sh +++ b/tests/helm_kustomize_compare.sh @@ -185,14 +185,14 @@ case "$COMPONENT" in ["gcp"]="$CHART_DIR/ci/values-gcp-enhanced.yaml" ["azure"]="$CHART_DIR/ci/values-azure-enhanced.yaml" ["multi-user"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" - ["generic"]="$CHART_DIR/values.yaml" + ["generic"]="" ["dev"]="$CHART_DIR/ci/values-dev-enhanced.yaml" ["plain"]="$CHART_DIR/ci/values-standalone.yaml" ["plain-multi-user"]="$CHART_DIR/ci/values-multi-user.yaml" ["platform-agnostic-emissary"]="$CHART_DIR/ci/values-platform-agnostic-enhanced.yaml" ["platform-agnostic-multi-user"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" ["platform-agnostic-multi-user-emissary"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" - ["platform-agnostic-multi-user-legacy"]="$CHART_DIR/ci/values-multi-user.yaml" + ["platform-agnostic-multi-user-legacy"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" ["platform-agnostic-postgresql"]="$CHART_DIR/ci/values-postgresql.yaml" ) From a49d43c93c0280af8f2940ad5ef51b84678b4084 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Thu, 9 Oct 2025 20:42:22 +0530 Subject: [PATCH 13/26] update labels and comparision script Signed-off-by: kunal-511 --- experimental/helm/charts/pipelines/Chart.lock | 15 + .../charts/argo-workflows-0.40.14.tgz | Bin 0 -> 29164 bytes .../charts/pipelines/charts/minio-5.0.15.tgz | Bin 0 -> 21320 bytes .../charts/pipelines/charts/mysql-14.0.3.tgz | Bin 0 -> 69215 bytes .../pipelines/charts/postgresql-16.0.6.tgz | Bin 0 -> 75170 bytes .../pipelines/ci/values-aws-enhanced.yaml | 2 +- .../pipelines/ci/values-azure-enhanced.yaml | 13 +- .../pipelines/ci/values-dev-enhanced.yaml | 55 ++- .../pipelines/ci/values-gcp-enhanced.yaml | 2 +- .../charts/pipelines/ci/values-generic.yaml | 326 ++++++++++++++++++ .../ci/values-platform-agnostic-enhanced.yaml | 5 + ...platform-agnostic-multi-user-enhanced.yaml | 47 ++- ...s-platform-agnostic-multi-user-legacy.yaml | 292 ++++++++++++++++ .../pipelines/ci/values-postgresql.yaml | 10 + .../pipelines/ci/values-standalone.yaml | 4 - .../charts/pipelines/templates/_helpers.tpl | 47 ++- .../templates/application/application.yaml | 4 +- .../cache-deployer/serviceaccount.yaml | 2 - .../config/pipeline-install-config.yaml | 19 +- .../templates/database/mysql-secret.yaml | 9 +- .../pipelines/templates/env/aws-patches.yaml | 2 +- .../templates/env/azure-patches.yaml | 26 +- .../pipelines/templates/env/gcp-patches.yaml | 4 + .../templates/metadata/grpc-deployment.yaml | 19 +- .../metadata/mysql-db-configmap.yaml | 14 + .../metadata/mysql-db-deployment.yaml | 62 ++++ .../templates/metadata/mysql-db-pvc.yaml | 16 + .../templates/metadata/mysql-db-secret.yaml | 14 + .../templates/metadata/mysql-db-service.yaml | 22 ++ .../metadata/postgres-db-configmap.yaml | 13 + .../metadata/postgres-db-deployment.yaml | 48 +++ .../templates/metadata/postgres-db-pvc.yaml | 16 + .../metadata/postgres-db-secret.yaml | 14 + .../metadata/postgres-db-service.yaml | 20 ++ .../profile-controller/configmap.yaml | 7 +- .../templates/third-party/argo-role.yaml | 4 + .../third-party/minio-deployment.yaml | 2 +- .../templates/third-party/minio-pvc.yaml | 2 +- .../templates/third-party/minio-secret.yaml | 11 +- .../templates/third-party/minio-service.yaml | 4 +- .../postgresql-secret-extended.yaml | 3 +- .../third-party/postgresql-secret.yaml | 4 +- .../postgresql-serviceaccount.yaml | 2 + .../workflow-controller-configmap.yaml | 26 +- .../helm/charts/pipelines/values.yaml | 21 +- tests/helm_kustomize_compare.py | 119 ++++++- tests/helm_kustomize_compare.sh | 20 +- 47 files changed, 1262 insertions(+), 105 deletions(-) create mode 100644 experimental/helm/charts/pipelines/Chart.lock create mode 100644 experimental/helm/charts/pipelines/charts/argo-workflows-0.40.14.tgz create mode 100644 experimental/helm/charts/pipelines/charts/minio-5.0.15.tgz create mode 100644 experimental/helm/charts/pipelines/charts/mysql-14.0.3.tgz create mode 100644 experimental/helm/charts/pipelines/charts/postgresql-16.0.6.tgz create mode 100644 experimental/helm/charts/pipelines/ci/values-generic.yaml create mode 100644 experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-legacy.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-configmap.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-deployment.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-pvc.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-secret.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-service.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-configmap.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-deployment.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-pvc.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-secret.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-service.yaml diff --git a/experimental/helm/charts/pipelines/Chart.lock b/experimental/helm/charts/pipelines/Chart.lock new file mode 100644 index 0000000000..02e63f6a2c --- /dev/null +++ b/experimental/helm/charts/pipelines/Chart.lock @@ -0,0 +1,15 @@ +dependencies: +- name: argo-workflows + repository: https://argoproj.github.io/argo-helm + version: 0.40.14 +- name: minio + repository: https://charts.min.io/ + version: 5.0.15 +- name: mysql + repository: https://charts.bitnami.com/bitnami + version: 14.0.3 +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 16.0.6 +digest: sha256:addbacd4a800daf2fbf006468d352c6749df1a4afaa346d6d44abc62773393bb +generated: "2025-10-09T20:37:41.761712064+05:30" diff --git a/experimental/helm/charts/pipelines/charts/argo-workflows-0.40.14.tgz b/experimental/helm/charts/pipelines/charts/argo-workflows-0.40.14.tgz new file mode 100644 index 0000000000000000000000000000000000000000..56b93174d4148ac512c8fa90d2919568f88b0479 GIT binary patch literal 29164 zcmV(~K+nG)iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMZ{ciT3zDEj@ZzXDh8S&h3RT7IV0+1>Xzw%d5+l7+^kSgImV0Cn38PTSqg@ zr8mb>_^*e1hQs0TU~dop9S(=(f48^yxBs=hxBqN=dvE{X+5W$V+k4y3w*Q5O4~RqK zlM0FXzlQg2tK7N2$b;jUNh~Rg#i)l6=8{gZFK6l4qioBc;dn~K2>sbZ2%!PGrg1Pr zI0#4pZx9NI@HtHsgb}((lZ0`JVjK~XV4u9bA)M2IR6z?mjbDB#I+J=C_XC#cal4trSZLPO>TzYwpy!oDBDfbCLT-cesmYgRF zRFZ_$2V&~o*uflndqZz~ulJ=1ef`uvCH{Yd!<2{z767gMe|tE5w*Rcm|M&KG*Zlu6 z9vWfA;Cl#dpuqqgu_$6Ol6YFf|6{#OmyVV(n3U?8AGueY&*j#436ggDWhQDHiv z)0AsA=kp*Rywg}n)fpTIC{3mu2Lv6RA1mys{xm|8r^J?jNk{?(5Q~juNn=y8htM@4 zNwLCVF056L$3#nFJBX$&SEQqlpL#t#vz(}*@WWI{!Z{0xKyrp9B6pI+0U~!)NE%N| zB^aWyBpmxv4?2vIA0X9nH9Y5Jl1rKEbWC_mBmvQ-MA#N56T(UClP&D~M2G=S*@uB^Cu3nn)&|SKaTp_IHUMUwi4Cja$g#-0Va&?l1 zp_=ACk|at(EC~oetDuZ3Fja-+;MrGYnD#f4Gm@)Dy$$rg0=( zdKEni)|7_&3Xh4Vd_kn5AVmU}OhwO!m8=UnWKh86Wj0^P$m^%*c1Hafq5`EtQx;J& z0Z4~%wMJ2dV>%&1ik_Npv*^nRX6+oFoth%T(YfsBeDUN1pAC6l9I*+)sbs>(p(@2m z91uW3^1ri@KPVj6`vo_+<| z)6KS-gmWS!r@qwuRkfcGu7;S!;T+KzVI*ig4IOGhPYT4Nu#+=IL!g#&O*)$jHO*IH zwE!nle?6yhK;x+ZK16U}s-q*l-g+B|{GQSq9Fkayr%+9aMTsoE4-|vuEDQ!ile# z#Q#*3iE?~ATQ-A|byEqV3`GS>5Pm~_a_IXkjpYdJ2TckI@#%yTfo^9+&IngipbIOM zRcRzs}odWy^c(!GFYe*=F2pL;4=G;V9K07?_Q}Zl||0e z80G4`s;5O_k?Zwci1(7j9=gC0Q94G1CBgz-dH>d`r_nJSIei7KRt2|`9?ldV27v%# zFH=2eGKyaDnB5R0%v4u~(2CS?NGe0(nRX6~EsRYI38q{PCk{nEL#kPV{h70B)v9PE z+$Vw%G$tXt-7HX-mU?+z)o&OnX8_L4*ec}J60zORYe88%1tau(isv3>D?mt?ppq#V znMGK`DRRK43e`-jR3L3}cRWgZWsx8xo?hu$D5;hS<~R&VNJWGK>PxJ|*^J#PIW^B` z5cxP()63I1R#S#0;2{(lN*bx9>p;r*&d(tmgJKcVh$_rx1OcnYb`>Qo5D4>(D|k@# zQV9t4zgTHkdXYs`Ql-hDNyr$`jbX!%96mfd1-c|$@$wvdt$d*n^C>|%0(xZ=Xv(6E zVipjfR4cR2u_Uh)-VQei_3!PW`8`g#P``VttSD#ls^pr(3H1czB1AGYCGsf>sgPRS z++yj^G8NcF~m4c={BjvDRnN;V&2H_^DJ}+X57jqf z7N=%rEO^-F&sYE^bpfv1;#&(m%@o?!_(kU}w0f8Wjs zCm96A(n&<6V(I90Dx_73H7D+G^)*XSRYz#cRCB;HE;I1z??&(%4C8Wyei;5R%t5?h zw}?$7iIK0p5N>5ZBv@G^pkXQA0sf6_ur4oO87XOi{S@MHG1_c+uhWSj~s)-rc-?LuKn9|2X=HpQPY2IpSw!gQ#V_pPU;xQIv zlz}iGU?ksU{`U_>6x9D!r)_nsyQ4@6c5c@HGoa)>{^Tpnm>KO9$rv93Z(BZmQid0jV@am-{3Qq?76;;RA_+gz z#v@3M@AnQeuBiUI5|g%ZCxhH=HQ;2jmBF~T9S6PK%tq0CPy-KbAiI-81?pEz zP$K#cK%kPFoq^b00L}QUlr06D4g_80J171^CbQl% zv7-La+p+f5FZ#_0ZJ^zqmYSEBuWD*vUcPbw>{aV0EC8D(P7|kk!h$3Hvc0y?SzL#F zWhLx-`)Vbq^1oC&_PyhRfDH>#SYc-Emb5Hb$_Zya5u#!B{qP^oP*~U{KfT>YJU3gW zuzhs_CI19#ndteFBqT})oojp8V&WLGN)Z4g`3fTI_0fdPKoOt565*CS~JPH0Rp=Jx#TY?WfI{AfVbI z_Nr!2t{H^u+jKV_^;U4-*l#qR_S?20Gv{r2YicPMZh5z_>c~Z3)i$N&xxZBRgaBpO zI|fII?m*pEQ?Ia*I|qd-&iB={?+y4+&CZh!&xw7aS$TwGJk^?@V^dx_`dAK-RHh2{ zN}eAcIgtYjDuB!Pjy+dK+|EdhN?YNIF9QL4j^xO93P&a&*YMw-jWLRSa(X-}yYo{{ zL|JM{h9)$WN`Bc95$V`)8SCMaOmC2A+dyx4&6FjIQG}&GQ`|jo7hx$m9jB6b);;p( z?d8b`olekIUd%Hq^h)(n={Ay~D@DRr3da}`3On#_4W*u&_!uutGa-tbNkD}4_aBO8p32ctBvoPJ4X-t1l z$w0?lWwSS+u;D(Tn)E|9oho&~yYF-&-Uu@Ta$L z%^Rd$3?tMZW8N2%A6`i@dJz z8p^lAtb>-YAOlgTeK{k}zH@1p(S+~-MZ&QJF+$t*H95JV!ZLwtLJ}QApc)Y!Bb!Y=y@7UiL4`WyAj$Tks=x&S_id88{Xo(cQJO~+y1mgSJKutzons{PBd4y1FQ6xkW3_EseFp4aq3yd zWpFLlD1?t&(G|Ec0gMI&^5AUdwB4}E<^+YC>IB%cBK@>_=Zk3)r~PnW9(l~IZNX} z)d9;n#y3amPos#M@Lg+^T^jF{sEsw!WwThZqx0u=OiyVrT6Z~W0AAA4-NO%Bh~)=G z+ND@=O@TnmA`6Ntj`cYy=5!zMr!kccO=N~GCsL{HY7XX(0CIOSvLf~bjCG~ksH(sO z?`>%9g&3V>18L8Y{%s_{$&HftRTqj{y%Cj&z<0!A&Ex&d_nz5r+UmtIlK7gSG(i}J zOe&YJRM2ojO~9vimf~26lL4F9(=Z^pjn`gpe|i-S z`Re1REA*tICo3aL)i#dS+;Ov^9VVi;M# zJVQYHi7KsCaMlp=K{VTo1O2|_6IgA6#&##o2I62O36E%u^)Rpiyh@DwM4xWD zreR0|4;`XE2xr=OKr{}3Or6aRRTFfZF>$L~;3Pmaibz1QB;mYC$Mb=YOYeC=k#T9x| zIw)E0*D*i>Yx3B1!5)-Swo)GRhJ!%lyMLNYl^`uzR2pC39!IgKmta`(nA_}JTA_(i zmm`^(z$&)e9-*Hf+k`;<5_~sgzmr#?0f&l<0Xf0y^)~imh$OSCb6DBQ#rr`0iL=Os zEQrh_v7j)ylN~^nL$%C-K5|F=)R-08FRk=S1q!qo9s2I7HSQrM&_3xOBZWOlL@`ku$M3{sSqg(<~D!p(QybnMIJW;9CLp1e#FolvckELyCpF zvw&zaLfb=cXXxz?y}h9my9zDZ*uVjrQt3r=`=NKQ&<=;8{k_6E7VfgXJAI(t)jB{m zY)irQfsTTJo1IjwHzFh}?29h3g?f7Cii0n=bGwWNnUP$r)>zyma~R5J+gbzd#PyWF zV4VJ;_;{#989Fp+JOF1m%SXk**;o-hdH>?<@Yl_v5^%9NwD2{JO9x$T=j%WrXhjDA z()heEOc`G*pPcOq5Z~nh_njqcwrY;f-l0^|(4Hku2=_@W)u(pA`cTo)*}Hdn?X#@% z2<;E6jp<0o`?U8OAarQ56v||C!Hmfa-%!S_gHJa=Bk)NW=tgBxdO;BxizF?G08Y7xF{9bsn$o?DN;992O}9i46O%cv zofq|)uVCB!-uvcXWaC^}MULvY-Z1}4H_X4xe))MCm#IYY(Y}ri@1uwX#B&xccowWL zJxeTgH5lj$SFrbIV~M={{E=#7!jwj&754Oha&wXmHT{sOZAzx7j_H4BOft?bLf%pK}nCrVG#DR`9yl*$>2rOC(& zqbEF+F-UtF4ReC7bc0t<(Uo>ZT|I@+S|Qk#r&jTtrP(+L!JeW7!d?1SnvMM^i&LY+hjN=-m>O2L<5MM8BPPu$k3diGUl!OF$t9DmA-7r-uBFRcr!`LijaNyWTkr$a4$cFoy z)eg>lJ4N{sw=AyZqI8Zt!f^`WE7rqdoJPfSJ;5Ca^93W>DA?yLaD!V)ci5_uSpz?3Yx$532K+62nU>KBdo5J_8O*w407_c1o9&3#j?Lme72aiD*`_Xs`K``Qf8^iYd<`8%#^;7yX~5X2}vaoGsr1v zXwc**xBlLzkGlNq;_9~YCI<6{r8g;btU8C#@dsad#o=sNdhW`z?ER9xfWgZaQ~es` zL?kR0aq{8l?eR(P0VAzN7GAzml(lK^l3NYdJP$)Pt0xCNJ zK2dr_JaxlvFD~Ae{5c@K)oUu8#JpITeKNwq%+rnsWxoqn9h3^x%wiQp+JmK;#R@5U z;X}z_we}=ft=jT!rJPHp$1!K}8LNl!yPd z1#)+1%g14;6bjD-7HB~4c$n?h=Ftys{KG|I6B*^9XJ$=2G3#$ru>r4FuL2l=c_!Oz z7c%D)6C(YYwS}x{^ov@P2`7TY@~PrqCJ~BFz2)!*AIA_c0kF}O-jEm^QJIN!)S(=D%kx;esy;yjSA`*=W z7c-i;iMa~5F=+F&9&Hp5U+=T(4nP}a+dv~wHryd%G-UBq`=vuRr7_ZOU6Z}$360T= zrCe;<{+vz_(`jO|aCVRl+ryb@pMbzPxl1UY7ef;cLrHiHx!?`+G?QnN$I#dh)4+66 zu*j@JJEX#4rVzMMgq~eH;fI(;I&ek-0vOH!X#L@5_2NjCd{YUbu!$rfl1hLhe8RF! zI5q+sJWP5?Xz03OJ5VK8Ii2W1F7(L$AuI$pGn)<{6!r`OIF5>lJR#oHLvODo*3+Yk zoW>KTPb?IAbWIa5Bq2eZbqibN;l<_UnPG*-E2Ev|s@A_Dqg+J-{-o1KIGyK2raZP+ zWR#}P4$2`AnU6!%e|w%8=~;Xaq|E1JO71L+RVz^Y8lgd$HF2&=lqkIKeW(4l6Yslz z-Fu_2sISW$D>H2Ba~vsa1YLKWOfMkLF^D`UhF*Rz@*7zcR1K;JU*C>@BYt5c%)R!` z@Xi4w)AW2ny}Lo?JuwQbzVJ^=h%+;h;Hfj;BO-BtC9X}8SCH7%Y;cw^MhIs~{B#z6 zZ5Ft8DziQL(=}evzTw==?-pPiDKnUgK7m( zDOL7BIAk*@kf>8f2O7Lh@bB4eQ166J221_IeI$y%UF@PKWa?>aBWBWY)Wcsc(CPWb z;bt!tL`Q|WUJ|WHRkH(TLFidNTF8T6IG_`V--Y}^Vy}Bmzt2|MUV!vucKvni-io@)n8ib%j z?&HJD7f7-UmOTm!8*GM0J8GA$kA=;>lO-=yyy2>nCCDgyFUj0BA2#yH#*ZIS;dU(a z>C+&hF=hR%B+#LTb4_fLx9oV}J>hCNIoyp*I+_3mosLxrR}Y*M-9_!!KNBECN1(o2X@L(n+&uk)H#bFjZTnJQ^uf3c7pp1-j_ z)xtP_saAz37#1Y<`5bmTbwsAf@iNwbR-hD}T|?+UA6>N5cgyM4sqbz_nVzEmOgK~U z{%c_J$0B|9^e4Mn`SBzA-~EptGld}A=K1t#tMnS)BrGsL{&ydJ`qa+|p&0L*ks93G zU`xyufWD39gQ+hD4lnyQ2LN$vB5Nu?D8h7lu9?jieNE=* z+X|B3KFuH*o@F)6F&5-tPt7k=T8h(pzWlZ3qQ^QzDev1va}IlUhQ;84#Bl&;EkcF^ zbZT8BM)`Z%WjxigiQ!zgm#y#G!sRtm9m(?YE`ez#*26#4qJJK;F?_UX;yMy5{sY!^ zMy}aX%)CO;TxhVyTvZ+L#vvPfzS7ZdAtX$x zjOVI+g#8)pgj>qboEJ_aL2a)+%_bZoN+RlWCfG!R>z%}dRBXYhZQ*N(78uxV4~xa7 zD~#jXiOZLZzG2B><@qU4XWi-eJD$gKEs_D{Mr)Q`un$%LhPj4t0`wPuIkazVJ%vnNjaR|gA;lzhMU-SpDO9{RIl32o(6hYRR`62 z-pCpQ1#!ciN_$tlgF7qXZu4w$eA~16%VQ^hdC{E8>cTLJUS00;y3FOBq2yW1>xTY) z-C%Mu`Sr;|<>TH|b9kbR{|G0r&dudHHdBRGnIfE&Z%%Duxb=y>?Vb+wa$Cs!#;abo z?B^^+w^&~ZmaR1R%3&taT$?wr_zT}F=2Ou) zzYkQ`c-qixe9V;p)o zr1LcPh1q}0fY=%OCYo1fsA{q58*__81-lU?-AQ;wFkLh#-Tc70Yev!u@nuDma^Nnb zP6^PbbZ@>b^)e2W@nE|3?bD@<1qV*WV#vO=60M=HYzhVq4HU@tWr2L#>37KnGFXN| zD>5z$4;(Ya=4*)A;k^&V8K=HQQW?P+q-NgY7{EFmfU*K~nFU6#<5q06xHig6p=xt& z6i2gcb@d>}J$5{7m*8oNA{+-wzYzJIx^NyJId?uhHOj8ZGL~1g+MW|$5C=N@glR5M zLt)HO=cvMJSEXc&)j}3q8@9E9{lg|p$ldeB>;(ePM>jj!(E3p@LhtPtQz2mfHQ`&g zGaM3C9-=U0xz1S4GZwR7d;cmwrF;)IXOP%>^Y-%O!jpHhQf299I2;ZS_V(c4;c!^~ zcX)8H|F7-6{b$?Td;15^_Ww29-hVdS`xhGCzv;zKDkSFr8s59Da_9ac&&Q9T_BF)9 zpiiHA$0uj!Cr5{uC&wlQMfr3JGQRiHnU$UbP2xbo=xuxGXCe#dK*^mJ!RYjCONYcM zwc6Ouqh1e^qsh>{Wge9sw)*X-PmtuxFcg6X4BaZd`cE|FBvF+tS(olJ&-yz_!%(%- zM;@TNwMO^iPDP62p67A!h5#SzW%rOb`K6p>;K?jI9#v)$9O(|-1-jP z&}al6pa~6$oG0YRNDYfWBO~KXHm*v2ePdJU`1oi)i82k^>M^8+&NCkkipf}6R4KV#vnNP8g4tC*B z8eOE53B5!8LEe~RUW0P)%XgDJRF|Er)E~Ip4&)WPesY&soykob3jJH$vtn1Q^?0Ho>};lhEm*0&2TNeT^hj>$x!K`frzx>#WYhzb#gll7endCB z-k!JDW2sD1sqBsHRZX$~C1!YM|6mjh557MbPPTV{*uy{UVSjfr`F?M>y}fflCc8Uh zyz~7J+xr+F_`CiO-|zU($ieU#!Nc)hfP?YD4+nM4b!V<0pvNCfk3T?<^KpZ)vvWP* zhhd2N@BcqM_~UT!{|*N~3_f_j{zo4@sg$uDdyfZ`;ot`-WS3JTr#1yH@wACoXCt_Y z`hE5P;>`b^GD#kG!fj@?zxZV9ZhYkvZN~aOlgy&XPN7&hD7u&)fEfuRPt3Mpght9f zYC_(yEzjodgf<#WTQBD;)7F!K`V#d`Wd}^(XHmjp66=@dZ`3!ci~46%wS|+Ul#;{l zgO2Bbw}qIA1)C$SwsL~G9I%OrGRu?ht6_s6oD5vH8r2Y+WNNzalKr=`#Jr*gQc*^- z^d%jqr#y77HV|8JTJ4ecwY5x%8PR(YX zb+KL^8)J`Ek4q@+r6cn!9zc21q@dbVABEDMxw2vKNr$<7{D>AZ$*LQs!YNCQ#18#s z3S+6czWJlS46H@UVGAj)IOI8mj>GQI)BT6Oi8vQGw%QbQgx5F?=WxWp9_`{R;S|ic zJXM=gXUrMYg=25Zd3p81hzkRls@Vb4(I-F|5Nyd2 zxwAiPUTxT?4G@bFxuZgo*k8<)0D569C{x(zs~O{^B0R`jpifYDo*1Xj5))Hb%T7#@A03{ZX866p0`)7&6Qy ztd#t@dKb;;Gu$Zny~7aLp1E8GZt~K}s-rZN)Y+8F&#u8~5T8Y3IP?L=qv+-z21qqd z;_7Cf>y(5y%Q-)_aq*>hUA~^DasMyZ)0!eS2Yv+Gk_+2du>Zfkz4NTN|F^TZ_iTT? z|MwUVIKyx}CFmPJ`k)roQ~2u!hiLR;&JpWEPSr{p3-OzF*bPahf01Ebs{qM5_O zwuV6DXN!5EGs7^7Z!+f?)HiwQjP)^Sgb+4(roJh-q`qm|IWrC_i@P$~V;Tn}mS_)U zY96?f%ouv^Egevz6#b>ESg%q&K*O}`7&=p?UVVVLT>Yn+wI?j1GMmP}(`0`$2bI1;ldbZ3wOz>|gKc!vFKo)9I9xDVAiwS!m+yJG8NY|L^ba?w0xg&hE1{ z|9_09>{Pz)Rjnp8^`VirZ$%I{gpbW+P9Z3JU=y1RkPhoB>r#)o&`3b#XIp4);8kgn zIiK#cz!duH1-4#bW-_%HgFp2pzRsV%V(Grgp2Q}O>DZ0~Qc?Z3x(nnMeKubY5^Wh}sQ zj0laVA#o4u7|94CG27k9Ub_O*S7tvuPz48~!*I(Eq zdjed7oNQ$I>9J>$)}t(Pv_qX#5EB;i#u@cGBc&$4XJc6?)lkDsiP2NrT5Ip@T@q5C z%G2X+h%5!Hq;z&bxE+JC7Tc;j-J5^sY^l;t4PkA42Z2bCvOE*T`lPazUb$X9z*7>- zfsf(NAafCqd+4J<(<@y8umXqeBKwNHYYhe{?X;Va%qKCEUHh9bak<5#x?&s6$;=I6tNzu@y>>)zxL3~qv9rJXY@Pq-QJxkqoSl2CcLH{e^y|V^E9ddNhw;NbDlhK)QNO}??QrNG zx}kN;>c;objuFZWj4DZ#iVfsSW3|?j3Ta7$to_+wo?(u|Q0Xj@E$qB%LZ+rx=N7H# zJI9i|q7jwZ_GbA*QAz6pDJO}87QC92(Tuv!s@<#mrblOmhG@)KuK!&gX#O6B}WFIj_4p^Uv>4bBvurD3S=QyUeI zN0v@tC0Vh}hK5VfiZovs;--PfYUFBHlm7pGKP%R%nfX!~f3U56}p?JQMl)?J5{ zSFFw2Y#&JNtF-__TyV9QK`o1P3CKN>E<>v#U76!N zzq4uKNqZBzw3D@@-`^}Yk*7=V1~I#NvISp-EK`Mo%g~~&H)tcOI20UsmHkN zSdVep#fMa^Wqw^=zRK(7@4B^I2ctq7bQ&{GPB>>=bOY|er4*e56Oz9gxyc0E zEB}cc_SU(1>)bxJ_p_TC<@XB^seoAm8=GxSurgH(!7}#-T;*Pv3#Xp`n$MiRL-;hv zWmKl#)zr47o!A=vSY58Ij#A!A?Ut3x#cfw&dpfr7D3FfhC7_)65-ydiuj~}5Dtdg3 z{gET~t8B@IG{%1Y4n#rUaB2vjC9a=H|9%)JONAof_G^bV<^4MBvUEIzqQ zsH>C|UOz-TTBU*g_@fZhsJI(b_w^xKO$Fhn@r0cfgoXR|AzHC^-NiLc{!KWYI0ISz z_7E*jXebHS(GBNhO78M%l}``jglpPKu}t%v2qo7FDtFztMr>BuFA8aVowWu(9u&OK z!ZgaeV}3qp6EVIyVj;`ezWu%O&K4(<#A>9Aj_=az`}e`#>}1^Www%4?X-V)g3&;t? zHFs1c^9prwl}&?losP|c23tCR^Ptn3u;AyTpE>r)8R3)#7sO|AZsIn4UU||1-Z8`= z!uI8H${{YS&|b~=#s)wB9mA7Wj{o7~uIXPE<$u}T-rgzaf7u?c^S?aG(-IN!2oeGph`i|(U|wc@81QpNhq92IlZnF0#)q~{a>rqFPI;;dPGC_r9Je$`_nA|Xj(;P* zG(o4jsbXI3Ucq$h&_rEYZZB6(?n514Gz!|TtbbfE^s?k$N3ah%P=76={k!W#|Du)@ARn*iMD~q4(t_(5>}S+jcp?-xa(1K`iox-Mi>|ykPZi zw&OtVB*&*QmDyPwcLTogzJ$YS(d>Q=xPcKrg(MdCfo--C>bEUHbFEd)3N&Bp&c&%O zo9NT0QJ4Glmblm#efrcd<6k=H(|Eq8xSo6#MOX>#_o!c^t@Y8bPGLA&xzxupk~w&G!@`%YTv=u~yepjRQAhUA8X)sd^)ooCZ2ji(;e zsDn0L3EVVfV;nXD4cT-sf%xyGpiXSedB4QF7y14Rjc3|iPzFTc2 zlu`bzE>i3(?CRD;zuyCu%ga|Cu#qzS>=-31I0^}l)1(=pgard%7y9h@_?*SRESxY^ z>hWd`1lVEWlwn?JVUT-6gN@R+47iF<`M{vl@=mc465kX7TRC`m{`1=pZw_CdT$~*q zofL{7bfa2?W4NUc2u&yn1G5EEeo8qWss*634imH5aBi{HqWS9N@c88X!^x|Yqs!B` zZ$6wJpS-y|z5MSljBu5yk>8b!sL!WlOr1@%U8bL=ZXTar96ot=Co{HENrm(+|hzSM;4F^%)tztFlM)u+XsC& zX?c0j35uC?D`|I8l{}2ksY)TW&N8Qk6ca6MI-O3jJI#b*T@)m%;)s7xUi zqR4|QpNesug@(Sg)BP)YR^9&*xO4dD;`?7K`~SPo_J`~JpT~Hb_n(W7&P7S!O3_NI z?ijWBNdKl=M{Vq`GzA(nW73@Aq9y3?v0_X1Y~ygE{Lp$p&>I$$9zt@SG>*6#p38Yc3iAq=0g&Gp zRq4NCICzdj9Q%a#Yd5Gvr^JQHWgBPbS)0CA*Xk0GyC{-_wI=!S&pq|Oh{aSgey{ld z-Dlgwa{T|^_WoM`dyL0r9-l`EtYtBVd|n&!Y0wu9b&r&reo^uU6vc@_tcgRu4nG zmsU}%z6i<1ag0lCNi5TJ=@3i(@s)sSzk5>C&GHs7PAIvPBv!vf zHJK?~k#KIb!(#$rG?c|PY`T$ri@IiQ@L73)t_evjljzS#7oT2dccVb7lk9Znkll%&A!a`3a=s1EFoOd;sIEPAPdK;beNWqoW@h<1B`H&g5_jFZea#aOy`4}gr>D1LU)5a zVssGUWS~o8$s+1EmrOW;#XU&l>zLif12{M>Mo98BKP=73?=5jySIry1hnB zp`E~iEF|NQz6IWVb#BtX?X!gZ=Pb_k(`D?yMe=|9Sy}$??ryLBzmM`PkpG`Y6c;Gz zA1Z&H1Sw^q>wKz8T6gT4#Ov+E{-sJ!%3{?4k;~G|$PU)x^fAS$5`9>v;%mU=kwyEL z=6&mD8TsGfQ174#ERz4b&noBtcK6ru-;eTq3HjedN&ieuz>0c%v-QtMmsnhvbb|Uq z744vHGp?*Bz$V!QmyPav5FPY{22G)?E3{|}P5MH!#?Yt=)M*VZh3hqk7BO7a9$E@7 zBehr5A?me=FN;%En?Us%1*&QlwU}2li?VKE-lgN3G-hUe(8sk)!F`sO|Kx_ma;zgD zyZHeY$^YSY#s5FtS?~Wo%JU`Vf02^@+0q~GZMAX5s8;s7MH(f4VN4tR1Gf`0ZjkzJ z^|fzcVZpUMzJC7N&+_trIp6=ky4?ReET8}0*;~v1$9cY_{QsxR{02GSDBGLmd4n|X zB*F6|+h-h(yB?V<{SEYh2%pnLUyXhNRup0rBxl5JM?Jp@7V9+Ip0>8UWQK(mXL{~0 z+$i0OtS+_o%Z$W@hR`h*%97&}I-(wZN5KdMSdxLHSyt~VGN#_4EFg3^B?W8(8wF9q z%c)r12C5L}D5WgvTtE=yo=*8zY3MKveaL3goj`kG`+-dQ2tiW^7@~1?xdq8*slNW?+u6L^M8B$`|JB3 zALaRy`kzZl|7=b05{v6|7>Nd*FfY)k9hT37Rn@_~#NW)d1M1Mj>l`eVNm{-sq-l!^ zA-iK%`G63WB&vDc2F=3sXY_{De_UK0!F3XGV|~=VMdTF1ybVSdJdnPpPN_DE240aH z61JDlm3tZiyfH)0%iWy?WpzK41v4#N3s_%nL4W_C$vwwxVG%R6uh_J4&@_xOV;}?Sn#@G2Xt9bsOi$phN6z~*$p^q+sADxx?r|Z%|)(XH@=;U)m&A&5aa?R zh2&V0>AVdPiPM@Jyz(bII^Fd7Z5IA`#G)i5phk47a7;QBe?x9MaER8C=&-u=!eqR0 ztU9esEW|D2fj&F-5R3Dgv`!vchFs$XB()SUmgt4m4PYBn)K7i*LGc-=;kfTIOL;5K zodzHkgsah9&vOajS2NFxni(iMSeaY=k@9;fd06xKmt?-Cpc25ZhJp|phirUrR*~He zcyCBHzv@G9uZ!cCN1Oz$B-Ld;7B4e3h>ah!!D%dEJH6^s4U~q_jNjruPHHbWb!)eb zwyJz`XKkpJ!A$+eiR!`~#;JeZ&SN@}%>o9~O(8AlxNH7b1^CtQzr{opN)DpjcFV|*O1Hi9_S*V5*cJBqr$qj9jl$`HnSgTNsWM#D?WMdq*xlR|3?<|Y}3GA#rAYL=;B25n2Hur@ z6dPE~M(opto!Dm)TPfnuAs_-RW&YS_J@>5*{AK38b5TE}B$h{nFC>p;NF#^3CcVn@ zHzz$i`<+~4_E}GUYXg6o$?sg0DWz%W;d##K@tWu=&)=Nr?A+^_U(bK#`PZgOp)jrFT$x?)}!fYaRgTu5lwXD#>GV|GeMwgt^ZoX{$zd}F!jJKM zp}_eZCrC9ar}0El9*N~4EZ#4QQYZQqz+2W#rV8*YVloi#O6B}g>G$_ugpYbM;~wGL zZCktxJIe9xePr*VW6uwwJ5>@Mm*KYz4p$^0$mD-yRToN#^HdvgV{;BPU(3Qc>4IfE!2d4`pFa zjLx8b`~4>x$_F@=mL>DPFQAOmJ^-W#Ku4rW64G`O>p(&aiae(L(@9a6kbVvAK)U-; zP00#~gtMt4*bap|W%;fkw7&Ia{d9R&%l|D%=j7iV?tk0atK|P4?he=azaQiI()quE zlGe$;FGySF?+enN_FE02X7V+6!?dOY7v%4j3w(X2$h~m+B6o`P3Zt;TVdN2?739Ay z(pdpmApf_YJ*&w7?Pu%zKOW`zqVhkZq<^>qV8)?I12CVPRRBAV%{oAogiTsN-uPM# zXfOCT5WN+-+Q|ROo#c2B;h0W{SjO6`J`4E&aA&7{|L1ULcg_DF<@xv#aU4%c#s^{E z-vU7a4;%~zy^kLi`&G1dDo|gqwm$M)1}GE79luepfW|^W%AcQDCc5dA)3o^U<67JM zGEd|B*Fhs&(`&Ld#{M;l1HCP?0wY+o{+}I`*Z=mjgZ27Zi@JdTCV?R|Q=#Q>uysW~any%k$r5{ZF{+-0BQq;rid%E8qXUv%OybkMcCG ze>>^z3?L>_5f6>0-t?MigK$hFwpQU!1t{w!zTy%uO~#UIyzS|-{v#aYDG7XKzfGu* zC0ULKEL#8j&-P2}e|NaOvtIv?@ieY~GwDapq%Xk(%<_L-0rFL_0K4V_7Gdf-nF=1E z*0C^E3pP$SAJNz4+FrE&^*;kYOob$Tz*)G2d;q@SRL&Tue}EI!aS~UYxcWkc%-5J31?MazrnvPY7!R|iLHD~} zMi0YM7UOq+&7V`_eH~YKz*`}kwg9gehe9w{_;qSb9|sW5hs-BG zKtix85=(-0YU|}#L&G`xusE0R17wCamkV~ClzLgWdhb+2&1}k@n-2YMmh)L@jl?fA zDv9cwu2r1G1@itLBT%=Pa%#^e8uC^9vogIZ-oRNH67D66EelCv&AaYPv(^2w+NQjO z2|49N{2uC|7$`>RrGoG`Ssr1{oSe_a6w}UUGkW#&)__4Y{|t@SkfZ)o8dp`Ixa(Jn za&XGKoS#C9C6l4b9N2P`n{SD#0b>KFzkJ9Q%$ zg{y*>09~y3=HdL1UrcqfYv&)KK`l$rbtL&qpFeo;X#ph8g43&oOIEOB_o zfj45q}m>2=-Jq8 z{WFiv)G}I6N40HLxRQLcm7xt*?H2Qp< z9+r=?*Pg2Hq}7+?Q_N&Xr;sw!n(cdDFz5@?fRh`ldi{b5$@u&gji{^#Q7!ki$rF^V z1A|NHY&p`E_pYt^*@!pqXB`PAEYRa#w0j$<_4|sQo#%LHZKJY2pv&bo(yirE zyKX`OQQ4%q=hn`GZe5t`V@y)|ypAY-xJUPz+n+HZ4?~ItamLhn`B@r;1M8dj|l zd#Bg{=F>tZkpZd<`v46FS*ZGtZHvx{B-IrL5l+0gyzCZI;US-vxK<;jZx5?*3uK0> zd2e!IJA2InQYI1P3%GLY#3nE~UMlNtygyy#xXB52LN}+dAlV zlU<^VQfbIUL`LYv<>lE$4WzOMzFMoUQkh7!OrbbDCLx|*5TC_?7@_T9p-e(J1!1he z?zmq`d{Z13JN&0_J{-MzcX4@gUP46YWJi6;Q_`=kdU*cx+YfIJU!Po@9Uh(3Rj*IV z4vB~wa;&?CpV>UDsSFveGo>~+&T7$oet2>6;l=sMPxT$F((xDz@`95|(ay6^D>FWc zZ@Tj0oesRXlC7``vTU1}3N0<&urQ6tYdBp~w;*Md6k3Z^Nj>6Q#cxeR;kM9OG%Xs9 z)^V>>!&+KfQh|5$4t-$5RrajDV!qr<-P8k@!Kj;&=DC+q@^7f$+*tEuiSmflV1BT? zo*c7)To}VPFYUhlEb6&rA>m~^%X#@(8pRVxKj_RVMNW5Pl?Iw!ta;GHOP&j!9!nXqmS0Zthh9w8RuO?CtfLFU5Jr}D0R&HndaXYZh+|DVQi zUBPN+Ko;!(9_&6V@BeP^t@FP>&Qq^q+v#@q0aVFvj5`Zf$mqufPQZeLzYIc|6u{id z%6sKOH#D_))k=5L2KsZ>!MMdF(C_sXh>hZ zZ=v}@wE;M(HKjwn+<7f!;v_Lv>|#{)M!6iP$>wp(tl$PTy-HM<3_mRSuC#AObs?!U z^d6oV^2aho)4XMoHCWWRQ%w58vIpXGMq|PYjW-u=c52o3VgaD)%(S}A@{#E+r706c79&_X-MmVi;AcDM{MA{EP3Bc;%#9&fXF0aU@pW#%bqLhD z7?x;s&2GQyrz`*01oBz@zdHw&`0wq#;X3~NQJ#7QtSSDx%+}X&-(N}Gw`JfDAM~9C z3k^)1PFp%odL;+iE3CLZ6l`g&{e*%M&y*p9Rkg2-!Ku@zjQ4G<{uUk2XU6x|F4o{P$)gjOa9G zoIG3`3*`T>a{g<3Z*REX|9zBa1D#aZ1C0##5AF{~Cg>dK>6+ zMgsRTkv0ve$N~K z!MEVQ_QhyCVL&ufDNsyEKmrtygebx1QzDR@ah6VJu-p}V90wkmsNpzJPKwhu6v4PGw8Jv^fQ!1z=LcQ8B_bX>%$SBBeSQfqB1&fFp zH7?LCw5OOW;lR?(WzMS6qw`~dKCn~@8jufq5Qv?}|7I!jaf|{g;0(X1U;${ZZO2Uy z*?5jpp+v;hV4x=6y*>KJ2@VCh(z3+7Q$_O@d%YJh`e37=G?{W75ac)aGaI}A1OWd9 zca&eD2@3e&9vWl5p`o|^gE!oKnvees-w-q= zBt{Xt(TgVQQ09KQQm=6ka3X{raosf(5Fbvw=Jbs@F|$xw$wfpUj8Q>tLlZHs@dKMg zBd0Gv0{qLYLUqSbQ-PSCq9i0(07+^}_hkrOva<|TBOIqV4CiRVS%lsf@1FnlN$D>7 zO*_$yqZ^8$wd{Ig4MM$26CpXl5i&v=agwl6uXlBIHO6ArGuS8-J)EP#HR^X?(_2}4 zTVpJU#%q#>;s4Fa8UV2KuU>nqd^(=4irr zHg)JCA%5>PMvSWgRi6m~cVvPNoD*3TJ_0d+y4ufU)9hlRP=GTNV^Vb-2JzN(= zmb=K}Y&4u66>=fu8Thxd8_|mwyBFMJQxX%7rP+8@nlp4V3Lbo#< z5`pKHv3rxBG__quYPQQ=a1NelBVmqt<)D#!$}8UxHz!sD=3f)ORVt6Hz#gIZ$Y-|# z&(W{hH#nGy-ou^zAw$r^KKj+tB|0sN0;_Hm1(jw_z*W^xPBc3KSKL)JAD9&Qc{W+j>87D_a~I7AT(h+!N!2JZh}&WIq0+~Ft*i9YBhetqK8tr;FTEMya5 zrECf-QDGH$y))f;K}eCgzki#k#l$#z4MOBwQSB^B@Ra1bK*{aK2@d3pD9u380*SYT zRC>?Wj810*?7-RD1j!W#gbS&3+8MhARoBO1MJ7@uq7ymu^bBMiHRn}@IWR*ijOr}y zvbb`NB*Q$#J~=(Met8h3S4P2#+3izwJEQ&#MFhusI<>w@0;gScOT!RhU#VIwMmU~l zEHf`InO=W7ZFN4=pi}~rmkfoFWKLls3K&I{~# zhHBs&20auk_eWh^ye*8DrXpo^N!BuSLa-~S*EXh^wv?;H*!Pus4(GK+&h|7Bf?}(r z(E0PjBUS0*;;n}&1E5e=C2IS;v4J!~if53L%4Nj1Kva?_H^AjB!Pf}a8N0^p`iYV- zfHe{`Nk+ZiV9_cZDLw!Tih<_!G`9whO8n)!@7}(7_21uphawg$k(qNqk&Yt@dlN#h zVyw(Y+ZuJ(dNW3;Am>_;DmAErn8GVItQ8E>96JjHQ5YsyrDz#|yCkGOm8Zv7BXk&Q zvkELmlYCW;EshJo)jYN^88a@b%~JA$s}VxKDE##hb0Q?Cz7%cH7>GdvLH6jkRA`>3 z0q0%?^hD0Ev}{wAc%c;~7=ul(Gyqji`f7x7hmh4!GVqo&$Qg0;@QUa$t0ou0+TnYH zLmFr|L1jc#zXSq7XL#?=phbr$!5l|K5-xhZKhaAvM}H#N?D-QNn`|iRFX3|v%MtzA z`*WZk{a^b};V<~r`_no2{`Q7&uGqDk;R^kU)XcyiSN;AK`V(E~-)cfk4F?F&h*O;t zrAlZPS$0A-HSdQk4W^8#X1+))h-A zdq)zk)G^Hrm3Ha}fHE{s6tl#9ZceG731Api0oytvcf?N>A2)7UPa}6HIex2eGO1v8$T^-Xs)JJK z<@dtz(@19ZL=T?&~{A2B5|wI^PI%X8|isiNfv;j+NfqKDlDMG@D|SnETdAs zBOpca(-@k}Ge4C9+f)SQp|I~^ihWK3r7K~*cvL+DLwm06n!GPECW^Cq*SMGpD;X9IYkd>aZ@LuNr76Zqlfv_;=@rl(3k$0Xsz z$C3p96RgfwMaK@HUJ_XY?xlBC5pWm;0;ZunADJ$Q$LxkUY}R=6khxje3P9L)(f$-i zOFAsVHaEbc1XJM3U;=5}f?NpXL!$T5TDts15aYYuEcUu3D!5&qD-xx8^;^WsJ#K5L zGSJzwx-cUM8YdW*xB}VixQ$8Hc3WHF0xjpBT>+0}fr}QM+geBz;Z-$qCZQM>xv^Z8gIDFX@0f`X&juXo&hoz0(5Ha?Gqb(;-j@7TpEE`!Knt(nYg7cW zdvhEt+N1O586zlKZ-!wWH8objiiK$uSEu$Ub8Wc(6s4fglpSWeY-;~ng{o@5#z`}W z%ll@_*7c|nZgmy(bF>vOU7%E*v)B%a`WMu$Dtmte=kCB`&K;c>{!mpM8&jdY>+SzX zxuyD~$yE~iKkQvwbK^F$e%G%+DGxiYvZT2rn~AIToN_!Kk1KXY$Cu56YidN1kc>4& zFa#)#v$g;I6uJQr1VCLZJLf#a4~fSlx^W}Wm#@3y@2MlWG5U~2Qle2HP;Z)(#GAJ! zkG9{lCaa{sSvi)9A~S}a+`3uw7aDyBXy zJde;y8sTL-6=mh6fepA8$U%Mi&oMkqoZ3k1?hgOGP9BGh_fZi`?Rm9;Q5j&4Smi5Ls%DR zn;^%(U6a#auF15XACVng=<1RhegF3D_hDart7lC9um`P(oaQr@D_MILGk7WuKpjN# zo(rmPcmP@?79mNRE;N?V$ecxtlaV!3qIzPEeur{5xwdIFHhM+VeI*fVC@Xamrz8YxOi$+TwyAvNk`ujPQxOeI%PlksLF0;HW_iE>svJoS!-w!saZvn92OR^90g zCbGrIG$VJVYWQV~V_*$x-IFE}omJseG?4I>k!Zj+h*GT*C$*&)Z@|`5sbs*3i_f2g zU23JWZEbakkU-%UE#LZOBc{GWllKvss5d8vWb!Y;i^)M34>tjCi_z;AmwC?TmX&@x zPB21>6*C^~Grrq&Iiu5huKKK6sQ<5$s!;!@RmH1t5V|BHuK&7I%v^(I#TQD8s)Z@1 zXxwURDI<#dRCPkc$bZa}IhsJ#=}^_FY_V!!=13N8jY{T+afMQEn>+sNih|+>Hjhxz zfxUI99Z;G}g)OujP_&GD0>BA^OzBMZi>WM_GR30_5ijLVWPn3OadlFM31WBx%eTz}Nx zFy|$&)cj(i$;nTze|imE)n5%)^?N7R&D4JlGz52iR;07UfnQ8(Xp^qC-zGck50O?$ z3H@BhTc@dB`+?MOOADIM21+}n85l|7oW8lZ-bBBNy0dHZ{`LFUu1%peU)z#fzZg-{ zo0kG3_^eZD?OGsGCChYRX9MoE*9HpdWB&m2yAUd#T@rHysDpztZ9~FfJ{u>Mx-fDk z7BLYPD$)pzRsF^m^+G3#0sM-RX3LyP$0>^9P21kw;=4TIIj>?)Kq^ViNRcd=4zZag z6$H1;D_$2FNwJ|vkZ1ICsZ_`{rVYF>z?i>al2CY`@B*SFi5>*wMR1I?at^n|zg`8K~ZE*Q`{XZAv)qnki{4o4+`2GOSYg@rDSV^y9PS`E)h5fA} z?Eg2QI}g;gi4;lU0c+pwFiY1&rUv2^$~&|`oTcST*5zy33=uD)-=HCK)Yfy z(~cENFW^OHuM4~%GaJ5ec0zuyX-&zTN|FKVY)FpDAGG2KkR+_gKsnK_WJ>`K#wj(e z2wN;@#u7>M$JmOUot#zbjB<++v^~Cn}mE+BG2G^L}<@#YJMAB-(ibUf#!Mi-; zRZ7us`@r&i%YoW)I?WC7&7QOVjB)Y4(jlb zzo1wonj~cGz7A(|A!>uq1Nx$_mg;1%u{7eIR}x*q9&d1Pq4IX@x)6dx&VZm=mXA4= zhj^o_lVt^gQ%RnKMvY3c7?Klp9w;?%N_BwOOh5eXo=FPLnILpg%5Yitb-=lHsu*d% zBLSvi?XVA>aT_nUGxjLQRmCK=7Vz!oT2BJvT{87+Lu6=GJEc?@h}w)M5+qTt7pDB4 z6&~LhTuCFM_*+(cqXJSw+X^Jb8hqiWafYh(Bcg}#Gy#uzVmw~J0MsLNv^DX7tdO+O z#@PjvO1`0FmU9kktbN_i@pU{tH=69!z3;WclPgh_Bnu4mm|1ZF#F@yq;S}=lM`fzg zn^CD{sp#)D#w2Ytw^UTp0_xryWQxzA3CS_9sv$KC)bu_eptoSup_!JxtZ5$x#0ewvgBsb^I5kN(}yDqi-F?)t|U7yU2) zIqX+7$vzkP;}z%fBg-j9NA`~Z581Ek;#mC33-uKL1{2ifANZT_i;#t*d0F>%H>=Hl zHlN<+t@JW}?QRAgK0V)=9a9V=$!fO32X=~zhH0G7{_?uS8H~&hDpmuhjk6YUX?7$|3bmPQJnp36lKR(IU zXXudVh1I*8<{=s#)TDs$0|pz3wTS|XT}6xnVYm&Yx#?NXjd}FEKD4Tp1#Ip4s*7`BIW!SDuosC&y+xq{rp(qx1~Jk~J@x3BfmsK5c<>vL z6Z-0Kr=pe91sFHh1#R(UuJnIM>LRB?82D>O3!_<@h6C^+4Smqd+u@G^=p_OKbmpV4 zEdc#54eag_`WT7&U?RG`9#d25_AEMYYkq?AuS`wlgxWS*Y!BarXV^`!+iUwC@*bcXs#3i0G#0{}gpx_Sr9| zy7tJn3LyE}eLqMtrG|Ef(Up!F@wAw;LWlfxFcu|EBNKSnVyDEXFCLj$#lfh#^#DGq z2K#s4ZXfmwHSiJ)M2;3eY{^ea(2i6Jo(5No4cpe)5@;4n4Izdm zg^Eqq0+xgcGMF5aNt!cS$O!-{CqnRv%@Sfz+d}XrZOw+P3xcd@p8JC^vC>1EU?iMn z4~uJ}t8EKyO+wrWVFj6Tc~5RjI5a=-RY5XVQIz-_R|9yctQ%bESD(hmZZ!48>E3Ah zc1mUP7PChbbfqPr$Uru4&QH8zf2b2RCu5$GW8Iim`?(n6}+w;3MzG>)}{ z^kjTX>ZSoHn-)*(!6g{bcIt$-x$V;rrvkofl!G$@J}4lg-XizvHC7N-Wc1tcUM_M7 z*#u9kQLHJ%3>w7Cl-@HCA(pffkp1S{BW5}xqMm|*uR&UaO=8sgR5yf4on}m8k9|kS z86Z}gK9#g`vGO!yazj|uc0G7dbZ@qeB-$AlQ!_tG$g9*44in#BcVO%k zUF3!207>IxWkpgw4g=DAMi^=8S$y2j6Ile|A(~NJi*@-tUs%!FG2L?^k8=jOmV6$U zNm=3-D_4{j}{3z4R|ek*JqR057kxHDFO85!c8HAUZ$%9O$YUkYY|>>hZh%R4e` z$KC2tu{AJT9UeondfZmd%z@u>NIqj?rOD~HB!wswe0>5U^0^rRti6WQrIRk+ZS`r? zk4}5%s>ukr+I0Bl;+nM6NAU8d>2rd>HO*&(VjKC>;4uyTO>pO@NO0qj z$rG_Dx971Q?~70Enu|XVm+G?J7xHOtcbDWgkVkhG9Jr$;v8Q!5lwqC?sb@cUFBRvRoxA6xXn0yK0!9&qg%ptemc1j+tYgNbTvm58tpe?C%BH}8N7;xQ!}5gpy- zp{2Vuke3As#@Rd_AQ*11mjMRGiSG&u^r-9y2)sRS;lCv2Y65&wP=f)CLse4Y&-sHfuLlL5=viIwoxfe`G)UU1My4y(i$td~=w2lSL{Hz{yBy9%gf_6}NeZfo}FiLSn=k!~kaeTyF9w3L~a4IV6sxV1Y zDGpUzlPaG9EAWQAeqiN4X~kxb(QIZ9tOTuWsa`Q|7;}<;y15yf#q!AuQByAl)TX(| zGgb^STP(QRj090}F3IiHh4GER7#Ji1laJkl0d8QJifn~rHBkj*3e(iJElLMh>av%J zeu9kPt(gn<35b2&R!xxmyjmR%oJmivz8tWgjcPZ`by3}bS{=I|9SeYWWKS@F}gr7EacM%sXXU% z2FcqSoZ2f^kb7QN;=t!n3tF8QGd>QG!hIJj2s)1d0duBwfD9@+=&fN=mpk7Vo+J)0 zxhVgMFweH|2mxXrSk*uJ_kZ99*KeUr0FY3u9tP?bd`)yw@+zqw(cszI5YL38wbWi1 zL+eWJ4&S|SvznX_FAK=;0+?+?1)B%dlrC}XyDyNsTQg~R3=@oxzToqCkevyih7kQ0 z#4J}$5(s2AbV(PJKTze`Ceabpv1}-^f*eEZpL0Hihu~0*^O>ii47)9tgq)pRo3$UNxdNx=HeLyp`k9?1Ua>)A)Cnf> zcC!1Ybp;+<@Cx$+a=gAfB=b}p64GF=q84p}sn~nHH$!+n(HYrK zu0Bk(CBhbIswfoD8)6psq*BV($|P8ZR_?u-@?wbrlt~U*)<+uuq0S-A4G67hMRjN` z7UxS!2D#k9;xc@AkM-Q&OiZihz`)Gw#(g%@73fCV1+_>dStUYGLS`>4$T$RbZkt?W z@K?-r*Xt2NOaU@mn+xl08CFL`d*{rb%?@eGlo=Nrz9EzO5m~+)z8=2O3CfTJi<#108uRIS0%Q?173F95TjExjeu;?^11kCsJAa)L?bRiplgI zr$KjKowc|r@D{LsNMLR=)T-8c?V*a1l`w0lf5m5jKH=)MB*zdSy-aSUpc!e~E<`i= zHWxqEGGtgXP1=0CQ;UbvhMKlKI>9e;WnYRKcVPSI3Xu1d%oxS8olC}Rkw21$g0Bit zVQf^;b))uA)D}0{$tPaO(WKohj6<4`0cJT-6FX_;>6!F;4PdQFt=KlyV>wM;q_mXq zOEa+}bO{Ygo(+_cH0njLkPTg^YvHIW4x<4D(4EIj!!MENcddVklBQpmZ>CfxZ@xMO zGg7|j#Q=@F_u`(f29l4ovu&UQ>&16w23Rrwsvmwe55MY4w_gqN5?}u66fn_Q|BlG6 z*B7)K^=xL}-BV${dHwgF-t|03%i3jEGW}p)vy6_^`VO#_TKQgwuVpPcnTQ85s_J3@ z);!dK?Utg0GMkQiI)CAI+tH?c9F-jZGsxMDrjKb(N8dHJU!W;^Eg}CY8y>U(s1Doc z5~JQ4iT=sMFxtgH0L?+#5FPbeLemj({?&5|1-g3-#R**t(G+=ylDVTuCBy$-@GQyy zVM+Cx(p9@}T+f0v>U9(X+mCWm%lp9s4x#_+GTVRs{PpwK&tE_P?|%L_00960yJsEZ H0M-Hkh8XRO literal 0 HcmV?d00001 diff --git a/experimental/helm/charts/pipelines/charts/minio-5.0.15.tgz b/experimental/helm/charts/pipelines/charts/minio-5.0.15.tgz new file mode 100644 index 0000000000000000000000000000000000000000..ee5e93736690e6abbc2b275c0cce90b011afcbd0 GIT binary patch literal 21320 zcmV)lK%c)KiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYcd)qd$FuwnWpJG<_+1Nd%WIMNA?Yif;`L3=%%!gJ;Jl$EPn}aS_jmxbA1^B)Fz?{aX%$5Ri6*5TtD@1dfe|B z2q8*xsi+8Mgr$nGlnSg)XM`m~i0RFLBP5L(KuR>i&schVx&_B+6qpyLEsLS9?XaIP zHO)u;h|dOQEt!%xZWJH&xBEMfg5P_dyB>q{4M}n;KNSO5y#62VJbL)3zWy~!xLg09 z;aOXQGr@0IOo1Tn14|)GNJN2;4b{V#3%DX#HoSs_jA$Z*wKZH8r#Dmx7Skd0dO^?x zr8vlZBo$M+q6kJ!DU9hF4Q6s701Z z5P6!ibfN*;f`?dc=<4+o9+3nAUr+@x9g{p!z-DAZx6D76WU>V$jm@vKJW0-Y!lHQ} zW;~|4m0Z$T_mHJAi-@ASZ@NjWFbn_$&A4QWi}?_K%gMa&n1f{z0E$eOuvvhtXnF|8 z;}fpV1eG*ZK~P|JK_fwx9KwrVf;9*&3+v(cP}dJ7mgv1#kBfP`nnhQzq5m@YudP9p zFq*1jC?Y}iK#&6dD37kG+U)CbnMKDdj7+p>SRREB{}w*jT|6vnYnBr9L~Q}<(>_Q= z(wHPXrN9NmOe(=fxuS8P3l04@u(k$VO{oBxXBijzm0^!S1chD+$_(dz95XEiNn#g= zp@}HTV+yaBnnFyogwJPspt#VeC_2s)NmW3Kx=*p{5Z=7210`w7712N55AuFlzW{$Z z&sao~WRCM%LK%1Nh;eHB&4q zMck?@rrwX{dw11mJhzQ6sBa3j0g)`G5fNoIY3dymzl&POsiH#H(F+99d^Vy2`2^}s zaPa{4&yKzJ*Vf>`kc6b-hDEfG6*bE+uj%V1zyN!(e|CJvg&IQdZ`<43y;Yk3M(|gX z3OL}Y8~i%EE=m~VX1at0fh(%8@6wn?fSw|aIT(9Dz@;iGa{w7 zJaExOn(_QV=O`}?OVq!Zgb7Wl9^BZl!;~wq4DHa9F@{bnBWkT6?1Yhp2}|?W1D;Vc zl5$`HKTWxGhxu3F{5R z)~D;e{$$#7y@6VL?eB~BD$`l7*L-j5;k+ja=q1c}idumk0eHod1V$8$tcZ0dt;#TF z3Dw$<9fJZhO*iGMqEW43%(Q-?bt4q6sESA}CutgMfl5=|W{lE4N0A(HA*jrF8l$kI z5mHM(aMNhUyO`laCTOJLYiL>q8)D4(IEbiF@{FkI5PAcuB1?$)de95Z!q(qsQMJS@ z8rXG9)YPiuXFtL#o+q&$n3)zjG_JOI#bc2wPv?H4aXS(SX>Vg#0A=~rYxF5 zp@*VK(dyb1Tf<*47JP=|emsVRjumLOGv5MMEN_&P8VJsm9>+Itp#P&WgbTFb-8%~< zrel`U7(vW(sdTqcxd}_m_!uE(x}wZz#Kz1}LpSKasL1D>yBUHLBhUC4aPdkYN|9)4 zRBQrzaR=th85Jj}zwT73nc*!NU||pmnk6hEatJ$*D>dlrjHGeFn5>y3gN$>nmF^g= z@#%%;k0=mIX&b0SYRO{h(y2zE_2>g<4;U6{l-5kJ8=`3c+Op3qA0;g6M?&Fhk>(|8 zPnz#&TI$2O&nnwpE(0Yf9n6Tv+^|uVNKZtp?S{ra3CnlZFbV)Pje83PELsOz%$FGI z_XhnR=zQpR#f*^9;}*jK6^f0~DwHilR_o3RoIj=N6&KebO(QW!`H3ujO--4I!;Ap6iL@Jh;U@@OW?0kzqDR`dQFbZ5FiC=$}8v8v5Arbv>Y#)1`dQuq<(9O ze9Kq~5z0F&m8Dtw2)2TNW<|y7SBdPG4fO*%6PjRTjZYt8TaZy9nN&25s2!Ix_eOzH z>ER$DY*tm$+O?xxQV~va7SlliSB2K_Fe+KuTxC^jJ1GF)8a37=uJe5@v~n~kjZzY~ zAgPBD5jAzr>2Em`RQet4*PE-c-m@PM3f^moC@xzb(XrN0OlJy_JmPs8>yB7qh!~hL zM({?2Xi40HvUOqu>WNKstee@>BLCsP`nss$jLopuR;kz9vNFxupc3MFN@gsAOz;~f z8AtBzplKTyF)+>;E6+C31z;LVb8TOzywMm+C1AEMpv!MTH02x#%Y#<7Ohq4HW0LJ^ z;l2OQi!JL+5n7r%IN$wnw5317U*I&O>4hTE^(KBaCEP$7Ri2Nva-h&Ej{ES6`Lcz`?P6SGNqa$d&IX0q}I11H#gOb=O z4XEVUzfB3-Q*?c3-h&-MQlFAEHWr_Sl_mLvrIuC4JiV_VC5qk9WbWKphz6phr+On{ z*Aym3&X470-%Xb7L&=3+XIxTGYqJJCP9L>nL}SC{c&?;V+MJ7a<0&WcHzXlxq?I&N z8$E)WU`>5J?+8G%XUnjQ`UmEZK8e)AZa?oU!WGPUKjhQkmF zp3r_369*cygs3qWvzD$K2(9({ky*Owgr%=r3#vqhB*N8i$t-(Oyy zU93{~)s!ikFsVYb-x3OvPUsM}`}p5L6KIAje-Hpnxl}mq0Ct}>q_4>=OQ_vD0j&~Q zie1x%j0$H60Tm-xAykPpg*>op#;^{nMcuwk)j4;|#kbR3d^_*I_$(^NcMIWeAvmXT z|FbWMwKdR&zT08al0V}y5UI^0&G`e=s24kEJsF3zC~0DRMab**s|$R?1=5`2iL*bk zTVy%cO z8mPmIJOv`r;L>1o=RKPUo@a1!dU-S~U>z7xT3h02GB3cfahsK~hek%NZky2~@^oJs zx9N_N3Rc4Ne6S*~bUhU=hXnHHfjP_$u|&5X`) zqj%@A)|J=R;3<=+>1KLRnA!013_3E_YM&I?V8dg9;e zKVE5d{Vo0<-`T~xAJ16&HfCg!a;aD(f6<`lJjd-tV_5<52iQVK*Fls^#b@5uFfra% z9L_S7!BsL_@@s%t*fpJ*U3L4@N@$Xq!&%l>_LDAq+a2;!!-=nEC-|GF^$v^_O&R{ zoK0B^#dlD%SRZq?fr`f41k-{3rUb)@U- zcT|~APkDs2*WRJ7>gBgumu1f|6tff;5xq73^zyHhc?XTxxIlxC~_<jS!TP|KX#c@=GfTEjn0>oz%)+roiPOHvgeMTf@A z_uNkT7Vf#2&EPb43&=X#q}>g%JD0k?seM>ig@~ZiyHh{WEWa23V?9Iu{UB2H#kxKe?tLwO(8g{9q@O+m+I8_RANM^NlYu2q_Ju8ND|bGfJCHjZZVazC^2 z*=$(4AQmj(5Oj1>4+vXvVQbQL%i8|^M_bg+&rG7Ih_PF~zU56sqDU29fwgi%x4ft! zv(YMSAg$~$2X%d~;uXIriYMu-+-eGyx%LLcYWl@!{cvW}3hHKo7hNGatCTw8fP z!O{h_Y(*_!Qf=t{q!~&gIUeh<18s?`Ml#=}vz9nDTDf~@Ug{OSINYB)Ik6*IT zed^V_t+BR}p8DJNxoVZQmGCUHw$I(x8FDV`Y~DAd`Um7;&JJ3Mb=ktDKNHpfH<0ZP zrTK12*QnU$)lO0T%vW6`Rd%zg_-@ty7b#Ea+nB#f^^CmDabn(@IEr7Yqzt-f2|pJT zss*+MglkjV7+RdoT}=};Tf`(`O^u8_pQT0on2QEdO62q{)^a$M6)ZT=M(&K?Sk2*@ z&YkuUbBz0ZMbc_Jl&D-((J+Qbp+> zmHj6ds*bF8%xGeQW8)ZTdcy?AO))P*ThN5wXnw4?7soG>xd$ zwhi)?A11~hTQtX}FP?vMa&-Cf>G8qQ$;HtSdQVwIQ%O-$@sTp70fybU=4C!gsp=an zdOomk&QJe&adbX}9v_blNnK8})09REHcpQZ z4>m(hCPXoe0X4^JMc*7zS!ByKLWdm+AtSTmXbuf|MpG7tR7WjD{b7nO89np7P&ahVh^iYGC{(xi3A(|>p1jQS+@vaPZG0zT^y0(H¬kHJ+X z+_Ze}g%-AbM$4{exLC5U1!;H_6v!PhAL1Slyo!OXq z3?@0UBxbckd%Ek8Ns(hD`!{(!q3ZpPn(6j&|6!2vSPtR>s10Urw$qAd{m7j;?{HBB zjyAk6S}bVw)2S8i2u&>ea6E?l#=5%iB0&|0``DIdMCk9iuQ}rVBG5cfi{m2ZT&!#7 zzSVi}7osVbH14~&L~E_;PMIeC$Z4NJclV)z@O=&8{uc7|Y6aY(!4Om2iN;18TGpQI z7a25qRyYxDUsEMmB#knKFrV?1X<^@`O00p51j#5+ts83nK+iCqwZCDp3HV0NRH}Qn z5v*?sJJ1?mJwcd zTkgue9>Rldd1c4dy{dswY({ymY7opEUYc)O+p}P5?~O1%x;!{6xWLj=CZmLpDyK}O zk^`!uxIe=qnujKk!*M;c;h2_=ek;!Vfv#hif-Py)&5WpNjrR!}r+7TB`W|`s0Trt8 z!m;$Ot1=cDus(Nt@;D!+sfA5`mtQ`4&_s#{am#X#Q$%PSA z{uuz=zO(Fe7`%A{jz+Cy>APGpJ?O)`cYWR9IYnEme^Qtm@80$0P1LXDeYxZ=48imQ z+r@8FF4Y#A+34k@34i4*Y&i@9fSo>=W2$g=die5W|Jl*r`i53fB9(yFW3&LlgeCy}i8>VnJ|DNenYCD3B4)m6=fsQBPQT!eD8*phe34`_e~l5ClKj zeVdfOs!oYL)XP)XOffw}-H|?gi{#@?lT;LQerS6G?WmfmZ`7ArWcjn3%|q*X_Vl=- z41~9~%x96=`uZ(r(KUt~+1asF^X%Z|_oo+^FP9Q@h#lUuy|2=kwMJH=vX+L*$Yj%0 zozN;r)!+5tEuJ=c{1C!0rV)?n=J5By$DNaJ#p%CG3h8D(dNLmL^$aJn4VAVz2m;L{ zX&USL%lM^#+UdjjBuPsb!tt3_S23U+zN?*oYAFPx`7o$V(oM87Um{tyw{&!Z;Dlq^ zE28|fug7OFBXdyXngW4{=UNpsxx{i+MEe%jR{u6k;2+MI43E#gzGDP#;jOf$R+{>c z!w5dz8m|(-$6Dt>u%94UMU(kf5lo;hd{je5Hla|(+P^sb0e#Lkp3$yGD&B)uI^DS* z91J!*#mifpNzgHen;>3fjw5JTjXau3p1KxL|2S;lwXnIq#0gDNwb+#ev66EV}w^FVXF<&>KuNI0k}Y zmo#>kQ9p2Mkz+c3rXW;;4qhIdo_u@!-OIz{^Sz$Ow`(PP!Lx(C-XNC((VA(Mg{9`8 zAcS@|AdH!S^*0ru?|Oz|B$Ap2yP)(cOm)q1p-cYEB(=nCMPia3hBBcv3-NDAIL|b% zdPSJ>lFil5!#lMmsKVXYgg3#xi-Yfvw86L&+}pprJbHF^d9k-0+rBA9 zdjbCmJDb5h)AgTw-B1uJ1h)hkLB8ujmyJiSFVX8r&09 z;W zo8|G%^MfBWb?yZB4xSzEt#6pgmx>hgK3F$};Ojw5Zw6_eB(VGSU-T?k^wb@MM7z$qNh5n2oNMj)K8Yw;SVKz`cJAHb5@YCMnvdrQ+8wIO6y!#fh9UqZx1&@zza zc9`zlV)y%+v`eO8l8+d%F}C3uge96d@0sZoGYAEQ;h2kv*5}p1>Za0rn#GrBeN#gR zVooK7-{IJ8fxKo}rV+>aMr`%c8Z#Vwo1x63Vm8JkaTO>w&0{^B%{#EX(Fk30+O7s- zvWnt479YZG7 z&kb^MV~&I7(E^$T)w;!eAa&T8pN$|4bzyDNUbjs>P1+GL0hh}$>1wv6DdexVy@f-w zN=pkGT0*WiT-C1hI(arDRXsAT5ud{EavYG_qee0x-BNzNq zL%elvY#FyTw}2!nO?T15MJ_XH@?hSkb!n<~l8OGGoyU%8^jgbPyWI6;+Iieph85xt z@Wu}`jVKBMs$A40PyIM-*L|~lT9g?#s1-vI){Uq_)@}8xl&A9< z&kZ}W-y6e$a`jTo3$4&s@-SRf<5@EWR3e>cLTlVIn$j6XZHt5*Y3o?@kMwH*K4Pq%A-y%EOyCpt1yTx21q>JlTCa(r=5{?2!OxCa!&yK-8TB4 zrqSEE0{}jjKLGLgbS?n|@7c}xyRe+`SF3Lz0Ejt8GZoW#)$|TrApkoVi#-Ik^8 z3H36L0$XX!>>tiy@b?v62D(nY^Y!k>Jf#IOSPE=2%>Y_F2xc##@E{m@((oWyF}j)u zp>leJrDJX#3Vu^|wK^hPs2H=w8*sU|QW+H;4Dv$1I`x&&|O9 z3>F^~FQT#e_W0?M`L!FE@Y_;FG1oA)*rDaBaw?)bqP3189vdtz*7CY*q5rGo>1HoL z()x4DteCnUan&pyJ52Yn+vHBc$gG-)(_>P6NZdV66|G5Cl{dk=yySp6)L zB-E8n_rcRP*IE3!a6_Sr+5)6PJ6fZ)dRS^ggY59V)L&Tq`a?9pCHkK|g8X3$;IjRn z$Bp>ECy$;yy3_wY$8)Fuwch4(O_4d_DHSa^HCs1|P8=IGClh8g! zf(r@g(8E)!Hf=K|b|jIaLiPiDxS;a35;{GE`&+%Oe{6ld^=9i`VAL0*xcqSR(@Uee z=)W$Xo$d7oYL*R(`~fdX8ow-(1-KRbQq8j7owj?Y?fy}<-Q6!i+r2Q0&8fQ$J=ZSA zTPeD@)#EC9>TabiyWW~QZliWdKpRuNj=~*SU&voJSJ#CtO0cn><9>7Pt-k%6z5l~|YoitA37($n(NQ?2+={&T_erz?Rv$hgW%}9SeKX#j^ zFwETw&<)_4e)mpK@XI!{yg;e5YKQIqdE#G}*#8>E4?YdJ%>I9{(}@4v*?D|t|9_6> z&i?;{+y8fV|DE0cN4ERFh+QsCZ=FUziuqex{GO#>LSJm=e`@m{Z!RwP@XD7hyKQ&Y z{0CX{#!2$Nro5qsWp=zrwB1HLg1V0ep*7!^4;#E~TD{uZwWQg!D4%lv|GR-DgONP9`v9Gzd)5aA@ciiT-hbH!F4w3&>OgloN>tyZD(x~M5{bt zCKVJmczt2hkm>4o>+*|+e zUegl&BGu1ozH_5=dVO^X#9H*uW=(I@G?R4cy+4)G(_?QrKwZ%co=aP{!II)8_B%L9rQorlmnlQ8v-Wr)2aymwT?*H&Oruy5`o_0MRb zc7KI(_2=-iLJifaYGMK^zFcv=neIZcyoBYwSJ=kb++pl zHi^r**jhJd-mv0L*&dk5b;b194B-uiNZShU-oZONIEjMw>pJr9-fel@6Ux8+xA@fG zyfd?`p|UnwN61YE4e(3w<=Unl{9k??&TpQ@{D0@+lZW;F--l1O@AiK`%k$>V;48Rc zvmt7CW0p`g&*Cw4zH(_ zy3fUkhE8C<3A+m|8+KD2wPqh49)Hrzdgb}`YqN1VE(TIH8E#ncWW>GlZ8^d%D4SQA zV5u5I?>}VtAF@}2Yw%bZx~0VGuTC2ECSNZpmxq8za8U!r$}+k3I*9xyGTM#r$NJOFcpFQZk?7hdDJ2%!DURjW3f@gx^n#!1K1#{+`yn@!I=wubuOoauA z8huMTyP@KM&oaFddtdME^dE-XTkwya{*!Q9_n2g4#1f_$m3_U4Eu5m`{>0&ML65#x zG{q~!2RAz-s>n`*+}oReVw>K1{N8QbezSDazy2#GQC=N||No=6+*^r`SDW|p-rdeg z_lm4KP@$N#7RxSTAM+G&XnUMIA4u57vEyD9sGQ+7xd6*E2RHu3U8 zebGd7eT!A3W_oA03&oe7>%VKVW}hytG3JBqyPiinuld*liq_K2lK%)9!7_t))< zi7Z>K`jU$R_B)2GynFZNjfsO;dG%+AJsk4rnhH3ckqL!!I$=_Yxw|3_vxax_XQ%M#;WL`#m1L!{+|Pgv9s_7lms z>^>U;%5iT{vUHM|%ZIJSn2>+Xfoi<*)+k`6U6R_#FUQJTa7Fk=`-jFr>XP+$!xl=G*PN9*yO&Qd za7PshN{L)*n?IsLb&|eH$?~)=UK*^eN638_oFh?x{|qC<3Om=JDuU^ znGgL)s718}yCI622&`&#^QC`H=bwg%%FYh={7X|>>uK>yg_eO`BetNrYqA`w_iwR; zf!i;u(ewfu@W-L*_DZRbv1zT7BGy+WgT;|@b)j!Xu!~!b6Zb)r7 z7vs)j=s(4+7p=@`x6a-+8yhi;6!ePm>$P1Z(=$5vLZ!NAX08Xj+tB-g&U?^vxElV^O2y%VQF7Nq zFBl~k+tf8u#rH3Ev>sMQ%i+6Z#1`%U)c5~PB-o5({W+O^q#eLz@t+SK)Z;%M?L2vM zxBvG!p3*Thgqxks%3)&ry zqNLqU8#ZpSas{STFhwS{5*Ic~^WD3^b%>=*F_N%<(ILDI8w-2G_-EGpmlmV`H$+lY zRD0d`=v_T zLc0K(DeW*N#rt=o3}EZRzYNf5^kqR#hEhQ&69c#oUi^4qdZ#~?hez;hYkq)_+Q%Xn zb<0dux?23uS=t-aryul~C{E%IOTgoe()|1q?H zCG!8l?!!j>=YxlL^8d3uZyLuo{B0X&@)xn6TGXG|P!G?OZmTWPV#^3pBa6?tvC?ae zU-(x!7IiE^I-gVr!rC0hI=)`hrc`uqPobh=^dS4WT?;D5;@?5%ajX{aYUvm}(*>x+ z$T$yclb^Q;mPNoxnsPBm9}pWFPkwphgjc5$&suXzdHaE#6I%Qbc|uq1 zMg>E28S1dWRxeOl{?rAs!Vm1Paj%D-zpNz!$L!p$ue5br1FBoToM&_hXM&E|>&nLw ziLPlH4=XRho%X7~?q&?tM>Dk{G+qOw`D~=^15_V9{@5}Oz9(WeY)@9e&1-ZuEPUE> zYoUCD7jYL3a9CeTpzTh}-&;QZ(Vs5;&)ui|IrKmMcU}M6z0?0b%YzFY54HJPmA~Cn zZjtox*j&&2i=O#b?~&Cy-p6<36v=-FeX3pjJLPqCb!U0~Tj)a@T)>96>s5oQ<{5?F zfw^M)_^gMUbsKb+srjNRo7TUqP*xVz%VtxqmVk*|B@3$f-KK;!Ds?$BZ0x|Y4nO52 z{)QwZji~7PUFmKbD;mY#t8CFN4wCh{s#hzuZ6d6WK&9IlGvhV}OO?Ye;c{EhN%2Lt zRasha#Uq{!;qu_DD$zIa`0la+KH)Ly37VR^N~X4qBPLXO?7Ky%x-vDzp`rsRA|_Nd zAmQRaFyLB1ELP-Q`_$f+e|%!qSO(S48Lt z|E)8-XXN!oE+$K`aHY!mn331d(;@G4;>(mIEB`hLxhzSL&LSsf$G=1vP^2*e_ zluBKF+mN@)0B*C=^pjLDyakRctfiKvlJU5{Mmp9>S+XS1zt$mFd?P;j*W1^Ma}m8d zNx7hSMi29c%6wKqC$cYH>NxD+N?T{P0!nndSth4s_t9gS&stkBQ6Q@~P>mW{)I?)+ z+NPqxK^D(Pm23DB`ah?=6{pE@ls1F}zMR@jY9=un)YRp7?;yPBiv5Q$BylXLlrXGYzWZm# z*6KBh^U$%|S-NT`8&+Erw%!q>er<5Q0~E48PG*u3L};_^`j2-Qz@)PG^85B`Wd$Yi zX`0M27Mr%VLtkgy#zD8X;rH9UZSID&u*Npl*Y-{3qjs3~Lrq4s)~xrtMQlP-X5jo( z&xcB%$Uu%NYk6%;yoY_rD_mB+o6~~hHRr)-q~HHTV8G1K3z4%pjoTb z1lpTy^u}|x)yJ{T>E@n-)wMgM%4NA6vgZ)LMGLJiFa6N;roB5RLE`!8>E%l=YqWcA zTwrr^*Rx3OdrrsAuM4z(|1@g9ZKo8>xd#^&z6|ZN{fmo#o}M3mVW_(h?GIeGrlU65 z>CwrLFTXv0deqj{uC;4b^!}}uaf#xLD#4;c=0D@9mb~_$%wnX^&QG5mU4DP`{Nm;Q z^ULpFUjB5}g}DLuKKOAKM@@aS4V~k|qm#?y%b#AJo*kVWAG$Gpe*Ux_DQ|6IBW4_* zi{zHgJUu=-xqNxtN}nyQB#fr&__lp9jhM`)ZNue@Gc#oNJkJ5PcTq-!EG<yE}c1}N|v-c4QHDp^B{9=4x{&4iu`{Mlan$AB6?~JJF zD(elAKHIA~sw(c#GjV!RrvOIulhamQf`=@^e0N<8z}Ys0q;<7;)Nb_>vs(B8J%O}}u%n^oyy%42$AcQ>j*1@3KUWbt1Y zI>{O)Y%`^aXTUyQfFc=>S;~}0Fz(HJqgH{hctS>^tP?g;>xz}GCxmcg*0|v+Pyms9<{)yv&#P4T^#Y!ryl?FG0%OuXKDQJ zlShsG&pVIq;(tEJQ(KH|m4jbPa=R@$;w5i$wu{~JP*+=i$p42V=Vj_xcRqjQM1ROAY^UC2Jiho`VYgN=yF)>>poOq` z7N{!VMw2W9O;T8;9Xgu*l=qZuYme|KT z0p>k*{-5zUW>Vysw0M-q6Z+u?ftK+9?I(@%zfT@KxV!)Fvpmh<>%(IDzcJJANdZ^7 zqR#K19b3*W``;4@J)=Ch6!?R)Cd|g+;cAu2&77^p{6%a;Q%fr?A_mF*_F}}b^#$iN z+D5MJsK&=QjqwpVSabbL75>UcAJYN2^zSQfu? zNri0Y^ecjqt2fxTrOfkJ@JJKQKYa9Fc>)}n-g+tfH9xn{u5l2kH>Js}FQW-lPigCgUX8!XJ4y5hSL~(qT*!s;JHO*c$@Alz9|U8q;`dM8{mv5Z_{4KjUe; z+fcXjKYY}5@WJhThmJ=sJ_UCBb6v$`tuzhrdCzt=nlfYAb%tr#Nyv`+tL)nP2bcua zOij6S4yWn2geAG4A7_8tHy3ZY0blE8rTTkrfEVQZi8tC^JD+63y@QOp4rBT5;@cd! z>VkWlgI5b-taJdY3qs={)-B=^R1{`JO_y!v7m565*{VhTKS@~80=dCAXOQ3Dr`b!`#31Khy4>ls6Mnu)Lfg%V0M#7}pY)Jjh&32fY z#@nutx_y7LqycHY&81(tN}>LE*JV^;wFWo4$83q%Gqr;S$aH+uz*}(bh*?|m!Ol_< z1Ec4osf7JMGaaG4iGH6cd9(2Ln6isqx5$LE=ys7heoM{!MHUYC%9qU4NBLl14r+Ie zv<^pYFh1uw(}pPZbCBbRq)Z(wIE-o-{)}8Z<=PvImcyvC=y>sQR9B#RAhmwRhFse- zy(`InSS0_UQwLbwN~A{H$GTc{JlBn3*8$yyBS~ND2}k?1TB}G!{-Qj} zy9K>4>cKBwF(M|?F9Y^0)u7C_|93e0ac}*<{@y(_RV3gqe*ygl){lOyHrBp{+<|Em zkrrU}UN1QV=uQ*VIDi-4yt;ByI|z2#bqt`n*iC>?Yxwl?lg2IDAXLw{4ycOE5xGHy z?RII1_3HJnt0{|hD5hMcz-m$iZXrpa^YpW6Uca72@a~u#vIQQ?(xl5v7?YPlm6Uev<7W3)a0fkS_r8R^6)I7SYjrZZu3t|0E{Mq63N%d+? z7+t)%9f41vQ)?vd`0^cJ{=NCKBlwTOm23O8o$R@KUDxO0(G~1IC8J*2&~3T(Y5;!+ zmfb+?lkx2;$+U6rCAfTK7GA~Yj*;In^52t@H)#K3aP#WTlx-u9YPK!*IW_Hzir4;A z>)gv*dmrt4MX0}%9^S<6BXRath1`-*mrlM6nfGVyUCOydsCUQJ@3{K!tDZM$|5mL1 zcRi2Li2uEbKI+;3Kdo^7>&fGK{O`l84f-4O!rszKHWtc%WwYq_8kcmn6zb|j6|LUFd)F6q&q%bA zlFHxRZCC2lT3#k+>iploD&k}Dn5E}`w|5))U$%Ge`2Xj4T8?oxBLX|FdBl8HD>3Jm zaMLfGJ)66PPBZdbP6O9Jmx~%})gtKiH{9}aNWX>AWB}alC%s$U(DyM9$`iu{5c6tq4 zpsQplb$sLwO8Wd`nhBUi;d?ICArr{k zeW|4}Nhii?31LQF*Bo;Tk0;Gox@z&#ZQY7Q&f)AMEB|NrI7W3#(}x8JaI;w2esKk{^YPhMNaTO`!n;G%L} zyS&BC|MCN`{Fk@7>%URD!_aKG2`s?=oGK#-Mh0IexIjf{aY31v#kFo4u$h~Vs<}6s}*&=QR>Zju@jg5h|dOfZejtwr}+*(e=4UQJ#C_TY0?- zLLG446DvWHK^bb3^>D=1mQ4*txpE#aN6(KkSn)>T5ha3(OMXq$i`p_+Iaj3MGpeRE zFM~aCIVcUZ4da5tx1{jaE29W(3LC802!_h8^~wofYmorAO#=??o~hcV`EnxhxL6p5 zw-MlHf{)(I?H5`f(4GALH{HVFNFWEMB+lKI88JRb zyRxNL`=YAOW!_a7`+LsV2aCKrMP~t4pI;yNV|%6={vSmY9p{Ps2=_j?XR-fh_raql zb^q_9$2)iNU!UdaI{y(fsRSG4ipKT;2&`wECkH%FD>pjomlAVDD_2e%ZH;n*W(kW( zt!VLfxCgU5@!Rmg*ut@3H&mWcfzBllcJ)K0f%o6r*0qasWIllMr7dNWjh%Cj9p(Zy z)?L~1)28>ffUsGhnUCttR8AMz&u$_d{vE|DH+I0fcWk4hf>**Jqam(vrj&MArK`*AF&lvLW_y2^g}2~X)~*(_dpQ}*NU zbJnfQw(|p?RO-0h^}-gpSoKW8zhmS{O((mRmG`0U*s^}(YJ+7S*!HTcI9>BBCW>At zK@^?L&D=C2Ey}5Xj^+E+YA)a+enwI5`g1W6K_^n&JnoGURMC&A2$Eachk zTt6Rr1>E90H*S|#{`NO?6h*(Z+c_E+u5=h?Y?5+85A>2fn9}H4=CcYqk$ve-$5q^Q z8=27h#(MwK4p9FaA}NaNUgbDlZxfI6$SK)<^jPMz))q`O?&=Mg@-vbxZDM&ck*-@y zy60o*B_)k!{cB3<%?rjc?zwt{vyoV}Pqo94Ej3#%{%dVjQKR)ETkN3akW9juCm2ugaQ0TrT`Wg9^3+V0iHkTD-sV=y?-P>%z?_f&O z32n)~(t)dAe&JGDMr5Be0e)K!Wlw4BBb#e*!Rh%yCoZ ze5veRmI+|0ufvGeUk!38uyo9$YbrvTDlx}h+rFHdWF^+KTMY>u?txy;75nquJG_#c z;N?WlJY`k0RF|M>e4NJgwJt;AxTw-7IHD7l7Dd~@K~<|C4hFv5y{ZcBE4;BHeDemb z-dr^S=n@z0n|SxG-|xS912P|ZIPRE&SMRQj2XhH#dH?e(}}iS21?WA)gVJ zVnb{U8xvKx+;=uDwx!+dcVyc`yjHQm-uz)5)D2i~-k23pt_=t;Dw$xYQ0^vs2t#5_ zFc@}QJ`jeHxdykpboHKGBQk5_OJ6HN_QgbAq2}c0(DL&bb*KEXjHPq|)HeEE&~MwQ zx0P*K0@Z4@Iz`H%*V?8GF*>$hrk;Oi-S?PuRqqKUh1st5j=3pT)0R zuJ5n*dG)S-p*?JW?Vsn?MA{xlQVxX@#AadY9+2KZ&9Z@?BPyinO>bqCsvF`ekw4J6 z{IOgcm0Rkn`IWl5<<*w$qgm5(o3*-9I(7D@b|W_|@#46r5acU{$}fQIc-QS(XubCv zmSSI&{O}7y z-KC89z-4PXdT@I3?eTXnk4}Dk`R(!3qqeSgtzBzx_HVU}mu@1~cR<^aK07~sc69mu z(esO!`_C`Ge|h=SSr_I8-233iSsX3PKWET6K0G?PJih$t<>}ec$?>5Z)92?;+mZ6t z7B*tW@wrHD+04`9qm#>*$F20)(h4TnIKFLPOd}UZ2j@qZZNudovS!GN^j^BXi-NG1 zmukJ*?PILA)$r?Hb*q4ALB30eSTaTJ%eI4Pm(AY(@w1mF`_DR-M-u_UW^IG9N;7BY zN8cX*pIbC^Cg_;GzFkun2d8I87ppZQBc9QQWM8&+pFKNTkN~u{aAz}`t~ynoihWJz zTd;mZ5`TA$rZ-IRv<;#!$gewBiIFT>Vj!=#XApg_%$BR=u&msXixQ~3Td(OsG&{{3 zYM#;4G4wZ~Co$nquMJR9YvYj2ibz}qn-t!|u34{v+n@onz^eg7MoD=2>I zi;zF%1<1w|sJwlYouwYXTe=2!{^q*|TY8uAgiq!dnV=;0(#!aL?I`lzJzwpTW21}J zZp>NXqR^G~&einEB_`l4ZZ7R;plhRIfi+lJ9F6UVb{lZ{21ljvD%%_N4UH}z{!N)f z#qH@-PnEQIZ~QGB6y)n&YoG&07jDpcd&;3PR7b=6cmkB5E6G#=#n-bb=fMHZrsXX>9bbKX3NoG)?BfQ>>{`f{Y4CSW5dr|M24Fh2nw+YjD74 zGoHea2Nw`CA%p&esR91qpda){{}KcIzk4yA4D|opPkECLNVJbZ_>l<4a|y?XM>6Q!tY<-=#gq(8Il+Go*5LG?Cr9TO2n)ts zfL1_7N)?rX`Sg3wcl|Nz|NLnG@Yzv+7Jrg9mc)NQdbInnw*Ge>KfH_o{w&WLJY(tc zDXxVa&BpI(G6M_}2!a=XesPhI==zrp438cT22%g*&sf@L-2C`2ZO!k=o5^N-sZDtC z=NAc!Xew!0g3Ts0sjJ2EOM37Wb>dIh$NIHiJyFqa?*@7`8*j1}-Z2=cH z1QY}vQ$f>6OC+T+L!|ofkC7OBjsLSP+VEsRlBOh8 z9_cS2Ic_Tsl%Uj6oz|Z=K}`t`6CaOR#7L6Ny#Z<;AE^XSC&?THHJ-&-LM}}w9fh=E ztp_g?JkBFC*ECD`d`43x`*1-iyg0-+WrcrgveX?cyJlB$%p+<0c3reI$TO(~B{SP! zt>h0_DiukR(6koLFr0Q^f4Ol9^;q<$YL@($pyNF=ew${ti#TA2d6Z+0QvJIRZ-ZcM z4USV)bSVffi&=&d=SnI;GU?crJ@v;^cu@|I>DqAM^#d7DIiN49BY>#N>88_hz;sFU z5uGzZG4ZTqT7CziT?`2`YuuX_8~Fn6A|x93gAmSs)KhfBB;zR+1>Lh0Q9)Ld%r$1! zh2Dpaf6|g3EcXBCB*>Tv8mTan?CZ@SM9$)&c4ORMaI@3j`5V$;LNuy+K1Nh3Z8TEB z(4fO!AGki)p~g-Nn(-jGy1E(>ISr=n6$U(+uquZhPMCBOLRZmH_1q%uxQVa=#=iAd zwCV1!)|jO{2*dLDwrMvEB~``t?wO#@%x^h&<&CIn-yq!~*L9lWmRv$FP7nOIvY^+s&Z-S&|B_~WOiNG}Lno&iCG>j!8 zDU7H`V-V-UkPi-yX{>v+VMFn`9<4kx(vi(b0*cQ8-yuaPikn=JIT^(a!EOR+KUx-n||BqQzUbhl!!>_Zg4rUU@CiCzyCWaL==e zAaRXKf8sd~L@udW4vIUmWC+78GL`~?L>oU=qv*rFl=)1UY2IUGctP5!US{cpBFwQ?z~HKgOKuI($nSn8Jf?PYsPCpL8@J zy+N}po;hM3TZPAvc62n&Y}v(<0$eI%wn-N28(- z$8O08qNYfg6s}y(_^U05nM9cFT^r38OObAI&=};e;j7+$TBNIYBf|6|6jFf+w~vim zi@+I=(JGXZO|)LFI1o7DF*PO(272hvknnV(|3#{@wM3q3yUVBoMB&zNAO2Z^Ea;5i z&=|&o&wyZ&!q^d%NSBxqlbX=1c9FHr6 z1m+`HQc46R@!WQAtKcq7b*K?LeK;`Z6f}PICWGCDpAAM2`cNu(cA>34>Hd|0duKl$ zU`$K<60;Xwm_r3YaLK`t45a1*WahYmo&{t;TY+o%1#yLNUE$>#){vXodlY1CO$3$Z za3o8UqGEa&+e#66Ee9mlNb9xa(j5(A8YM)~&>|2TEZ0U<&wM!Nd58i1n_B_ej#R-_ z>No8c#uf5-C31Sz4}xzien+kNKZ3vLe;ym=Im@L2l1L6J7pbP?YJWzk5bB{1bib{{ zJ*4TpOtj=}6ls~S=+H2SS1d_@reiK5>JR!?BuT!4jB8122)@kOgcK3zaf?*q0DA?> zyxb{O{Sj59UmfJ?@a?_%xyRCppi=(a8|ojD8>0TXH~hKhb?4{ayLUZ9L}(pXhry2R z7AQfqJ#YFpqL8o|Q-*K+s)rJdy1#92Z`-k-aX}#yG-A??!B*Sk;VmggHs|!7nLXol z(Jci$P3C}Q%($Vf)FbD#*ceRgm^rbj(TRAP(g-C!QhDEq8}r?vA2XRHWL~OBJXcw+ zfNSm{bKHF!=ZRaH{TlDF8~$jyQ4;HFQ*u-F%2kbp9s36H+VPlLr z9}zJk|7t5y*c?Tv-}2X<+Rf^mu;9UF@QA8w)0NIQy41oxt2g)U)w|WIrH#2JG2MdI zgtZz?UuSr$izAVu@wd>|Qmg;-Kd$Ng=dbm58^G}v4k{YT4G|1k@LM>fG7>C9l>`9Z z!nwW92i^v6hh09%&M%D*-ix=vTex!bbp@5D^UGVnb3@vC?AsS}D(yE1&`s}E17P!u z1?aBv>sJ8=`$W#biY!IOlFN^YaUTE^p$ z#Tu%bZyLkh6PjK&3D`j04ZMsvg%P9jY!lr(sT(@tcuURZUCe+~B10*Y(7Cjbx#$Vs z%ZQo|6dz=Q-4I3l*L2?g9rq{tkx(DTa&%EmBdf3*FOv?6udQm%y}fF$S`|0XPt~MK zHDw~!Mx2_%!M=1mI$C;ol8&SCVGK&SWy8cYKvP<4-%``IUk-CfNGhdEL0qs3Vt|tuc)B7w;|{R6+5??P2f7^uTnVJ-$DY5-`(xEEL@JHwmfcXoDbl3Oq|@bGAB-8Scx`fgtPb|Eu$?nxk6J(a zfzG3ZlWR*ETdgzU9JSW$w6t8AM-y#eeK}W_Ib79L;V`v6?hdU5WxE}O5RbH0Swk^| zmDks{3ni~O9=mbU#9ZjT$QpQ_#AdiqAJyHc&hW>@xryARwH5UESG8Ynv=s)dP?Wf4 zW|8vE@Dg8wf}K#~JG&}fM7AHzH8Qjjr&X+bLBK^2SVq{WwAWIlq%8Tt^Fr~nr`MtB{y|)%s%2Qf-or?+W^2Yeuo}IJG zR)@c@k?Ywqe!?&aLb!5`7?iGTu(5+dFVhr%bAE zu+}P-c1)%3s4_ts_dha;6&SSBm~~p*lEyrsUax-9z!N6XxGvv2B}LPO=`_lkV$2r+ z^d_bPygaOfVh_VWsB{{&slEuT8~h4Thuj1MH{uFaYh&s56s>E0C^;hxLogAQ@&`KK zEB@LlJ{IQ$_lm#Dk3L&7FLzqO?Ur8+lyf@Xs~Xw$61u*#`^s-#gCCiiO%opWdIF0d z8Xg`P@k$nW>XO9|SF-4*U!~gRvYuHuW^>xun$mL@bhKXuX7NB*xK{jFTJvHQV%{|1 zD5+?|2VL^__7!bh{?^nH6?(RTSzs?dyFF>w0DLHw%bSQqk?JIL|9cy)``>kE_c9`A ziq4PnTNl;)1v&}8S*Sw${^uG3yB~QC6iqE!2DSOA)$3jqu8_Yjyow*RG#2CZ2^NY6 zJx4cKZKdE88Qxx5E_FnMwKaI2veuBE+Xg(C-7an5Esb@sMa$}jc(fv3jV*AfHH!MN zJ!YFu78~;ZrYE0eJjJaaA|;QQ**|n1c;nClIQ;h P00960T@~jA0L%dZqm{OC literal 0 HcmV?d00001 diff --git a/experimental/helm/charts/pipelines/charts/mysql-14.0.3.tgz b/experimental/helm/charts/pipelines/charts/mysql-14.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..eb620aa50b5533466526048c7a1649582ce4de89 GIT binary patch literal 69215 zcmYhCV~{36v$n^!ZQHhO+qP|cc5K@_wr6&1?bx>YoIUS%-iWV$bw+1pcSm$p-B)EL zVJsBTe;VMb_u2)Q`!^L=|1%obL$qmC2H`I|&(|}&xn3LT?UK!(`D~ll=?}L|atx`} zO0Cf<8kv(gii5tv@BSv-ro^T>R|6wM<3~^-;AdfsXJc7B&FPj=8X{mal;~+O3lkPa z0FFnXXT#j=>|@@oK@SoyFK_p5?|Lg>C2`%q>G7-kbj~BccO8|tyL9-VlESQ*Jdt%AjfhDgM0L=Amf4^LQ1twH*0jlsu zl5fc$c}v4qP64Csyo`ObfK#uSN5PzrJ4fG(vUBy^H%}4pnmIuGE=0ZbiC=m)Iy3DwFxwazii`p`{Rw?z9vkY6Ho) zjA;2}JbGaH_GMBu3E~p*0Y`z3uzGuNj4_i~=bA5}F%?Iw2NOGhf#OB}9Lx=0HKcgI zeeMc&cU{2^d7U7pa82?gPj?w)%dB4*tspX}l$SX$Vc#3t`)u!QZoap?UU^&M#B(ql z;lUHL$;v(0vx>c?7{ZAdkqUL%Gb2ct?(6#i7t%w8+VwHthfswAhQW?lzePbv!}Hrm zq}k4*LYdHR3oX)PJW-;hKJbN(>7XB#<3sHDc*pl~LexsMW*97?0y`?FZHV$GlEn{-yYlSvayi*GdY9CjD z?Uh?08v9JMR3P+++M*e_tyBqq)g z5mY1{Ga4ZiP+|+5*?rcpK(4U(2njl)0HNBb6g876rXNbhk&T)_L7(=fh1*x-6uCns zhoCstU1V$%5n+SAmvDMTsPUrOL(a6^Z=nP9L zreHX83+bSzqwxEv)3U}UPh&KeEDB+ocHJ-vG%GU}s35yi!n;Cf&Im0?xI{>RA>b77 z2>9*?$mYr^X*?Cn^+X5z1OJ7T0<(;sCKAe(9SV zqiVIlyc@F#bV#g2-GcBcoS`6<-BGIoXrEfEn0tGr9{a$ys>Z*i?Zf)k;rcQG{Z zc%I^bqF|F}v9}FQ?jqjY;(8*tK4patJ9u|ZqE6#O{gVH%_Z_qkB@4ABoA6RG8XPc* zQpv9PhEI|X2q|bucOtNyzvTWU4`OEq)GHM|yr&f@3{p885-oOwy){DAq|m5MKUcm! z&z3(c!iZdM3gd}yHJ?@Fnkx(Z+mobed_LZdLwvH(gA~hHfKaj)h{*84RRhV&Z;1_2 zI3PsfU}T7=j6$OE&9brnq_L2aISEvR+*C&5fO|Qbl1(f_0?L)ZC0HA3t0jscg#QI7 zUj)qvT=wwjog`a+wg?r2-x`j{QAWPq8El!ZlDjX8^vqnr0PrjbV7xgE{GJ2gcFUL+ zGY{j0sR$55pGijCqiw(j$q30OQx|qZ=2&UykTC@W`J>0uQZpvy?9;rLc33;N>C7Yj zSr7<_6}>`dt0>2A2NByzzl!5&eB)vC;Nhij>Rx)scnitaT_=_3>9BF4YmZM52bUA`Qi3-<)uc^dQVYUmP1C|9co8^x-K^_}WMX3}_`ZNEoly-J6iIv)_A8g|s ztg9unK4xQYz z*xsm1-hqX* zr&dLYo(~EweI&BLa)4WiG2MD+C)^wIsL%dlR^{&*vUDIAGznwkgBT^gBQCe|f%u)` zkg0&Yk@uH-1Qvt_uq=_I%9Y1U{S0}xl3@ZhcjxtTn&R^=eHRAFe+ zM--Z->o*v2*@Ntfkm?c9rW$V2K?ShB77VFbx)!{_gCOeFHThD1=kst@>NW^d=*?(h z3cJlh=n%bi);tw|;8jQ$xC=?5Nkq)+7;N^ughTDK!&I+6&Y4Vv=pkiCn25$1QgxCI z5351N9@q&MDj|w2=o2ur)^v6e_Ky}O93$-@F?NdP0-t8Oe6Ldfp@>k85fl_Q?cgK~ z%D%bDbFSNxa~sN> z2t|YIzM!mkQ$KJP{Y95n4qF5?v~#VDvjcwU!a#P--&?tKlMEtI&EbGLiYK zHdijZ_N?n|txFFS3A{RRBmC*CYIn_KSf#TD^<0osBxH#wu5C7Xb2B8@v{3li`!e^R zQFQ`#*1va+S-~b!W5|6#)^-LK+@ane1<>9r5yfesEo^+|>Ctv2N=bM#$7~jwBtD^! zeRDNZfsxM}ddcn*0_Gv{t79)R%E}u1VrphfzQ^!b_V1kFyKSw3ej(~G#B!J74BFS~-%%i+M(bJnsL1E5g>phRlAOY*&$SZ9 z&Z%pg@S&fwfB&+BT|r!eTYS@R3#To%=Y%Q|`+;g!7Ajt$LV~6@YX!Z;=4nb&Cv%V2 z{(DphUJTW>Yr5!-FG&^qB<}ZvxmVO92oAr;;~MwDaHpG8ae$$%g%sYyoaLU*1Ak%u zs#4c+NR^k&a_W3sjqw@D^x;+Pc|yuf_6W{yvzsnEe1R%w{P-A0*6%nS=r?-)J2dfqugd`Jdgruf1VTN<~pP8 zrqyungkqjE2Xs685{NLinm-FlvT({YtxP>aE-;E|+p9UYK7^9CtT~kg@=SRT8$I~k zdW<$UH`1=NVQG4y8fu@d@{ zTeV%A6o)srR`Vd02sAK#>txvckaxt0VM~5V3x0Cp}lXca<71zyeqq z)coh4YB)5&-A;ys{KZ|Tz2#{tyT|-1Ksc;CI;|b_BGD)j=^Y_Cp3Lqb_`v;~uim0W7#GxjL6qFiKHiM75X=*W+Qo{kW3@0MiuFS4K|pz| z9B3D3HukX)6IzI-Wj&{r^PlU&s>3I=R{!22D`y1wlIVqE8_5Y(X{|_tJs?sJpF@tN z+tR}TSirk&C7S(LGwX+Ly-JSqetvnQ(HUYUPZIHo zkNylc^Mz&b4h2WtvYF6UyG=-lOG1+viu$lCoJvg%a9HMuAs)dWS|q}N0r*I*g$*kf zK-8^(OLq{+HPy?JVNfg(<^3Y_cZB@jFywuM?je`>4oUmyRK@`=`|EDhZosFf9hZuM zPGjD<$>VZyocy!+IEDX{MS+3C&*iQm0Gn%T+|zM4-=B~F>^Z>y^Tw+G8&N-7*B(*a zeIijXzlUe+VDv?n{`(Tc4il&h;m=81H8#rs_X@*I?yPS=k3qL{V_j zB593^5&XZQ-z}82KRJ<5>2OU7vnQ1tfb5aaBigViyil%{e`d8y7qx<$n#%1T#|Q{b zKdD46XGW;n5|GQRK-tQicF&HEs&q{`xn8V=I{b>I6%cUKQH_461ATd5hnKKQL*E7L zST%R}${hb7Le*aDL@~n4`N@qd>`(QrZ~MOFT7pI3gc*e7q)DLd;cVC+ELfNwb~l2b zJf7Sjykt*L@pTlTGA~vkBG3p&h6m#uY}DkN@+$jbhW{02DflZndJ)InAT>#F>DbeX zUI$rcgF2M9+sP?-|XOQn-LP6BR7Sf+E8@VGEtX$g*sc9Q0Q$GM69Z zA_{c8@p;bOcCxYxXOtey7V(#yebL*cNC*@ZK)UWx*@oj513iMIhhDp$?1LBv-5I?VQnp09ZPFS_TuQPEbB&O*$ zpB0D0eS2Kto+~$W4w@a4we5C3s`3uO8g!0(5`nn(uKGBs0B;A=7S4x>Unp{3HphqcT>sLf;|)F>hggB09R@#l+rPp$d2P_~s{Ov9CJ68UDIgf4h+b$6Je5(@wx zUr)#^KOf;?k+(I*Q@#_p=n+UVK2t+PW5OS_xCTMDOMUU6K)|O zxQJoT4$j+-s~H}j5CD#s-1TpkSF7+au`c6prTorYTs{bg4^Jp=x&jlC4_lREDb)VfHaYYUm~e){UjnCpbGu^6DX%1!zeqsx^UmsqJwO)WOcpIHy?M<$wK4_K z%c}G?zjDUmyvER`%o0B9nd9E55-yQ$C3cgf*%q9=aHgcpM}N6q64b z(k@M;w;;K!l{%E^`G@hI`*JwJJd_{YZ!4?Drtv|Pr``5P?@v&=I-vyf z%^p~8wTu-=VFpgj0k$5KCouEzD2C4A`d7GRa|acCL75Ktd09ypkNlB2(ws_nKO0k9 zV@Y#?Cvc@JvV5PGfmD6yBoFlriIHufB$YW>tE2m0trg_O0c(+`WQzh044`+x0dRn@#}}1SqRfT&?Dz&Mvgj`99-OW*=5wEr6I{L$Y&MYiaQ*u zpS))XG;lgLcbG2_by#p3r}CgQL8Y|##jcUQsxoL>_|hW5rf zfflQd0uP`b z9)@~-y1uW^K!+hDqY^oQt1Dq3klzK<*Vml9ts0>7Ju$GddB!l=vU6RD$^pA3)B~M@ zkt+QG5Q0S>%aVP*jFzMc(-mFfO8Z)+&K_x&Bd9@%$Qa4QT)zvkp}j#LduLfHPzM=u zg9%Yd9bSgAL z1IB;1XN&bHI9vVeB@w#-Y5xuJ?LB`TbhLT^-y-=YT)5lAS|3AMgw#1&}p z$A5Yja+mIOZGAvwidd9H>%HSm zD8Fl}wfotFXRuH`wtyiE>+76Pc70;=+}*sXN>Ai?Xw9Y4+f{`RFNaVx3C2Avvcm(} z*Sbfabe`rxstAI3@+5R+LLXHs1ae3-MgiOzM6g6ZPIQ#+O8Neovq$h(FulIGH*?0T zMjxn%yUMqdir8oMI(~qxabg2SKZ*Thxl{UULuGF(@Ru)OGfkM=^Afsa&DHWk(T3XA zi%n!nTPxK@{#l<%s%XF?t!#j8v&sJ##Xa2lEGnUbYqpOf+e#d)F=0`|b}rdw3|aSP z3|F`SD8gazaapYZ39E)=iyYZcZcpj6=OKzBGO}xLN%01GhRWlt>n@^b+yK!#ym^Fc z)vS)+@p;5f6W-_&v!n=`RY(#3ReUL$3BB0@f`)-7y$64V66ZR%fSEKOC__eqDZ)Mx zjAglYwJo-G)LGqIKi=VX(d(K6u+6=vZ+41lg1>lGgfw&F2X)+B@BD#2F7Da^e7;TH zm!9cBwL(Rs&vS!f$*E2cB`@d~IO|H^1gV48?Pc?1ifNWbwP11DDzV0of_U^(#$20D zp(L7IF$W0h;KOrg3Z#XlYqMe?x+P3v$c9MLOU8hc<>Yko@B3tgCkV3)-tmof+>Q=jfz-`sp(XWv1^|e&I~ic`BQHHz^n&TOyp$+b5+^*TevmQXO@+l+BOdZK%fD3&=0lVN zhT=oKUPd14Z70)c$I=rZv&}1mM=a-DSQhv(JeO70V*OO_)R%{H&TQG}q4fe69ldIo zQR%`&tjS?6ByO=@5-i#g6VuoRm^~Z^a(o?p)qT>941HRXc0^Pk`3W zRmmRXgnKtjHNWNl744)i_gjusrx^?|rW0JGvoK%*QTd&Sq}DRDTKl&Gsw!MB-+a}n zVeY*=zFh#u$1%;c`x6~3UhLi3#3eHbAtD};50{abjM9V2A@$(y8M$GdB&U70SQBQQ z@u{=i5y!47&|{a&qSd-zBM@UR9zi-L&-QQoSL*SJqWmt}I9CiZz`x?FNP${-*I~ge zrSCjSv?Vv(2Ga-+@2IKFU)6v)Ph^Xl!1zE=NWo5mO}Wp6HZQN;9j!M8ctDkYz@5b3 zO-WQhc8q)3%!MW>;OmAl=LR4v|wR% z%(C~0EQt9EuRiP3_u+hK=pxTOL-xPb38HHWuUV0(3OF{EkSTF z9c}(z*$cLpJcA>*?>(3^=70?rqQTVgL_hRl(w~_zxX5Cli>j0-dd}RYVC$Ztnlyfy zyNkMTwS%#wnwae}CuC#oNy2`V`;*E%fJ%m+b96IuhAq{ltE#-M+Obi%iahv1}{DY39x&Tf0p*yv1 z(z{6_Cn9~TP1hlNauwo7Cj&X=B4xm;*EbNGrxFk>`}MsG@{EJfF7o;MrtkfA->Lha z^Vi7xyc}x`M$8-$Pdg0=FLUk3?6_{y;&g^zk1FcOQoHTn!%>pI!>7c8Oyw#2Ib5WQ zp?~s+hrDFdZIbfTk2ow#Yy@GN-Lz>~4C#E*Pn(sSlmrY4C25FGmpn7xeS?g`F*o4m)p5C)OslIzt(zGTE-(wsWflm}S4 zEI&1w75Lfn9)O9_Jz+#{Fm}|!%m#%kJQVmpWZco>F!J)BT~d6MpEQ_CJ8XI<5g>Fu z=J#e)RYPZ#UDa`7BHUhet`r zDw?%@i_AQDQ45?7CZHEkJn_-j>DO!P^q+aZBw;pKaTvso7N>PL*xKkym4_lQF5vqU6oCaZ(zAn zw{*^}HE<=HpZ%&fIF*{|cliU3S!v+PiOV>hoxeuNoYdK1(CGfgrCTMb^i^nq8U^4AJO$MLw1$>z#eh$RS8@rEe=}AN=Cu1UkIy(eA zLtLK{5X+CIP6K6$JrqJ66{11f7N%)@PnU=Z2xYBIZ&XRGXBMUI&g=q@dkwsCvHsnm z?2_#$R-?XgWa*WD9_nC1++Z{WJZI6+CO^&Ty8La|X3hrS|K!RfR)$ zel4VDENj*sBQ8x36(OMwcvjnd7NsZ4Ku#J`g;7{9S_?c?Y|qukZE7iC;$N;9gPj_)_yj^ zR6jPqWuRzLQK2{-PN*r8R(z~KY}CsB_PLN0`5-lrpP)ml#L&B`@N;i(Ei(B3^+sgX zYzfQDwJIWWse53hKf5f>6@fdD8+BHlcA}n2o(&AcT7hDuBnT~13S&)oaixWpSoJ`~ z(Mq5G^S52?lnK#8_L1l}RVAWqcXJ$s#yM?mVW<-tC6R-|2gJx+rlndx$5`J=((2w| z$=r@`V0hn4NI&}bLt_KA#d&YL!$sfCH6NgFx3Qrw=Jbk}DB&rb`gM10+tG9E+rmFw zU)kVWh@0?(JIeD7qXgC*W#pO!HY&V9w*a=}{lmZVFW^BL<~U&FV$*VF;eUQ2^z z^|^|Gl#wuPw|zOW!!iK1n5<|ItUP{^80jsNy=5_5tEj()e%bSK-!rKhz-Xvgz1~J z|HO+TA*5?2<_{6v-9b+Pc^plpWT!2Y3AAz0lf2N450v4Vb)$}Ubh-I!g~|j^rA$LP z=(1uiknK!91l#FdVE}n5%d%(++}g3*#SY%+EhP{?&SLWUHsW=D!G$Yy#!0Ho?4Qg^ z(>8LYRzAx(_;Asj#aZLsvv;a|h)maqqYC`v#vZV0>08EW|Kr>?S9=0Ai-K#vO~xMm zh)%_C8Y7Hlo@xKC_GG^sNn?kK+rC!jac*8;Y#b(uM}(cYrG_wbQT2@)RrT20CFgLy zy414F8|0&%H2D$37hzJI-M?d|*N?imuBk{3XeJ$C7s3ItFqYuche+`g+j|Kpq5^Fd z!DM=~%u8;23M+Hzv$R+i_GwMbT8&xLcCtQm31Pm4z zH;WNgzL|3~OyjDC)fU2aqq$u$BBTw}3;aD5(f+e=rr?!0EAG71lt6vzjdz#1r+A;yPjF=&`aliR)~QJNlsRUV`?oQ5OB1DzFP^jU71DF%`Utl?|zP z>V;f$4dbb9l^JGYPAxK3Yg*M9BPX?06ceU5K;JbB!{Zygylf7il?8%|w)66~1NQBY zH#8spiz`L7GRiR@b_I4S1`(8FJJtGNiZ?3SmwvGaY1-aq_SI}QQ{(C<3`qjm8q=0d zW{=ny*2X*TM+|D++1|VgHfvY;z}@=6T1QS-5!~CywsN?9B%?v~Dm!#+lb+eoaemET z%|bkHj0JR`a`+__$19}lp?FLr^GT#l9?70fOb@Q8VWKXkev*LpHCJR|YO$YO@onG> zMn8({j&1QMWqlU%wR_)Ptdx^3vv2D{X3_!JqLn$C5sk53BmFLqMZjpVGI36my%&lM zp>u3j)XV&RHsj``=K^uVUCD9?~bTF$Qw$H^TD$!Hi0>B7%;z0*&k| zFk!j1peBj)pRMv$6nzs>3|+Y}NvN-Q9wNo|dHmZ9#TQvZ;pLK@A9ADkp6#f zZ0~egWa>(|(!>OSx2`j2zvW)}-w9JKF1ES&~mCb8PufaH{H7d@s8S z<@n)WG$38$mshiU-FubVX$ovh+g5MtF2lOjt?QHj>U6i)u*WFLX0jG;7Q*(3*kKe$07$tM$(oVH}vFi-!4R48O-ImdG&%bsnBQ^k%S);#=5> zzphrxDp6@1=`Au>Oq9JqQ8!2%P{6uIHprUS6fyisE@=#9EE{7yXr-aL2R3yP&$dn9 z>Vt8mNqf&OlxG{?qOR0a<)%uj4V`3YB!I9hyTFkiO9fF&`Rj=Dsf$Qbvj0$Vjv$;B zE^~ZQ#8NOkEi$rpRAHSf;IqHxo$d8J>=H~)St6`Ts_J0=`y#y^)V(tWf%#qxpoTfO zlkx~KJ_An)kj#Yx_kM4svhB>3YkpV03+Fe_5o6rQT*6+ll7cPltNQcTAT$#O!P4m? zsDJ|572{-aKT1ZLa#?ls>}t?lYBa^nIS_tP3gv63n4)LIaq`FGiRT6LlbB%~ z4-^ILd0DlV6#W9ayA``by-xU9 z)kt!$WD?hJVKQmbX=8`CodMN9-K(Xkhv#|;y;DGqV%kT_w<;nzdn0=^?a}eP4+yW8 zp^Pr_Pm7R;#CIGS_y z!j^z$&3z;jG#SC^m;$=Rb$O8Y9Y;|43-=hF`JQ7?Uyf0NF=r>izA+y>QzQa1MbiU} zwzK^a5!~j^%bC4gJW~clA0AtKDeE%G30j=Hhym*K{WDhlFHPJ^27!PvJEMzq_(_Xi z2t=bZrriB&Ny>`vFMITp@_5u)#*wYAEp+EzzC%ColXS9N!>CU8PP;=N?=kOq?rF8Z zcg3ONnI&+Yi3E=EwQ}s4x?-L)Os^dc{93@Sp)7*6kvfUopb`TBg8OEW6DE*XNXDn) zL6s+T=2MN)GYiWKaQ7O(B+YN67g2AG0lYIH7U2CWy20AjQ1K1K_&T!zX(TQKoangX z*_5g0KV3zozxuvCkWUm8GGInyHEO_5ej=W+^U=wSeJ7ryqIQg7hHZA;=7~|_`V`ab z$&y7VIf*`_VzuMdwc1{)djb2q2im}{w7Jh~dH0Uu&Md^N19ake2jA+I!Ov+PtEaih zm*xKKe6J>ztNdIS8ZD!m`&QMye2Sp9U1NB3y=Y_p^LvY+|E9>H8+%B>xmo6CiXDQw zptg|#(VB{NKU7HezM@Z~b@lp-}`00?z`S%l zX@sMqS)RzMW*_B$%km^x7BQl5--rAueGM!;<&yuBL2})xFeGVUz5)7ovBJ!Gf6}2V z$7dL$y`v{t$!AWsXLJNx>3GU4#RhAE`MryY04y%|Fw#wIY8BGC9w?6amrgu0zOy(! zmvNMsQpBAf!Gm91l9^E*d$0q00fa9^&JR7T=uZ!9Pr3m@DPSOXEQL~W`Hs6j z32BdLc;Eihz>wS`F2L z(#G#JmYwnCm=PlNu6LUERmnD`R#nUR=5;_+rEaOZCBDCX~*dZ%^0KcbQ=NYyY` z)k=`4QHKuA5OW}!m3V=NTr&tmK2T&W8UVX6_nl{9J8Npw&72Jxst zgz%%JvUK$Y;MGmyfjotAN-Xymx9d%N3{Zj;U>dcTe!mF%z?Jyp?<}=6L;>hUXNvZi zEgs4EDx<#GW0<}e9SZbgyC}C+K41Jg{b`ey!Jz61{Tpb@8XHqa_7e*|$-9TGZ{al2 zYAYtEu%4>3>RoE@!xf1{A4A^r^a2R?7E%O`1-o5upx^t2@784@w)i3Qd3kB4Y1|Nm zUE6&p)@1!eKe@iwvgkB>8bnfOzB?;*FgJNG={*KrD?f!!pCW?sk@K*bkw5FQ8jEo^ zFaN$1>DzpehO2gvP{_Ig9i_pYajPBt)9^|b(_7Zt2s`lTM{59|c~mlfK;lU=xF5Fj zO=1VDvw}!M)ZA`Z}IkD)F{v955p&x?J+XWf+PS!t3P`)-%ux_Eu{V%P)e8kDBJ%)PG}b9CcZ!tl+!_;-cYV z4AGr!aKrVnu3}>1d*}8C`IkD&5X?_|I#dtnS8sHQr?Vbc$icI2?C*-I`NlAdP>#kq zj-O3|_QOCl&VX#NjK-9NBW4=&Ny!=~{V&y?9C7sjV25bmQbiR&hy4d)XPFmqhzQo~ z?|llF;h6bX9b_*$VJd(F(V+LQTiC$K=dBTv;@uB!>hYE6KE#`9+WkY06QK4hM;&lbia*7 zG9?W)K`aFG5QycTIb$c{(fxDAvrOjX6fBM#dO0tYg|yroq!IO8J)@6Q#-qetsQ{0L zR7ba_h$cYY68;0aV#)L6LBj%meQdpxK4q%Gnqjg*WWNMwr-s2%mqL{4ztwLv+x^Yf zt-$u#z5X(CK;k}w;Rk2?LdUb}vdj3c>u2$a=Zn2-v-QSijeTU-<%3p_ch?M*(I!YIW&Y3>AJ?l&ZV z==K8YQ4f~?4R`muHL35qjlea;;F`yOo{rZ)-40lK{@)xbfwr%^?jwf(Sjj0sEQ`fp zqdKrt0oj2%Fo6TV@cCO$HzuiSzV-E zp&F*&SF}XZA-!TBA$(8x4^j-$d;+xB63)MjiT7mSkZ<%6&YK0`A5VdP5=y3CXBup~ zF(8xgevOnAf)C`m?4>Q3Cyn4eIG>XPUr=cg587I4R-vkZA74+1H@GP0+vcLKZpFLRB_+I$GC!X@pv#ZwTh}V@=Y& zOE$=(l#4vyVN(sCPt@<*z?Z{G`0(RYC9QU*kX5o-&eS9 z&=Y9P0x8t(UzT>KDAzJx}xo!R`_r zAUj_0w@h!s_vlWl{M~pV*kD&X@Xn*?&!Ls}J3BPXI|fr&{!#3z<6S%M$u3ocA&~nm zXwNC+-P5<>>GdDWb-SPiZ@40fpwVRbn_0b7fkqCcCCwD-c*w(Hc&kk zDX)!#%m}I0x5e?)S!CkpcDSJs}GFQ{UrD#2ar2dk!)P=6+%Wd>+V+(Js>; zy1kkI(mu4quX_z3m>9*23 zT;PSy97Xgv{1%DEr01%O%)|J%Ue4u=qQd9_M?v= z^t;y%M)VE`m%yMg!joisuW)hFFJ+m-arvVP zJFu@?X93!M#KY9UI}t`4I&l;1KnL>#@b)%irHzk7EHV_l~~B+mBq&!(=N*+JlDu4cy_@8$lJ$WBP6oY1KVbADoR_ z_8%kWa;A?^h&AoKcSv-P+a@YlW_sqk`8FDk-I3(2kzDee$d=RGT!)>_#@j1kKI8t+ z@b>t-aRmQwnns52tgoiO2P!~rw{j9tv!BD z3OpgA+17^HL#7@K*AP{Hflk~i{ww)nAsudcGd@Y2&qLWixE;^Z?9Kw9?>VA~L1q{r zk+6KIC&f4!4lm|TQ;-z<9(AiiA1I_2^~>s)d;D@;bmoKgPyQ2)DR#0L0qL8tJPiQ9 zKlT40f50(Cx;ypng8re{!ShUsK9yyj!o7k@BTNV*j(Z0NzvF{Ax7_C|$pHKcE;OQ{ zT(S!w^;m!lYy!kEu;|2az&h;zjG1#1^DkB;0POj^AH;e-_xF_KdPj1CMv_Z$aVKeA zz|SOE@W}?B$|rAHy%G;v_NeU}9-`2v7=a4Uqdi(ff$?7%H<03f5)UHw?~hd~@A_<LbX+>ImvT6-)4UMmFHrDZlNqNSvzMaczb<$l0-da%>SxwH*|6*_9MyJe~s=A-2 z8>)un&Tz1Qhi+kUvldJXBDpW2ev~__bEI$EqER|sC0rfTe5)B!1!^$qYs9xIw%xUpA+#~ zejtlI9nc!y!(nt|V^E%pyf+EDFE|4AtFvowKClu^`*jPVYDpZ*d=H`uK=)R9s*yTb z#ExdD5^b2H^7#8Qac`4IU2<;a1gNQahrPbmjWBGIguLQ1==?+qe_21g`HV&kYOR2tmOWWu-BWPf$zTZLMOwPLRk{vW z<+pqA!RHyi48BI29b@a;tAF1Kns0NXOn0pgpOmfn<7e9UTG5$Sov-5Pi)_&OfGIMj z&+jQzRkMYlBVwa}%ov`?>wOcXf~k-FAtNraX+EFp!T_G0`|nIP=bZaH%)-j?a^Q=v zSnNN;V1b7{?bkl%av7Ubi8A@3<|~Uz5RcBse7pt!bIl1~D@c`SIuoKaYy*_OiA<}5 zF@Cbpq)xX2dUFKg-Yrqj!%4c&23}QAwQ$=&LsGI&u)t?JE>F(g1Z?yL=(Pn4wofA6 zi}B#n)`MaA@`f9ZjGXgTxL7lJMI1=k??8x!%M$kyySpuzs$rY!Wsn_H$0wQqomPAJ zHBnRh`tlvRSkAJ$`njyQ{1CLUAx--~lh|Hvb1{&CHRUf3zb1UYkv6xa{7d|HWtV(8 zht4pvje1jOkyNnD3PtroUiQp)L|Z-^;+E%H)6eM%E~3xs#_vZ zXLm*~XbROo{-W+NYGyH=H;kl@KAIHgSavb4 z$sRMCS6BOGldg0)6`1@)NC?<~Z|^>Uj9Z0{P?8g2IFn68gDW;bx2-P*f1t_c-^4;q zwul{xJIl(22!jAhrE2bEKL+gG9}AZBX4-iJu#SivLOTqG>WE?5{5)u2ksq zoj<$;|Cb*C#zN(e4i0H&5(i<`_QV6hFK6B#0xh~lNBD7Y*-VEaaBes)4;s+wtS@W% zX$9b*TptE%h+o8LI#HKm-010h*4D=?qu(B@NVzjC>UyTuhb)f*yRZ*PztM!BIgjWe zDHqP+2+Sm^jyY%JhXGS@#JAI6RQYkkA9Cxjg)@bX3*8tqAs%ZFrgVApWZ`j5RA~fN zrQZ|AySSB9yg&30YrVP$YKUA6#*FBQTpadUmfovItHpq;LGLS?vp`dQEBKUbff$zj zR10v#13qzfHQ$jl_&1a39%^|aIgr@zll3(;R|@KcS;- zm$-(&a54bjN-1@DAJb;t%TLTa6@_r zB6f-50o)dOeP2%#9T5x&U-J@U7dYdWh+?6GYYG)LKUqNgHY4XfKiWK9UNndsaVxIf z=!P5nvHSyoU{!Q4vr7_diY`K9UnX;6-L9^C-#m67dG4Z?ln#g?ug)s{AXXJt52ib` zW%a;~tx{D&QoKtT!~0^$q(2Txp5!f!t^CBZOAu@75~rZ{PR-sZX8%0kqVp$b6eZ5B zP<4wa;sd|m?}<>@4Eh(aqGe)nJ~hKpGF*Gd?Mw|*DtEMdCsXf0)=YFo&U z*K3eEVSha67+#;K6Kb=1)BmfM+LFVsvUUGXOAUvKW^nE9`5yprK#sp8{|wqnlQHr- zpD>x;ECC5w*)gmWgpoF0LU0> zg}RC^Q-w!MQCZ$b36JZfuh3JWw2=3N-`4A`Uxi0UV_Dut0gvmYxTrBM&|Y+zG9De( zWqBI~Jg%1pL(f7rMwh9^qtRiOx3T^@&O&`gm$C79qnrVpG*yAHub=P<27et)q2Fc^cXpp?Fyvn;!rV5WMU2D7W8r!u%=F0XS`+Dys_3}3A(X=kN zqu+R4uXN4hX~i*uxm0B>Tpm}}uPJEoAQ`2YYYXG#mQ)sOf5ccuN(Rj#jOc~RF^p_A z{%Ft(0=X4{_h?YNvfV7lqq%EqN%fWS*n5PRc&&`bH40rU4avAzRDQX+W5q%|Ho;&W z9@o$7VrBm51Q|9v%{FegZh+!iJuB-oE*{Y#83^M@hTfdnS_Pf53m?*tkLA=QhD?L& zf?Bjc?iDV`D3z(l<5VUArO&Cxpl;)oimfFA)pK3GXQW)F9*=0?h-cYiL|%(CF1E&q zyr%0?mz1f;Pn=~;QdF5t1rhP`ONUKiBjx?X9YDd4fmhF!T|uL}azv+{mj zz+;mQyL!J~FGSv!KQ`I0>-Ot)!sN>Pbpek}Htf3ndd={-8a*oW>+3i|huVhw^&#I^ z-E-=Scgh+;NKFTcH$}^j_PF3Ywz&oP?G$fZiy!Do!^(aD1w5L&ZWjQ^{U6b_$J9WN zv#W7{m+7jj$1NOdix2zVp63^~KhWo<=QA3Qe~Hg2L&YKxe7SXG-C|EUDbQrg{>aHl z<~O`&G2OE6r6v^!)x^X#hBm)!^1iC)ws@qZ(U;y5JlgcAYX~n>S#Vj_*W`LO~9+flS zZSk1OP!8aaMIf$=0*|h=MmdmHKA$PzaU4f+vLvTe&L6GPd2A68rtjNK5;!DhJb5(F zZRN+XL0T^Iv-iza#tM&=T*iY%Oqq$#c@wf6FM=q|yb$k00g-18!lZTyB0fMAZF2jm z-Vm?ALqkq<{5hZ7(qj|%l+wCh4_6fDSs9PHF#K*7HnrojUYH8cCOD($xghcMKaJ^- zI{>YwB4U|cGL+pE&%o=+lF=Ecs!WVl}p%cXJbRlbJ7FlHW%dbtE>9$c1m|!&0mtQ*N2w*6R_?j zBLYVW?Zz^kXd$J~1C^ghJ!x%h^!t72Wg3Ta!!9Q#^LC;b-+yih!L50-Y}~(<75*&0Z!}PIZ8ykNRZ{keui$BLR5lI%! z(%Cs^QrRry;P`0>EEz4&NI1pR3AD!*;P+YP@_1Hd#1<@)+y00c7>qGhhl!j<5P5n8 z7kxU=UthWM15a%Wp_%@oZOrZ0OAWle+u=a%P7qgj_NAg0>QvhA*b4@o>#z>YOlrk4 z_7jTN^HdgTb0<8Kf;(M63^_jswKd3;@e}0@GjX`VLdWdM#D2ikygfVK7jFrH=qt#G z3sRMJS^6!62){x06M(=pz$0nA$A&Mp!!Dz~91Mu+pOSYY9wy{hucR%QZdd^xTDxV! z;p-^)s1d+-UJ^KS9dTyEP95krP-QvFlS8-gd>0lNOu@XIBQh0Rr|;8kNBrlpGHtXO z5Dd13eZ%=K5AA#wgjpNt#q%e?>0$Ogx|=79W9U#Y6yBv5gvjLpw+CT{gT(^jyaki- z=Y!X;C`O34TfWRDP`NJI*E1Jvaz3iIW!RxE1GI=;jRVS^NlRRF5*o@4*g|`#TJFrf zN%(bT-*#pHL45PgSup+P8%7GwohBunP5*<1@WmSs0j=9~x)zvi%)uqjl6Zz2>=;-F z(8b2aj{$VKv>gK9nPDdND;&KqK?o#-D<5Q;B&gYV;kCKs$@OcJ($=X`32o2WbkU+wRcFewmIEJd7N1t});c zyU#m2ogHMP0ZJNwfmuJFO{R$KR9&#s|9$6qpLb>?3-qbxtv?UbQj=tWz?PN&UD5Zpq^-mUdc}0y=P6j_w0_l_pEXE_Bwl}dzZ*6 zW4xmvBVTB#Gf=_@;7^M4(&43!f1J(|SaCpn2n-Yg&gKi86XlEkD2&g&u%kIhut0tavOmtw1(86L6~eUAeYW%ciyfqxlMIM8lDG|F zSOOVvG8VW>!cX54%&x{V5(a9)?UMrN$8=*veEa3GckKPC)B0eX*$Jb*~WX=45=+@yV)eMjF|FekG z1I0)VXCa1j#OHqjnoLocB^x8!W%}k2T8v@Yg}B(%=axR01{Ojg9#I^%i-mx9WT0vB zz(Hue@|4tSmeSFi9_XT=@e}L9uF}FAc+_fS$d?!jkUoD1@~q^W!DKSah~0tY7W(NX zu|Q-59&BVVO+hY)){WUiF%Nkn$fQ6&I3;YOI(~J0&^xBtwP$LZ7aFB%284?fkTa9P zILH7Rx<9=Xn=mnv?}c$BH#aV3(D5wPy}(Er2Xea&L^*(Az9f@f4Mgmi;{8iOwpEN? z3G}jhCMK;^lh*hdGF15q_6?}qw;4>Im5CW_GazVN>Q9TAZX~Z16)LeAl_7Z9Qu&L# zu$8ktrcv^FKsHU6|9Ba*UJ8xEPPTo#FOh7vyjj{}cTe&Re&FjaEav;$))NI!`fYBg zP0CK&YwHf^_t1fDj%1>}kQQDp0a$Boce!x!y^(!dtZf9*Ae{LW@qCH$m-vv#NsNvR zz@0$%Nr2d)kBjAukXF5tKfh;j2(X&Azi&Tp?~uUpMCIqBH*F0V2GCaRZJQfV8I$@( z?-mVR@lRI=Qror?pby}0L@t9p4QHbu>Wi_5$Fx8%)C@DNp?3)&VSHTQb1QOQJH-bE z^Z$YyGVhx>xcr`S6YB>OLOz2#)iY>c ze!wgOsnCdB+NcfupslVrP#Ka~&VahU7$)&V^mkaoq9j68xF*eo3jH(E7?&-?I+mbg zKw@8;(opJ{%qAiY{w9eNVxIx~h`9_CDWIIcO|> zD|XQHy{S@9XoiqY<5mmOU)u;ww;u(~dT zc$Q)msl_3FUm`Cpjlt$g<`>Nc6gXmMMp#86_k17fVlr%$gr$f3LP}x{kTcs+@s8Lq z7)AP~SL{$C`A%N4oup*cQmj%*Kw5va1SjC&!y)VwdBiKxcZys2ER+e$2DqDZ@9EOn zzS?s-g<>QsnSG`36{+J)<@7`a*jS+GWEha`BiFT9$YwMQ=ymgkh1h;xoT`P@-C{O?IuDG}No^72Vi%YNpp_G$@_S>jgx8kg z3HpX^`#yT(NZ`25k($#HzslS`1l6u$60voHf=2F)j;q3Qf4$O?wAI#HO$JEajwz3CL#Zb62`@!~Ga{?le-Rn{|P*Sxm?&`Byj zPCRrT`beYxn1(>&A00v`{mz_3s7i_mbuCR(A5ln!ZlV)n%kDc&=P>wWZd&Iy`#6d| zHJPvPLP?WUg^HTYAT8WSrf4j}Gm8$5>CiGgSxtL{wKy4h5y*YMWr(5;(lV4bC82zIzfB@0!_PIqx(dIX!DD07Dy~3EII#Jme_uN z3XSIZKAt0e2!rzkm1b(HrW;v2g>}|dML#M_=@TroO&lK4jV{*FV6IKD_VKkkCNSFc zEF&>miN5D3vY@%+1JdQqPBC~yPt?yg|7@!c^C`f#wpg}cwOzo1=V&jOcGSUWIWfcc zt@AuHt|0f^Wg`MgOemt~_HdB?Q%olk4V2XRgEzRi@=zjlvmH;X82L@X=#Z_G2E7M#Vh$Hc z8Sp_&i}&?{MTnj_hq2_h=ILYKmJ$)uj1Gn}9S>snR(UAUsir=<{lJPnsHqLkA^;g^ z8!%P)EP^?~JE>2^PTwCI)=pt(q)rkOV`Rip9V*V$kXz<>E#JZQ(JzQfT2MX=yH=>s$b;E>Ko%0xl}O1@jG>o_hV&KP!1+RG>8-Ev0;xUzmqmcppk+R#AAF$ z*kvfVHkAX2K`xb2BLL2q%5aDT>Luo!P2=kL?W-KU>MXbM)&Q~)9ZS$bUF_?KCnarY z`Y;>T)6_BqrJapm6hpHP6-tM2v`Z+or#WS0s&;QbZIO%a|>9LCdm5{$-~__qQ4`R_>8FT@W??D;Sc9V2s#P5o|@SrVZ` zzXOeO&_VL(qD2mcIaup~A9!>Yw6UR@v*p->DIO;2j)9kGQ4Hr1!i?SJ2( zx%TEn3A`f(Df{}!Pbi@BeF-oLffrGjsv=1RqO{0FXA zSgu!BR~?VK*a0aV1U)K}c3+(w9>3`wx1pPV*x2}mNzp7q&^MGz6J-MPu22K;Mv3Bp z7-O3RxS8Pb^e_W@mm5AZ+2DMZIq1=SLc!fCfRbY*HV=B@q_-)4IOv`9S{uKfoc;0b zFK6P{gLm%^-khBr_r%+G;_&U8qm#3fw{PIrkK*9XpT*B7Z;o0*0&*no4ww&*JjH?u zv5d}!K4{U6>tl80OErKIMX(S@QjE|?IUkyBE9mzdJj7dwb9Ko{QZVJI}v^Kfe18%I!Yi+1ve{*tv(kl%5%I ziR5=X*vH*P<(7Tl;Ya9y@3fQm_dF3fv(t>L#hmK^B+GaBA;lmEnV^+yCN>xx~+lgb0Oy&vc=Qf1NObX zCA|#U#)LRJK7Dt5cyM-nv`-VVpmo%Cq~{~@xD@K14$}6;<|e#@7LPll4*yle%s#(H zt#C5e6@Qj<{)|U9R6v8aYl;uQ!Rv~ysdm%yD`?4N@VCfYwV+Q~_sPY*jHm_PY`8kn znm_Mao56;<~Ii#^ohp(9t4MM~{cIH6Y7a?ngN6yr_j9f{P8U@^S6`ao0_ zXV-rHl7UnpVwF=>aeyzCt=9db7*cA?6ZsiCrWr)YC(8fD{^0pS1~b@5IZ54_L@ zLmfD<;(rzS*df6>K0U_eqb!|yCgv?rxZ28)&W92i>*zj)5^i_IF)k}lpFcX7EH%F` z{{D|r_f(*_>X{B~yys4FqlNxZ()A;bCpn-C8i{q$Ye8`qNs6!u-pS#<&^94eT$Szwv3}E5 z2mVPHdsb*4_YTzsy}zLtz}YN8=N?6Int`%5iG8VUe>wk0fO(mb0@ipjIBps@r;pEsSSDhcvK5^)Kdl*DD!2lgATQj6L7au`X}I zZIOmjP8YRFx%v-O!v2+8!`iy3Afg!7xwBcQV~e^iL%a5HF%o|8bz)NzniiFj7A=fP zA($oeMrqfgX!l-e@j>mx`@*pR|7JH?C6O1eRdxVXSmTQTbv?!gh<6GiUgm3;#NS)F zZ!nHVosgR3`P-##se6WPIQMq-aH;Y9YG0=V)}6hTRkTX8>J_!+!OQd6$bbpZ2Nhq0 z@yO1krB)ph8dXCoO^kwJjQ;di&(*%`egxlO{QKhfo#!2NNIeg{sJ*+_PIgg$AbeZ; zf$EQ!D;$g@{_Ifng%O01FY?)M9MNgC9WzXT((aRK0!(v;8AjUtO0T|rCiaDWxiB~R zc4{I!Q{6dz?JtPVC0%nXRew;3$T^;;MKh|dQ{pM&GNQwFv$17fdO-tYt-U>s{k9oq z7YD)PLTB(xbZ6G{)Kyv_+n^OMpWZ|XoR;)IQt3!zK<`x~que={?tY-yAE~xpI|>d( z1fVsBaRMM_3CJ)%9Q6LEt;E0;PM9i3%A?g7ICO{TtF^@`3U_?zd38Cjwj2iqbiuIw z6G%q2%kPW4IJUwNT{q|gSH|N+ch@W(Fy?-M+K8%Rjxk2JBTn)QLyWhfTHHbI?pHxN zH;SNtqWq=lj0wAuJIBXSn}Vj=pi*hP+vCU=uRT&G_9Gqs{OeefO&NWNSSG>&ci54w z;{?3pd;ovFvyoAI?C`Ifs2r2q<9#CN^0-Vdab2)msPZ_@W4+BFKl>Pb)BDZI+yt6e zOplHOuNVe?`brYgswyH`zH)7lk8jxwt*A7pF`Y1v{9bc3|kV)C$fZ1c{S;%o}I zvRY{??a(Yjw_E$>MVu@ggTY$$IC59u6_HrcL^VD&EZ#~mTUZJF&`M9BLsR*vQ58Uk zZ+1vDc7_W%*Zv_c3+uPcYYUs^t8$E_0_G+6njV&*NEGF$aXN2yd@ z2GudTjf0SCq`}l^7!*GuUQ=QUCm{o$cpBp{$|r0Y0*OUNmq8LoWZ=`OV65flz^-F{ z%pu%v;Or^RVJ3`wLD9VZ%FdVLUOM84veE#MjNR>dI#n7DoFODm^uI$7Jx*bh@7tIm zF(|nzT8-Ene3d*@F*G9=VITHI_Y(D%t`hIMIi2ZZGg^d|#d7{2tW{bE(_O~6HC1#D zEgq!_s*12e0WCFDYfCKjxrIm4+RhrNFah%Q`9`!_{R*zO!0C(v(&%(7X>){A+oT=h z=rni>Z$7$R`8dsGd{l=kMp3YU(h1_FTmJ=xqIo@ypw}*EH;l~ ziaiGfG8%O@Nx5=>yVfk{8)r#Z;VqJWf&xp0B;WYjcZ zW<%S9&R+Gpz22*W(vf#G3)7-q4yvFEMSFg+q`My<7xrQf1mLMj3C=(!8K11lv;Ye~ zA%h~}(%P?qo3qmtBB#(GFz%;V?7n%s+v7K zxY206n~&8KhX?iy&}Ip<;)rYg^1SNpuwoF!NzLzIz9CQ?8^R z&Fu??H}&2ki+$ZS^(mh^4ac(psz0H4NghKmc%Ju@j-D&7H66h|&;fR2@EAJlMo-&P z$RiYVUQ6`e4AKcQPaU<#Jky3*y15h`Nn2E=4Lh_qlF2r2uz3?~cs7_;k{`aomKj9h zNWR=nY~eLGU}p6MlQ<;KB?u^oYVM*wE8i2uhDMgox#Em0jzXrTzCD`0>gM(>diXPD zX*w=+>jGZ3gq_7yeYn$(0CgRNBaAm4PsGm{WM_V9mG=Iy0FC!BWE|V2*zx;s($~?q zNBWQZZxh(7y!wH|P~PF+=zhf)Y2CbWs{!q(;S!Z%8s!g2ZPrIM3HxGi=lS>fJ3|%w z;$81Rz1P@-!n@z@jMZCfeEZ@VzOOhsGIwn|0)q->a1E6@!XY;gDGw2yWW?CHw&X~^ zY^ku3mIy)d#D6=Tjybzy9;=F0Hm1+)%vrqN26K^67b%U&6xm9*(G>!{oK-3*Qht?o z%0_?FCw;I3Zhjo6*+Cdkc=A#1wFHm_e!`px_>Jo}IV@7A=V=?htNa;V&-dFZQ&BsM>99p+ zgElshwo%HbKijIUt{#)sx7rEit(NBRPg-IiefpEY_>e{#H@s2S0@k*>=%bKk{O;4o zX%c7gAPzr%no@ce(!Jf2bZ&nUo?X)Uz~Se=DP<==S3p0!OIw5qKypaZsvNvhDf-{?oB>EY$mzyH1M zo<~&LQmzdr*hKTlrufG{o)T0%n{FeYYom8dc&TKlPiV>svX5Gj_z3IoBkH@b@+MO% zgRImKl9%0hnstjOr$X=qeylEm;iX+J!G(c-UjF^>W+2v7GJCMn7C_gj4wDSlqKHPA z$z8Eig5VIXQ}g30jegQ~Q#?A(s)m$JriwnW3>C&5hY;YF#Mks4s?^{oQ%Ih8pFS$t z_~X>0rgy)6&G?56xed!5V;x?8!)OeAqN;`6^E*(Ciih#2O=&wTXDA+}x&r*o(!XA#H<^JtV<{f7n0}SC`h}Q0G;*H~;S`!hH-qr07Q`oO z@3N}x=PVY)`e!@eS}d1e>{K@HCSd7g^QjanD_*Kz3VP`Zl3KX8!UQY!?L|@)N={+y z`Rsgyxn}T?(}LH1qVnviegwG-j@xM-I(8DYA3JAF<$ysnvmIltyERZEv>jxPCCZEz zN~9|e6CJxQ19{a|k|N6}Dz#+SYR<@nZU<|nYAiNn-#<>wR_-fE?OXsOPN9TrQ8VJX zDx~dc9S)DMtU`SZEo8?R;81?mAM-_`6=3!ipm~8Pg+mpHM<0`mOuZ z_Hd&GiORgOG2{%rc5xCevtT)*>}J*wTO}OqQ@AL1&E=@rK#_3nV+i=)QeW|9p`!)k zjWIWrN;2A=0acp%V?NpXhwZqu4FK+Ox-`C$7$H zqiI8rtcL3greEWQy^?EbE_9Ta!IJf>vR;%pUaNXpbH0j-Wn^6Cm7K#yCM_Nr6N{=- zyXhd{p1v;fjhX1Wz4GQ*Yr;ZorlC7k+w&|A)yYaK&aA)hce+tP%V=uz_D#R7F4fYf zPA^Gukt7vqXf4Wbs9-~Ff2(n0qiw0AV zf_NH-@o3(gVv1AD%LURn5Ie3e{v!C3u)mR?9?DyV<#n@RwRI=jDK&+LKHZe^Vm~w4 z@$rW0npiEf9&3hFzdB=|Pjzr4%H&t|jy6Di2lVC99YA?za-jcPD;W)RDMZ)A^s7x~ zOiQDx=yg8zZvReWKTmNz;DFUt)@EHuf`-{vFuUm4N)G15FKr+r%*P&A+Z3imy>>ip z@xZ;<{e`ODa_A~b`H&JO=be+}9dF2MB`fN0|Ad+vs87>84TjFfVAox(RXWuGUH=l+-AEqee#H9__P}m)s>=vz3a!?Z zom!ydC-Jn@#vP6eC>K*toF*}H6t)FmbBAQeU)-vyixCR>0c7K8th}kA!BA0PA>R?o zAz0>vbuHF$LwETpMM&hyH4JYPR+f$8ONDz2Y#qVmN5R5LBBA|TaKeGTZXUdPwW-*z zGex?bvYNbO72=46eYQbVqt=c#s9!QB!9J!^lc`E1M7hLPm9AWsj)f>pe>hqyH>tEl3m6MjiQm`z$D#I!3d z@k#x=$A9PqocT!;+Q3S^)gdTbm0zsr^uoO>+X@S>3ztVOjxFO%&*x!$UX|&&tM#L_ z`_XixlTE`XuXbM;&ELf{EM!7Qjg7$!rZ^p?CCc|I8F%NrCPfJ!p8kR+*a;8`Ec-1W z9FA!!-bQ2eB4s>qV|a2R+Ay&;+!6BvSle{AFZS}^+~&S`vAg#=(671A#ahqGwRU&* zo~vp$@^Y(H?k&S&w-%FdVD62EcOiD>?=|eUxj?baNjSPstjhGJa%y$`Q!G+cx15g+ zs?(+HUU(!!Z=M@f=pCjf_2$~X;-IA|^r>U4VQa^&0OGzC&oT}1jDOw=&g$@&E}qpvHdsEZ18ul~RtMa02~_~z z5QVpoX%Q*UU+VWmTt{~QO4dH>+w#h*t)SIlFI_#WforgGRs+~@)vN}z;fg5%#nTw; z-Ab!v_b!%;Dqg!@iWUO$WPW9DljVQ_SB9&wJDbIk=L#*utv%)BvhN%=Q|?4zM_i#7 zF61TM=-E7@)Co1WD|f+$TH66)U_XYK*2{&OU*ly0L{J|ULoBPs8WwZH(TdGtuKqUk&;IU+No-sk=Z?6;9q8s zE}fm9VqzC37+4(|KBdb*IPnV}tW&@k8GdpAcKZre)0XY5>4hWVoMyQvI%&D!#PkaI z$xx-6@mo$-bM`3t5Bt?0tU4<5%w~JDs@f*Jy&4CDu}+4X3V%*FBnir^ggKho!m8on zRT0Pa6HvuHljo(0!H_8gy z#I44(%qS-EYA3qg(k5iHV$(b2y0B~#7St_nTc@R|H(gTRCw%9G!e37fsVAMw1xo2O zZrnZeHth{O?BV-jf6nFDy9}T=PX@4_62>Ru!jaiD?nT{JOeV1{992ys>oEJ-gTeg z)RH585q+kI1r(YjBpQLW!kHfHK-0~FLFQ$0@{*mAp=0xXU%bJD%2r3MPfNRk`}hxD zh~5z8{uDk^X7+qN&^@xRtRgzBM8iWbVV1~5f#63JzLWfBjqfrCEDylr*2p`EU_%ZY^B zx+Ct`C@(wpRVAP1F15K!?Z6QjHHeeb+|hIS$f5KCaF(HJV|55{0?BrFI{d$#?%wmd#pBd&L`(~fW@0PS*ZPv7 z*(8GOD+!>_Hr4F{Ew~o8MLqcc*?ZURHgaT5w14v{vT65}RP7{1U8~jGGh3EbE=}9A zMwF|%cXcf(MW&=|lT217sY_LP*1W;_KkJ;e<}uFm>=!t%a3TUgCV&L;LZ&F$#q7OZ zR3wtkVIKkbEG?QLqXDv=<&`~_T>7Z*~Z#hPJ?)v z5~FnFj=AQ^vf=FX?hq*}GP=lS=J-}pE&q{a`)8*6Ft39+n$mvA zc{S}{u~UK6X|$Z(feFiKI+7)&e0JWyNUhra3#+}Lf5F2k@MXKrO@@wVqm{7sIl7+l z8ceEkQNj6EYN&r0jP{ZUpy)h0oDQAQ6eFFZt-gm>Fs^*_-6ysr@t87^Pm$2XwLE1H zEmx9fDMWg_yAPu@Ch0rxt~x~+?U~}UfKon93nL`*1oiQ}L%`w}A&B#HTaR#nayxoBRlwk5Y3nrL+QsD%X*Do9jPj12yOww5D;?|~pGBzC%N2F|! z)+-&7$l(hpo9+^j7XC=scPRm%EBam9f?2R0Q^u3cO7o*VN&{O7?{QvwjEIjZF}awJ zmbxDb^)aRDsRI|vELIA-Fzv}Dc;V9nFI0weM)*Pt^X3B>=2yIgFeJTk31Rq@APmdR zm8S|~_yE+;2xBP8iy46omDTx?pbQK8GdTsz12ZfVg0@0xf-_93pApb7RrEg}NW-+z zvw#|=Eq)5HhH6_(czp`J`Ot=vn*LeB_2q*bYQt`UH`KbT5a95$?XwJZNZbnt2~&8l z7y>bs#ZMdrQ8JX|!XU=E4Ke^B&dCIt+d#%7mrHjFhNzKU0Ed`Ev#7paK*ZcuD3FN8 zhU9`GR%}e&u!z~}O~8oi%JDmgMpU_>035Ndo~sQWF{Qp3An{>_eP)P6X&JPDB&r0@ z1(PV5i_JiZdR;R^C8jlf=3t2uYt;gms46s{4Pc@?!+yt*iHg!%A86uSo0x^bCZ-fV zS>QxvY;2Hm#n6eSu_#kR4e*J%<<|h9m|yqUQRb6-!iBC%9rMb~M8) z=2jTz;3}3G$tAMlVD7#F34(N&odJ}v_%eT)JVbRo_CC1_xg0U$3 zdCi2fXe*#DoJCvtb096+iku5;(N^f(Xp6Rjmw1a$Zw}3jxM(e+Hq1q94fCKbS__>I zchOqx{K$*eqL9Zp+S_+&8gV9ptyeN#8Vn1IT#;yFh zleH+!g~Vvs^H#!Q)ajTVjj@=L0z5{Yl-Up&i^<5vWK4<3pfaWu2}T5|<5e zqR69TEof{C6=*Af~oZ>a+@@`4QMoT465VJ8|-M&cF zM#DhVkK0I2Vsyee3vy#c5}yWk;|xSU9(tpujMawU80Tq*HUvi_G4*0N8r0@WD2^$K z^WZqzdZz%%Q8Jgzisfjl*@otr%D~EajuN$5M8}klEVfcbXvb_lS#ZbP>I_hbxniKO1bweDB3RRnQ{S%ufX`Qrc11ff%Xmfy!b=sx(wZjZ~|w3pY}usWNh; ziK^<@k@lY9=#hD3&WIl=6`3GNYEU8tH<}Z6vjRh&yxQB>&>$PhB%uydAr2d*Cql2# z(M1Saj}_f!P?U0QIv%C`O`RgleOgQdI{Z)xm%&t}p3x?QwXzHXCd=5@yGJZ_==_c+ z2FM$nP*BIPmKQ)8l?H!b`6IkS3B{xr-xv}S-kYUV`W4TS`EIX5tRhy8T7w31DGE0V zOiLYkNSb@%W3p&?3YuoR&LQxao+zQF1j;7RcM`(zzd|HCY!0jsWq6b>S+@~qFN}g; zn5d2}ydE(j{!m#D-w1~L;o4^xyhAav|AoTH(Ns#kwxNlmaH8{B7WR;ej;44xwa?Z) zR?w6OiA_7k|Q}`+Kq`d zRXdEf)j&J81HMC~V_II~FzovSazg>Ffgi5WSu#Ix0HACUc?u5^!Dta2u)STZUSOBs`1Bs_4O%ZQY;dtqo; z$xzn!@kGnx=OxjE-`Z&K%T}U0s&(GM74C)^{ta&u?GSi7k)Vh!CgagNn(0smLkTi9 zv6&E7bRjvktvehsI({TSqMyh}&5ZQ{|B_5$BvN4`eL~9xO7l9gqblr3Uw)%-HhLJLkBg zC+G4gSD2t-Mm;9<7{UvxV)5APX@1z2>W&ec!cgOJs>aV!Yr}C&E#m5%rFQzNtk}n1 z90S#T4t(hRE4;u3bj)8fw8Q@%UZ{&`=5xmT%TTPoaAsAaY_XsteGD%z-{Pe+S{V5Y zb`4GFkAqAmkG@<&HgZCyMvyo+cqdr!Y23ogsTvc954vwja`u3nWRnFlny8o~T6f`6U|kPK#$DF+&Ek ztjYY5n3cx*3qvO+-_+KmOuC<34och@GN=btOolc~Nn8(nib>iPz^EMX|E3-mUs+mM zy^ea3R4P=QBC04?u?1Tv*xTwj`Y=Jw5j zJd({fD;1^jWHNNva?VOg7HsA%_Yaf5-~x z;=?FQ&M~veKY`UohpjUx2u1L9WE<~uAPe#)1Qn4Zc2-<(9tOy+R6R8S+Lfzkc3`_I z{l%uYd0b4B{Zd-I8czr6E`Rfq+ykUbfRbLS-91h7 z?DfI7dmrAOo*f=_Pxp@Y56+JE-W?FN)c22j(|-|6Qp|fu$SW zD7+e&PY>TG6y69dLlA~^0r?x#jwAL*aX`K$7@}v@M6Bh`poQRhe8$CTS5Yt8JjQcL z;tt@9kLj8vQp0a64B1Df2mocNE6u-dC(`a^MZuL7M~!&^GH zfu(`%Kl$-h_PQ}mq;VK9^wHGrA$Dp!b`6~l^03VE#>?SNY9ZBGl9Un4eAbWhRCSC= zBDQgkD>VwwqttetXBu{%XqEOu!YsjbmIX`l71Rfd=#N}7?m8z2Zw|YsC;y-w)=%2l z@q*m3G~QH_BT6b+?Kp%jN2pbN{Jwj5`kq(a4r4dI_6LJ1G>Bq36&UQ*O2ovwDqS+B zT?vN6>4f=DA`B3^sY=S(YN?j@gDV8OC|fU290`I@|79!X*uq;i1#QpAp@g8?@Ml_eOnm~2CNRj|z^XoUj&7=vF%Zo-@V zM2pVy3H6x65heQ=EQz0E@jSb{5DK_iyGt$jM9K97mbGZCp_EV{Ck#whYsSS8O9sEZKX zTunS=MW}iSf0<|_w5vgn0+9SJq5zDu<(AMMM6qRch66!zj%uKnSwbOTag1F-xS<}? z@~>8e3%Qo9^4hMJ$d_oX(8Vm=Dkai#tx;oJm9zODX)V@#0*%T~WlNOzUPSJ~!G?9Tj9F9c4Y*Fq@IKSa5=ahy-*x`d zS=*GC0Fx$~Ty9r^pYoK~1hJY0@ljBtXAgauLO=|%;|Kkh{y%@>>lO=A*PN%PlVo!0lX({CC3?9->Fu|w!YelNO;(OZ+w%9*1+ z-4|dk0cfUYJ^@TyJxe+VuY{y0>}HNjD@bphSF56%y{)d!8Oa!s_Dx$WiJz;d7~HPo zf9 zO~Q}UoK#nX@WLAueNNEFvnkMOa)w9>L$AfIeqvb#2|4L}=t`xfy`y};sUUrCI4+xR zD7sXi8+5lo$t=riIQ%7wO?-e(K^u z!&rBgWMe!tJEdO=a6P!Bm{sY@gj z9(4t2gHhkV3&IlB(}0#wIS6|n3(`sVQqJf+4W=$aoSa$sAs%=)zE!BOuXMPb{dnzr z(c}WQ{yCo#&dy6iZA)x#VsT*0A76q}Urr(U<^8!;)p~rVt!0GEM~HAK6phd0+$2M7 za#~{i@|m8IWZSYl8L@c2mDxL1h#ywxW^u;ImFDbq#jP}#K1b9_bAfZktTY!ncf?9_ zv5yn4va`;-1gb;{GY12+1>P%K>}LpAS?A=KIV7c}fZ765TFRd%9Hphm`GQee3Y|X` zrKRA<3PicFfQcf_5aE9NE*#-mW4^frG$B8XC98=mP**e~7cWhRqtr}IJXJCo_8Ib7 zSNZCk2tA!57ovp=l`(Mw!b4Xj6unOgN>1)_BV|}A`+7Ki35FZv$PG^B=UWA17>LOGwEGVLmEz+_RJXYDa}cjL(@MMrdPu} zmFSG?5caYB9)Xyb^1I|*Oa#o7CMV2H^5vE(kn&cNBFHxH0!>9L)A6sM2)%65sbKSy^YW$BS(OrBq8!{BN^b_)czGZo zpw)!wf9pkZR7n1n9SE~SOzBhRoLA%+q5<>@a(u<^D~mG+=ht7);r~8;a-K6XTJo1q zpYRKXIfE8|rP44RG84L1AWy8Q0tqVbAAUy<^-G%>cU$}%d61apX*!OJ5Fb7ZF6UHeRF3A(sj1hcQ*gvtkp?`?U~|T(Lbyqjh(!7Mc-Lt z)*SKQT8VtP=U3qU=5sdiO%R~u{nKtE3%Hye!sx2aem-lyz~t!MfoYke2z!?XxPULe z(u8VyQSUmKKoZ!bCvTNY5v+Nz_MA%xh39F>{%DFN|JY2EH(xkSDKU5D`t*XgZ#ua)wM*(3wOxCV z0S)yWV^Gl*@>|GP^ghvthh4bnA$8)?X!EBeZEz_OWgZ&-h3(s4^J7Z*>in8^0}6t;Z-P+L*h4>h?k{`pf2_HG`RUiri|x%%O(|&}C6;=d zGNPeQk%bS{F=M=_KxC>ZHRh9}VR#6t-4BOOXU)Z}j@LuY+8d0oJ?LauI(rVQ8;`v=(^C3Uqg+`XCD}onG6QmpL~n(1SF#B1{iZQ zo149V7a+rCd6|9vFbc3Qn{R?_{#xZ;%0_cBxN-SSyay{Sz^S+SlU;&SNBZR=Tmv&E z-T8Sf!zmXm7h?XoUT#<}#mlAm%U_Bm>GvRuaRdMV)6+R(VD|rgJcn=L6^n$7_WwmZeSeJea@3>C%K~Ye`M)<*kdCky?6%tie!}gv5!6cLqh^BlZ=%VO|B>KEivM=0{Rq)P9- zQ6CBz`B%(Vy)usc%ivBRr2lcX} z{5u)K@Bax&zxN^vx^?*a04fAs5f#x!3p5h1&CEal@Q$LAiQeq zf46t^=HTu7H?3j+$w(zO|I>?3JzI#aV z87>3FSlK(?+yCysU2m;5zHWc|)M#Mw!6iAG2|R|$AO)tA-y!J08?mk@DOUWSb5kLD zfYU^lb=ketsJZGu&AejTALSH)8Q-ydNg%$%IG%#wfkM1-@V(GvpIhpcqcP})Zg?rXgJl!prD9kCMw2udA4!!A85WyrRQ)v# zt(}Yq+{=k?M3d&Stzk|D1&zhp@MbgofqtsU0I@%~?80h;3hJ+)8m{X$q*SHYY9Eae zmw2-47xNYf?P#)OsxO_X1Z>~jgpe5by6L_q~{8L(qI0hJ;VW4gZz7gyHJU zLL7-T_?GWd=9zWg*?3|xS*?{ub5qu;ZOR`A(cX5LHwSN zHVT}&!Dvc#kx_%TD#^LwltQua3#19O2;TV=SDm=!6U72$&AaTDL_TSgn2+1vY^-nZ zG#X!jEpVU-`)z{oWO$u(z~Nu8(RgFw!bfuq3+)3GfBki`zY@Z!A9z=z5SFUmyRe^L zA%X!oO$xg9uq1z!78JnmB%?FLoV4#_*Zm>Gjn8ugF)NTTdaivDzdkJ!ufg0x*G=Fz z+I4mtT-)&20$fV)8s)CmG*Po8d=0V}G4*P#mO%RQ>TJC68Rn0OZHR~pcrD1k1ExY9 zg1Q8kHH>VESD7h=>!vrQ$}QxdJ~{5dutthIE-!H|@|#o1gl_DKOa4B^5JDs)0b|tn zFQ!)tsWPSbaz-jAf%9)DVVw8hI;Y%4@~bksu9XjvV^tmjUH zF;(#cS~PhSu_VYA0}+!V*Csrl9qHSdy0OvV7Z+oq!tY&&0&P-N{iq27zEai{ppaJC z+FJ2VuO)$Ka)teMh_`g%`>ym&R3!s!#w~W?O0vYHb{zR)hkCz)&01<(Hw8P(wk|oP zT}1oR%5qxBwXeoO!kH567qa#hmkVBh7>w`)jjnH|;Z3Ds{lMMFcj?*zynXkN?*Dju zcJluH>DlpKxBJ8Ulh-ewugILm@%U$&pO8@Ke^UQ)#Z=7U(7`J~XwrY#{FRXHSo>Wh z(G`y8pPC>K(MfE3({cN&Fli_V?Fg?CLm$uuW8p_e?d`Ojlj#VSd=y4>LL5U%?v|Cc)Gr^hI>W zOj6Lr$h^jpG`u9NK(@jqg3@u;TaMbH!uMF9VXPo9OB)wpe>Leb#;5M};-!S+PU+%| zJ8qg9*=$r64$!0xy%SQ;FQpp@Uc6K<3wbV5Q7rz9P!wF&J533!&AmIxBc( z$ee~y`ws5#6wMpZ7V!u+{X+9TlG8Gr5Gs-@;IvFJP((srQKm!jJbOv#%*pL&7EMJN zG4j}JX{}ZiSv`UcEx;jy6$+79O&HlHuwY>kom}l|l|N7Fdzqi2PXy>Lw z2Hyh`xy2xc$kdjcxEw2mm``5HY#lNuEaCu3v8*u7pr9eC3h~(=y(9qZ zbDhMLt*fv{B<9eIgQ7ISX+E?C9$Z8^VZSBa!g(&a6Ta?IN@T4KRE zh{ZjZ)9=2_Swx0@gho_l4Z+>1qij!SB}m?zww%?v7HXw=`NxfhrI`gn%RNn?AakYL))cN^Y1p+~s?uuore$=bZ06V}Q^MXQffU}eT-wWm`i(`* zMFqp*!MB?qhJo7TZ~U54~XuG zNXoebWz3_>PE(`Ez&%5&(B{3Ox^-u>4x~(7vKXG=EsS`Tx6}5Walj<}2k1RB6fY0w z<2U^9*woy@iLWM}W!?#nI6RJ3wt^RR&oQ~KvNOwZ8Y~(mhaVN2rgS+K!78hq0`q)P zI0<{!!i zBl3viidLxd5=oSwFOUpk_4zpL^L;_Biga46aSvkDoFzPb`<92UcZ&UudDkPgX@svdz|M$-N*2dEQvxsML=RYf@j6kNv3OSKOj%p?%gB_>m zjlGLt5FnBd$UWu_lVst>Abv;`_Fo5aG$ribSJOVM*Gn5yeKw}g>IlWEHX!p2gVA1c z-GEAv-&hnRb4JVl-jdzB`!Gr`Iw<~L_KNuWt!B<6Qmy5;8m_x#7b_PzeVkK_3ifr9maqqDK0uK(+uwe2PT(?Xul{`^PsuN2%WJ6Ptd z*f^CZ(R9>vwm0Z!Fzik*FM~U$={A?kd8H$xL{L1rv!jZXJ>f&7#6Y5+;vB({p*?eY z0X=bj6i!Z{#rJ#teuQDc;})|9+uI(XSQyZFvE<)cuuRFiLkpI%I=m{mxmS? zPfx>HK6#H_|6?%ULvK8{6{tY}>um1m`rq2-R%faIE#g^>{%6<6m|bUI@R|U$FzSIU z)jMyzw9@>}tTbY*hrlo+Xzp+B{O-ew8j%Jx$frusRMaF%ZK`j>kE>F^3TK0c;Ylhf7a~6v&dr1CyKb(#xtPI#W zG2Ayq_Q-XCGo;KqzCFp^Hd%YemO-ELZMNcK&?9-YeA=w;* z$m!j%nj6x~x}1mQdWz>r zW@N1Nd&*@6>O<+WLmiYSHIyWqgO_QCN8E^E|L++ePuuT_a)Oq%2byum4hX=wX^(W zz@occ-&8$%ySZl^)hC5gQ=Rj95)S-GU#q2;bu~b)r%w7o84D>=AE-F7fPm!lO8;cr z;$(`*s6gfE0mXU}41EJR`WM?~O)p>$Vq#Ed`%~z6l@jH(&G%i&ItLAta2yW8tNZR4 z5z9dFMqEHppcQtrB&m=2Y&1w4$oW|@6H7F!rpIama|2P~gQ-GRpnKj(tz;P-DtT3g zqrh+#z``V!7!WDPzfGUPmZdl!HPoPTI1xSX=|0!yUEdiAy+*DXW59eDiZ72R%=_z zJgF)`(bV+GYwC8Z(IL%UKo7skGxGBa5vtS(_GHZ*da}Krd~OjJP70MHtZ#fVJQFT1o2rvh|bupQiUXTE`)Xc z6>A-7xuN^UjvLwXIBhBt%5KA!orx3pLo4JmkX|qtx_7u;%=9&rPE@3!7}{{6iD@^j zRyW0%uhi!(N+NL<9Zydo6_-YgHHjh74pyW)PRlGKVJ$Fym1M4^j6x|~VXBmy8pDq; zMvgmnnkw_?0d{-W;qAz2<~x7sLzu+Wc5FD#hH>XX*&$1_h^R{p|DvaH+#w2|@#Na+ zz;Ev^`HfEdgdgfOTk*&n$JZeUH~IA-j3S7yD+SjqE>sl0II6!zc?SYh3j7qVi>i6;X&LQ~;nEuCGVmxNtBy&x06 zLKg84`(WE!mMOy6TDeg?)!ztm8sV2R8q~vYNbEOE_M)H`n~(!!2d1`KWZ&V zzSY$Xt+II<_@h0)gf?FFhF9CIN$} z!jwhl$!F44vz$$7;=uvAsL zbp^id?_s1|-Bl$^`u^L)gQL^4{k^kq58ql?KFhCi1$i_ZG*nH}QKwJQ7lyf=e2d}c z%k!a#6_QVV65iBcmxRoO&OsGITNJ;fG#l1VZOW{bi00Yk14wDil*93| z17K>+w^2CU)jo*mo`3W2lbE>blQ^LkMPD-H{pjCw1zsJ=isSS28embO7YG4DLok7eD6gWT5I%35;c_6dF)zGmQPAGZU0?% zAbR3rkCC~oi01S401%?bGO26$m}gbCoop^FvXPvU$_LpH zfZ=B(?5-xt?rPx&1!hjkQkQ#S4$NH1smxm-W6PA;0^5x@%!?|x@kf3fA7czzeK#k4 zs?88!sI`{;v_h@9D{5%aN;nKmPW}V6LA-iBWL4gnW7IQU+!)Vu*i9BA5xej}) zpY7lymhLlediS#V{8zg=wd;P;&sXq)q4@Df$GvoNGn0CWB{)C*WI3iZ(DQr+6bwg1 z;a~JX^Z8J+NEb@6sDUqf0g!TQ-Y5KbqupLe?OO8RAT( zc3R{EoKZ08Uv!z1-@dSQm_J+TrDegzv@UlS%pt!Y_CETN3rrK;kHcUzaakp;D#kP! z@C^lcP7~Dc1#O_jGc|%&Gjl5}h##iSHE<$G!F_R0)!HOVP>qowOPc|i*ye}fYkni; z^AhtxgMuAvGniYCP?&MH7x{$s;0^MI&2Gyt)1U|P>#16_dDv{Vp^qzmuer*c_F;|n zw-Nnx^J+Bu@#b5=zL%Dx2Q-8!)y!iL(^_MQL$7zW91WE{R!1r&ly)VDDkW!C$10Ke z3A$Akkyfv3#{5UJ^_<1_T|~c(CVun)r?TaaHb2)h%`eD%?rAuYJ*=a@c@JH*dCqQ} ztM8a|cFQHU$(b!O<$Qg1@OSj;L5C!s{iU`$W_`tG`I(sIj_a~F7@Qhmd0!5~o;N@k zg58acwe9pfNTUen&`X-xI7ksjmdmb3xajS{cnfbQ7)M$!*ni7zzVnXxGRqo1_T#jS zBxi?d*OSTEI9caBZxYqbc08llsfDmb6g)DSf4MweU2Cc*+NR=iw+(wm{cbbZ6ACX#2TqVt06`Q2dGh3VL zF?d;xT~(W;kptBjHU##TJ%k7^0!SU}%cNUw|AWt%8wrb2x|KXD zGbQI`C>C-S8@$E-UZRV$p4r{JPUhQhR48|y^{wqZ((_gVdiOd&u%Z$IT{1>JfKXRT zuPoRNvnc=WdO<27G6r5SH1(!-&XnYxuoh50b;Nu4DxFhf)4`sdgjU+jL@_f5Oe=R! zU!z}R1a$*ld@v@r`ozZA$ zMN+DTHY4~j@3V!(Qc_1MBl2Z%P#~!XyQUadip<7GwZ2%oB+ag}hs!K7RujFp3Z>gY zWsmqX)T|q6fP*7(#!zsQ9{gfJC^Iieywp|?;nkAWNGi>UqLqGjnpcDH!W*Q?*x$L! zQ?WZi21BqYO41Ctxu*Imv6ib#IO*qOWBlhmdi)1&O>pCzUQN-Ti{n3UZfWs#2 zTg&(li+GStA3pv=&T;Bo5g&PZGPt`EkU>Fqry2UY4dam7ShH$~7#4A3`jLY;&PL7? zj@)QCCJ}8TJYU=POi!FdeN=*}8;fiv(IT6?E2NN-IrOAeWL0)0f@CHvqh1mwNLNZF z7NcVBA4lON?1h6}=XC#=ztfMP>WYe3ee!)K%lOQXzWztyz`uYFfL-;X*8k4-+LpHd z??9I2`oD;0G3&o-bCN>?@R%dX*XY9QiFcnw7^f3p%2$jjD@BN+!ZM+u@&iHH^oK89 zeb6+urybg9dhst<4`?lN9Ytm2Y}H6tkqR+0Q224jCg2(ZMBH^46HO=AVHEsAq_#eO zgMA38QueIIOYW0dS+eo}sBeJm0M^@jmVg5Jzlpy!`MDSwQS~JEvi}M%Xh8FJeWIBnt*ZY(fpn2a+eE7SL5s;qs{l%Y7oLP5 zBHolCFPl4Kk1t0?+>5-ie;N#Z1Y1wf7v$Gv2(pwjBURVoD2(`=PO?fb%TQ!2V@7(| z>l2^E0Eld<#LTBT)ymQo%pdB*QdMMXX7tbvjAn0IvJ7>z(Lqr;PR^d(S@t{+7IL)G zPAE6{sL@JVK?YhozRey2cb2QNIDMHMR(Dfk!(Wfi{Ry3&+Zoh$Mz^w?&sSi`eG1$59E)Ey;lBC3>(FT)(2YcdQhP3!Vaz~D)pS_bzga5j zgU(^-X3l8Z7fY;3cF6~7%*LCvN2lcHr{yyY!mCy`{3XA$eugstW!WGh?pjOT@Ll^} zA2>+y{<9WPVyh=V!RQi(kWd(L1jXdxQWj0&R?Gr4nYauGwyoUGa(mb4P2SH#EnxMY z;N}0MPH%wsvhR7_{g%`H7>p64PbUEh)pK!u}&A5>6WZZ47uK9g|TyDlD;l7m4z)i_4ZN=76bAkr)Y@ix*5 zi?ym>VyitzGK3^kPtFUL@P4DpX5zItGmk|@b=Rwbq?%}?rB%*;=Q z3+)M`NzS2O)VmHQes2Q1E2k;l=`;yFqX;ux1Fay*(%cUx&O=Xp?`Z;9K?yo(Pf#istGE!GsE=K*R!&Z_+w{X2-SeO25QLSH{jlt|Si!y-iH2iD8r zV^o7bETwc4*)_ulrq*=?L%IpVX+HV`duE7|Ry3m@xN#%@wyt(rczf2ph}uo%<4lSI z7sQyK6Hk_8$IpZlt1_XTH)q5<^aM%FeHD|6R$1rbvoqVow%UpP8Dyaxo}pt)LEem?x~{JT+K{y!y%3 z6NDfyPh_2e?+vvsFm7VG+85B=Ma8o#S^olByohLGrRtuit;Iox%7Ch-<`6AWf~|@v z3gS(aRl2y6$~B#mK(ppJCZdp>dO*#Xs|v_pdetou?GF`!qfB)VI-h2Oc+M-PGT3uY zVGj=bTv&Id4$lDo&I*ZHjL|9)QfY3^fWlgJi&GCUYXwu8TeyL?7HcVSz7xP%5o4?G zXbn@otOT8p5sEf|b@D)wF|qpa=h%ZDbL30z?G1xobjdDvmCpMF;%llu8covWUV7~)~pk^!jX*k0II#!l%c4veg zhOLTO_Z;PNa$>jpGS5e!mnKrPKw3#&QQ&yS!p3?*l02Q6KtmCqtmui@NyQiD)TzC^ zFjoS3VtEN}A(!A7Y~uT|_w&?m2p%?QU&LbwksN@g=rm8SeMfM$>gS3t>R~3OQ1!MvS`-&aotAMi9Xf`81trg}(dBlY$-ztSLxV^vY{KWJ zAARXhk^YC+Ezkm-^o#o+qXM8CAbTJ{NtDx@@WH{f-|C z{;S8)nHPco+CiO9$A8^i;(ssXSuFmO8T;F=$S9D``5>+Z(9U1FdZNb_Q~LRlp-^X{ z=1D=ImWE@|n_7Uy!bh%r*i!Bv!hp}M{;rwmqxMzV6#t>Nzz_T>jQ^nT|2tco+v`jD zzldk?`+xHqU<&{-f4Bz;qkkFs;d77t5KRX@mwqBbg9K08WcTngffU^od79qb_|b** z@yefMyMdUAs*`qNvo={O%+}WR$JsyW@}FPsZ7(qX-%e*^JrnBTvlY;e89@_LVpAMm->K zwM2%auKQoh!L-f z2sD=OW(WV0h08SBWjfl^~oGjdcMdp2+f+2@i}F);NjO5^|%Be{sZh!9(!lIe9Aml`ww0f?q9eJ z7Gda(=dcD9$A8ZF|81`={r?v7Jc9k7aaNasM5||C>bHFX$+2oZB2JPNy1P-)E{CQr z2^qa;LsM&JiEOmttI_*vu-TVgMDjKwPx^R)G0;4kh{oO0s-7|SKMDURoBSX5=a2g) zo&x#5zO}iQvHx!_?f(mT7VH0$Xv4r=vz_rW$64Dc?{V0F9mLUej1&LWw0~vugI=C8 zR6S+*EabPa*5|O@Fc|IKc)lsGEPnrooMqYmDW7WH{b9*ZeDk+l|DVGCP36ufzkQqg$?^Y6 zqD#!X5iHvO;cqSe_ty4u|F@9m$@zahV1T!!FfZXUmQ4C5<~vrAt|#n7mK?CEzFbEv zR?B#L0sU}&6iyJE_I|J5kD%4)?1FGdbGSuVLfYi|KolP0j>CAaQ+yLqcR5tuEzfMr z2#E5>v9|TUj&ncp6z%^uw{`qCc(*+NTgbDR{hw|9W;Wn`!E0jQHOppiY0>?HcXSAA z(CUq#vA?CcNZ+&YU zN|$TY+`UcK0xQs;Lb!(;e;x$|XGa=Iec;6tm-RcfWE9@w$uS() z+adx<@`e{Dq7apl1WLT1HCYmn0GkAL(|JBlXe0FO?e7 zr7B{H^em~Zg=N_v3+{3yQoX~~&9zQfod!xh_08T%IPfEVDVAQ=L%du|eYC@WLy8y) zx=VbNfQHg6(m$E^DwzT@noU_+KuewkL*GD({>4a1Ci~2U#Gsw_r_kLh70PS#%|<2T z%=J&gaX1LC?z>|QDG54s45~n&gu}BYsf*cc6j(e=W??gdYuq z5zXH>k=OH&eV8~{UpVT^Su$a5tN&d`Zsd|#Yq8UpE2Fu&-=cYzOLH@+VZg3(7g ztQqNnCLL2S>j9z@mC_cq|H_^Pe@6< zh=&g(X|AzyKVq%01J%R9wl9KF`@)N_rT1QRl?fZ``aSKkcEl`&adzTK zKb%g;lNEmYksjCs|)Zi+V!zifVNWiV55(X8{sP8Bxb5VhLg=&ZI~RL+gJ$@ki)pk`+9n4Xi+S zoR(NdqGrjcE@Lfa6iVF+)1=(A7=DEDaon-fR2iqKrR!aXw#W(=q%5z^6#F=@n92)PPGJwe zWEIx^00GU>fS)5M6)x7&*%Wq3#PNvdDpABeA{4rb1`ALga<{v?$5e&Ah=U=;{@@4~)OaYz^7x%f=0h?iNZDO{vx zyNMN5Kh1g#3+{JP`s?g;THkBvaVn=~QkAkFV0StQF8$tpZ{U9yh95QRlW+AjL91$> z#`vQf^WysI@bqZ!-Qn5mgI6EkXlo6T#n0NJ5m3Yf|2oY}4@R^}kZr2)?E4qfs{-a+ zHOp0N2q+~MgKafs(a@EU?`Pj0zCExuNnF!z8a94B);wXNVR-LFr6QQxBsG z$v08{Z4?f7wGXtI0J(qj?@#>8>{q8kw2EwAIX1oA582k)H67NtgQLAyZx7D8-M7VT zc8{kvpv9)WJsG4s)_nGnH^_1>-M<6A?eFdT(M0#?$dbPQ_VD27^lX1GpQUmt$fMcN zbG#`AI8!AqS^?P~x(6p#VvPkp+{~R#D>s{W!7V!Eu2vbn40wDB!a1L z=8z6~PGa8FFO>(QoA13SLuYLcZKC{=TDe^> z&$3O4X6?Vr4nR*_(=u|JHR{qltp_rc_Rr5;ykO>$mu(}P6pL#hr=xNsrK`dz@n!=S zKFW$Z!yRH{zs7WOZo~XGl66-LU3WF`fdWviWT?j*H3y)&>wsG%_{;V{VF`WUE<;gvU6%a65xm%2)))4*_Y z$R>~Z^pJhiPY~H#{S*-&v0|Tj)4P|==f4`v0lV%e{TL4)=>H#oblgkFoQqU1%Q58a`2MbGnSt~uoQ!`??fa$$-`_Y?%$WtFr_nW8-nd*Zc-%j(9TX1`6P?&023g(qv}t zUgQ(#i8sg_GW$Woyk@EIrA6q$tW!!Q^BAeLmJIz#6!eTTDd}aFL@L7NAAPZYGaBbVsN2lv=~wwQTnvr9qVdh_*F9$2 z8P~BZo^MB-*$q?9*JlTRN3V~AlO+2~Z6nP3icRt}G07d*Wp7#@1IT?j2z%ZD;Uso9 zHrBS&@4~6*CEI7axBf+IK${^~WZgBa3fhU%^<;vo4-ImJ@uRcL8LpXoz%ODIWyxu3 zR@kTK9oc18MSSdsWBEnttnH)gWH-cHP&Wp`W*H;27Rok zxliCyQXKUCo>%{Mq@>)LE+R2L$YRCSyPLP}m3B8a?IP^bmrljL>0%L~UHU3#z~HOsF`VCaRv*+Iv@EC)NG(XI`?TuIrkG{*$BHD^^BE^9WqD^GlU0n?XRIn~yc z$-&0r_Szr4_6P8(pMTVtSGQD%=v{o(g&q97#j-M_!4|B2ljaiuxJeWp9iZ1S4RZ+y z))E-ZqG6?Qzg2j{3P`Jx@G*s?mDs96l&7@Vx?(-mK$#IioA=g2qCKf&lvy=o%v&I- z2fM)-7n{t6PPHl-J0#7lvW3g+OxAM?Z5c~9xysJ>XQ)>%7cX4}k@fsygeMFuOgD2`KQ2xwDnl^Dv^C0b4K zAN2Sipf9;hqpBhB6MqWgKOv5{8vmoSy|uH9|Fnu~4=Bg~tua>@shoXe7K8c0E@FE`+c z7zTC|C%=gF!j7pB5v)Ngqad8aTqFD38xZr5szG|gim2}E)}LbhzxbvH|7jXK$=vSj z;Hl%WnpUD!F>}Uw>BuzJG#0ayH)Clf#<-cB%{857XMLx&2LJ1%cN5Md12b_yIPd`~A#<7jRIZ%)+lxS&9rs(f2B9KU?DN0W|9uNzpsx}EEdlD_K z=e2tLG&G}rveSsxAPhfF$Fe6)gaWCMyhna9h#$d~*QriBYrHGI5cquYO*d zfdHsIj|8&!)kXsQWg_8k{(Xi5n+-%+mt~uWX<_C-JsS836~V0`Y>Kavdc2eYOBo=^ z0E3~Ktrd!R&_4g>Tx@T`HZ2<;EZv03vm3X~K?J4Ou@AtnKwr|?PHW?t2L@eIPybecUbgX4Sv_sukh&yE*n&v;7 zO(xp=EMOYzy+yZwhNh2eZ#Fv`*U4rWUtQiPB%Gl~&Z00D?sdO(HS+V*58scypQpYQ zMm#fJ({QAk@Gz@oD#Oc}$N3qndvDl1FLJka6=rsh%|;P#M#!`BQZCItgRs-bRr{$x z4bigj3RnWRhKgSaiNyqdPqMYDxO|w@b=`-N3~qeWyMU62{gz_lNr?&0Iv0#}uu-fn zVxT#DN78(6a(~yRZU06Er^$JI3v*(+xFE?AQkl$2$HLEFh@H^7g%o!mWHONXY*g@Z zd#++%iofJaB_zwd4KpKnHbFa$0wq#3_1|v0jahg96#x65V)(bsO!9JRtnXO2z)ROZ(3vo?m}`Q0q_oYv(2yqBNa2m%+eC5%_Y5 zecrnUd6624`u-((Vc1MVuCmE5w}Zm;l4HW3O1dQI)+R0w?S2Y^HahQ02rX0tTuGGZ zzYTq2^2>1NZ)I9+`F5s60`r{s(EB3_r3h;x$PR}fUK#cRsWglmvMu764YnSm&k3>v z-^0>hC#1tB7lEiEt6yMAqWPu05@c(T6Z4X<6gPJ>Qu3XZ%P^u(cy}qiCv1qB$a1aP z8f*=hIB(M}1<&0HuL&&2Ii$i){Hq|w!!eGcOui$zEQx?2EOA`e#<84MZfRlzQQ6cy z>IEILTCqpoKr_X_2i_p~gKXD$%i0(El{ZNS7u?Zv`OZLI&>sK z6$6U84M1lRKrxfh@dHR0v5jcbFsz@~xyP#H=L8Kbjckx0J4GoWmT()yIRmehb(E~l z1_yLwwMk-@A*s3ALar1nY#50Xxy%+?HNvZ>S-LPQFG#{Txd04fGP$N#Ofxv<02#_< zWn0%;t`i$3mi?ATWSZNudj7W2w9XW@Rd) zd-fAxE2(1(7;B}}2F|oY$zj2quC9{_*%2D!K-MwxRdsvUBq!&VRe|hNs#BvTtaI4R zl$oH?$fUL6%7t-qPd9zX`%=Y-w>K zbvgYX#(l{2-@^+;rJ;<+$qBG>GNQ`t61nxBoV|;*sUxZ?KZmA7ru#36&TjFvs@}-V zBx!%`oNCmQ=IP%ic5Bob*-?8RR9XrjIT+kKU$--bDf&ir+MZ|Ubj8Lh{P3Vnm3AG1 z47Z8{LXjaVSF%-X$;CZ=#v5Qx?;5s9E$0Ux&q|o$F^wWTDdMG(F6_m)*SPlBo+pH9 zo|J3oi*mNWO3Jm|G;G!^6AAL=%%O~Moke0)EMks(J(LrE<+Q!5(y zrAWO}>p^9xbboavGjHd+S)*Q%ST%N-4I!S6V3Ye(O25mhgowA?Oz$S`0Cg@XawttJ>3k{hI$nb)zYa#+X-%XSL6?3UzP zFB*}*T9l;L4>xAXsalw4DZyr*H%|Zka5#o8c0j^7qEx7E5~5xVf50c0JTc^dK}$;y z6JC13z-m;Zi!Q*g!ZlFz-*aA0dLP*M1tnrQok%a6*)lNUzd{($=08VGDQPlA(c3|! z)lb|0Ir@I@?T3S|1OGXEO`%zL|2ZO$B$4GeYGSS=c--?}{rsOU^woL!;}3=RKejg4 z^!p!Mn_El#hlM;$lX%ekA2~Nb3zw*pYaep0%QT&cVK~Tcd-7Lwpr0C*?X#+7uuzeSZ|w8N5JW`|9gfxjFLb z(^u8AV296`ttH(L)_d1dWZuoT8M4gVF}0MP(}T5@KXX^x&}wGNR8M!$U3y`M*HT0I zE}t3Pr^bS29GXScm>sS~)mk)!K*tT}Hh=Ruc&9hG3M1GS4KvUOs#=8#(>ty&ef015 zL^dJ*5$=yGHhE?St(Y!?1X2loeuP(n43UKu0Lp-W`^vuzBfml!Qm!TX0e|%UALw+q zVf1m#5LN5h2p0H%cQ)a-y8nY;OaJeMJd4@?iTTO7D+@u)6v`v{v$BOlyl2j%X`nQ3 z>`|1oAX9Rhm0!v>?(Q-rnyzYn(VMx?ahEm@`&|LfO6`VxaEDI~&r{AJz6FNHlQag0m2u26j>Pg6aCmpVw;Mm29-T!&wlFnL8 z;i(dvuEG6Gj$*|?Ka4Iy*<=>y0O7SpVu-e3fOu3R%+pxeVENx+|rbKFOu& zAt=o6GaHQ=M#Aq5yZSjzb{xPJieU*e%uK;=&U6&~>}c^nl3R?t9PNZs0kNKhDkp0a zK|Ez8L+!Zjo0cKf)<2BZAkX&FAt21s)n0nxTA}o?@<%W-NEm63Uq<1u+$`sU@##); zg+RkBQocXZM4ySq*`~Tir6q1}4dj&wRG3ei68wafEhW^YQ@LC-wux1vHYKMK#cdK; zSF^P(KlglX9M9S`*;ltpql)r9_K>CY%{FLd^Dk=HW?k!*TIXNvc`wT@EjikAn(tBx zw_H+9P(C=NcZqIq5YYJ-Ll^rzloJCBs~yYN{-f@@lm>MilWp$|J)7_Vnm}d0(ntUP zg<(2Wqa_)y=M3#s?7(F%Ge$@;@?80@IS%_)ie}@u8T%!Vkca4+qEK)7veKZdaWYh{qZ2g2vIAunE#zlcK&ay?=0v4A|Bbc-#}#i ziGLMF_q#CrFC#AoZIqwh1g0eZLR zybUfQboTxZ?_?08(XF_vD4dSb!J4t4GYGGOo^t`KxbKe~?g-BAaKtSj=JYy<*`MP2 zf1gAOnkMktM=*)~!DXv)9pYIJS`y;jws0s%J-HF|kRb7;e?OS~JiH(Ld2S>J3HH%_ILk)H3<{W#1ih_Bw=*dx}#T-KMvzy zLd#m%3lKw=V?w!|R-dq<@N}HZI4(7zN~#x@i8vhi4Z1{_)PL`=dz48sBF~5wn}GDp zfhnKDmy`ax>oj3d2BGsq7!CSOM#S4N{76@nLb|yMavtr00#AW%EsdkFKcxlezd5Hc zm_L61?iO!1oqD5Rrm(ChshxeO5Jo2*XldzegTC)a<0y#z)fXI^V>tAqUf>NJzaLNm zAp_`1#*~%E2K7Jx_y76-{@?%0IUJ%D(1We_Q8)?EyW+zbN+SNa#yvxfZIMgiR5co} zVEjgS#mL#4Ucn4@*1wrtzi`&q*0)|HsR;?lzX=C7KI#L`czOZt6Qt1L^yQ@=Ari)v zoW%%22nYuTfHC&_FVG`=8U>Sk_cH83r}o92uPN?HlIxZ~iW<9^+-JCBF-PB;1UG)# zjgUNUUGUYNIupjo?fW-v@zMo#fQHGBU7=_Y8NSOZ2U;RG9Z5Ixz5czM7+-^5{Jtv> z{3_@V(0EjpXix~?$Q#8pPP+|I1m3`(6EQ8PK=nQ0#Kt^au_JKgE`tcVdJ+tM?7XCJ zLQw98Sgn|QF;+AV$J2oqNw{+yF%2qIM(6^^5@%NABgH^QJROsf8tWZF3)AsOt#DRtFAXRY{dHpF}eU1EK2<(8RlD%WXZuHS1CZSg; zJVHI^p)bXBFnN*b9xD6VyYWRw(Pg*D$HD76qv`Mhh>-Y6^Z?%5jT5r51ypT(4N1Zg zP?@sVfBvuko6-57|LgxjQmIYUW|)JJieX9s|M-J&JVZ0rcp8DU^JD1dE~0&9CJD43 z%DxR95V{}Z7j$7yT>8w6)(bA5@LM=Qs-QYP%uqRX-iqJ4YQf}dZLPJ|UO1h#);7om zKDN#e#Y+JH=n zAb-z?Imrb$@#DA9sB}g-XgMT(pL!d!xghX{f-hf)*#Q;6QUHRME?DE>_hBwLMrnW) z!P1h5`-CTw9)~3H5C#JocQWyMAECbJ((7T~{qGOhq=jirtIx_!;5pwP{Js6RgYTiW z-B<75#q3^XDx3wqAaohS390G@L!>qsB8Igu-~?zg!?cYFiwD>i8AG&%wHP*5kYyMW zt6bph8Ao0ZWj7NN$&AFBfsdg=oRmP(6%~&3O~>vebg}FCmJPWU<^+zx5IuAIuu294 zXw6Lk!%Wm!1T5}dVMFC(38{!RZD*1`ZqsF#xc$w>`u5H$TaFI7z@?2@L)uw#i0_!N zl1O1m&Vo2fG9T2QH$qmY6#_bzPe)oCm^J~Qu**o5$|z5(Ae?(2@wTYAqef&MM?Qsf zgHC|i3|#uR#Fx+MwAMHN46jHv+h~2$TKhB0<3*q}00YD{<}fIDG{!D2B;QS@6If5VJ6^UsNc8SYUhrjvZw`CRCYH-TNS^Z$*xXua`_EGLDpkqF_6`bdEm;|Z=%KR1V zPbDpY>*3A{(=M0`N&@OC9kxrANocvNcv0cJb$=uIw>p-;aGXt zpfo2z{W4`0cw?3)nM*g~+DG#Oogvyc9c`jp8-j>fDM^+=QAPxI*-;3pHlazSjX_Nq z3!Ake^NvU_3P**FDccZH*kd%L@4;?;1e}`Jl(G#;3wwlyBpaQamXvNtTG+xH5>J4j z%jqC@Ov*MyE^N^a;a*eq%n?yorAE}$km$ne6^gC04LNzWw@){R=!~#Jduowd{rbf3 z{>keZhNGT_jP)ysrnZ!A22ina`Ni+v z@u4WJLM!TN$o1H(Z;pHyM!_#=7#ldpVc*%CPQtk74T?7mH8kX=&W3!CC{DdhGm3|& zj>#U6TZ!gsX-CqKI8lF((J`1!Yc{>+(}3$(*;(Z=S`@9rl5GeM4VgX35_yxV^Bb*= zKUZ!?Et9>^(F*rvuy}2dvJKfYW}A6f9)+U?M*4$kYih6bvA?< zR`6cSB%o&8SGKG?xH094JaZc5sfdcko(H$@Jdvk{cB+$7X6RNBzaRUf_&Q*A-*h|; zmYTjlVMC0EUoZ=YC>N9LHF45npYr0sUWB!8na{c;r?7Y?{`eAb$d(NoAz0u{t9LKDZCd&ncp z!4yId1Ui|I{4_EEa|gK!Zu}AYo{Vsg;Tu*Y&Sp=9AUM2q&N;R<@F-*Pb77Zf@Z}AWx;LZP@p?=NO6}lU1>y|px%OdgE&Hmo>ey}OysLr{b4p{b z80RuypAylGV(JRmXkG<)HCOj8V$kY7$f=&+C$}h`jXP9a+++8$Kn}4#A-5B3^*M`v z*%G3Y-2gt+NJ1wfSj23aVbWFRaGfBYuUnI_1@*O((50(|hlX5mF&fGVu7!)NhY>yY z>dMHZg-fFaSE$S(IX>>WHZ<92wm9q2A6Sm7~1FO=p5@iKOTx~ zKdn5YG!ab`;WCY!(YZ2A&H&m!tda_IL02aJH!~ZA0n2h6R>tUWQ>ad50J^ z1ZK^wkHoNCdD+SI8Mg7FFLI$*bbo;2_nfu>^)I)m1lWM3l3ypRDrsdkLa6$e)% zo{opWVCp!9(Plj_dO^;fbgJ~yA|J&Odz#R|7z=o~gk`uWMFO^7k!li_1=6TqSzszn z;W7;y^_)@7mt4u$OeElNww3xIYiTo3AVK1w>cdfN#AUV-upRlmv=9asHhv6MN6m$b z@;SnyD{0Ox6cAI8%2aaEt&24tdIc@F+||Wr550CNypm!qpBsA>Ie>2E;U0+_E7@HR zkb1~S=LtX%q<9a@N(3W*u9h>mVYqN~)aHo0o*EhZB21;Q(PFxL=jW+6KnCsODH%01 z_vmZ!%}F4^tHn1by^t+UqY`sRy2x#xLh023Y!O!< z)0{57cG)#;mPUjs6jDY{Y&_&R5e|3x>(J9W*|02PxogkU+U8iOQJ5_Cg#=9=G-Jsq z<0X!gq;!K?-W~+QAc_9Y?kIeFe!(8ufX zOgyst=kg_f$=uJckc_-h*i0CUTaq5x3jG-x3?BM4%a6Ze7cEo!jhlzibJ-^110}YE6+rMF%S7{Gu|)s zF^$MaygR9NTrwg&5IUr(gWS1Bgk9l-e@SQfthzntF^x!iQqF8m=Y|nUqS1#4zA%Xb z{|0yTgx&xzt_U%NJpTxTHo`TJHOnx?&V`y5z_=wJ?8HO26PW2J+GpUx(8{-(iF@ zPhmmBi^NQKNk)v(WNexS(a7n2=FD(7wqAmM23MQlwfARf5a`VF^C_Y}L^iRr= ziBE09_u(+69`LD+B4tKQksVsJZ8;4OMmOJkOi2FbyqX3B^!B(3VDm$mg*RRl5C$1p zRG7KcV_TjD?17W1MHQ0q#1%Wi=6RKZGR@~!p(}qv)Al^^*`On#h?#NelpCR4Jh2Dk z+WcakqCJD?75Rn7x3IY(+$*vUOwyH+c0J-u_FS0G83W)>2jkVC z{8~iw+1^MF@#Hi#8;t(aq}rcM7yK$Xx^kKsfinksb>EV?W$H#Cyi z4;VTl!kCKGH8ekKB!QP5wNuD6V(OM`^Ly8R?_)e2p7NEA4(9kf1U^d^JYHc_EQaj1 z#McOFH1NMp&skOyDCHMj$!3P<`gkr9_GM3{7qFEU$fekC0({w7!L7lw&Nmx7+mhoi zBcO>;kx<3Z8&mv?@+`m?!S9o+6f=NvJ<7uPgf||Gek!AmJ<3}C^c09kdlaFQ-eQVd zTo-2&L|o!xV=ShqpDaQ{#74%`)(f9qzp8!ZRh9R@DGLqX-|4GwOIv+qodnWMFSy^i z{8K&YVdg{1Pqt){P_rQ)j+`EP;}BGGa)<4O1QN?Jv{V1xYjoscDvPAXw45&5TtxTM z!uHe3Gd?VIDZyVY8nNMYFbT%Wm?zfBs@wqPYgBPJHRZbD9L-&|Vl_<0L*0J=>RfaMd zZh8Nm|k~>7_)$bG;s2qlU@_iH6V|6OU zJzQ}&H^J){*LMw6*H2~Qadj?gP^skDrO8#`WOKmxj<69x$)G^X%c6l_SdjKPVuOP_ z8)5BK_B9Bs-UYtK@FoHAbqrgZp%+V6eX+r592gN^21B;6P+16(FuH<2m2hbMHlIkI z1})$>+&~5mLb}fgIsH2iHcU`f67-`Wg~ z2{*}%@=S?=qz>~l96rmSJ#(Z{a}F<$!pSi@E{-Pfi8Y;5L?MW9I!I@3>4fk{Xx{4= z_R|mCL4l=rq!YWA+zHr^jRrqgq$9&3mU?p?!hEOwe?s8I`92&>hrW9F_yX}6Vm4k) zw~0O)X*EX_wW-f`Rw_F2z5Wk)@bI1nAP#;htgH)ZD%9}Jn*e4Z2+X2#kI`_)d{@Ol_LY)Ee2!dp z7OcYt{h2ZxtY4owQpai?0O_#@i0EfzN|dTicvf}IO4*izbQQGnNOA(yoHvxl4)1yC zAN~7x-WbE6?doOm_en%FRwh@e&_c*A78owM;KyKOBZAS-R><7WW>Lu!QlB|8vClxN z3cVc@9rNF?pG~?F`;r=rgiTRW{lLLcaNN*`g#u3@@(|8NQskw@nwIZm?;3{~X)3@= z?NGoicZJ6NDZdvMv6D)WY%zykV%2ixAb{^$Z(zU@F}lxEN#5ptMejk+Ge>gcy)t_k zH;3PzV=EN&Yx%5f2;YyrpJ5h=+SAd1V$$x^p4$iHS4E+@=6h{Tz9z(Ta|3UBa?P$`i+Kf^sf6;7m6ng=mU`Ovy54x?%-9!mb zh~ck|wu#-BO6Y8Z@o8->YkoRXtxtkb7t*deE)TbDGJBM%uT!DE4zJG?B)rgmlniV= zRva2<8XHg%ZfRx4%WI%oB=#t_+%XU*lvxDU1iBrGGDM}t>lk`eNq#+yZhrtLFiU~! z#^gJz4AjeJ^M%%z@QOJ8D=q0)S=-nPXUuNidHufgmAQiFVxpg65j+kX_BX6LE$5V3 z{pj5^irCpL2Bd5xvkulPjNK;{^YZx$-j(l729%kfak_4L4NiD5ro7;|q4Tde93gox zo4>YD;K;pQm@}lG{QFPhC z`?oDFm46^-9!~qM&muIkIG%l`nd$xT-!P`zvGBu|LuXQ2sljw{S=C6+ofUDZlt(glXA6%(d!~VKlHB%`HMUNh4V%+Z-G_X z_9C(yFjb$$`ja=i%)UeLZZa@1^8$gc`hdNl1Rg)n&~&rWQpmdWer`YGsrhwH&gKQw zV1=hDH!%nNG28wMW&3ODSX%DCh9m=*-M5$YXxw;PPFLF*%k^1(gRaIeE*hzBY- z^0(f-V$c?`4%2aqIgjllv-bUnLvawcUWzYB6D3GE3AyW#n;N+0EWiS_l-&0`y%Rup zGZ-iMeKs%n%y~iOF^9s@SI{Xay1Wtk3}RTt@>Mb&ozElAsBsVWdQlK2A&L^Ke-1bN9n<2*UbSD@Px-CT!c6qecHhpnTUpS%;G{@_? zP(Egl-N_X8eb5`Ri9Z!U3>3MxJf(>iSZ|-#t(V;6MD!3rB#Tuea8m z6-?0NpPH+GZ6J>iL4M$LnbIL{5Hm64-}U@4i7A8{9t3QwFP1o7M62-+UO;$(jyD8M zJiZreGEe1ru+^YWg+9H#y={3!Vk?ZU+5?s(ZofU;KRD_hxIiZKW# z8Hz&c-Qoe7cNO_;A%dFGSvJ6{P$GtGqmNe1;9@#ax<}9i?UWPX`Z02vdtK+S+jL&- zbq~8Q8b2JKe)s;vsq@3$$;sZ)>ES`wd4J;Uzdw3?czXE$2>$xk**p4&^S8sJ*DoAD zV5Y~rF@YZ8fDK6nX*IeW7fMu~9A=R5IOsu(MpsibAzq>5FCABmfjJoDK3(N%(B*J0 zxiigaHU98O&+tC}c_7wpwy$bSd)Qj)SnI5%4FZ#n8(%8vMSM;5?MKsE(mB@#CP^%D5fz3nbSRf{dbr1t$UapDDOIZ$9wzVkpv)7ng(A+rlLU0gjp^Nh5f_! z{%eo`;eg$D1g#q_xhNYU{lCW_gR!unG84$NXASx9T@LZdbh}5JM4$0vn*XFj2UjCd zrE008X#Q`nbu#mRV`Dl07x7@s@Chg~%nO|jI@?)XpqoCeWGp&4&PnFjBVq-?M!-kU zut9j?5D*(J;E|6&M>jrm+)m%)BKXW1!A6?4dn>TaUIuq!gYie$V7`ZaCT%{(j3it` zQGbQyvGuxphKEsjqO;Fpv46kc1r;EQ8?CEg(x!hI`Hj}aFHxKR6)&!@+W3$7Grk$M z(}ECxayllpDQU2-|Zc}Ie7d2O>5YHY?)=6|LFl&58PBc@ zSF zI+Tn^DDs$Ue275??V64O41QQSh|TD!gLZsgwac$tFgMOM*fw5_br;7nr`&ZfA;_U{I~epTi@E=ZLZS!0&G@} z4v*gKIwxuC(*Qgxc+i*D0~vPl4W{|@X*ID(jKT@OcFN+!Fq{dHu91(;#Jh#G@$zoM zpr7sSm|3Y(q$&BfR;%%6G?Y1yY)+s(6rqy3oN^Z_cas2d zA6yw;RK4wzgQ!_!O{|t;S-sNapw7FzB>rI^@QjY?Pri@wPTD)rym1c;Uzu-f@>#Sz z&EDdqG1&9gUV86_L%AoJBV6?y`K{wnp?JVe1=3*$lu1NfqGCY*ZT=~4|0&-6Q%p09 zd&tSWk6gbhl?Zc_CS5W&wJNk`Bxl8@9(sxP3gi;~!%WoiI+R|1$-Kc*@&!-ozoA)b zPUPGF{NMk_=^pHVI5|B1hm(km_nAN|v2*Q#DniOm%g8j-9l=I5!o>LsA=>Zhos?W-JG{1BNIsK+1m z(w&GgrFU%X!m?Z32=RfTdWQAn^F#&Z+?mFg{)#RVhGdpH6X>84j^BH&RGi%?C4h|7*YuF(SRrPiXccP*DEzE&=g z()x*;A0=fu9ZDyA_`rnmL4E`16j$|V)NzV)<%30!0gG_Btd|aq)_46^y|ke*M$Idx z7P}R(p&Bu6O_^r``3~b3%vny3$xPE<7zMXlgoiO{?g@w&+D=)49f>wznF@@9;ditR z=Q&7IXIXiv2or5C1WG0;nK0pTMA$Z%!D039eh@CC)(uG+p87F0nCaVt3RK)_o2b9zev z5ltIls46zpvanHBfGuG}swBP7wFHRxLRn7Sc6zQ^7QT5Q%Oi*uieQEtt36J@Be)hy zz%6kiXcHnPoTq%#A5GDKWlvrcPk87+xFk+SyrfQ(Y2ZsBfBVu)Hv}$rPGjw~qdgZb zyI&YI68)Y5a$hFJ!Sz#tCAeDNL6fL#B3XfiQnM`wmn#@#gCY93fyJO_k))OI{2KW; z@NT+KsL@ldETywJ`T-?KT>-r#7y${|q{nTzG^DYczcv0gV#Jr~AYY~d=_Tp)@Hz@di zYUS7Fsnh@JxK;}&fNSo54o(hFZ2j*+_i&^CUB`1*^uG;b=CQN=RWdI5ol)7#$0|p; z_(NIfRUL=z=zZ)^8qFSUFKTn%xvJ({I3)H9Qi@b60NoDFGbD)Rfca-7hBD3|lrcAV z2IVy4@Chia_=|1*A^&SK9JC|7dupxszyHk!HOc?C1VLRyFQ8|~m`{hXX)`FJ?H)w? zfxmDDx8hggw(tzgGq76{=9b=-n4q|garLA&!Kl>|y|vXbnZSaA50MlA2+@Ho9T|dy z+@h>{qE1+$!^wCFq5ec0kUB2yV_;K8)V7_nu*MoP5gD`(qkb6+!)&%smrTgH+yzXP zsQvzo$0UShhrb%F3@m9;?h?ZeHYI2QhN9;_>zz?E0%d72t}WdN6n)fWahhwBLa9hZ ztwl*{AMjoP76|*vX%jnixG1N zJ~9jL|26WTPJpvY`R`!QmjCuoHuwK)dG3JxCk4QT(keYzURos|H?tE#sdY^gFL6~y zi2t}vbcFC5@k@+hCIFZIPz-9&J3wxXlG{rj?R_3?3)Aj&%-DWfB!iA{iO1o+A0&)h z?C$gbqW62*Rv>Lj{75Fp^LwTC#gJ&fa*+hRGJe5F#t3!@K~$&hGwK-(~E- z4lL~5(H{HMmJRaZ+07kU&HZ*i*xddfMQiY{)-dZ8O@ZFGdf7iZUIgfalOvDNh^ofOoljHEjS(r}y*>pH3)ZM-PFmmPqyL)sHo;wq6r2nR5 z9q$qlkgD!)UcGwp>NMg?VT{r7@bmH^YY}ZKVxALKM*8Kv z9}P;di|wCtwk@@G*m&{YOhV8`@i55-ULE(Ws?{3#k4N%d#Q5{SYURJ9y^~{0{@Xh| z*x$&1>v-;f{D&Ykj*4D!3EzLVDa&-Q&rj4JTS~2tUe6WlTd5= zCN)%IJHjuFSf~-CngjSV;#L@oxzcf6;8LXW-EAGEqg4H4R8!R!WVYS3? z)?gPmXAlfzVBdB1;Vn`bH@nG;^PkavC1h+RW>fX#Se}x0=p0LFKzWN_XDQEw4y>~f zjxzZL7Y=ydJvfm{_1G4Ufb99cJ8vS47U~wB7Z4D}G#9uSUg5efXMc$_SyHX?zHE|BgG;Fw~!>mA3NX9ns zAjMJC(np*R5%*AmDccxu{K}r1^PkM!xX=l(=KR;)KXA@}CkK0*^WQq2J8=Fp^#8%6 zpg#6&WER!;igRWStz^|!S6c2Q;b<}TVa(fW$Htcr$|99^9x9_75pT}?<%kRmo43)L zA)Rigzu;-cC}J8&>;2mec>K2Ywna}(mNjX0WfTVjpoK(<&Hl2kr-uJ?zo!KifHnT# zdwU1A|M%X}{wDs*TAn+=|0(+?++U|Ch#}sp$~1u2oc-()f~CrgG2)cC6LG5+?qYbs z&X5q>0X2yt{mOVOB;MG!p9pHl|LE;RdE{1jmFsOp7U6%hrg_F0?kHPrp<;GVPL15X zRQk!PokRhqJy5sx*1^Au4{h3<^ih=!8VxB2R=vfgHKzbK+M0?Z2WuyHc%Tvq(#c9X_#pi)wOs7Xp zgpS4Vzm`?jo;d=k0he0iV*4;G`|F`5}2!j_s%Jv z;LUZwEvRT74!9mj6Vx_`)-Fj8Vy)nMulw#!_#0T7`ImT;C2c@ORB`0J(jVLEV|z&x zgqXgNjJV(s4@SPjWK2B+J-uoDGJT781g2+=E9y0A$MMebE2#b90gNxSSrd+`@ygK}m)URLhRVv|#1Gi^&Z<*H(>s##GGaIksS|JFO=JxF+u#DNE zX#h#pg8S5*XBFsQjDIbA3Vz*OkjwotFf_y6We&mS`D?i3gZH~kz$sFG@rRz3zo|Z! zREaj%Qdd9Q!DRLWi0ViQ`uO2BlhkXaeBmK|nZ6(Mn`2MjfbX>T!ETujS$7bk`FwZ( z;Nqy;PY=FI_P#nk?j5BUd*>Iu!~OGvi<5(+-pN6C@8tYsZ*Nxs7+>a}JwNPro*y1Y zYDn1y@BTXoJQ)4+@7sUd>-_eoFWcL1WAW<`^k)aZKkocr()rih&QD*8Z~8_1i#F8w zvQfFMe-d072M0+fa6OGf4ir=}dOya0TB>UN9cB;Ee`X{4=Sv)Kwe0UXJ5`GIpISMw zc~<_^=>IU^?uzw)2Zx9Iw*K#UZ)5+rmglbM|6X4e`He{LY$dkf0misMpt4LGE^=-b z@CW)*U+1Y}{O%j*fM>;OkWGr_k2hr>Gp&_`YT-;;3c~KHAe<-nmPaHJd_kni(8<4D zNa(QC;mtE6E(+YU8k558{NbRl;wfJ{k=+}6B_)$c){LX)+Q(9q7MUGo_AZ`udQ$ zfvQ(QdG6L%7A3E^tQtQne-If^3@HX-6gUNC@Y|70a&;C7V|UCb0qao&Qsco%WU4H^ ze(mfv=<~!oZ-qM*Q6kNk42;NrzE=@YQ?xl1F0(Ys%1)FI`Vm_J$ik9AiJ36YdHf-n zbtb))Z{LZ08tVRbBI3VAy+M)<%cX$*?Xn&B0b)_JBAo&<>}TG{EyFLJ%}DCB>5diRwwcxA zDr3q9x+J$qA~Sg?q^!hq=)0iW=jmnAyA3zOj+mF>M)<@i6(rbSY{ymlG|zq5N`l)4 z@9fk~oeev+NvC0Jzt6?F+qfUH?}qG)J6}s?BN)2Y*A<@X9oyzmR9#-j39W0XDy9OVhz_-N;;5GwAI655HtnD_+UZdXbyD{6_uY|W~3hAV%YW1i=)u< ztso7R?`tJ!&GdAEDc!aR%Z-FwNh0<_r^q!v_@}sHgU}Hg8K1||F+X=Uoby|v&Xl}hQIa5x))tH#N8;wz>mvvhcsa`cg;T=7jhaN&qV4Jada#i_pkq&3A!x^Ek zR&#uab7ciK8j_#L2?~dR5?B_^Q;LblWG3foaBV=*t8#hBqH2zVtwvsotyVGM@K ziWE+w4;vgQ9;ze79LKI150d0|M&)heI$?2CJlOl{XwP$!Gdp&VZGslVd~~fVgUSdl zWK`17kRe=fhBNdY7DUzjUN&97Uw`UdS`I^5TSDO}s7Y``nU2!;V;&axJZ0m{^1|E* z2%Z#Wl6Q!>R{|U1Ex?xg3=*mDqK?hGmHu;W(Chqv_(l3E{eSlNj*lGsZ}xL@|Fe$g zuH66p5fzt{R7N!5cHb2`F-e&`jDt{xq4P9b&}ZU?!>0^0%P^sZt7gz8etL0{y~pSR z>Zg4L@seCPSWR`JZ;Tha>h|Hq7&c(G-(#pR^!FBF)@2i7+$zpE$-yhaCNPl#J}@?1 zATu<=m!S?(`cO+{6FW(W7&9dvGcp83Et+&!Ddtxe+At+ai8i?;Ycl{Swni zGcn>k8-Tci5MZqE(*`ocq)5|UNOQ4kCWZY>j-L06ICZ9j-r{IH@?5f09t@71dDu>V zlkzRM1lMV{hgldcL1CcNY}i*fPch~rxwnCtlAmS5XXURf?Hj?7Am}n?P^i*3%19uN zwH1%j0SXRYLr1fvx{TO^Afsujcz8THAfX7-bk^Fuz%4|V(uoux|QAM=CeuoSA+iPf@ap+IF5 ziR=YfTl|hU_ItB6n_F$64>4h`XW{Exw>L z98^AEQ3q27X-*0-^Dkngmx6Bla5(MfLxnir44-QDBxC)J5G6QB{Y$MhlM#5pG?A=S?npOJx_%LNMOMS=K z5$zcu*Mvas@gT)&&zGQN37* z$WMvQ;ely&!4OK?YzZjX(J*mj5{Sp?K>$w!#do!BHb~egG%G42VYwj|)04N|C=d6aZS z2mkQU*kw6Dx?>od2Cr6#c7-=3UT!JKc+@^AY0F*AdaXB2Vs-;|k@|xyDjOF2Jt@y* zxL6Ed6&5rg2~WftoFf`tY;j>vShjYs+cyhDfb))_vUff`Pa$L)Xe1 z%kyqPkKu_n+Li%E+Za!jL|~>J(Ss35A1OcMA%)%tDDt2gM2*rPqVwa!e%iCL0?C6H zUi9g#ZKOhcs^$0N!u zJjY(gf~|^DlpzzBoL>^AWn1O@*P%S7O7f5f`A8-$33Q9V?~CW?7_FJ$pn2_Fz6I{c zY!_cWcaCv7H~RMjV;RUT%h4bVkC)@wY7cArV{y!LD2dUh8sIiWb1$N8*jOJA##c$y z1;a7;^ZV&AEwUba5pDCZy;i9YI##bAt&Cc+M%l|UXX^@A0awUZ{LcOHVDKDsLk}1) zd;23n5ymER+X0d_ z9FsGoEe8NYAfTTcrR*3o1G9A0id*{$mKXzy0&sZGmdy|DK#2+y37N``wNG&sv_lV*ewg^fdYo zsO;(2QDM9KmMYq#?Ef>xit$j$RCzvAjb#6*|!nmu!Yo>JwmIcze#A zvc708nNF@^$C|=uQk>`O+Y%+M`bp8dVke*;I5{){h!nS5Vz9m}Qm~e(ZwIVOpn}zg zR0x1$CRG<`K_D*3rGjG>2&GG3_FGQA9hcu#ol; z$CH*4N8s zJ?$CFF;AF7UvmwcS=O|R-9`>#hIv=gihr&f&Vp*C0p@9Puh1;W0aak!$PtF>RsRi) zFGJm;_@?Z1mNjMwb@B+Hgnce6-0|gR3}KOVMt7uH7FM9%l4e%wTw(prFtb|BzL&GA z!LON7$|Zd>M&H*ek!H~U&|P>Y9Z-$_=kVm%_WwHUZt_2^=ecwGAE6Fu7}76Ajs!UY zc~HM_1iWj0_q0%WKGIuvxxH0{+Vcp%A(__QuX zlN3Je3PXx&aX?X-G-#|`?e${{$G@~yl(kFJ$SX}H2=a5YXx6)EEb*+)6+#t7Yi2S& z8aFfpBVUl&F2WaswQI+@4W2VIuU>H>MVe#rIsx?=lc!rP8hZ=UUCUB!sc<824D~8% zRrN(P(Ge=`AS?&5l^I3YaGvmpo^LDz0HX8)^{ujXq-oTho;KkymMjd!CDa*h(pOGY%@`KgSosNwzd6B zl2s37wz;2A(Hd@NK~<}6_>L=ErAm&^_e?kky%w*wQpMbAS05cj&79k-%r~9pmrogV zY8Lyid~}(w!2avt=)l?k_YV#?`~NzgU;X|ssdhbAtkBuh^QGJlG=O`as~^$q{k~!q zgP397=e=SLw6f?Gt7!I~?E1AAaVD15a}kG2?d#tZ>E(Z~V7)cJjUx+cj@!7E-h5ju zWNkO!tDTy~ufOG#t^WG^Q*ZCJ&kXY4I4>ucMGAW7*+jq^`S19!>)ijcDjWH49namC z|N3bUQba#+G#ko#4Sz+`{k1QoVk-d~FJW2xCi;JDgaBAx*i{z75o)@)|!qOjQs zp!t-9C3BJIN8OVwK%jOfHTJB*{ZM-SItfh(EPk23Z4Rem&kXi4Yr{)EI*=jGNHOyr zASxPK@C-pG>wX*~B*djO$)0fvKqhzESg9mSxZ4us*v2DT3HQ)Ne^4Gl^&A8*iQT1H zLAMcIWBR8Yo!%A9EcI0-M?undb|HU{E8ry5nQ1;&LR+}kRO}tk^0*sC^o*R&ZndVV zCG$Jl+jxGK=k=Pq$K(0Z?qDoF=O{8!xe#MBr~@L5YAjdM718t8c~$b{uZP z$dB7m%ihoJ4)`jVH<*zPZNLRflR%(*x?)uz$Ga~=)Z~91B;K<9vEx7P z9c}#o*Yn%~{SRl3f1aMF#VCa=NhHt6Lcv{YKF#|&WF$Os}3~HEAh7`)9X}l zJ*N-3hGl3v5p6iOx@{X@L}w(JfV>)4VZ4&@IDV&wA7}ZlNNO97FH5;o>na@#Bo;z-Mlf}4=4 zH4J5(_e0#|!H;Rw&qog@(M>YKL~MgRA4fUEOT-%!Y)wSY0}C=H?hU2)uY$8=P^M7^ zav`gG8x68|>EM<-wPzx8-FcBF??_VPawjVJ)UjXbq!&NIc}69E^1 zbW>!Ll!}{@=>n5eSz&p}oR=KJoD9=R0xtUkbz)}1uJk?(FN2*%xR#@BjEnCuKE33N z6cVJFYrbVIcHa{JfSDEknhXbCekrUdDi~>7cDX2%pPgB^F&I6-zEn;%E7!T}67YZa zQzQQ;7i<}4lPq223{WlqGs*w>z>@#Fha3Cv^*nb*{^yx2AbDth0YNkR>0?fNw|p&LHJ@bCYFfYDV4fj|#X1@shYQKbE+Q?La; zW25R%2kh6g%TcagfAc==O#yzc+v1f{;F!Yr4fn-w-jBfy5R^Ugy|6XsjFlw5<4*xW z48HR|h6p`5!~C-mdCiA;O5PY@qxfgDCFsYN*X-&l%Tg5Gq(Bp=QP(59sd>BtVPcVa zyrKuYmCWOnI@ZZpTFs{sS~wH3+Uv9!3WRs=(_B`}-U1V+<_s1)M9#w1vXU9Jw6l<( z4?2~75|vy-u0aFz%M3ueRZcInw90G5HOfE>_mg~_5Aw@fka-)Uo>bBkK;I#j8>82# zDr9Z?l2MmBnw!$#RbEcGxL7kxWc2JLCCO|TezVPFcCup?Ew<0fjPnuOniR>^9^{z( zTqNJo1B!N8Fu(W^E9$@0!$GECHDZ4jwZ4}bgY=S6NrWKbkosf4)pk#u4Q7)dNaVqyz&q<^>V!Lgfqf{*iZ=RY?8MC&7FxmuQN;csXG~3Y_4z@sw%)hgnP)NRw zr(d~QXl71M!>7m_oU@=;F%kJR6I(S$*$8t?j=+~|g!zq)U4a<;%Pb9w3fgv2b(`a! z1XZQe2-BEpQgXT@;)K1eZLzIlhlJE~3CV_Ni|?nnw~Bh+-OuS*-s#vBpbDs^XYAv^ z41H&AEJ?dm+Ou#E{+*;&bz$-WJ+)uiEj!~iUJy>^J^s(K!_X3KS5`Ep*uQA8&Rtwa zTfc^Nok^ECF{rqZ4zkxWjG)ZPX3QOM867-csnX^1~ zB@tNaw3VOw%4+j0@u|80Wg-{cV=QI>TIc_9;>3U5JK4m4U(a)A?tlL%@{L62N$(x7 z8rE(h&7Dd}iZD(l8CZegZkRC0h{@>Vm(L<$c~GDLO~l{~Z=_=Jua4kzhl zTEqyJ6#6>RQ$U=Z&PINgZX?PJQ$$0KDHX$j z(w;GSm5-#Qsvz=+TxPUKKhixFl2FSAv(dzCt*yUZ4Dw`hd}x1{-Qi5R9ew}|dql0E z?1TOV46I&SOgh|#6I@|LI;dpC+5_g!GtRShDH8EK zCxNX>TBw_w!!4JB5|DD<*FDvvN9ynMnH`+D!d>ouqO#?lp zOla=L^iQDal)oO~H zXQ0WT>?FM*7!(phQH~~a%W5_Izht}FfJxQ!N&$8LANvP~w*P1M;CQqDuj9EZ`+sEf zHegBp6g=Z0Y9Av<#>$4UIlW-pRe6U8h^Fz2Iu~&|tB;y3j zz}bljA8(}W@u#H3%yQEL>gTdHTE=&E8?_5p@_RVIv;CNeKeF*BVyD&ncg`8PL{=Pq zeXBIDd}ALbY^P!W=f0BojtL5*DCdEnM8y(q%{8K<8*o${=GUoe4$Q)LFN=ISmI*si zQ0wQY>$%B`cTql_NSWk3f6t{h?(zs*GE8gdx*MH6dD46yxjy5}L;w-tdg_(-{NdxZ zD>YZ@H-bRvzLS13W+RimfT~G@B_|e1XnWfUjmd-AwaqTunuyw__5k)LIPCiavzl;g zLPQ81Y_MtP#_DK^zpPf1=_H3nXIY)n;+-a2W9;D-r$5$`UI`6|kjXZhj4g`98J9b$ zJ#&wue8|>^{knBg{t-H4l7BHBJuZ1h82(#)ppKwI_ep4g{d}H{rti5cb7>CpnG(Q9 zJYQUtpa0A_;0{@VtN^$q$fX2HLfO)nV)uF4OQt23!XKr|?Ff!EAaIoPjoSp<9A$lc zUpj?+U=|wqI00XNLSry}1NF|#Y7AAdvw-UiFjTdMrAtZkHRkW2EgbHH^GBEDBB%@@ zOL#mXU4y*d&&ndbOo~24mA*+PjI10*3Wus_Z|5GRbQ#of|C18#J$Nv67*Y^K7>F#gj|8+cfMgCXfKj@5- z0^*Sn^`21eRpe_UWC{o8H#9JXs36QB#1B)1|FfCe#KxFF?1Z&Y> zGvGv>m$fj8f}kdU_f=`1ndbrb=lr}OkWjJ$G`*Y_0jF2wqvjyV(kln6pD<~Yar36s zrps4=#`-H4hfKM)Hw8^sST$^H6Pu(L(?N5*(Hg6UPU^UuL8esb>s*|#zJB}!&anBk z=%q~|EzXl(6`1_{baIu;&{@j@_sOfLO@Ztc{ptzM*p9{IC)}Tw1RmR=MA(joK(hL> zLN%=H#F>?oVyVdd6HE`BR|nC{ydOQDPV%ys4B(cr5%5>z#0&Kn5yyNX!NQWFGH0_9 zAV7ob*X#YoM*Rlv-+J?o`FiPsbFGu%o|Bo{eM3A>p*RgPm-W$Bnk&8>^$Sx`KO^r)9xcQph;Ni8 zeYiW~mLWkHqK_Zd6%4ivs(BiRotDi;(eK#6#4Qd$z`6zVqO`aG6L7%DMLq>PsI(|@ zkP^OljhQ9+Rgky=K+S)`1gZS;VHpjxQ8t{8e6#Vz7Y$D-oO_o9R^#%UV5ar&?d!ko zb$VjsAZv&mGYJBM9HTAHyCB%xokKWRbC9%*i0%qQU8GO}~g&i6){_-3%pPFs@iY zR^bo8ghsvS_yB)q!?WoHggtL{LWjI3MGE2y4*-r}0?d-jCL(0z`K<<>yJyu6Km_$P zHf@ZS-2s0mV<7yu+(JHJhBMeKT3@!J)<0VF4=PcJyO<6Jx6vPoNKgyi{9WnKKZ{wGE<3Tw#OUGsuRSFhZWd=DRIW_4NBQD2tZR4&0{NZnZPdfj4`?mk~ZRe-aaJ8sfOWE9XE*){bK;Qd+;!uR$Etz%eO+`V`CDD~ zl4ATNzY(@)ntHj*D$jbKbM&w&#)Hog4AeTh5Q$X`2OdTn=V*?~CD0~2J+|Z7?p+^R z%0=Dc zVQyr3R8em|NM&qo0PMYccN;g7C_aDdQ{YEsuBEwZl9FY6hG(=~-9S~LP$(1%g+fdipHjqr4SUBkNO^Yw z=i#3=`|0=l{nv+w@^8Q2&;LEx-#_@%{^6Vc>%;z=*RNjvslWg7u>bl`pub@{NNl+eWhoIno8kfALA@ElGN8-U-o006)P5uBp| zAk1Sk6o&GlE;IlE@$V4rV$vI99>F>8nR0{u11~^h3?py% zwoqZv@9w|q_H7)8hov}vZKOFXIoN;G?YmI_wgo_leH1all%p8>Gvpm~`&%%^KOxF6 zi3Z^QU<<}^`cpyK3Q&xq07X7VY_MhM^2vNoqAdU@iV4G9@}Ky4(Cbk&#f;O1x8E&L z>5K8owgBL8Dm%{u`i+-Dyf*|KbU$=IYz2t<6vtdX8mdN){`(y4=$_WfXpk)KxMdU%CnGgyn3Nakx5c36qQ6LKe9C0+Im@m5E==khJ z5Eh3RMtl!o4pcnICy#3 zjic$-9a=mP8p!^$Br5x{<)sYT@_>3rQ5<4l@^8xnewZ*J;&X^24snF2B@N$E0t27S z_rO`?cfnB@f@?gTaR#mtL-Za6-7NqT8fFcfVm?d8vRy`kwQv*!U-hX(*I2v9DMCF# zWXZtTN=0k`b+_Mrz4f^@+4EK6rabLk*sOw z;I)|d@DwpV0Bi;iUcDZ?noQpO_2rwt?Z1L(((k{1GY(!J9K6{_=wQ6x_y6{{{+l<4 z{{CzKaQu3L`tXha3XNZ*S1(_^?E9$y*R3gv5QQ8C1JF6>9~^r7eQ*Eu&HlmQpg-vM zyRY6H{_WM9!~X04?L6zH+K)B z=ZGW9#M@zrAVYf~A{+regBUtIKrdLf~3!Hbd}#*}w!$zLvU|0VoY{NGVjv0P0S=>RTS3Zi}$|=L-5o z9bTUvAKjdu3}koJV8;>T2nMDhB}Fv%S@DszjFT(A0#R9kE=>3={XK+?2_qk*UqnNN zP<2d^4uHWNM}m9`c8tg}a9axKcDEe?0-6v5AUaPXg}_f3Cv$~}6W~(>IVz$wRokM9 zE8Ex-VZ3D9(p6)zZRx8Tbf!#Obm#+S97j_n@d_!K0Q1-kI;}9IAZH6=pCYN*V91J; zxR7smyX^>42!BKi)|H}T`2HhWq}1J^1;EiJh^)bOyW0g+pV+(0|e@+wa2T=6lg$a)kPOPK*}J#tB~ zUS#bw=mE%!wN+e^Q1z~yQ)!X9juvJR_EA^5t*jSzhv_mvlyQ2js&cB)wp3GPsf^8( zTaC7+wAAQ0Ci56_Y z%;BB1{pM-QLIfT#pJh}7iWsNZ=P2;*55Va7cmODxlY0bc5*@L34ADLH^(`eyES-)M z>wAb4Wrmr?-ozBZFeDEsNKK`)6V?@y!6Y4OeKin+h=3g!$Ke8C(+Z9RMFMXmG=laJ ziZPf{zPl&*?<24iBPubAs#*5{7(T!S+uZ{JW^f#$0PF-rSlGZ+$O`YK{j_C9Q?VZu zt&TB&EIk9RF~4P5dw0ngX|Ff9GIqQrg4#16YFS04EN4K3?no<-I29b>L?9JvFkECS z1m&pcO3S%BNy1PXPaMrf?;x-v*AvFgD)~i`TzN%(x083~QuUgX~(Bxs| zQ0WdoA%a+&VaHk$)EfcGDLODUyCLymXhB3DIE5#2cz`3Zno!mcr%-$JnP`U^KWdEF z?Abv=T%nLju^>sCB3WjEmJ1218Ilj@aEw9*j)DM$>MJK=a9AT&$U4%_XxS*M;sAX7 zRDgMuZB$9X*2Yy^=oScr;xrNi1q2vQBf>cL!JGsLXjx$#ax_UohPZ%|mAW7Sau<^B zyDo~<0IYz&Bgl;@86&;q9iwc}3MC`Wa5A5Z@1}qjwgAb&D+jJdGXN79GE_&90d}na za*_$JAjT(VC!(zJ*>oW)5A?1Y3lWOzkU~0T;7I(l5;{@c;KI=aN0=`hWn^1PReI?= z<(jn1sug{vi2y~DY%^t+jE!g0C-WXH+56~mIAuLcL?y&-x}N`so&W~m`th+0|AvqFX97#jrg`hv{ ze9XWLq2U>Su?JAdWFutpv&kjlLy8!R_zMNj88LV1Y}Li27Z9KId=mL6=JJak=sgui z(A$nF@&uDn;HgJuyk~tm05(pA@+wxl&*1_v(^l;#6Ej=h!=+tjrjO?dIr5AI^c4Kl4{#obW)g{4 z@#)27?!1xeGfopEx_3bm@F0aU#COuY!VuC{0FH!Bs7yx^fsjmD=4BfvQ!}eoqV+84xI; zXBMMRic2g~^9Z-h8aq=xYIQEBdK0H1Rw>|!F+7a`Oc&R(=c}BQU2w%`h(2J3_T-9( zBLICLvCIS+heA0SM<8aho@;B^1Pr8@6-iyui`oTe(?|w);7PfsP3WkYl6xGW)FE8q zhRBiB+OKNB=3E&07B1&IU9n}gEn3Q(nvQGN<&pzw`9%AixCG`gK;rp~y z6`pDl$s=2hhb5@W+rBApakpv%U@z}A-;;*^Z52mu(@}|ayG8LB%K@6iU_!}U4N^r~ z7u+a!i6C<>_w!)Lh_SSpWa=GSfZH&DajIKy1v}4Aj)sDJdO6WmWx8V=1@GfF^yG%6UU;9AdZaK*X6)cPKWh=B`|MBMSSOP9w)zSR4Vf^`WK41*Cs(LJUll3vGqNU<0- z*2LjJx2z~b1odZ_BcCS}+01sH@7yvDqX32^LbrfWaBDU0R(fIV@&k|+7pSf+?P-iu z6@9q9?J80Wi0nNZufSrVb*&QXMVBoie@00}5_U}HbIhe3YCV{@<1xK z8N5dh#Wp~2NjMsSW9f7fI&dndRWVb+NDvAGgT;)$VNoEJhTR#Thud%89ebm{UmpK(eRX;DZsZ+bU0j^qq$4fg1q09_lZjBV z4AS2fBweOZSGC~+M;C}OI2D>) zeUVgz=pKb~#XEx(Wgc#6la?^)97kwK(FA`0H)j7ph_!@;BLrgga>wlM;ECK@VTkXR z!Mnk8^luU&kbW_yt$-*-IOA|0GqvUmu-rPcBqej|y3*i0cY8K_m9f=?!sPj!sfPj^ z2LMq@sKSu3MMJWhX7(_o{aBc0G()5*ne&Is3Of2<6wTs+RPls{;7luKwK*ppH4sb^ zF;VS-SUC`d$?`(iz%SBRp&W7zcqN#5 z#*4+_$>ox~7nOp@VSfQGj)LXk;wWeVH(jEv06u=Glp574a7_p|OhZ$%S%HMzln!Xg zZd?$0MZc&ljYlOY;-UB}jMT z)1l)4Er9lCXpW5URW6>{t$`>yYgEQZt`-_hjbY3{fMSY#$Wc(jIUX{jMJpL+l>sA^ z`hhvoD|>KrJ_0E6=_1z6*b?;yWjl#VwWWcS)U4NIdoECFNdHg+#{7`7Z~cfLOdYp( zou{gI%p=KF{ey>k1zEG@X(j@to8MiZs}U;$Dzs?GC|6uruAlcp#ht%0qj7hM=cBKVJ5)MK<$7bhWtT1MCFh_Gj7jo|Y zZjpxHsI4}+UQA8+0PIM=l?uvP=x)uH5qj~(b`=_M+Y8WrkIm+CJm0=Pytu8~6;LQW z#g#?M=N@J(L9BPU|GKVrMy35#*M0f=)uG@fo6S|2wYwZoALnf?G>`d0h|9fvEhiPz zsJy>Nak>y)P;m2S$Qb@NI>%r|Et46>R-F}cXiO%zT%hNlq<6pe^-JP|Wbwo~c=s=_8THKHS_ z$B%J56Z-1s8Il5T9g|Rs{smLWWI3R!qR2-KP^fCmpz1)iTzHY~+7{82NQ@xJZb{D1 z4of_eGss=V0UOy3eYNF3G?_U z19?keaJyZ{vanjpnx86^pA{w?9E`djz4nz!bN&q%Oy2ynE#p?e%7cgGI*~r99>gRlpZg<+)3%zMt5>I~^!{esrXs?rNP3N@m%59JW8Be|07|M2J zt&o$$whpx@5IO+Ac_!#Wx|pTSC+$9(XpsXzdh60RvYdQUEHhuEqB37FxjNbZ+d+5# z^$8q?PH3DO5(cXv(|{kf1SRTM087rQ+17wYqc|>@CEkl+Z?LC=i-8 z?JUAP7&q#Jbz+QEFNTN#y;YH|cEF*cTm@FyAhV-0 zbjnJOnB?GQr2Y&;VYFvqeLA=!G}EDR4p~%i3L^&9EEtlhjXn>!nG}^NhJh0-0Seh5 z6Vp)r(m*g&22NQ6LCsh+i2&q4#yWv9A?EyuvD}sIuBxQJ0hlk|3!LxOn=YG~JVf%X z`1}9$#yIM+ne!C+GXgp%1TZ3O6dX;v9p{vdMg%^7YQ=_Z;N*r-%SNi9b5lIyd|Qt1 zwuIOebL-C2R<{1rhF-8E3=T1OVaxhvC}No7dt?Uk(CCZ<|C}B&)|bl*`+gBB7m6Divc|ph9uPhV*u-V}`=8L{`UWLMQ^kcsUIcQ?ywv z(yTrT{HW9*dCp|bZLd5E*lw3q90dp^x??Fyt5iXeR7V?B^X;}$6UZ}*IuaY`Re?Dd z^@61jLeynC>WvY|`mAK~TmG$$nC+m@*1YgkPyCso9bG zyDVpsMpeyQxsl;;Q(B%5KjpS7;kCH+?;pfM>Sl(3{E|iNsPJ=*Q~UHC9BqKh-P{|g zo$2zxWvU4YuVO<20ILz{h_i6lErTdLjqZPfl<6~jvOkqsSr&Ds&b|dwbV`i*hy>bI zX8A{LEY@iQ<*oSs>S`<1e=^`K*W0#$SOnx)YTyh-k^D}9^Ac3J#8O7IfG;Lr48s!?!o>*r zBnp^ZG-H{cCxkMUkbEj5m+ZPNpvQ<}5*WBc`D-nJ9M2I+xPf@1ey#;_0&$p7bTgxf z%}5x4@6`jq($w4V*;iA8&G!M=S5IB$ z4Zth;G~fOKc&*H$TmuGRzqHw@xR(m7`*t;n&0$zhWZlYb;5Q2G@*rD??eaid3hwfN zTZ(Q8V0k{qe!7(KzIt-m)e5>)a_x7k$*lu@IjOa)Hp6a|*yX^rkk;h@wv^Q6fVPy< z0#Idbr_U-)?)CNSeua$gzgkL2m92E{%>2jNb?ZEo``cM|AeFpAS!5*2wvUV8w9}yq z^|h4kutDT`o>C6A-g`NBuJzuaaTJz zE<}G?ZtE6l(|hBPj9mw*;z7@(!C}4kDH&ZJhui1-ud+GS;qw4=1aHb$fmvJfy-|=E z*}MQxWMB@r`b(bdKOVjlLaXDVb`WFgEL)B#(jylV80ckOE)PKpH1!J@LI#;vz>kvw zILLl0oCEM`|KI|r8|jkfe65$oTKoNj!wX#CojtSyVDeG{otSD9YeAD)Og=&%c=@1- z<-}np#VS}HhBn`(tr?8YDRw-A(G=+vCPE10j`4zVGzU|ebj+6}Od^w((Ak3dJXB(J zCm0i^r=6_xIjWSrrGZYMUp|kQK~Lg1L~|5z7|K?#Jn6~_ARAC_ZdG#xsDIB+Qe7Ss z>$rcuT9wgS!ruWlJ~8EK?z(t z6te6JI4(&fkCHf8Ua$%*y7Y2*Vv6r^h^8|7zX3~DcFDFN7+gi+LQwa8Hr$(ARa$R3 z+|uwE8p2PR57mt{&Dg7IE{ETTvD&@Ch;;)>Vi}SM$OPDL^!}1~A1}*YhJ0bx4Ji?- zw96MUYHpmaBxZSeUNdzDmm{gtm+sI`cQ(^;m1;j=4TStdEw>ifOwjD*TDdq_)v9{T z24ZvFtAWHg(YmJs(nfq;1rn9t-_Pvh#D##uyG?mmbzmkN`eK~2)GDX=oA znK`U@LILbpXTde_U7HY%q=~6#!{Y)S;@L)MI2#_<;n!&~FJLUxhm|BVNY8Lu2@z|t zg;~zbQt7qxW?J9#ZL-8EmDpUq%nCE^ytyq>Nb4Grf~iAfldrMmvw$v#GM^B%3hzYa zp498+a;^cWOX+p8o3AMC4P7Wrlv2!v!N`QGkcb(i!R?Z+0_M+9kc2pzdgxw-)byat z6DG%UL;}>?miE5fC#lsx66;VRPlFCRa>E*H($!T|k-3&~qoSx3Bf}2XnzC(*js-+H-r+Lx? z$E`><*iY4sE%N)F%q;Kr8#+K9{I?guIdXzh?@F`K*EB-s9j$MNE@Yb0lKo|E(LIrc z$az3DWxx37z;2l7!m79gBPFJJb+Z#tGCfZdgRbaCEewr@fvt}!R?Z81^@?y$M~7AQ zWg)$qjK9GAh3t-WO4fBMrz-(?ghJ$VLcwJ!1hw=1>KtvOa%?4ID61r&lQ7LDb7Osf z0(lUNB`*6uBO)c(HbuRtTbS>G$Sg}zGFxx7!En2 z5#$J@ulMAYT%f}^p?9675(dlscUW|#r@Peh0nU?<<2XcKiYZ%QIkZ)_fFnA`k(AT7 z6#8iBY&EA_&gCXqgixTeKWQikOa%~m)xnZOt0+pnZHqXMOk6Rkqr0-TNgP0qMw~*9 zri+3hVoWQ0A;olIX)v?_N=Ycx+dKKR3A!!}6pY-?qICv!RbKie;sS>d3=3~>TOssx z>u?s3=(c8H*-F=T!<0|Pep0ESrDI50Y1=2f-%sC0`ign6eJkn8T=j0oJ{ezc>sa`XLS20hR zJI5J=ax655VV2{c-t;syY;OY>OM#04@zN6@1s0AIwiuHS)>da4hWjEQFZRHTwC;<4 z8JszUY3qk104JEyL`p6AE(xZn`M6Kzd_$ASv(hWt8&1KveEto_U3~?g+CJ*(i%9^i zB2H#DuW+UdaC00TsiO}e0v9;K^JK2Gsr6z+eH3vxmC@MJo|6$G+6hOIJk4unzjAlF z!~s9Ni(rNYerSVb4j1ZDfB=CctKN04OSs*aNs`Be&#YrAxpw7GNf$`EmA7fgWGrcR z+}NBmRSdV>-!|xUN$3Ksk9HrDU=$<2TKky-%c(kDE1MmRLszN_Gx4-HIm=wo`l}af zkl6}Cz0g{zYL#pk%Ccd}wqwM^vPvH5fQEFv^`3cs_016W`n9-Q(u7#~fWe11R z0;t#y17s&pD+pw65^XLK%6Hw%2W^(F+7+i$nE+DymW-cR@>VfGP)TLDh13(2%V7<} z(58)5lcm5fL7W^_XOW`nAM!Snf5}PC+|i-l&_tj05hOc;TXqK02I_cgtjh$gsXMl3 z<@Al{wUVMix`ub(srS;uq&@k?_o@sfLU~Oni8zXal`x!sh(+7va=Z$SbfyVh+1b*g0e)A6#7GaIkkfVoU_u!?1dy5&*` zm3r$0mZaqepd(1COmAd2&g7*GA)c3}NgV!ktoNB+HDuMROQmGtk`cc{$Y;`{7enpw z(z&>el8~iP0AQ9cP6nG)rsBxob?gXoinBE1dkdmjGEPZm$yli{qB3(?)QiQS4tF*{ z^>t7<0s*?WGuEh+Y{J+c{v-#3+3e2S+iFNF0$TEzaYEq~Da%%L-AFyq0Rt5oNTdKe z5+&4ofF?LX!5*-ID*5^k`_X?|b+j+G<8{iabvk?CVTS#gx#69GU?KX%SGl5u?y_8~ zXA_V_nhLOE9gp1=Q<6p%C1JP+B2qvno%{f^Onu7GBe^OxhrY~l+Y~#ZT>y5b@c|Ii zI%T`F<-7 zeyg(4*&et?F!-5bj;cH8ckbknmwexS03uhDZTd){1hUnQMB}t!TRRc`(ZiLNpups5)zDL7x68BW%VdKpYf!9_V6g`3T3Hrr;I5r&u?F_IoXFVXl*=pecuAiN1K$EROcN`wPkAn45IIhGdTA zB&Qi2t6zVmX&oyX^qkxAi^}cj^wOFM9-CloAj@MDiVdWCY(%kozQ>jfdQSSdjL>dA z^J6>kn@#`N4(HlAAeTV1ULwdP5UrmLvK_Jwq=anQs>jO<*^YomP7c|Qh_y3BwjGAg zX(GRvG?DI}z~{*o>CjM-Ffv=weEBIO3+>rV-pDd;w9On@An9|Yk8~H!OXiTQZGA-+ zNmsbr9b0h)EfwjST#I=dv2{kt+9p-!l+5E>A+cl==DO691@x5}6CGCmaWhO7`>QU^ zWR7Xii6%>S+rQvclWy!!bH2&?hCOx0$tLP^X(w~Ue_4qqo7z&$K3UqFFE{_BJG(U{ zq0AXj&uJ)2(ohyercEx&O)++3*(jY>v`k6)IMT6lQp#Kto-i zCa83`X}{Vmm99)Ko2t?^>%N*~l}@7@GFE0yY?rgL4t8bY%5o&X=hT(ONY+QoUsuo7t11!?1HGOJs> z$y}M0Sk_CJS4L)Og7)@?F%=5?xUewWEJ%Xzr@44RqFUL}X-@>G5qi8NQE`}1Yf z{0{neNo)sF$t_M9>#R!Pxju+2hdagh7=|DuQ|!YK1j#%GJ|)pFWNf%-@?LgRP&)2t zj+|Z4NRQqHm?n_Ih$F;O7>PO%IHho&!NCkfgaZgz1YL<%w#@oX{3tOg za*zej6H0|7Sa}G_Q54E5mu>#(Ka|LRQAZAkR9FEgV!4VVlm)$3(o$YS8{s>pniGBu3nk=CdrV36 zUu4`@*9 zP&KXgBz{{L<&xP#wi)8m=bwKg{oBy9b8ys86B|h_vI>e#q!(EQ?}{l#Rz|f-nvs=} zt(t0N6@)9N8(FD!n@%~h3K5Tzc4QSoR!lvz+A!Tn`jMvGBjswmBL7Hv29k;#B$Zi6 zs`8Mu$VAdE7fE?GlCpdxRT)WYa+1{crOJRVx~XJ}GOsxioY8%wqWx31VQ-A19-F1V zJwN^0iKp+yk_(0*@H{_6Fan;3C!X|oG2p3P>U#{mSQx8v!~`a+DAT|k?F!7Bp6YSEA20{Wc3tMdS?^l-Epvd|9;MZNtF5Hm@y*Q` z9@&XnCPR>9pt=))G^+M2=9eJt&ej%N#a3w6l1FT_(6+)cTgTGV-CIn)S&=Yw9a`I^ z46R>meFbSltB6h$ht?HnNF6Gf^OVU$Q#qMaF3 zZ)m#9Aaj83+fVYjL-mU+d#GFYpsa41LA3N3Or65q=usFeT%aT)n%WU-BuSUcdmGfE z9D>_nnH&2SKxR~pwJ8gl$!bH1v-v+%ky6e+RTWX{)~DiyPv5G|(U3&wb~hu#-a8)t zbPVPR;}(y>97c&euB8(<=b{0|B#DY0Q5u2JqSZ#EGarm>Dpvc{cL>ED8N_{0usokf z?sXUh13^MnXU$MJ*ZTlEZXT1U3(gU|*RbTtH`M}NPC`)msHI&X)QYp0U^ULAahcY& z*zp(Giy~FtTdj8bH^0}v>D_+4XWC}BKi?Il8}}_b!ri#H_wDl-N_OeX_vzbof89QP zL&uv2I~(?GT54In0(IWftqrzng&_+XlK*5OHd)w>juf(RIZ;0nZci--TJur|{ES)z z`&j53V|t1rmJ95WP`<|z0z8oyX$dg*nf|>o7DpaXS(!|v8XH_zJ&R~K(uXI1O_1Qj zJ)&u%0Z!CLV}Pinb&MlvJnqO8Bv!OmNc=26!LCi&<3b-HRA+8eB#kB?^`hx<(yhW}%9w9|Ve$8*&rBzue z+EKV(W{Q?{DU$3Bb-jf@M0AeT!8dnveM^)+%Oz6CN({tnMT`Yv0Mzb)B#k`COWdMB zZ_DU`ALqj|cG#}c0WrZ57r!_GFEVeG-X?efNTf0Xi6ye2u!!E-GXpupdU5cidak>Ei&@<@(T%W!@8{J(0{r%<9#p%CtU7!JZvDIx7 z5-^ZjqKyf|QXSifhgYMsn=6gFN0`TEC=8dO!GP6Nusa-A62alLHDpnC2hZmTSD^}U z%H*L4YpuH5s^tNmipjh!UR_)t4tRiKeFG0(y&gc=?RL9h7$V3JhzJ)lkF621r_TED zJ!Qh08($~K>RyIyO`+b zMmh$-cmRxyDL)aM4#56_7oahQk+*-~(U)pLthync-eW5j&DrFVaOn`p99bm_6-GcF zgcgH1O=qJUo_aM5Bh2b5T~(Q7mX?F`6l!-f9zoF_os%b~_#R5nlwxA9)YGuN4XloH z&STFP*ShMDXrT&RLN}n)u9_y8$#wo0b7@aeaK8>0Tn*i+7p|eqS9*s}U8|li@~ODU z#~prEca5ks9Lo7Y9wIwJ4@^iHsAMaF5O!trH*D8$rOT~|sKwwIh2){63){k~lz}I@ z5@i+2f&9mck#P;fwcuAo;cO6t2~Ti-Y!*-1es$Ti8mx^MWi(%l(a#>}{b=h+x)q&m z71v!z;qLu93!^LP)*cqtLYi&b2l1U+LT77=W0e+7nMLYqM}43#U2UI!h!Mpq7;1Mb zpYBQ**QLcYY8{r8Cvba$lpOv+aor&e8HF0^D>8E1f$B6`t3czn*H;@_HAb6MefNgD zICI~*N#)X{aYDbj7kHFh1(AxEn-Z-Ol`!7q5wu93)sq40E*8Te<3$zPQOwvvQ)*H% z)2XYV0)!bzT_0AQpqg7HQr4X$VJHhbS$wyE&L)^k&z7UR_GT0rM$F{3C)%G@V4W&R zB7FkZX>Ybwv%L+D)p=t(KbSCRB|ESt|{avxbcSJOH+F^0YoRz2dcMI|8^;>&kttm{l|`tFj^Q1C=p zC1Tuf2`B?gWJ~TT5qMOBCVfum`n#j!EM-%AUB1W=!XJx2_G1d{6ZeC6!Wx(07 zbjzgKdX-ZY(Q&@S)DU1EgqWo)R-|Z;lXFF(4nAIH`JfiFGG%odn5myx9TT(I^vlnB zXHr-|>It+XlJ$-z`1$Di^6c{Mp!iN5QHd$?&7#jbmVdwBJ=hmUs2IO=&&~%aN{ZL7 z($}K1V-^?t+MaMO5U1E@xhZF^p1N97Kg_mcIiwswVfxNRqZ2068Bm_hx;0KI1Ffwq z%F?^T4B@FpSm%ezV`|}jGRKQO)1cP)%Y0BS0US=-0bAPb=6Uv^#?=h4wxT>)lDK?|~P` zSC^Nk$2Tu_!A-8RMag@H|7}&<1*hkuQ_u&emnVV(@%H_AA?DWaudgmDfB>2Jh)Q3F zq+J-BI08E_j4j=Nu?Ny0`{IXT^^4t(?U8X=2C{VCZjN#Q5+?I_cg~N#J3a5@-v)^J z6vxW#*15D66NcuFiz91jC8oDW@7|s^U>f0n7cdD+QQ6wLJ2i#JMO8{?y+|w^yj$g4iY*H&Pa=i=Nk4)h+F zYca@pUE={~&T$l#+{to;IN%zbf7h;V*SK>Ue;MR1zy|CidrWHF&Ny8Lvkj~ouYDOG z>(0$~RJ}4V#j$U7JuFb01{{{Y`AWkK zi*0!hF02SH9IbGfO*#oVm(A2Ghja_E)$KEReNcye-8)q_lg-{7ee~odspKK@aO^>K zRfFBkx~p08Ez4gEVD`?`Zbv?Q|CH=G3HxIb2nJA+M1e=iILq_-lU6O}S^&eRjdH|p zbesJB`NrE(JL#QJg}Z4o0pmnQ5qSY-GH0e_gpo-U1@Z)uO_Uw%qs1=Ugpe_IK83OE z_AeGucbRoQegs{`)NcLQ`1DCDDvR(deKnXFiWelpT;1DPIU;qLhJmR|<>L@@c&^P% zDs4p3W-&P#igh;pBDv0Kvbr?56-1@m-ZC-1`j(5%Dp0r+SeL4bg4aleL!=;_wBRX< z=CNj|l1cmSVF=F57s&O&YQvE%i-8i5pnTunn*Mx35n=Wgm$?qkOoj4xl@sD8G6GcE zAuz_>d9Yn3xrWI0%~F(_<%{{N&daOTNwpzM$qdIEqYo}wzAo3p*v;K32&F!E7)VAW29RKx0=0mUgo zt^rlj3ER$=s;f&ti$vuW&7w$>EOTC7tMAI2&C*+BWfY<`e3(f(Vf2cP+k#YbDz*Z> zdVFmWabZ7cK?;thh~@cE0Nb9T8HothsqRgjll{y}@v3*qv!PZt46M3KDb!!cSk@{8 zFsrlQI=YvOwE@qr3sd2^P%RdJA!8k}9N80{O`XA#g(RC~^=bL*Bt&M5N+p<31qEGA zIr#SgBYScl&|CR?DkFfbCdP7rVkA=lfUZ&ggo5s|7#Q^-H`VHDTdjzVbu5X-CdGnE zQtvdbYT9*6<)>YkXHm|8S$Sg3p{<)(a}+%0#G0dODJ`8NbE8^1N0$nl&LMOqK&^A( z9Rf^x>Qu%AP9fG=GMw9TRTgh)6oSO@Ij$PbeMflld>gAVBK3d{5p=z*87 z`+e}AZmy|`m>%}$_M@=u)?J3~5tZh>Pcg?n4DI&?b33i)#4uTHlT!QfBWR=^eEQ@B z(lQXj=~URQuqZj^8Tmi z&mZZNseqpH=E)ll=CJz%&9&x8LvQ{_gkRynOkm{lho?*N6Q#uV20T zQ-A-}{=vbYKz~E{Yd;C&kp8K^Hm-c*zLL+!EdV;-s6OuuKt~uQgI-VczNaOD&}mP7 z-R|s(;(QS!QC4|8)U)i`zs>-BlsctjCx%O(TJ;&HIGW1(>N)12d|~M}2S!?QLg>7s zHY?L-mWmcIj&eTgTPri2Fvo5!t$C{qgim{#?u5_E=(XDG(tDKf8H%{k!Olvip6C*` z$>++b(tMDCN6veTs4*cSf*Bf@GQCs!&yE@A|Lg!5%yA^_80uEaGf*a21rVfZc(wIV z0j#Tzo<3lP_GH>Tj&x?TERSDE{5$QYmj^?NL|9BpAAib9q^O6Qs4BMPWGg3MrbWi) zT!lX;mN^Omg;7A}V88E)^)K|rS_+2o3`WTuQS1Xn5Wwka%;nmh24%0*-fTSS0{|(w zx)$0&;kkNx?PQLL=)8!c;J!Q=BsXL6ztdx|83oHVTl9jopf=O&#!fA1vZ%9qbM+3F z1R`^0&L9JJP8w3UxVN^88pp>*$3>0>s{LQksIYYSyA4T-ZA`Zo7s(*GGL*OdMfxP4 z0o1K*mGgXtg8)Uj1_02JQYTlQaL zR#+D)wy`d%Oaq4hy`qp8u2~!jP_5WTQ4~lUq{BiSqTMLZWGH&(Oa(%Kc>l_RgNQGg?krg@E)QXsEa zO+kF=Q%oT#(mE5(Dm_=llYEcLd|}6FKyIyJP{_&ZCNedtn6n(L!h*QY=U;7_U5qFC zi5ZrulXUV)1448-QVWrHTeKzq|J2TtYx)1O$hOO?o6}L3f8d+(F*W%A-n`l0FZ%yp zKKuWk;&c4N(e(|uJi0g?fR7*bCS_NyYd(G2Qg44cy&j!iUD^eH5@U}6K1}NUJ5Xr zMuc(fUyvZ%!Tj`T>m~&x7TIHjq6`p@KvogVMNM!NhWhv!n8ACbYpA;k0vUH{QecVi zDNJ&Y=mW)^qe$MfJlNXGuFC?F+T3msk3;QT*Y*Y239!%2lu*?)gSn%RLwCS8yWlqv z;V2N?I}~jO0MaSjC|Uuq7U$MmeYKTI@GWy|jTZoYKz_n;G?gs5OUB6OA($erN_bwj z?<)jFZ{T*bJ>XBDfENN7$6Yt1&%~sBo3AFxIWi{+R6yZ^B;Wz6X&^x3Wa^M+Qzjm^ zq>5wkA7Ye%tM9?h52s*wb@HFU^FWWpJdJQs(-!MlN)<(%F66A#Wixe$+8DCg*49}B z2o029#TaSGT-mK-fXx|64OcfpC)G(WXp>5RNje%z(n0MSd)wRItd^_Eckh>&K#Z^6Sw@2K$LyJtZVZvt`>W^r_ zy3)04DEJXAgsmaH4nL_8vBL%7bm0OhknX-UPr(lQ)nQP|mCo+2JD@zn6#>EICm4Jy zblC&ez)nVwy3WRG zod6o4%J%M>j@J$vr=>!gto&uEzbI9)>FL}+YFHTv=8UQe_f4>QSKWr8?vd%qi(~{}BbBx-omSDWnHX5B4tM1! z8CFlJwbWXhTIDIv6Kgt76TY%+T9Vl`00g=lB4;o;yV!ot3?iUOq|l5`x_Ntj zI(mO}a&dO~et0w*{d{$O^6fV}_N1tjt+twg>Lk&mywY;$5%3EmkyuN9+xgfHAcx(L zA1i9-HDU*JKYjYt0l$GUWa#xF@Pb{3`<3kY_|ajw08-7DFPI}@@@mBpJuhxTlswimn7K3CgR*AqMycBc>r6MGR%mnbs}+l|yreQtH#KRJSK-brxVoncQ<->Y z*e*b{j0f}d=@`$WbKrTLFMx-|FC^{0Z(m%ZdqiKfG*DGKy_#;6?@bSSD7yc)Gklu~ zPC=GB;Gf#9;CW^x;&~uuzlOl`W`xyEkLB|g@ZNzoq7J;kFuMYNx;d)gRukeDw8m{3 z{Fn&`@ZzOr{8T7X6Zz|tgh%M(udMH)Y!Lq^xV*YK9e}fmTtvcbU74-P)L|pBhT8-4 zEZ}wqxvXQBXq^=oh8=*dg`v8GQV@VBl{N?aH5}-B&g7=EFe<^>39ycLnSN=JPwdmD zPaq&jSTw-dP^pLGJT_` zyvwVT)Awh?+(akup>8=m+{p`bG-v$5>~PGImI3}NIW+G^lvg~q`gE&vz$3W5oG0COBA+`5R*1`N)II^ZeI z4c<-n8h(b{9L19(M!7)23&-xfZ|hcjSX#bcIO;fl0ug#phn+*6Kov*87e=nUQ8=dP9+QN* z@r-H<^8qq*aXa8MN|LFX5#_D$+UD(ov&bAAWM~hNFp!sHOG4y!BaGxjL*kxFi)r0P zuOUmQ4=fpOO-nO-(J&r|pnW3uLXh|6dya@qUIv zAyi!+hZ~47sE_}6`Rdg{{`}YL{{E}y_>ZUfeEit^%hoZ87Zgut9DGL!416-*180%1 zA|$TG3kI%{P|hgmZjFW~|KpuwA4Lp#X90@1ka#)(N5iAzA5Oi4Zhz}9J+)KQ`wO_o z^MOp?HNhbgrpdQ+!E=9x1|Tc8m7YL1Q?W2C7U0(ehH^jw7{}`1e<1;}9LqpOX|b)J z5#Th5jYtY6Mq!{X$B}ufkUs5%8Q2*wr1sRic|zfe zRr_vNHBx8;l|rD|Lh`chf_B?(o;3klr>6hh^1yW7K$K1WO|eHXMtsHdQ8Jm}51`{^ z1j_l_M&9hj^sE^<|9#CK7CPFKTY{Q7?91!y^6gKOuunXY0r=pOS#3@0XsYnTLZBo42=WP;mvZ`s$o}h^ld)MMa7Ub z&T)3ANqgqyA|cmdVGdLiBUi!P22UlR7KUT{-m#V%UfNT4XwkwHDV!UYK|EHMZ8Cn< zU5PY`gB9;vPG|~mRViu4pgYv@#^!}*gLJMzRUSvDOe)-!s3TcoHel1_|sZ5J*RmD#8;mKp{*o;$!cFcR|MzbP`JX)vrs;VSfKX>K`0mKYhvtfh%; zFf`%7%6Devh}LOsn#PMNHqwW2(jzQOp9u24&LcrPK7k=(K5CN3hT}q2Z>yly9y;26 zEV9&*p^`Z4+EY#y$DEwg8ejX#Cdq(i-~Fz_VG4!;jMKBKHV@>BF^+=wg`Q4d#SenD zwjz*&y0r?hd|J-c7LBVhdpEw-dd$62AbZhrpG{(Ke9K#Pe=aNWU3>0Z8%#xL6@_3T~+~W_uIae z!`MI2_er7>Y9tJ8fp&J|k*Zax=iFkNCste1jre84C2~L`og<_~$7M0U`b(!3kX7U@ z6EvBmN@q22a=tDjN)m`80mm0r-m6t-;3ci9mg|vpUx*?vmo?YqI9YmnMQ0J_C*!UL zkYh!mrG@e-zNKc>8aOSd)tDl|0#Da}wE1*1d=^lGwoJ|yUX8<5Kw7f*QF)si>o?~k zbt9Qt-iuk}{N!k8XRv5suUzW$kQo<2(W<_~QEbTqT^egBgY;@*m5j>-(6^aTvulZuRi$VSewBNb2IVLw1bPvo(x`cWBXETx|-N zLmv9@nm4mgmBjW@U??%xUsm$Xl19 zm@v!rZ}3|QI2PLTtm;4cJ6p!3hDV-@wRj7e!(FBdTS&*-P4^LM8s z@YB)x*~!t(+12IXpOKiE(<;Boj%uORf6^_}7q3{iAxJ+WIGX@Dc_cIh>S6?$;6UcZ z0)af`6e0Bt5I&Q)G0P)S>a4P;yPULC_hQ8|)^*kAHwv6m2@Q8^CSG}Bf*Rj_M&ux6 z!5***$_e!EBmus>h)f=u7!y7Nx2E{5B=pv*a|@^ zQrF)d9b3yM^OzH|q!7pvR^_)~u~~gOP6|4PzJ5{c!K~6AunI~}R#1%#%}aBANXNrS z9=m@a^sc5u>h8LZNr?SLFD8M-xMsr>@T{+2qr(K+K9QD}l6x(@w~v zPmw&;2t&4+n3x%uTK2p1#=>M5{Qn;52YtDIb zc^rbBO!FL>9@EPv98Fc9THLG9LabT~<@w&Na|eI!BfU#~O3#0fp?{aeJ)e^37c%ZH z;C!uP;0^izU%lEt%$@&!bMWf*^ZDn2Rrl%B7L2jJUtj>n zko&XV{r=V+j)DO=rX>0=GTxda4g<*HU`t&GK3F>0lNV2eWFEsX+;VOdFVC5id84+J zp8H1MA6^pInC0f5BPtKU0V_8@6ThAFV48DgNNT{DOGyCLm-*X4cwL}(=c@;FJyBW!I7Q zrj%#KZDNB^S1wrhgtxtU0XSDLI56w)cd%S;EJnT%X6DaOkc3JQmCRMN-jsA2R8f}W zIr=w=bXoE1e@Uc;y`e@y^3#k~Vr?p21^!|vSE6#+514U67w346 zc^#ng8VVXqWLnQj5-tO=x>g(3$~3Mxnw}sSgg8PY=DAXaI1BdhxN^<3_}L{X!pAfH6DQ=A6A2M5>vHSGM%!g0T&G#-b``Vb!P|sIm3^cP)j_jzRu?M@;VYS4)b3cE zzI3Ol?1Q5Ff%711-tg_a(dl*W<;T_;|3U5}du))V#Vdhp=j|)0nY(16qCpisSw(xo zkU2Q=Ice!0}dxA@W zH!F>FNM`UFT*4*bYL!aKi;digS6Cv{-ifz=m?uIdNx2Xm&hi6!mAbXLE8;KO=ae!$ZZEFn@v5FeEg`AymVrwnil`O1pU**LR6>`PgV&&smEPWuyV;o|>&>YCYw@>e;kCvpdJ#$uK zO`1hEUoCy1TF=Sf;XG`H(nz3=y^Jb7I4P_lMGSG!iAaPx4dmrJCw<+GC6u%$PnK^3p$7Hfm2z2~na4EAf=_|>1f{oe@j z2SV>u1lbz9!1epT{ezcpa{Ipr{Z}ua?|*-i&(rMx+MzDFJ>V*@tdc!t5ZLpF+Ty*m zUpsrjB`G+47(;27V#IQQvzmI!d(=JS@-UaQOu_xJfnAN#9pVk|;96?DxU;&U4by4_ z4UH+;KK-bY4+ZCswL71|Zofe<4NSegA^Ttsxj#E^mH?|<<lS@?GW^=- z>rH>^=KuKKUpw~ePtEiH%Y*$l`TIW(`){7-|EKtTsq=q+<>-XNwpoa&h*8@zNaMqX zWlO8)XtA#%TFlLEuu`m7gA^IBm+gzP?UyY8Hb#mS6aD3e!L7LBULwS;;9Y3v7%J;2 zAg@d7vRTi()hD3Z>{R3c=C&do8cWPdbzSrY36wP?PsGTJK^w4I^$aRc5?d2im-%~T zd5V5b5X&qq4Z79t4E}dP7ZtN`>cV~qS$1Z00_Y|Keu>V{oC}rye#UYUz+w;7#TT>eMQY-H`c~&AzeM-l_ zHg9`r2CMbdK$HKs|LS$#|0`ZUkN+q5Jk9#ADg9C*r>NRRD_y;%nV`y7Y#U0*DlMVq z)^J~_w^p_8D|6MFZv9xEV9d>Cr~RjCpNh8w@Z0K3mwI&VTTw}WNxnVRi0S*;7;e4+ zCHUI6;(2EM9e--)|6In~RSkiL`G5b_{(f%$-+%RH|9Sp@iqDrc|Ci*$-H08qkp$My z2Ei9$5RiLBDGrdCBfZT{lI6LTYf1e2r^Z_S@1~Ajc@}8W|A+erIsN~#fAGBidy>!7 z@BfwO_bpEkUy%cN6O+Ok`GLQlso<;A@M|5csE{mcOUtI<;P;`x*ZkD#e|_!rn&y9_ z{(o7t{|{fjde;9>@p*dvZ|*YB2Y{I8O6y(dyZ6^u^cwE7j{j10{1(u6tDCNoT1B!r zcdV*hBrK-?;nnxrpH}*xtrrN`wEy$^Ft7jn2X79a_5V|Ro<{#y>HaG1K2k{m{_wiJ z!Ih(%Um6-$9rtEB!DL=2BqMr^aI`w4OU9#p?(A!P5u*V(4im-^JsWNTzz`Dxd^DMe zrH318mT!2jZ0Q`Huj3nBs&x9Z7E!K}vn^$XTyV?h!17$9N5hty-2sXRycFPYV%2L^nAr~zS*a=)uc-n@GD z|2@g)Y3F}4Sij!WeZSW;eP89K(BmEGTW41JT2At%U0!mG?<_Ls_?$z1t%frUL-KI? zfg>8hP*qhVxkmy&rk$DH0cI807OIClfFNQr+C5C6>9$JyC6M{?fJ+aUtYH$$T~_QcG!*vK&OUQYIKxcBG?XCiuW9v_)n^ zTf5a*8r5disSf~f)aHoma@}9K6e!7{Qi36Mn7Nn=7J5lK4~@5M7p`V_XZ0seBE_c> z-4@!=71)+!6_RYFS%_O61oT5JrN6^zf;v@wTh3oCR+poTj3T!Usd2GFOH4TaH+N{U z2fmTxMNF9M(UkH@fVJ!$m9SJHSuR)Ua;zGXppvrHdVEbj)e-auS`b zHUu798yhsssN-kn{{}v_>;E`d|MHhM{{Q`#`SV||`_J*8PxARZ>wh6w`o_iF7C_Si zKdxBW>y!N?IKhl2u`txXOM)rlkFn-|;?w+ws;~GM|5xu`s1V{gj*jjj4ux^9>f28o z+q}M-LF?=XSlc;HZTR6`1k?IErAB)Jjg|?way5l6j^EOoxc95*{c;>I>?`uVd~PGA ze*ULu6CuEj@xKRo|NqNZFQ51SpXBrT;(ud`&LYfD8bW{z#WorRtR?(85cmt4b~hIY zTokeASm4~JR{!gL{yB*-Cv@edV1xcYJS@b2?eF)We_B2wk_@&RNT$YZPljXhkV@EckbJ;|c60{PdwhAafTeHD{g#Tv4rcYYyb$utGV z{Hlw=DzE9dd)?)b6f<`3Z*HkpL6Jebgsg3Lv6eM*-NjladMFBFf+Ozi`jrOHwd-&k zaYXO!eKhl=O{W)YS8eMx^_5RyjBfB8k%VWJoyTnms(`a1RTRnI#?jQ;L9rgLIjmb? zt%wbMy>`>odMb8BrKXT`v)SbHn%8N6 zoTWg${m&sslO$wltweye_W%C=%U6em`yURU?f<9wJc0ec_VxZvdqTFDR%{H7bdk~X zhR_#b%ojFq zzDg>}b<$Ur2w6!6_>vU!7TGJN&oof{8S?K~GVeM0uhdou_U|(AJ91|PeAB(ra?zOSlUsJPJ`T z!n>j%=-i=2tI~I9kr}8h%B!MPEo?hf1YqMY%L?01H#1}$JewQooP>zxXGH7aW{9`3 zY?Ahvgk-uH#S}qd$1qMIc5a)mEV8L~RZ&n8Cb2>*V~UBqG&_WhiFq-rQFv7zj93kz zEe*_{p&$tnb*np%OL4mu3#T$#_cWq&97zFsOQDa3h+=u2IEezgx9f_P($^Aj>%MAn zmwNsl^-@6;8XIyf9drvVj5)}uY%=7hkD3WuW&8;vL!T%w<)Zc8t#md#uB>o2JTCB4 ztS;?H2m4BQC$>%92R{_OCzFAs#dddH2$coFPFZYlb{|tAx`NH->asfvRaxG|V{>wl z*;|tK^TO(M%_f7w*|q`jNxYC(-P?EEOG~-V2ruH0RSK8crzZEXcr#c9jBcD3FF&1C zF{ILay4=#)^<8ytv8;0SUF4~Oi6QM}W^T13qM^xhK+kjVkWR)m#;6m8NVP zJ_;IP6u2iFJ`a``)id^V8O=iahp ztlsf?T&Qe7Q}`e?5uRWla>VK}^zuq81C%wlExbAwyG|Ib0G?{t^3@dhMoocVA^j5^1GqOsHJW>fF#dlhB#D9!_znE?^J)YDK0rEUz@vJEde@Emq4k(A1O8+h zpzWxxePQYKf(=qN!n|wn=FJ<>3($R!&E_kjvzM_&MC+KspJmQzAy&(`qHd;nN>N*Z z@@fUW@*1TQ_7iq^>7td)ah(^YdhMB_b9S*CZP#XsRv@M=0eR+s)|F$!Y^i8-UD74k z*E6LWOk>@SV%@ZX4d^QYRE)D_)wAb`;cT`melH& zP0>D?BAXPb&C{I6@Wgx<>xPXoT`oj6z;Ov@!=yT}*I_h@?#peA@6K*6k1o#MpPYX8 z?rl!nOJdaBH;6`%E_BXF(3!xHp^oFNRr)4C<7C>(kMHpT8eo zUEjRFe0TBP>2)P>SwM7E3e)!Ze!4n;cX9gu z8EQCs?bTQ-VQ2^$dmuFU9*23Kc(J$N506HppRcY@-hY2~ep<;+laQs#-eD>hTZ(GV7&-sxk@k2?L>R6d=Kv0Aw)O;gs8uSwZdvShhIXa#pz5W8#< z`7=r)lCWbkpJNU>WHKr2>J+#-`upYa57$?hSMNsekFPE+&Ti^QRXf~mhtTD_3x@*T z$D^gtN6CDYQJ@tGTT;apNIO1%H@Z2!et$F^zP~)WSdJqwPQRBq>D!qhwhuHIm-sJ) z-aL~p>zi_Rd3JO1-TR~Kx6O<#L%PyrQ2{U;6F|L&U6fi>HCGH3qtol3TJ*j| zCsX7!suTlR2v<9}ag7XB(tZ5HX$wuI==Nvm|7Y)AyBo)iMbUiLufTCn?${GccE9)W zJ-zqU$eteG*lr!m-N`; zRfVc@CRcOlQ8gV4s?Xua4({r#f|2dK^im*Ig7os_m(#y`0+pd5{* z+w;aZHMggG=8$u0+Ig>n|Gks;W#`xH_SIE2#VTJ%jvx!h*H8#n_M$BW!-HgjeWFM5 z%jwxw4fU)Y%MeEbm5$~Y{bnVV&I?c^Le)IBl3dlZSl7M8kHdO6q&TCyI_oKlcXDaC z)OF{y`|HW&(u0*G5wV`)VjUch5vx2qR1Bwca(Pv+oT&G%E7>O%J^lIQzgOv~`s7|% zjsP*yY5#&FL`n{VRbXf@J+^iQ82mk&0v0Vg+{?42$9s9UveNYhXeS{SbLq;bk4o_O zI)OpiVk@E}Del+l=qftue7d+e>0F(Dx|o0DPW$ROd14pq?)bE~RFB7)2lG3f(L$;^!()ie7aZwXVMPVFy45iko$~| ze7rF}wm%=AUdds5j@D^@HGNsW_v=pks=dr8I#9r| zuFp?;z4pfw-%X*jubyLajyQ)Sw8&2E{o1}fIhF{1_Nje*eRlfuW&85fqw9<;_QsH* zu**>c ztWfR&ug>}GLI*Y_P^lbW>9Aej>|*M#dDE4{)W?P8$5`HO<+9B@EF`f8&L$B-+$?ph z1+F)4cd4Y;z`Yi>h^JzeZ>@3*=?Tb05_cIL6z?TCZ`S0gIosnJ{2kqKR`><4Jd@qr z4Uw{*!H(7M?D>-l^eqeimtjfC-Ct07|5v|h1~U&=u*K^(<={X(y2hL#7*0X+vzyW- zbFZs#Fj5z%!hzfuAF4%TXBhe}W^J^T+5-pRJdzJ8dNrlnkr7gmJ9G=RZWhH8Yjnmd&$2*itaU|DE<1+_;#Be#<^q$^OfHYAsX74^3VArHr@Cke zEcL(TR&OEaerGPFZXaIM&bsk0F<1{jg9nUPp@q&*B$^%p!4kB@1inM;kZoj_Wr#sg zO9bN4uLI~uCl;X(>n01U+I_w_a@>R2^KzZ6V$}Ord0y9&jb=fgPks!1;;qmOr#7O5mDT`QAM>oowpCNfoBLlVm znIJZ8#!wWa=NfrN%ZIkrDV#*U#Ed~QLf{9v!}q}t7DbLec+XXjQ<4B}+WjFzGeYHa z-EmsI^{;>F?v}S;S4LquZeO*pd!IgEc22H8U!E3N!|c_`R2zBp$1oLU{)_WyLPPW+ z#9Yoq-JYzDWNOlaDewIq(3VMM)wq|8g@~K8QUs`VE}3DN+CnfXqnYdRFv%0=$M7(z>2U@7K$>R% z(LGGcdS1#hAcD9@gBv7@#2{8^Qh`uDvQZx7k7FyZ>vIk3{5@>trF}28)N(C9mMpYB z!9VQsKb9r1w%0Xa!1u8PmiPWtk>F{VUp85-=y`a_sjkbpzB|qmYto|n!-{b(WnP`F zys&w7rilkPuU7Q9pdMkqvMBW+#UTpd1H=YfXmp6Uh#-OC5C= zTElgjzr9W0OD|q=oaYNK*#buKMGfK}%P8Vm-<_(F{IK+Tu5Ksyvym^kjD_JXzlQUy z=L=cRmt4KlF?cTBJR7Y?VuIyKhltE*OQ?sxVYrfS`mLa2&04?wkZXK783L3!*A}g0 z-Xif_ad_T_vbxIE=bwOQSUvvL-nZ?k$=Xc9+~bn7WK0gUk%*#RX-;P$0~{tYJK z5$Kk6+^`#2zz_yP-T|a=G z9+yR!O!bq~2mLcayr_`mf->uIQ|s7;e=Ch@;p0?>w3*(mW}mGed}(hh?Y6>z43t_y z1UXMf<8)g)Ca^P#=m199NBVb(KK3 zc=+zk8};wu;bHOLSMT2*{^8}D_lNJ^9KL_|_U#`IUmd=A`Th^!aNUzFD+w2n{o!!s zzKWCEkRs9a;HO51##4qzV*!3<6ovtv?1NJhv_LzGz@>b_!6j0`EyGr$*FFAg^9%=w zaMV1NSdWJou_Ms#wmZL`G+(t28$TWV``?X5L7vc(L{EFgG+e2{S3$>bgO)|f^mHmK z^B4w(_+6q1L5^A%_Hmxv)$vpPP6u#>EX>Z5#w@E|?>ILTWv#OrMw&ZHzOgh3#&PQ= z86ZZGK)i+NK|m)lC5VV4U?$keOEC>ltI@IkMd>DWg_Qgt+p{8*Z1)GZ#?glw0zN@( zgbIgiNkKt`c}e_;TfE>h=Fc)<5hKmZV>?kG^9+mIsK6#We=O~1CAw|k`GmT zx?2hc2*^W@{8PZ_M*fRnFgERD0w@a+(_aR3Vw!8o1Cbu%1r?X_QQHnawYxb!J++}? zV(YwP`OF+=xr~zw$VNzL=bxFT6%xlZG@EdZD4aD~y4_1rwO$%B)MWxiFt((moWM9U zS`S&NqW|;w{|anBEpzz)yTdnc^ZftiyLYc&ZTbIWluhIRj@t4$^8Q&?Kg%PlSnZ5b zNi-qJ#rQh$IQ5gOVBS-?oRDQ;J6)tFqBy-y_ za6#WxjNa&q!Ql`GCHG&7?UdHYBYu@ed-3^{1mrGOjGQW30dfh+2yyHGUiH3ql2w&d za`^w3Q}TW1*@+-8ouGQ*Ju~(7cV4p7ZrMDf)ayBk({B9?s@h29YmYgma=G0$j&NQx z%&d#b5mr^@h)YJ+9u6_VVrm7x+{4RC<-V?I#YcZ>=k*o>VZnwa`DjNeyA{Ug7!gUVnDxJv^>>c-)qaFBRv1IzeKL63&tct*io=r~f~E`>JsMfB)w7%kBC9G0Fp< z|Cex+%8>6;BA?@(;VKg*lwd&_CZkqB8KR259;7OUZmy9zng&B)l_p`Hg29GgEKxMf znFuM@NGKK*t&RUwbRI3ri`Jkh`C-<RN^B4(80HM8_~Ho` zuRrQ3V}BA=UHtH#Nd;12V($Nyg*zA5nkw=dtk+w%X% zD38bgcW0vIz!<7fIar>Xc--PiEKl?%k7RnH;&Cfr2x^Au$^fy_a~|Bj93TNx&QC)d zHAm|~*U0>zsC8iz8Dq@wEsoF#g-2irBaRw{n|V#}W9s()qkW>UFr+_AePPqL)?qmN zoY8-vK%h`Q8B;DUkhr7lCL4i#c-rmcA9PQTvj;38ZT^`f5u9Z ze}+l&t@?cu#}S$!B4G5<5f99#=Lr|;b3JUP72^%-6FI@;2)s;P+$Qkb5qS6Z?d!J< z0ES$(x31>w4ckjfg}T8upg<%HfgPqdtD@rTU=VX}&OQT1^y?CaUusEKa=AbFeI;Yq_UmxqIZCO7q#akiWp=r5&Y7lZZYKxWwF5K5SQK7`#m$N3}>eEn(l)i)E07cM6EEe6l$_4gZNLXZWUQW)=}qs` zfr6ZF#{R0azn9m{9pCeePT?BA4c5nrD@K&Y`}Rk;^|Ik?lB=@e0{$Tq#uO8g)AN>k zPx!4mCIYcrS7~aW_^kYF?J7m?yafvwjL{XIAeuN?RdSDg22|0`lA~fsEjcD5M=j<& zT+?0pnqDiMgWjz?HU0RqY=Sed9%h=eBY3dMjwzw6sNQrQ`e0UR8@cgRXucJ>P+U5@ zD>?TBzlo~B8u{M@u;xfSr2p^B!^2la`Tynncdxeb{~x71LI0m0G+X}65z9#%dGR~p zh&dwnA__qlvn#ZMPW1MzHt;D)_Di@f=VtyWe#@0TZkW@u*`2&vv1h*M(1Zwkcu*X8 zUYbdFqm<_KsMDP$aR>$K2?hlk8E@$Pv(mKPb#2r~F5PM)SMd|Gf`u7sAy?c=Bo)lb zkLF98=Fd5o5Y%VZAeXsiyi~ z$vWLtH*&xaEt`6<^)uwC3&j}hqH%@j!-v=BAxD-YlrJH5|;=%k*)V~eO=KTM5;MsU9HNaaIS?h z<)R&7$WiV9oIYGFx_4z)6-)LX{s;Oz9CCM-6g-_9Piq=~+KK1jQ>WqFc+#CFcW3P& zO%?535N3G<#Wb$l@utl#LmUa!nC2Nf@*o7m9Y3~IodZ!(6@edQ>`uE6Hhw}wuk*gLC^@XB5!pb zu5lV}S8Ja|Od~p)_G0#Q>iFpd4W(`G*24wc6AmZK359hdBz;N1mfv27_}!9jk!qe3Y-LB0tvE zI_-95U@Xseu6u)soX`fYLzRwh*_A7*E_0`!?d|1TsQgf+EGdUxN#=d~Asex#k+_f#b&u7k)D^4EFTtqkm0gb2405QQGFLWiZ*Bu3B`%qPK=Yg~#DfpLU z)~aZu6{nw<`_9*GxHN}%zDkzbd#+F=zpefqi; zt2TTDutK+c^JY=Jf1r}AhLrp$RO)lg5P(hqS&-gT(z#bL2LV*UT76b0tvr_~ejmbE zHNHxT+6SU?ppb<5t0_%zFu>%1j|)$mC@4I+FWedlVsIcP@qya12S0%W8jFJg76eZ4 zfn#CR?~9ek6B>edhle#=TSUBrnE102*z;4-JN8_jt$A@inwqFMz|9b|LdWWK-i2a( zq()zO%aehACnoVsW!p4^cZyEoAr7EGyuOEnB1y{{%BJhvULA<@DkDP_9+kzEQFX~j znrgM1Arx>++P9>A-Gr!C?kz#!+*q6j&_BU05VR*4CL^$0gF}_r2hQ~?m>_U&WOH! z_+GQcjh@L6lW^fSL31MMNG!(SC1_6IH~aU$vODI>%&jRaHYGz+0@QHF8F%hwixU|0 zF%>(x#}Vb?PeYoJa0lFjzkOfw0l+uNMjSMm*~!hzxx~bqHk38~9VmtG#;<+h@Prvz zrE2RnFTwlw@4-QcZV&i)vSfGeiFmf7RpjRH!m2zFZ1Gvt@uu&CbW6;knuafZw5e?S z5tqL7z)L5%PPJ2gWKFxv8SUrFz4J%3#9({}Pz3&WJ#ZEdPK8$2AXNEC&t{~v4Wv`> z8AxR-729c1I-Y`TmUz1hQAXbVMo>fAH$^{CG1&zjW~vm&>vWhY^syF)8IWlXb3vDi zAYVjNvAAAhbuGm2QiWLnNQmKxP%dzAPQwbCV3iFknO_SXnKl=iic#fMz)H8b{(+Zv z^T;b84-{(!zdE6;3-+q9HIH|;gH=F|$ZfgE@$>1`Mf?2p`uODM&mRk1L5)bKOhAl4 zHZ_sNz>dn~wv*HOYHAdsK{A@NvU^ogr0kfwpL$mxFHd^^`|P^=>GJCO;`8~>Czq9j z%UqwUBHa1+_vO>s=kt^63Bldg7;@SBxfW+U6$Za~c+hl}Uk?eUtegQ7C zi?Q|aygcciop#z+r=Kpa&p#bkV8RZM;BA3e);3j333C|iT;cs}?&Llwd2~nlczeK^ z<+^r4*+e#JGgN}a6Uk=)(mDIwyE?hNZg;!a7wz-v1ZZ>zq3 z{nWdv?5ASl=9*ghZN&QSd=5hOH=R`eoSEbhS$P4j&U*9CM<*BUpU+ONi(i(W6{m>r zfE_}2N)?}eJKaN7>WuE{tashLJo)A1^163&`Q_yDx^vq7_2hEt!Nv>?5$hp#i&zK8 zW5oCZheFLiW|Gi2_)jP1T906jY)0QNccx)F7hW659Ygd55 zU)LV6Xwl(bo-IAz%d?f4yF74*Sj-#g*{6?+40(|}C|hhrbo49bqK?;AbkzBDadFa- zJg0g$&GglA@>eX@-SKH}sUD9p59W6|BTtyxXPBTaLqq({bC_qR7blA3oc`5+j7~3$ zDPpRIeT66JH%feAdv$t#@_#;EEPyj<2WuE_JW|MgMn^vSzmM(D$ER0v*q)M_)*Ms?;C za{C3c8DW2In)X~Lo*MUhGe?zQwA)kXeEv83w)u=&e^$u%g<76SsC^}6(MKx%Kt z)wk_OA{Lsx9I!r(tT@{8#w!c}}}I&!)=m!H(&FXV0Iwpl?}tjtp~3 z+}}Z^3%q{Q3}znAarOSkqhX;3dDmLDTu4r^ly^Ka(FvS2oyjvhrm+}g``p4hk$if*sxu6RAS4!98>qF@?C=vPW_ zcu}q}@7B2K3;nI)M)_T@aQHM`<-AYHu&PTK$0}UrL-boSQ`v3VY*#V$JC_||^Znvu zwnqImXs?_YsOCPY$H)sYOsglb)NNb!zgD=+>^y9-4|v-|^UhBsnjQhc612nwzQfI* z?e(kq8s0AsQGHmwey*ogl>7fGPef9hCtV{zK3fm)xe;VrUHoQHuI8dQgl5vXKb^}# z^M>|2ychK3F2ftWv&sMMn7@LZreSuierQ&V#S8<4&mC?)5gp>$^DUvhhAux5cHMzDrTC&P~=^4^@XuDN}kNvw3Ozt69uz z?5HIv?`)~IqR#LOt>l`N5vBE*QuEaBF|4RHJ+6U@Kfdj>tmmZ^St5vgG`K;cNG&RJ zC|2TS?IJ&p@u#lOHLUaZF!_}Bz0^|6wftDJ(E7Ocu*?5gdhFU>*MI?(y)IBF z5*n?Olkyck$n?)QxdJXR;p`G{ zZ|ff=uUXDj=MoB*qp;E<4+?_%ghN^`>!`phz7o$k>QppszjZ&5| zeaNbeh(f^LXiAaD>TcmHh=!2Y{M3U2wT}MPAj=xE#(=pTy1N@mQFy-pk`d}nwcZf> zNg%|@@p(NQNl**}i{Z1He-5?VY#u~P`~-$mTWY_R*d&J~2_y`-cuSM;vU(Mn{Xgaz z!j&+$7%%>soT&<}(9-Cu8h7j{oSPb^%c5N-6*Lx^)eAVlM<+NKtCH#l`1`D+m2+jM z_yp;;!E%EJiU#rsmz$_AnGAz|Mvhxs1B0q@QA!$-QcR}>s;vEhy!DE{@z7_j_ zv!>NoUfIZ^)fQVQn*NJUkO&K#E}oIS%>!3Bd$GA@QhGkzq+_a0c`#^39+C zIBbKw`y@(GsK;g)!K4;doVqCk_zF;i;8e<-%+wv)1j% zJn1?5LK;_R4zxHVB+8-p%Am#gZ_L4ctIY|p1q>1j0l2eKHg%NB2|lsd8V2Li_WZ9J znKwLei$=}ywLpcUC3C2(To@tf8Vu}}0n0*+7_dv&Q@bjkU0W(#YLP?NrFG;qL^R+B ztpe{Bt#W@CL(AH@^-T5RctCZOzfC6jcbD(KYk$!%U(eUCBdssr*n8i^Z_E8hLGm?6 zt;OuLSliMX+}i!(F;DtP_lZ-1U+K4WM;+sz_m;u;)j3+*659F3HYc(Dw^lf!Xe4F( zb-qjO1*FMf)A69_?`2@*QrZYe5j-?)rYh??HZ7ld6^&O6yn<;{$|LFZwS>G+&3Jcc z$*B9wlpW{)K6$m5{MZ(*Bocp3t9R-A2(B7W$y~E({h@bX(ma}@)bWH~WltlR`!>xl zxV~0L&5NgonBJBz{YE zrhJA0ci&CY@%7nX_$hz23HgVK)v!&KYLlMz`JW#KgB`8iNJ{~8JMDy&c=Bl2Z!^t% zQMaS70;rAx(=p86MNCNgGNH>%!o%bK7s5LK!YCU#_H zJ3_Z(?qwnHM}zBRk+KB#XY7*f^N3U>ev}XKI{fXqdhFk?SFGPbA13{(-I!stYtUaA z+Vsxy0@^ArM$A`}nVecJvGwQasvnTkb^Ga+4sMl)-{1;j3Q`v+;K#S$zZ`cu8PC6W zx&$}>1J9L1zrSmK+tWh{FtduFp`oAlB|Tvo<4jB@1R6a`#L^D*oJ}fof0HWvBIY4{ z1%cI$*#zetRS-gOkn*8Ue1f1mr1d}LOQM*wc{ zzKKc&!2W%kD71}MmUhlp;OH%EW;R-iz8K;t$wZ1APJ9 zUMog|#mGJj$X8?xRds#7jy5A1_GDw3$G-VpgFq@I-3Y|oj;@~G+8s=a*x%>r)G^E`fxwlX!q$F(Ux`T=qAcFIo545Wr&uH(6 zQg|(ICvW~u@q^btdQTXAv7y?Oie(}@meWvJaX|WX^NpRDoC6%6m(+dy8_N z-pukG{6x~^)k0IiM?>g~lu{TD1YIeFvf8HL1FKKOG8Wv@f>? zDLfU|^CR_78h>j|2sa2xKE`qW0p^31(d$B7M**cr@y!_IbHzZlWz!prM)8;P9(!=R9$TICSBQ) zAQYM&U_O&*@lX57;q8RXJ)Nu!cAzbi2^SELq_#@@FX>x2AoW|=sVH^YtpHRLF=`Dl zzjz(F$6xU;ejCvSVdqw#{PnJ%mr+6-k1=pL=3>;2%g( zJAPaD&DGH7j(FwlceOuK3!=FNd#}OA%|+4o%E!vhw&T-xf-2F*!NG9{o1e7}PXu9Y{N5-KFqgSY4^qLrI;>*u!5qq3n@HdY=#(pD$XaaMD6tX z0Mxr9ht+cd;KS;>%+jco3nB|1tv^MOaHubLwR zHKicqSW=O0>(?+@pqrreJ6(!BO=%M}=9S}XtHT)3V>qL85o~{wk)(Fic+1cuqu!U% zCO2aicMEF|da?jC;pYYz(z*_6%u#-3;RpjmdO7II(hnIjwdtlT(fXI`k{^{%(oEke zoJ(sIVUG-t_aN&Nc)H@~><1M=*Z1W5%EjzvwR+7*(CvOPxR2`1QtRqwp)$w`FmU|$ zqR)hVNg<>zD`WO;v88@<_KAO}12W|#1T^m|{8oo15`S*9Uhmt0U&Pzkm!IYq zsZ^ED(oiqiq^S<-vq$A=;RSzH|9}aQT1ZBPaEfD@jZ1q(o&as&qFqS()-%IUyp2=| zU#2uXWt0Bm_xtER{eJPG{}xp925nb?8H9lj2J3oXC*qZH!Z2k6V?}V-g}Ku@fc%SR zhBS#ufb@zZiHalX2C`_Oq#*SXSO^2>7o#zbb;3(wasWwUUE)f22~73xxG{qHflyk& zWlJwLnj$D=RT%+VBq}&6RmGAu;Y*!wFx#RRD4%Ug5OXdO4C_JMGE5BkQ-I*7l-(^p zzsh$j@^CTuoA5`)#gKXmVgRa z@gfSC156irl}R{RjiZ-Bnpwyv1b}Qd(u5W0{c-a67k5^cz(i(HsF$o*N_-rJ2-PX1 z%#Zjp#I*mVk5S6gj~FIgntbrDZK}K&T5!f3NEmENYVFR@=39YTPqk96gjcN6=+BG48lb#6A*RNxX2fhpEFJR&d{9e=oAm5Ba!k{7C8eNgATEoNDIAW6%92F981F_a z_N_3Vz*T7{ex?U{*OWY^+i z=|Y$@t37C?_S>3p+<1$5J&|#yG*Xw$?8Tfc!Dvq7%8AW95k8&f#U_N?Hh(ouCB&Aa z&bmDagNaT-t!c$qSKK%^?PCq;f|#)ZnORvV@41fY`{TXH|EG(S_x!&_{;qfP0PNP? zfM9sKS=3j@ERbLonin!ZLAS%MF&KNtvqwCbm65b)JiI`+dmy^6$TmH$?l%$=O z!>ZpTtX#GovPmSs_9G{hjRWOf*952j6m%!tLcj|(jhJA}@%JXF_$4lBX!`0SaTX`K zpzugcON~8ugHcOI6Y}LhEx`Zt&ke@C{ErYc?}Y$EB5Pg&>vB`EWXJ#?Y%}jQN*%Hj#XtRR3?j2H9ATf8zv0(>gwG{o9;YV(UQ5x~a92JFNIG(OZoawQX6 zMWjV#l_v#KPd_#P{JLXuS5pIj+Jgy+M7k#sZQWhXsA~!hCyO`A>GJDX0qdFPccbr1 zMhDB`WRdxCp|=Y_5}O3iVxhDVWr7@l&7o_v%8{|u%SFN7-%PgRWl zA;Fe2sj|AqZ@CdrIzXyr8YYe>hDU*pPgNM~OdQIwtHuVS!mvoZW+^cHk3*e@lMvRl zzO?mlMZOAm>K74Dt^!)2xcflTfGi~3aAW>*Z-&QWH038qQpiRWMopF&oE9_-dQFxT z3ZIH&mW5-iVJa~U4u?42=y@kCnka5Ksv{jyB}b)V8T4A}fs5tbAce$^o*8Qta7$As z{gnhSHAa&K7M|KH)gL>1un9qVTNpzbxoIjDRf_7Oj?OgX+jSWKm>4}6#FvBZflidA zbB?~0M4*Wlp`v2~$=3m+yeo6LXhF)wb7WS!9A z*lW=j%=K`t%CR*vT(39Tbvp$byVs9U2y3pU`6muX9qxjd|GpRD-{+tv7*nm506TIm zE3?C-nE6vYL=#q5c?onKx6$?vH`yhrI7DlegMkj(Top)jzNAJ&a&eyA&-`5`B@533 zHNL-FDqjgjO>q*73#U?1BHPB%0&k-7)wCY4c1lP)eaSJEVG|kg&T%#S*NwdjJsPiD z@Z$=9nRTkgXtgt%j>qbEHx)Wq{ChkPn5-8GC2@JK0#;saZkH<_Ym*?qgC{kXIYerhkqt3}?t;8viOY!flan50N(_OjCMiPP zk@pxBYO>%l)J1$9<&2f^ze6L7B(STfx^peNATvfs`kG5*lR?#{EEmV(l#4%5$=Af zZ;ZgglE(Vd3e|6SzhB?$J~vv+d+YP5#7RH3G$WjT#=&BdV^Un`RS%=b<0$LT&7^Wg z-C>=w%MA(bsiL&29D~NO>o`H^vOt}iXs(d$uOR$m7f~pA0x~Hg6Bnr!gp5($DVnXb zS_8ba=je?hI}!tuS;+jsl-58LY;Pd`^WtV0d9P35{k-?9sm)V)N+uVR9SP znB-r(d^?~dB2=OObngCCezz4t_>}8!iFG=Fq$ora$$h4%h`$u`U?N@Dzw^kJQIFXm ze0ggQAmRb}YCtFR&e@u?`htH{ybOtqKSV09MdwMK>)j$1Wc{{Ugcc30orerBtXmTL zWxAk4l+~P3kgi%RSEfIT)HN(ar>{l*1BnV2J6hrnWLVGk#LP$f@;VgUC+3n2U+km; zKp6!JO+sXt0Il+^Mn$luly$ILaW$NI4Jvn?c^(U?C<(6-Dy4L-YTue#C!wYV3x#7# zNETaV$2&L|R<##ex;Fd){qK!-%rzB{4j6_gNU8Cuh!R3{OU}6&E2%`gs8~INy0qzp z<%!X&U#_~;zt>8U=e9!>Wc7d=I>5?V;wJng?5&yu#wO~?!SlNm(t*R{Hli`KHmzzB zK@@Rzj?LI7EL_QLOSMxq1l%rp0yJvD7|H3tHm?QX*qx(fdmB z(&JjzWdN3!R6vjk#T`O0skmPaCnJT3-p%7u9_eze^76clQNLpIsx_otFL=0F7qQcm zi2m-P>`Q2OVJ{t=;=dnlHK`ibJrIKC0eb}&-J`HFNJb~K=DamB)Ma!1Zl_T=Z>nX2 zHK2mMXDZB+Z?Bi6C{7YQTEj%zl6@(F8X6+;LM?zZtxow%D7(rSWqoGy^WX6sg{X8m zN0o{K`M)j{$wmG>LnHtE-xADG zbc+!(YFADvbKxy{i&ae_k6eS!m{o{sH%QubKbE%JW_kUy0bPN&26fhrTm#AuuXooY zlXTykn)Lu(OwHLVy5@U|VacjIPQ zJL=~!g-@1@PC&>6jUq7lI8x3jU?_*WB30NDPU}VaNl@79<13)srwbYfPqY?&MrbT< z`Scy=nw-zJkr#!$Irrton4b6tZf$WTRSUKy-0#r}tE>LCY+eq`EoN0C_K4(~rbE?~ z;2o=2q+m8#g`fz8ZQI(Ks8Ex$ZuiU@t+eD9N=qd|t{UEC2Hn{D)4WKnD!j=4A9m5; zJ(1t>k?~S4DyP}eO_R@TSZHd!w10Z0~_ya_xxI1f1z65PS0MexgBDVEdYXRzGqT^GNCV$`0y2tZ8QfL~- zlGo=5bMC_xLDXb0$cRfBEzMSNsg1gQ{+0QyN_C9m*Oh!jK-X6U4i37;5f6)1c{QaQ zrw$vy`q_5$W27*=)5Kk_WX!>$nWgf0L|y8_F9=T6Ev!=jyP|_#(hYwI8-I=X9;r)I zs4%*E8&Qz#;05gfllum!09;w8VsD?34?p;R$;s43x@ zgNN4tR3Sn{dK#=X z(L^?-jZeB_CJh_+9m*>Z#ysQR-zg$>ZkB4(eYHctg!V}=`p6^)LWj#@NvWXbvoe!D zfbRRe`?g6{@1HWrX$uH*%uS=m&BZ$775R(mI$-%I2agju-zRRP5swnt5kG(qbRyr< zg!7?7^$*w(>((!UWK2S>B&|!z5KDmbZujkJTIFAZ1W66Zrqk?fvuK}aVVOqqVV4k` zl?2e})7&+>Wh<)i^|DeaJC(KHFAwGbx*mQhz7 zgq=C9^>XR0N84Qay zG=u(fp;|S5YONROu7xA4=uQ;2^{M;v_;a%67-OtcZ~P?5Q*qXMKCxyLV@%g6ur$n* ztN<0+HmEQ2XbrWjt#U6N|JwcJ*H{HLB0e)mXzd;*=jKpldQ^|H50uBtQs@;Zf=o6y z>`s%#nVGpSaq8XEu+c?r<7FE_RKY;-c5J(nNm)*)-g&iZC39$+E7c2XX_JJ(-%5^xn)mK3H9scE{!T1N#tZ{Di6*-31YHu_z$Nr?I_X zL^m3WR3e=x!`M!$T(gss7H?Zll>0z2;Xp%vKR2eIs2`2&`4wY7UtnX99`np^SM4w= zAU{8Yy3Gi^Q;i4Z78P*Jo8~%us*9u29B1AM?TyFv6-4t>W9yuM9-#@4Kc2(;$CbJ3 zLi*ICTW8zYIm$4x%<+v(w*ILcWKgdJnQ?%dVTQ`-*kblm-dK!-_T64uJDYc7-4KV% zW>xDdY{(oe_z2s8l7phtKeHzFvW%;;vyd>%EslHIsifoH{ON!#BeKR5SLZI(cE>Kz z4+McrY6Ox!0#7zw|wH`TEfta1gNJZ28oh8}Pk> zGxJRbD@KF|HTY4PG{7cyv|f`rDNG@ehIeq4y;47kM&lQU3&+x0u*$42?j9@5nqG@M zJS3TdVsA_~xA>|@|5_?B5uSxu`((6bU!Pj0pVfAea7iIHc>tr7+aVJbiSE7y29^f_ zy6=}E;nA8Q==oza6KrfQ@vnGfs{H*73 z07s`q^glQB2MUM`1FmxouC0UWDzbnS zs~znaQX6cNkXji}50r5AE)X=?H@8oOhFZI{=+Gs9;=$3{{gD@o-ZK;6m3T^nXzOP- z+Fzq8`wt>S=RON&hvb%4s)T!3oRpNYQubX0))=GHrb}QG8Wrs%@$;R8-Vw1LQ&PY1Lm#h;f#}0BhTWH%o5PN?BrSLM zp>RFd@3R*_fw^SsuQvpkon|B3(mtGo;YbUoNg-8!OQ9OQvnRaNqS(1~J$z{s*qX?R zhVB)csVersJVRyPL5gXBUGtX-s4oGMcUE(0 zW1H~>XMaVJE2)L;?zLiPo6;f+3Xf()kX=-PxO z=BNHB`1dPM3fDU^MP|oVhkgCCBVUifQ%CYeUb+qlnH5(4%&C>9cZUf?6=&g~qDN-{ z1-y8SpqNrVJO;aptB0*#FHMX=z;`>wS=(Z>RvW0e!W=58bkKIZ9&g7twW2Qv z<5wzN8$Dv0jXyCgOq_#uNh*}k@qEq5Oc3EOkI4A0j#+G{N6@~AKu*p}H5|`F)~I>w3&_J+wCMSp1*%N$&Tnqr z_|~-pnOdCMH(5Lbfc~=DwP=7RWi8F=`ZY?!NOSLsiN0NoMzitHQDe@I*xAgsjEaWzL zh0cuf+}5?JN2KMns*JoTVM%d>9yjxhX$@<=E3M_VZiz~~)53?j7iB?88AQ*?2v=KL zrFl+&ad&Syrw!S))Er1m39;|@@X_iHMeF5Gwzyd4&Re!#9O!;NYdz(SuV=35=h?m1#sY?sV|Hnvx*OYNd^#y#PZ)fdZ9PRpu$vKktM7ZwMsti)Wr1wi zn}7NXt;MUKz+rGX8V4DbyF%b(&FYWj5B%-J2|!wD4GA5`mN*^#D;Ac(rqq#9nuJy} zeEFEsN^$JWiR2%0EyEibXoxeSThfOD>o1%{$tR$RrRap2{nd>;>!paWlR0+yunSBIUpmqxW(erQfM6hpC@dAf!%(ut}E3Jg1 z4Yxs6udj}cRGDO%R7QeS<;iHt`iJ}wYFqJ~iLSqNBis*{w(9rmlW8#b$vpP6m1UL2 zJbhF|z~V}==nh24NVF5?ZCPy%(YY5Tj=}*6oNayflmz^aWO5A+voR*aNN{ziMQL~t z*pD0xE$>d7h{Cb7Ql?fMVO2pijIuqax5kFIm7Z#;1B%-7j$-ErzdCmr%wkwp0@QIx zq-mQ-HqUoo{8P#g5C!YV-Z}!c$x{&Ll?Xj`GkmrFVeD4o3_=vL#$WMLAKokR!yM$& z3#^RoN%B_vHrW&6Dg`EJ&J3ikF@xR=Hy*kXfe{CCgG;iAyLl&HE_01 zdh8&3R4uNJfn^i3T#NKyC&tXX4Zq!IbR09mmAmHbZ`ZuzabKDozVwx2F~|QYbjN+# zdD=Jj_G;Rv)hOcGc4~ zEZMe*tjO&0Z5GI}TIg(b3#hqJJWm88_s@|BpLV3Jv5S1U|KW1Rqp=C{C#PSlj@f7g zv_bz{*n`A0ypj|sE}y7Z?6O-EJk3NYp6<^WtWIG4b)scC6Y;i!q$nO@u5poZSTluE zF=KBV$Z&sb>hgWlF_{PRP>=tzTv0V$$^~BxC(pB?rEk~-%7TbyXZR_fo2a*&fp zI^RB}`Of|l=~NnrsE=-`M1z*I)^iT}bNC}@zsr$m8MRT967d>)Ew@|2AcR@#^N&r| z;R&l&vKQLMs{8nZG?F58IqN{q?%b+`_xKU|Us3g3Pdi)?58P#K{s zzh}InL&jXHDkTuN?H*_x5}J&(W=6|x_gs+Qv9z-(i*(M`1#(gjJnT2gR?4LKm<+Qc zTx~Y%;`Cz~00)FvE3}@^&G2{d@hX|!4@5Z&p^<(yvPcDE#U{*;s@Xr`(<2ytf1n7g z?UyQ>_2IymG`{<8IBv=TTI)^idn3^nwMlgdLRWp#v{HQ&!8ax^q<)&Uyrx2!Nx>?5 zNkg)Z@p?Taf=pvmO-J1?tNy-89Vm03x>P3{VyQjs)MCn-zTpptOzw$e|CKy_>U8QY z`NEs}@eVsvgD6YYh49KX6Lm)hhHV3I$VQTcUQu@`fwMlSGhhM~#{(NsBblfj63|ix z3s8!J-JhW=cEwW^{Ii;LzZU&&P(oBUz^BH!kyPb$%J&g(mrZi7tDhl2Op&>8?bnw9 z!#GANvsvx2mz@wiRW4gWVt9bJFd)u-{# z#j1^r|H=7QNN+!H}Phhk>lr0V0!G*2>e$fx%zB?Dudjv573FyNc$W<&+-(tqf&vNl4{&m06=u^mQ+s z-!i%P^{mdM-L@M@*(0+9Ke(E&(d%5X1mT4n%V_~diaOTCNnOivKv$#6)3Rk7!99#T z?>l3Bh5j}>SHHq8P!31cY*Cddmp>MJjkD{6p03N5!}YEm3iKEOKZ|M9^-m4#O4(in zWsEPp)W~qad4e;XfZpxiF0G8E7JV&mkkUo>sjiNn+MK?s#(KXvD8WKCKB~rE0@V0n zsetts4dG|@W0>D9u)`J4mZ?uos2VG~{v9mIiBCkPw=*a z&wssvuUE9C&hX>|yM>Qvu@a}^>znT`o-!8K!e;?z(dBs-0DYD`lM&jbk(`@Sf zNT+=%<^4dSF#l+>Tfi3&?`IN64co&dVi|R^H1B|Z7mR#@CTTPdG{nTN9ZQn%4?3jp zZrPrsMSCQ-D4^y3HP(_jrlHs77#jzoXv!)$JXuziR-PLI3?Q1Kkf4H8ev|B+@^cemL-$`g{f6 zbL^$+Iyoy(IFm8|epZaPjggHl_VU+zeArQAr_gh;_013^D2F3W`Scl=PDc3JDIm7% z;eBuC8dA3Lq+7|*$d&8yZ@LXS_u1p32Dr~7#vCVcI3XWWr&m^F&$gAeOSCTPfKRjR zL@zyzZ?@2e8{M!81pR35etheRak|tbo!6GP{O*AHWyC36th{)-nuWez@29vqt5a6U z;Izhsw-)gM0BeixA|S;X574h&mO5=@gGt+zZfSt0!QEHwm%FajYSkA*)6hyYFk5$q z0o)_!{)2IuRlS)p`4nL{GOi?x<6RvpHim9!LIIICv2qy3L|FF*IH=qO=L#~}-YNnF zh+oDNvT>Cv^|>u}w@5^M`jtq~ikv_fJD{n{=KWxgE#Y^iY-KrC#Qn>NdpV-T1^ zJ*jzdmiKoWy{vxOprg#fdt?Amks8fKjIP>ApEdnczTg^F*+dU z{HbfrmdD zQU_*(Yl_Gx5dfRrO|F5*D9{kYTXIrGJ=m%^kiFdNE*f`r$_ksk(-g`J_iAIYkzgA* z^)|Xg06l@{$3$^qdAcu_-EiLoi!E1AWC@RKIBN+wtTaM)0lYVGyDXU~`OdHZ*i5ao zMo});Xj4t?pZ@$xTP`mKKhuPND@D&yCB$dwj7@MM62jNS>gRvx;Quab4a#l5{`V1{ z%Qu*xxWU{oXj5KgmFlVO(OEeXz3xl3h^3we;pt6f#^D zJtX!kf8|fk!Q|+G zM!+ARRB3eDlX$CR;9%i%tkJv85nyutXeFJ{^oE=9R$e)1%c4z6bM3e>^G9{t%A+xYdJu zK4$ULeiP*C8_PPMkUq%CFc!0qc3>`YG2>Cw45W3L*WDV}r5}zxZ@#}#uRhcbaEO`j zuQdDcKVPB~`j`Z8=g+wOSNH=N--Mg*ej!B9bm3S(3OI4ksXb}}UX{}w-^}rUQ^u43 zeNuwU&iE!wzh`eyM`xe^4m@m{f%QT2un?2u{BJ4E@42ndfoa>u@HNvnHs3M-3P*Wu zbI;gNs$2@c-mzFeO@J~RN`cv!2(e=#$C^u9inW0r#%%`H3%Hh0{*!|$@#U47N^z>IT0gWV!Q1C4vXf=EGGhK#T z6n(r5fbCq@@wfS8iUbGtEN89-fe>?1boOdM!o?i2xr$4>#!Y zp(`F_55IA6)TLZ2%1Z|na++}$RI4#KcmKdBksmN2aR3uxEIJ2EUGU5Q2meaFTWj7==mhg9yEjwvwzuH) z!Djc1A?L`lz$R*#pcCG9ThZ&hG+*+H0K7&?s7LYjlr$Rbpx?;R2T=1csA~=Imp+{8 zee&*s#wuK5Ag^j_ehCHY6xD=#g2wKFNb3Lw^+}6yC;qO?J)ffd-O2cfxu5VdYqixo zY%?wQ_xSVh@~)$&d2fg^E4yi-k=Lzfink$o+sm(xJJ|mGlKD*sgkYHe!}192j#_hl zuYXFCorY3LJfd_Af!aa`eY#6Js%Tml`#LcRgO)+DXa**_m&AS|C|Mi%dMLD6OXIi% zTv;(OC3LmpEF^-j;RLp~KxfR7V=)^B=3GOfd8TeL5S4==bN-EV0J&vVEK*4pbj4$m z4F&aR!V$r`SH8(h;NH&l;E&m&rtiOQ-td-k0`G2!*#%iT<(=3NjQDNd)L!SRsAVGr ziD8`~u|C>JT98F+)D9v8gXtutt+wO9U(FF>$nLrw~Gmc#tV8lzJdZXP?CiO@Ozge!bAJl2g+x%eD( zYS0(Xdzh}C?_CTd`JK|P!srOX{SrDU8L}jNp=4F>mve9SF%eC(Wl3$d35KBpjmfsNeRIj(P%>T5cH@PqFq8`voa{`b{Vd%xm zQDFl+b`WDPL63xW3__P>FB}sB2@*-+3EoF_ulV^zgP>l@6i;%3Z8XMkExMusR+T;# zPhA)eVA*Dfll8uf*~5xe&5($E>r-s~$Rr1W4NMY=mGP6&UxTjTv<)I;qUl)kHl1`0 z+K1#1T7^KD>=0tZDY*Wmx1h5uZHL!%?k0f=gTuE!fQdEV7@&ye8myj!2fnUiK|aBv zK~f8CR&Xk*H#Gqr)qf7kINj^KD%J#l0N;LjJjsk$A$kzqjK#kAG|dbfPozr)a*KC_)2S4d+c^*MsXZWa7_P2$)411Jhx}uQLy=b zj2JF9JS6D0@m$*WJwvD$`u4>ipg?~87IN>nVsYy;TX$mqi%)oJ2~t(!)u)sv&wrL9 z_%sZ;S~)vA1LL&a=*(jciJ<8k-ssWNb0-?vCL6RT|ky9kRk=t6q?l(lng9h14Pq3SraJk7)vKjBN-o` z!QXS-D|h|X5V9kLeu~)T>x9(v*6RRP4hZap2#2B2g=PeNnk`;m_gsPtKmesz?(}f7 zI>OeYC?Y933M(!dcL{SFr6$Z$edm759L**QEhVpKH_h}>-CW1`^qL6h=MLU)01R^l{v7}wU307sk20jME+%=gYJf1Poo5sS1i+3 zg6OT+rQUD^4ZQ33qM?5pfpPTsb2oa8c~Ul1_qWz;F<#06tVEpW_#?=0w0Yh(Lz)(& zr14FT>XtS#{xcT?jGhjb)RQ;`dm)5O-9s1#&y+P>%r~Iq*fEz{ ztzYK@Zz}}spnAmBl%{JsFipO(c3rgL_Kavk@JOb2p(eCOByeq4RyTHCI#CA#3ICB` z=WjJ(goUK%?|n0?e=wWjme;J*KQGOpJb>~&t(FaRpp3D^9mHZc7|q_2fE-6E`6a82 zJAAm!f^?N}*Yono@yE-3czT$Zq=d;Udn^eoiaOQ>-`jO;KMKO+3W*AogMh+b=_}N6 zLBKJ`yWuAHFDCfCT=))N{8pHbg==>3&7ubJV-(-g(xzl50i_PAGF23lBZneWL0J?! zPi2pe%5l?jb2hmt!C|(}3tE7|xVXP}?6wDv94a0zA+ncm! zbqEL=iwHm(j)h+>dF`yi|L6La_h@Fl(Y2^?{F*WsQ?C*=Y)z4;{Ro$xnT;-|Z(-p` z#wmDem^lXhoDYl-?`md_eFMo|DcF7ZHK4~=5Xe14k_9=^e8$eC>$r%~kT|V#6mGQ} z^C5FiM9}_x)`j+1cQ>y?FCMd?t+|;6s0@o`9lrO{JH8?8ZtN)W6iTN>%=2R5oh&ay+)EwYBZmaQi0i(sxATks(?Vq zAauW7x^y~|k(55aQQg%$a}?9EYYPQRBt8I!hPcYt??MZc&Vyh!9x+!9&|RaenY^#n zzwO$G$C@=}=(2nL4GUV!u6WBGthTEyEwWla4MR7nz^YX=4D#e526|W+W~tFg2n);c zZf+fa{#0SwHas7lok92sgp-JgoudqU&~82-^PAe0-D9^nQ0x9MFyQ+Rf5==l&7l9N zVnE=(_RPN{T8&A_bmyWzTcd)URoVq;$Fi=x*n%E|r0rn&%Eep24=A}v)^=pO*0H?N zQ*_fR**53TR4_pC7dlJi(lRCa1=7;GPfScV;RfR__x_N?Up90`A#Zrhxu93>@*3i} z58m3sNiqUKe7(NXnk3;AG4qO{U9dHi1;>phClDzZ=)hfD6RVF1ohCu0>RWKL>jdtx z6eLW;zx9zkfS#ISp!G-|deFopNzQPT7A)}Y>O1gELZx^fK!v=~-~NntJOx<}#krxj zrULuyt}nj1<)isriC`FS%FD zvuO2_j~rQ&GAjtZHtn&9P2g-&&9Ly#@b(+hAYJK1u6a?*vPH}5i6At8bh={ok**ki z}oFuO>hNW?zI)NSu6TzJwgGdf5>fH)arfMpYKUYx0Ss+?f{EPA0 zvRG|jIjk2)?G&2#Rm!XX$<1S4JhXp44O;r+=B?M6ROLAN^xa5`V?5ChY}>Ew;|IK> zZJn>W?bIh{!K0j?@vj#lRw&?r+%^81sYki);P590- z%ijrMryA+rS0^6|D+9JezB(*K&eAw~=&Zh)AWKhh1@gzYV#3M^IuoJ3yc?YYkc>b| zk>TqPb|FuNLiUyE;R_y3_DTsp-}+`T{%4W-WOoKLUEb`!Cad3mrCqbQ7aVXi4oJkN#HzXV4(3-MBoq;K_*@h8UDjTg4gR}jXWZ3C#%AS32j zR7quC^w3gW0*&6~7Rq{9{jgHcx6!rt_{jzOs3=7ufi*2Cs7MXuejy^|AG&0R5I~pR zV8cOPvh&B(eCYpIyZ>3|;TljAyL3kNYu0YN=*~7;qpS% zs(RUdNyD$cxgVugDWL2)eY8kCa7Fh#4l0uGsn~J989`P9psT3tR1{%!7Y#V>n`6 zfvOybzi1`xdD$=bp!?utc{xd3Xl44ZQF;*@I^p9^;)0qWtzm?neV8~)-UMixrtt*s z6KJ|GtP$H936@WZqDqMxV>R5yo2azXwC{lD{{@RcbidwyuaR3>6y1OODe9WZ9+-t2 z4D*r2-m}gj%p~DtJ~%*_w=$lbE7&~A04ZZl4p`yltP`)v0~?oTDDTm5CX`5GYT z?2|RPrIJ)CRh3GmsxO3$tCee@O0?n}Sn7H!eG;0NW-U>SF|z#zA9I=PhK$Je#@~zjiwI0wr@#Tx@NQsUU!b%%XQ;WGd_wI}d1MmU zv7yGDNjBqTVKXQAN@f5az-rw>+}-zO%TeY2!QOIj9LuQOKiF%D#`~B2zC9{iMiVOn ze@|j1!1Dw~=GiwIew|oE_ly8*OM7$15=lLB;B%6Dlz8X;FH2pMHKZMz1AHD;A_gA< zIhwK*DtDJ8%nX%o9P$T?&*e~Qg4nlpDri=QfY^BW+KIq}{rtv13+6?#%mw$%n})Z~>=dB9?mhf9<7xlmev6ge`;4d8mvM)_Yj^ys!f)|6UE*!~hJrY`GFDHB~fYb1^D4gbbT) z$?{H?CBl%j$X-(cUPxlotW@gU4;aVk{+9Lh9&KQ`N=GT|W= z%bs?Q^G1IKRJPpU36FWmN3)|?{k+E`nFzw7c?rK1Ds>~!t-xufeI%7$lp|5$e^qP^+q0aWx~Z-_l^qX zrAm8LCR9vVgnaKiL410!_?hDcn9t3po`}*wXR0^gDU1~KVH$?_%#F1@D$_V12|Y># zN$6SQWgt3(I?2Yem&DFS|7zN#`@%PLL zW{C?4g-Y`2$B48><^{YbpW2{uLS~rTk^~geyaNx6BNMWM)BY*2Aq;pjUbxC63R(=G z2nuu(C$nogY%Lkg`+x^WG4-!5!405$E3RwG681{%?o*pAsu8$$bTR3Ak}(xP;DZau zaeL8>_@3=*r?lslMO311H~3{`=|;1w zZneero)=^|WIhZ9p8!XJ6p;y)G4b!K7w}~zc!tUY9yDU}0uMLgUj&`)Q7N@=ek+@A z2KLctbnzw1M<@G}>7jXLG z`Sw-|32spt@qiw*L#J^mK;)Hf!5?5*;>OXS(yt@pCg4X2Nth2=8{h&y@JQd^jwuO3 zCDje+d&nQ%vOFr6@JVm!i*C36Jul0Posg5@8xoSprve=X=INF1I0;~2YJ-n{1#Flh z4@D4@aQe;x`bAxLuTYk!N*9$2c6k|<`v(sekC#FvH3#owtrgY;y>ft?Kw^H#&?QLc zW#2#0uD)V=|4rvk$(5l+FK_FpQYX4UB+;n$77`DRN|UF?#7hJP8w3zDyIBfl5?tE6 zz4>2bS>nbqr%FLk8){)ie~agDR7KS(hklB=q`;zPbMO6?LCd33QYjgac3U_BU%-*B zT=f2Rjz9+JI`|}l0fny8l)6e~>h2*^*gxpryfS1J>lO-T0hK1zB`Il|nVx>{xBC*W$*7P_CN%QfX5+{ucrBU& zk`Qp9nrMs%KkePrWN)@CaS!gI(qseP3kjRdQ_*euU=rmSgcYug`|pD)OWcDysKgi= z2P7FUx^X-nQ;112M&_vftKgi4A>Mi`AT$8-Knz6}aJn&INkk@W!?pCxN2N>$ixFv2 zDbqo~1U^!kDGGJ4nsAL!^^bdBE=QFz;ew3lf+ILw;ATG3m5pbAqfrA{r$O<(ESE#2 zzJh(vgZZwO8kM{HzE;&PBl13Du2L{$(;+3n&w?fNW8_mEylDYdvOn6nU%6CysH37k zSae>%>DS+}8gGF4RN2mES+!Tb9U^T9657CwEv)#!Tv`&A=qk`e%1{)4A<(s8rHc8`u_@eoc}XMU5z-jm7domP?>A z(G-LyokOaAU_7ez70eHgIr$kWBWy#P|G=io)SA(e^R(x_l=jiDTA}iMOrs;87;IkH zeVZe0K*watxX_kg#Xp7a0CW&RqrzKM9??W6X9-gOuDf1d!091P2rI`;yaFoU(2ypG z%Xz5OR`oazjUQ`Nhr&RE`clYQt~8^vv$C=R-w^*Sjhz>AosO|lT(#RbRQiI&59xAe zZn=P;P?+AcqPA4;$_1_i$h|7}bjF&64sh4Ewg1+VjTP;RKZbn7d=f&CPGT!|l;$)b zl0sL|Q5q5v&hE_}Pi<)u?{~bo>+K63{l*7bjD@9MXe!~S7S;gCMMm~Y>n5jh`{e>o z|7Sp0xJ&?~J5cJ@J7E+0-*uOkbHL15Zwu}wr*ZqG$(DJ?OWk_=Jn~Z^Xyni4dFB+r z%vtXWxLcXnO&82l7xMFaw&Zo|JtQoo!T;g|`8|^f7qfS4!sf@doVVWf;BIeE4+2X>#nl|Bjaq1I`e^=@kXr7>{H@>kg3K8eYIg)CuIdV%1% z+I9i&Ob!mW4kR1~6jaUlfw1wc-~-a#ckjCQ%gDCm`P=vTBt8&)$ihNM&KCRIG=^BW z{?Ze_YPo>Z!!%0RgqnE%_hZ@X*LyM{(Nccsdc8HA!m=A!YNw0zzVhz6g8O@8z@iN~ z?yh%xes{eak4}Pg5|c25Ue6CHiJ;eG!ydl-l+f#ioCLioBfXf21f${fggUPDdcUVU zp*<3YJr>X?VM9iRY7y~XOr$*LB7j~`ZIstf{yg~Z&5!R7PTuSve%{FY>^)h6#ahR2{{N>ejpE`{3g2%l-}~Qt^W=56Hf@L^#k4Ol7^@ z;7t#)JfQWy?tI6F&D#sJQjXbsS{~4PUw2aG>Uuwb%*+qy{##HT5<%(|`)wvw($66S z0mS#Y2++@k1CJ3^(wa6D!b)c@uA`V=G6!s2&1TJ7gCtTbMPqzoe&mZ%D zwI~nBo@!#-E%P2c^GDV$iUdv%>LPKR0+Cjn-}7j94{^QQ1uX>~K~C@~3uqy%RMVob zVh|sUpK?lsKV}K_lT^^xG9ghwLLSl6yBS(4FAwN_iCu3iVp}{kLHUhq%rNR__l(lG zi|L@IoMD1iJ~gpxB?PI^TwqMYiHVN`JU05h2k$7E8d#VbREKFb^$-FX;!7oU-X9Z@ ze1RA{4{*I(#w>+OgvdN@salbT!N1Gp0j_t;z^Y)@yAWIT!I!20t9QK2-QVq^zd~hy zyAfa?>JD#;0(-~H-2J^wS}nES58Y<&mIrkHw(H$85Cp!Pg?`$GwVUn&?8 zAMv)KZyWE&+Rx?Z?ZWkg*3Vmn>wic~{{Z#1X?|utHUXDqL&WC`*fkm8e^04e@F^8$ zxzVf1=aFOq6)4gNEJB0&D)mO5K89;EO~`!`+GcnB0RDjZIlk@G29oz`U$(HLUx0pO zM8a?e5>2Z!^F}l@j%_5*Z!~pamzBEv@&M@p-H$oz-S*1lGV6VBmm15xU2lw=Y(hvM zP%&Ycl}%oE4CTLG1Z7?3#!kWecyL_L$WWZSvi7G6Oj#TQ6Sj5aM~B2L~Y~l0w823YfPh zleOzQFL$!8uawlWNU%VMB~QrcRxnwe_nedJT+GI47`BN5*B-M9;=>6SbpfaQ!w)<` zzq6?PWOor}kEk!`g2UI6n3WpjP0#!0%Uo@|0kg}w0p3*)z&Z2EoB$Wpxx63XBFwhZ zyRO;SZk0;4xflsclYw(C&O%OtJ7PMTYawdD>~f)9{(4bebh%c{)&^0z0E~>{gu^Nw z86If4fYZ(9AYqrmY&l#>O{B5E1ZuAzv*o}b7sZete;Mpv=5)K<6W$A``S~w^;JI&j zjivD;`cfZU!MJ$0EQz^&Z)s!R-Ll-}qtTl{QZZ#d{YGEDEij#3z#jU*n-c`mx&e@1 z-@W}5Q^9l)(Hm8{@*VEZWY@<;lAiX~Z&U}Y%@3<73gbgKDnKZ|?~3`2IdSVbyi_=Irq>_3!HTKF(j1{47CL~Qb-tiZQc+AfzMmxdOHWE}d;|ow( z6|4(f+gys^ulIq*eCnF-)N}#6`L&m?6Z4Xq1l*;Sx}lQm#X1w|fcOh`huhZd_(L92 zJesM0d}H^zIm`8;_}RQwO1PnUAa00#^8^Zh-aY)V|KYox>NlMbHWt*!4Z|Y%AstW=(S%Bv z_PwpXI$DPWZIx+6ri6v&iUdEu$UYa=1V#UQnh560D+&Mdgzu6na5JS{BGxP3B;GAm z}JSy{>>spNHD@xP6?53yiKO zcnCC0cVK~Bc?x%=v~xr*ViM0sMtyIa(CrN!vWU8Fp60x3pkBvZzHGS!Cea1HSbVFSWZLlbF<$c3~abVe+q?9^^87 z@-gv7U1^tkVF^tBWm8&QJ0}0K$s(?kPyV&jFkHeG^qAnFwp=Go{JP>@)GMruJzEEfjJX&-<%oV4bLcq0^OYq4eCRxN1 zM#48VB(o#x^C+k}jBD3IF5vV<3;SX-CSxiXULn`x(DrhG z)gB}YO+n3kxzl11lZr9$G?|ae3pm!*9xy7eOp`;x!c@@Xv7mCy!+M5!0pIFskl1<; z4wn~j`tnMclqp73TZG9YUF~6GvPI^|0w&dIge@n$Gud5i`=EsPC1iIAlWm0eJlS1B z?!7IS;FBdxwh`X7viqT+(3DHa?h+>32=98?{jgAZb((A=yqjeAL&D`HWOoUZZG?A| z?0#UFTvm3=goq@KmpFFkvfFL@pd|ODtZD^JwmNptmE8s8-s5r!iBrI2t7CV)?0zUH zH07$8Y<28zklhapl~?7Ht&ZJ|vil+7Qekqj8q|!*R>$r}+5ONkxg1He=$Eli7BLx-36(MN=Yh}>3$Ru>Tt@||t42GQcE2A|{TMU@aW_H_mdoY& zq{SqQ5-O%7oX;osO)Ve^oR675&L}TWu8TxTgIfwybKT|YnAGz(_Q!>OK|1lI#w7|E zY^~H@t?=I8a&=5%!kLBoq!XY=H!$JeNu;PTl{5g!%R3(5Zx1L~SwsUE%v{EKm&(;8 zPJL3Jaxc#GStJt@`4ntRi`)BMu8v7T5fLmJ&AZ1r)V20_G-M;K`NVNJ!!Si;5{Fd6 zfDXB!xd^{!!&6`d2*TlrVXn`eX^a4_%)u zVX{rA#)~qJ0*a=%hm;Mtt?cet;Y*1>cBHPh*QiZ++UrT0z!{yr#!Hu9*0X>9>lBC> zr5H#g;cz;p;Y2IwVR-7nw_E`Igj&#gr*nFG>hnnQkapa9Fq!p)+2_A@)Sz^KBu&Pi z4ZC$fq&(*$c&%d*#pX|}&V9DCu~7(pw$T*6>-SI%qtp2ZeBcS)fq(K8B1$ppiA)DF zVM&_ks9AAHd^+Y~K!vPCRwGeRi4k@R=PVfm0YUMmMT^)OPX&Ch=If7%Nb(SK3Pc_y zt%xie0nArO;Y0X_B@vmhbp5q z27Ib-+*BJ$6RjIK_adhjlc#u7mqrQL#Xt%uu8)HI5BTu$`0WmSM1+i)G>FjAK|?;H z!8$PfJK_n%TuL?w5qCN-8V*4z)yzlh?NhuYq*8(}XrM2vGf#_i>B43AqhE%Bq(X5)1O~UHIPjXE&;=xR>AkI~sr9Q)ZIH4s>9Gh%Fkz8eO_H#wLI)2eupwk|QoYVU;MudEiHKMGo{qF#2sg8#84va& zE0iQ%2Ob8PZXE|QCR2qu{Lv0^9So)6&{>i}MyTgJdYV8&&M45~kot)SZ`24>-A43d zUy_GYjb0{YNCq3PM1kxBT-Ft3GSs?B%I;%Cn^Qq%(49c>+HesQqg zhSm~6>BfYxDEp8v!-fNEtrq7fA& z+!*uoUcxur<~>EPJ?x;Tx7nJ9Gy5?f2@>F(R@6i}^j~5_#wt0MAgODXS#05 z0j2^y!V6Aws!@D>l1@OfKPb9YAnS`j5q?5t0y1TZVrN=#)`C}}azcs8AmF%Zq>_y2 zItwY?yf1xDS7zj$vP>yPsHui@}ksx$d6CI0NYGKMEJ?dN%d$2#$ z+60r?&R=&m2D=Fj&o#Ku1RRg3^m6)oUZJn4k}qoisLAU-KF~xhmkDpq2ftB2KdVl- zRLsg(0?F(lkO_$bA_D2b2ekq54xdRNAgN;h2{lk~6;ETqCPd6M1HxYv6&_O559H}O z45-ajZz8HEW6A{hTnH*-t^?*vij*Kd*hj63gpy}LC=ydkR)ITdqFWX}#*OZQdXa|{y)K5}D9aL{OqT1cfc9fMExkzjM7v3i|?*2>f zlad5#=VP;9_;AR4MnaT>LT?J0^QZIkb1!GA9v35ISsVSA8>!_w-Z+_rYbY`BCz|z? zPh#sjCh^=%;>C4M;>Ds#yeLefuP1>j8xGE$;z0?ldgP)Z`uTMP= zgmrz;mkCMeFb&bS(eNFkVUeK6qj0tcV+aVd&B3quYkl1}Bq0MLv2Fpql8W+g zaSj2OgMMOCsXjWWa$yPZgfpNtQd+B0G-v>66c~r16Otz56PS?LLqmQ3w~}&7Dk#RO z&NZ=Os8eb%I%K0%P&9}r@oCpla)|8~BLVnCs09s;B}b-%UQV7tp4$R(YJydM z=B&n|x%rjxnGf~j%u%8#L06cVq2~C?2IaQK5tkvnEmT&JN*LUaSP~3&#Yh&>p@09G zgl`xUF2lc%wDMH>p@7rE7emb1&uTN7Rn|Xb`)Co7+o|bhn%s+Xc~3s=X+ELET4}qQ zf8`b4u?b5`yBCKyx5}{}|3GO>LN=w+`UcIb5`=Wl=RKZAr5=tH2UV)5@s}~!a;7pG z56B^%@G1S4g^EASW!FDS3Mc>dU;kCwXi5X1B+w|-sNI!EN~U}>TVnU36dKGlS;?X* zZ${n}Y;Bp16z55RZ0q&HHmmPpy~Imu!BkmJBRrgC;R%+*v6%DYgv4@*=@mgbGb_O@ z&!=f9S%tcB=_R*TE?@uJe#Hwdt38$h>+T}=gj4De&j>W3DA9++Pq?6gp(mrREHyw` zQgWQZxd8RR>iqg0@~`DYwQm7(hE&S7mg#{woSLL^?~wf{Ur+SUC;F6e(%EO<(PM;k6G(ps@M$xhHILAspv zWNjzESI{|8*7GD~1LB{hv04djGcc;2b;A*96rq_sni($%Nr9LnbEnhy;Nw7X6>U{; zF9RZt5_7pQw0M*4vF4}3Ji)<1?-Z=J`V|=*d^|e-?(pr=|9*FJuzPg$^T)$Cub-@* zr330GA!r8$`c@&dm{JQ(6r^ewN8hci&{oI8MQ22TI9D2Qg z`aGa(bz}KyuQZH!>O(J7Cq?fpOYG(`zF|G|UgzDd3m($f_MA}~8cXOz{<8TmZ$cA- z<5Uy-++EYA=e{xe(=uucXPAg5=+rW%CWt(o^m;y#J_+b+^m8~_Gj|ONdqvdSX_?y{ z2e9-=97}K!=b*S|1xlS(K01ZO zew>L_ZP<9fJMVV)Ne%HkjcP<&u7ih!O^N`1+H>0KWH<>Ky?#={NQUiy^3=JgLThxvet|#(DMR4G6^GY55<-T!lji!hv) z6+*`3PXt3glA1=8vebD3b`0tRlr}jJWcyuSt)*ytehsBo@Q6mq^6bI$ypBU%n_us2 zzS!w+c`yDB`yamj2=My#Kj018j7v+0<|9i+0WxU?O#Od*-G#!XBE$3zxssf8-WJ5A zIj6$zR%^hvS6W79Vg~DTro`9G65o(Timdd_Om?e{d@Z78W$~^yGxdRFkzsv`!gL=c zBX%e`+U{hNbq@u&()|b;WUOsnrnGp6X7;(2s_VH8!rZSG4VbPZd0033`?r-k&pL?&i9QuCtOlOHKQTVfGovqESt={HVZ?o?- z-F^<=y+4Lu`u~D&caL}9?Le*}4OvL_!IVYzE+{m%R;9R7u@L5)x5vBt3h41z6A1)N zpqu^-?Z+l?6Z^xsp;dQIzrSuCotSH`@<i%=DZ}zOUYHwMf?e<0$uVBWtraF)lG`VMNUF>IiQbvS;1a zVh$t-^oc}Ibz93Qgrc^E@tMdB=vd3l09YbvI8?_eVjkh!8q+O!`|%x`&kwBbZ>DIJ zVJ6*S<7GRE*-u6%focB~R;}lWn$2Ja1D=d^qb(fewnd=>NG(tr2ljfp2y?3n6Ps=W zny@|i<$Efe>uI%yo*Ziqlkk`+Mx!s^SJX<9n3+lShKz>68ZtkhM4Bx(@}0t5PXXJi zL*{!WEdmx)E40Pa86H#(H7Mq&FZfR^9h@Kz^lzOW?4m!CJ;9uy=H)=wXgXO3 zVjG6sKReOxC`u#2yZ9u}T2)Xav%9+N&b4r1S{x+;?)3CF&uLK2Tk2R1QI^NjorE4s zc5cU;9U1@$<1rb~g!#A+Sp)>ee8Sa;Rp;kqhK!$tVc~u!rxVW%CZS>iClewQDhdVy zbWaI1-CG-kJ(P!JN>9uu*AQ`HuPp2;8xI^zRIbl~s4Euk!L!a?*y>nb>{RuW9r5?n z#tf)1&y!2?9f=`ldTI~u54FZXO!EEw^j~0)xzig=x6Iy>K8=WA-11k$3DqjdV(Wm@ zLY0gYkZ7j1f|n1(Y(g3oi6*-`WOk!Z^-fm&q}g7y*;{gG}sDq7Dc2?_aV1Hbg@zSu|#8wv4y0rypN z*-K3KvL`4y#G%K{7XEh|&lbIW=Gp_N$D)9K(yEr4iAAkk3Zr?`Z$dt*HXW!j>)N!Z z@Ty&brdz#)n}2SvEw+Ab)?2?eUF+B1AB|u0j9>iG`1Kbte%+eI>pG2Bm$EHYFaH-B zmf9MW>K0{L(-5BW%Qm68tMkx}LYLbgH)s7ZC&|AzyGQxi(Zq4AbXdO@e%5c!yw++E zp$O|_KxJ|g3&o3-9M>&2RxF5~{$ULdU`HI^X-?7v4?Wfq5$#PA9>P`0Gar{3wBMQUX_4i9UcmO4>(B`zUE^ zB<-@><0b_yykkC%B<*zecsvtqG)~~(eIC#L1GYA|p2Iixn4BATy7srOwg;dY!TtvO zk?$#j*x&-4S7kboEMP=vab~)6{X}+S;*Y5v$T}D+9&CA=T3U3?r|#N+bkv$p$PBF_ z`VNRmZPTKkd>SV@bNM6=8OEr|tgHrv*ZHRbz%`dApeAeCxv2#t>2&^6DGTO)gy0bI zZY0!I+;9M}Zm)1e@ zVRB9c?F39Ff(_E7fF3(0Cc&-1BOnoUcaLEIs0-ih9_=5kcYfYK{{G{S$MEy+;o>8S^^;mSyZ1R;O4-r+AIhA#t4jx#exW z^fo(=&-f1P!aFt)>U`#0&8S_8es^m7nFoa-A2A;WG&!d1=BokKeAx*2d4wT~H$pz*(thhrh&c1&Xw*5QGkguc z)9IPM{>U4U}XS(;W!)(sib#75>sB{8+9~8u`ET zQ245SEa3l}+x-`1{{MVy`;q_OMp=PeV{XHSR^aI1&42Zb{?pq>!@`hJu>-pYyL;cG zCW$I}iHEGLn1`gUa=Ev^vG>LVrBKJOSa6?8sdVg>75Ja>jK$Wr@{^*+&dN&1{oC?6 z{rvlp6D?mF_rDcma^r;P5aa6Tlf;+c2J`oS|9OAAwEwsIThE_A?*H2;fBw1gtg{>o zXu|-WZG8UR=_ueZWs@BYcA(C;N|Sg!Q8@RB_^}IJEz-KM8Zwzc_nY0ruKMqP zch}aS_xZCP^q+tBbcYLjP)%aA9S9n{Nd=2WNB)=wX~?3{zK)fne}4Ok`YCF_9e^Ih zS)`Jkjt-zTA&EbJt52|271yN)XLPm$fBp&H8J)rB&z(H%7B^7?m9p;qY~Boa+z4H2 zRsw?>c;@AF0g~!RbpaRfd&(19{;HbOD6l`DQ$-Uy(C^p}z3MuSi3qN=jtaoduA}Zj zHL*ge{W2;DM<#agF%=Vo@ZNn|%Q>reg@w!3Ol|fqboFkzkrmX7r{~0mKz|2Mb!z)G zGsmy(xQ6Edq@komsG44ufvQ^+pc;J7!F|GG9`e!b2f766oksZrxW6HmTcfwoRF4 zPg<|0gCM&oPcjt2&g*7V7ye+rFB4_54eA+w68&=CSuCLj>iA$BMU`V_`(8VRnX~Z-C zg0(fcfQUs(UxWT;Q7Rbz8Sw;G=_F2O4tcDVt3oVTlnkLe=cU39e1(I`NU74DqfV5? z+LJRnTZbn)_Rr4i8bT_M=!ur6C|gatTIpBm*i5`aBw2-Q>lNgo{28(jl^hI{9FMYZ3SQeHx_kTW#N#@Ici5tKL z_kS;6ZElwD|6aU!y#KqEa@X$voD5?>WCAaHvfRWx3lypZrSU?`j~%)GZ~o!!xov*P z+j!P<*Pfspcx$k_YJb+z&ew{c=L#YdGez)NFmxu*oac1-bshvviWGh225B&&iPvt% zF$UW6a(K>h%Nd@M@@2-}j5#lZ!G`FZvFfTU?yT2fRnp0niakDwiJ;K^&%U?S?`?LA z^)i5{)<9Pedt=%k(1i4z`vN7On(9`s6)uZ!6Q$4~uN|ffkqz$_!e1@KH^!31JIQu9 zH}!$pS0JIoZ+7<}OCi*flEi@ccWj?7RMD$HwseLH&Z@J-uA@I`RP)?o1tATi5#-%{vrXnB2@-#yqj;kjGlpTx{I zxLeDx*=89`PQ2y!L1WwNTZ-@17UNNZkwpX8O%pDC5>nA}WyUeN?j^DJ23I?eY&WkA z8-k~>zg5AvnLUd}8`CYUvS5KK4Vso_VdLvt(x0fnuz#0Z(COoX-u8k@NJNUN&_SN2 z!{o$cG7l%%(gG%9FKcq`Jd%srC97Sr7xgQqFG+C9%VTIaFi+A8S%5BqCQ=LNOJEMS zK`Vm4H2xL}KfR%)WS%qzH+uPLcLB;DW1;(JyhDA4#gd~s%ND!b01PP)0 z=jYE|a5Ju&Pz!2UOuBs&iAlFd^bgW)*E{otlx4mr>`dsR$&RrHIR1Aqv5hFvMMnP96JMcQso*iSHa)DXX0v!igXZOW8 zw@;->o*dP!#HfJo{bjTF_kXQD`{n<9{@1h5pPzJLwSXSD zfP^0i)lw(0O?WfCcaWpKraWy#Iam>gD78|811Ja{mvRAa)pL zKojaGGyp}3LoOz2i+;2T#XCPrGNLcvqT$@~(hOWGso21vJy~Y-F-aic$d1psI0K$$c`63{Q!&-DU}{-=(>@&S z?X_P=?Yde@)0p|>N$O`m*(-pi+>Y(N*;|buQ{7A)SKG_ZYw+@6*@e|N8LO+A$8X7_ zq|oJBPDJiW$eyi!hk7VWW#NbJkPDx7*I*SjwCjRO(u6CZ%j{0mZ+7fc)7^LmIs3*P#JV z(swLMKYfgKNV&o!_lIVmk_fbdtb_8dJ3EjidCCv166vs6dVE%^VhP|sq{0jfti3=g zBwcFW#bFm`X?a$&BdEKCdSWFs=y@u9)f3u;C2v)`qhd2AQL-dlGt_Jk+oMMNJyzd= zF;q0KpSvZmg32ajnN%n`FT9`-FsZZZShdy$0!H;V3l7R6b)=cHkd9~o6CMGrL{`Ad@<^i>mNH$t1jXW%sdEN*nzjk5!k9{qt{&_5bayS1(KY|K{`O zkMX~5rQ8AiABS+PGbEXOg{90zItaBE95&>)EXaHX5=HI62pF&9s+CW2Dfm`(u(?@h zSj#I(6IsPm9f#CIki{oc`Uz%ClChU?&yH$y6Iq}yXp;To0|vwBA}hxxx@vE8vzyzp zDkJRQ)Cl`ajIf`Lurd_G6ilq|47{w4#GO|5Fs5cjswgNuI#ncfXrss34s|kosUJd? zdxIVrn@SICOAp?u0q~)#8cD1rO@cdtN6~d9Dz!eQNslR|GJ2X|I@-Qf*N0^YAB7lg}z-BU|GyChAiOiivpHE=Q!*oKy z4+)#dYZ>;psldayo8!+Xk?LOu`&Q#hvS_Y43z!qVR%ho6H^%(Dm+*~5@N^JTIp#dk zIk8a8n><|`EBWMmn*4g_V;0h0wl1c=Ss8hWDSlOIWF9pZs#OW{0)~Wz3(gf^T6*69 z(nkNoq}Mm5|MfRFEAbyUw_iT$f45TZivH(Z!^VQgiUBIB!{_o|XMbq^OvnfYSinY9 z>LalZ{;dRQu#Tx6SY$Gzk}w<)|4iz8H6jwowtZ zx;AvRVIi-g|AsDh+yzg^I!7g@N&XkR&oo2h4L4I-nj1yJ%LeAzDzLhmSvd4p^I^>e z7jwxHo&ziZvLEvXk}4G|;Q|Hzjx#bB+eJ8Potzsj*YJr3m10d-J@NEPxkAS6=C!Hd z?Pg0^?r5GFN_Ua>Zs^GEEuZdd=>E@=yekc*0;n{Un$C;MSrv=jujIy8QvJVfscw1Y zoRJpBRk`7=_MpKp53!9Gh1tGuxPL|k?BnjRWC%w(>l22pfYpG;g8C$(!49TDDeePp zYd=WCP`8ryhIwty-i`C>r5&6dJsLN0f4%wJ)>_jRZ?mNfqjJ=E=XbS%aCi!*Zk&el zmUa-bRyH=Du_)M=(4CMtf1RHZuiM*e#S7IM7JkKKJw@BDOGNGARP4i*eOZ{J!Be4? z`Qn1@mBXww&i9ftuATEGW?i<6&$E-~tgql1;{Y~yA?4NIDWR$`mJ3KD_PY&;qH{uE zRM>gza(!EyPDz;RgvZ8Aqs~%RrJS>WE1Uk>cX7$0uwdubjA!xwHD7(_7SR5?y1Sn= z+)&*3X0Ke!EsPO_d&NOu-Cf;fS;q3CX)%&{8}3VUo|{1b)%e$SFTum+f?DslzGJH_ z8gW6T`B~5@Q@?M-g4$GyEE;n7_xZoW?)me=^`_%_clv);(edtYYpcI{=3oEA{bvuq@Am$i z^#1s@_piU1Z}|^v&(>6nzqP9O~c)|7ijH z9o}0c6f<8d4n!k$s=x~zCS3>|>`0}}8d`gqw)#HOa5yW3jzleOO1j}i~?CsYPk zxqv9045(PMo^7U?2m2;-@qj`iNF>!}#;mOY8)EvI1ZHFsI{u^Cbtf_Rp_uJ80v087 zL@QUFZxrI(d(LCygQPxNZ+^>0vJ>IFH#3#R$^9zx=QumBYa9v}g`vpY3SCp2aiDa8 z*TrsFtJZ1`w4gh1R$eQdE-B>S)j}4&m2QK+&2S(Fr^t>xc_At%Yp=jHFVJ0#U=hZl;(YIO4XHpc_~XeSZsC}KRu z6(3MK@bN4*&U3|Al8YIGeb#YcpmgmD5_lW-M-ez|JAs63vtyLaAVfyy@ zFPb!RF85r?Ol>bDQX;px5d&tPesz3kcf=XBf&VdIGK2n zdmg_#0?9@Z4LYg;8!}(1V|eGK#3|FKQG*z_dFPN0^GvaBZ;Dd|+?N}vTDJ{@YQ2;+s~Qve1D!zubhnHMKTOp* zRJ%e5#pYWzTTs0ira1{^kN6YXU31ynjlb$@U)t#ZCNB0(`F}p|zuKza|G#+D|8Jw* z8U5c;=4C>nfQUfHaj(5CFeGVf=J_{jvu)@`)oAVWU;b43=#^d<*!^GXcCGNmbmm;Z z+xz>S*OpUOqPuGeR}o*BJHRv2Gt+3hV;W98Io>dqzJ_Mj4w$#A_KVJOk?(*nWFx3+ zqB&#G8_a4NnfOK7-1XCBTWhb$81>{t+sTp~Sv5URtIytZUYZ{~s>bXI73kmL*s}9$ zr9xzX+x=Qi&Wp8*qF%p_wQ=tYE2#h#)^JV}yNXcTpItw#Vi%UKwtl{N#TItm3d>em zu3C^;cKC7*%6U#1MRmy_A0NQ;2-swjCYadYyoQmfKL#RA%uP?m)c2&x_TXl3P^y5O z`o#)!xJfbb!#sxKr0*$nlg`f+0`w9-ClNkP3OSE~PpJS6%&;|c1!G)aC-{g7&+lfO zl0`lRmORDq*E4*R8CrvgJ=+UP&Yakza^0T)6vG@tD2Fnk^C6mT!YYRvqzOWa>ANNckO z7kG*BPcjK>`K8{#^BC!x?eeNjetBlSmOrGrPENHf*ZJ}~?qg{#ZS;Rl0+&|*7wZ47 zw#)e+Hn+AO<3HR=xdZw?4&m*mSP20oaw20O$0x0*b2N$-mpS$FDki}Lwna8%<+iH% zaY}_?0ae^lq>&FVxAC8Ba+D5->=Sf*O{t6b1XUWPzO=_-sxjNEU1C(0!7LxTmZpm) zV3fLobqW16dtb?JaU%NPx#z2(aY%d$-M@9Ad(yr9q)3LiVH$=r_&p_|TGv47DaH-^ zXR4uN{Y3SlMlQ2g=aNsTdn=P$4Yk=SWD{mCA(AnZuxe2C=EISyi#H)L_1Dnh+qz94 zkStn}gL{@=a3J0>9+4Wn8SQ+WtBMsvqNNl2F{2e*r?EmSyt^jECsEm*dl9`lZ zp%i6qHrB$gOAoM2g#~E3-(`qxzRJIl2=y%$jr`fYq{U`as;0(#dJbXQBZ%t8Tt(*f z%)2b3f#M%OnUk!go^s{ZzK?UpXh4ZmnzBHW_+~UA5ghwW@PI_`0uG-b%Gl zuX{`Fh9N(9UsdL*C(ScguF(@v9piA4h6(HCm7HTGun2rU8Q|+fttYk-$_S@MGFLNP zx(pA0xw@1lje-p0-I*7s?sL$mGDlHc@liL=(sDPP@bW$7)r82YAkkY}3Kebjzc%`R z9=-o^XMjcVAFBF)zrX#M|Ls=F9nk+v$vQ2y;ms(O$p%Vl8zE>wj8%}ph)rn(v7nd+ z=6FmcHGgRduXx&kNajOAwVRF7)+3(4gc=i=71tOqhX<+Olm|550=kECBG8CLygUCo zp$a1NmpvXEeMApj^K6{F;=I0~#GET$Y<>l+^pj8H1ZX^_6Dmm9lL;4Oqz@yXLLx?V zV@v-pNcIPnYuS*VSp3+#hYpyroB0^O?XK-US3=^n3*G+a=A;YV37v2;Q-5!L{hoE9 zTNz6ix-T~0v+kM&sHay3_o4*u?^})Fw!Sma)PK4Y=8Lw2-YWI7_2T7K2EG02MFHIQ z*UwuHT24q9wjA_}5~g4Dn^DtZ(6353c=fV*;YygM0ZS*B9Cd$ls~I_$7#Z+2?3YDw{jII9 z+mUvDBrw+t8FSQ0k+U;H6wpuQkJS)mb^5*uh;o2UF|hH|ujEr;5k|W}@ygF>kUfzT z6q9-DE>?9sm-Q~Jp3#}iMu3^cv)C9{wXV0(|8$U`oAUqezk2?nU$+0gc=hU0|GSNH z2lT(9`?n@GV>X#%H?c%nFqK$`>LU~1s?P1KSO>c1RI1LR`aGrB$t?TlJO}R*s2;p6 z{IHS5p3>0}NvM6K)oAQ>uj2x4-?-!N=H3!Lw6&sZ>{ECl3z%BpQD3vj>pE<*K4?8fQz_dXRrH3oK}{@jRD3YIo;uv<8dDy)s2flI8$u=oWht))g-2h?Uyks z)ydyY(|Ob0>_rN#7_b(b1qwdvnboZZK?TsS%k~Fj=&H6`(zIgzK+05~3sQW!S@Anx zE4lwfu7%Bxxz2~D*7Vm{)g5S#0XzMAQ&E}goY7e;rgKJTSK>L?Ui}x%x{r+IHW-Ui zziPt)esJy^RVB+Wu<7FMjcb|Fr~0vnH2XRbWOCv;HQE=U?qNLzb2;UR(B1$A&o}KxC{MFTtlwb5Z@3K zZ*U6#p8I&rZ2bzwLa#uwPxVSZ%?)SgV`{8O8IG9*bqZ3boU<27kZU%BYFtDV^e~?0 ze`;X=C_*bUB^%|>h6Sl%O6x$q(T-BV&X{WpZbbfnPqvn*>Lto+{W5(E-k}z`{qt;ieHummOrUTLRTFi> z{1|wINp4CX!A>yvk!HZcF8+bq@efb<{>ySz7x(IR#ZZlnei@ePs66F7&Xgh zc&d!4udh|j6rSo2PYaA$4=C?g%`G3^Lfxt=(=)B(Nj9>b*U;@k7Ol`QzS7S*pLKi( zx?R_OAiEi0LnGtj*@@gO>|I zWczD%@fvF)U?1ng=hyC6^4Bf|oa)?thAC$HmHz%~)U`;kV_OjYxD-$`pRS9i1~O?t zQ$uYhz)ulTrQ=8`jG4*i0mY+8wmV@7X*BO7!{wN>2%EQ6H)|ZOvU!7U0t2ke=N?3RuZohsoMKISE;=g%3wQQw) zN>*(th zmn#>w3!L$~g)VWJuFPZq&#M`_;MYFw0zpQ>;+nc!q;Cup8(>PH`-S3r2N%?)JLHD&;`URtUOv zbOgI!yT5kbxaTzyzMC8C(@Fi%7QJNo(!1l~mp1%A52J8H??0Zus=WVr@fiR0R>~dV z|4yd+?9xNuDP=cEncr1R#hAoWAG@T_<1AXC`9ta_AW<^!&1Fqo5GV!)sSz5L^D-EP z@}|IdaY|iyLgILSV7zLF)k9*%y<}?7OjJZf%ceH4$*X`BO3XN?U?9!^r44ka^%V`=iO%VQ^ijS~a#WmD3_4k!&woQp*Da!G#Gz zgdJA38U1@D6wenU8@VLiWMQIm0?K}BPf5M3%^jgBcR=@mx;|#Xx-La#jq<;Z{-^Jd zZ!G@Li_L2O_pO(Y?>}y(+!glHl;OoV=gi!Nd#n&e(k#%V@?*xxaK8%=*A!UnXYpJFZ&P+Qvq`26|xlhw0y zK>Z{{y0)D|a5pl?n z`AhW=OcCGhb?t5 z*KUN;;HsVECo#Ou!{JPwT&y+F}*JxgAc;#yDR&E9iBLH%s-GW+>;9 zkAvUThbJXZmXacn_f&D=ci{AD9ED;Xg+k)6Umq*<@kU9$48&m&GrV}8}k;W`f*B*B; zTn9YUeo)Wyo5|HZ9u3*(J&D)tyO<#tm_3+95c5JFg>|5x6rOCb9~-q6Rwbpd(*Jt< z)yuWT8ij8Ld~8OW;Z={gyW!)QCL~5Lf@PUn6?{f8T(cxQtW}%E%`hn2!uy6udZ4%n z9+B2u`JNpl%rSvaSfT-U-2^$^GZvv-H7M%lM~MrNW3u()rCPE_`}_kdZO(t^M5z6E z=_BBR_y7I=Rwe(#tLKmLpKhhxmGj@vn#O<(c$z?}_&x|8hRC|fXdaW!$kWt6CZjnC zQ4cv!=JDiOXzoi+j8&dn!tKM;E_iu1{WX)twwxjLLOX?i%c20fJ@M4)?9JU6o?2Hcn4baGAv#V5Kn+W6_c z`m1+LqCe8!ZXAcqM}geiBMAxlXu}4n-$>Gg3r0e@@!gO6Z+iG32_w<>%y4>YI6Z@I zDr66}TQpD3x5O6-%7n=cz@&S|LrsDYvDk=3By2@s6CU7`mwlUwe5u6bit^g4nT?%h z$^jV;ktph>N3KY@S+f6Z$epc^(`l#y+NU-~ zDO}B!!;P=x4o0~VHgmj7(2A7TYF*SR0r zLXYTTOGUT;-~aRf!qMBk9}oAB{|Ol%_-G~w`6keDE(VlF+HagCX`(|`FbUt&aAIDC zo@0^>okVOMqu{Ck#hBIF8CHkV!@!?$NI?nwAr}*ASOT%HynJNfdXi}Bau(nVIcS!jm#v6 zlq9L3QcrBcBo5tSSTH4mkwHkM*XbPP__DmfGS?mV{8ge5lR@0(4sYN*i@1=YWJ#06 zo=37%jR|h3xNaD_fmhqfJsVljwRM`rYtGTvo_W<_*PahF4|@BFOyZF4n0Qy0@5Yzx zK)X79NSRLTy%;`(T;Omi7`PMJpr`4z@*n6q7+A~U7G zg9n*2m->0*8SE*YA@}*sh7AS=K3~-s7fO9v_W1Ymx|-#Lkm5V-G^hqPAHU zhw?K8d_06oo#5$6zr@WG`G$0g8Iz)fg+PR0Q)Dr`$Ysn}J+S*7h&S;3oJ>pOva7%I zI$JXrfre9NCNB4JmTBpF`r{8?!xGfeiEUI%Ckx44#xgpVJ3x^*eOjIBqDi{S1Wl;CLsjwlU)fKT)rsf1%p4jZjsX9k8%#c9;qBs%a5#W(iFB-B@DsaRmJzy_GgmVtxc=XwR5T%9v*Lj}4apDsxNfZdOgfV!5|IgOT zt+N03_VdlHNBi$>l)Ga8-AzBSkP$I+V*Y6VA5>v+aK;~ji&A{VxsRuFf;EnL<#>%Y znso69T6jolgFe(nhw`%cUwgKvEHWvPd0Q@&@bIj7-qEMJ#xvx?VfTve*4Ypp0a^~p zk;4oIKT0afF$vu_@U^F+D#+G=n$y%Qx9}#aNIsS2*50b*^^4wqF^a1F+*NS>D4JKq zPHYbD3l13=PoX~KN}W55ZX2>MB8)X4ix=IhMUYOsx-RDkX!J3zM zS>19LzkEq0YFYU|A%c-$u+&xX0{Q>y)ywkz@Al^NNB`ejDR)Kwn@B+v-YWuo^9|U5 zMOg#_ogYE*Jq$lIm+{v`RI-GdfopOMHepk89%X~L&S6oFkrs;4g>67eJe1mrI>{e& zfQ3R7lFZ9h_RKh!VT!hoYha1(Q0iO-mUMy|r#0K&fFo%#uBbU)0i+R^v!l!(OGoSBR{OFV)xVM?)@J z$tc}1lmnv4vIBRiUU`)&f#qtKvoW$<*U(5$EwSUuzF>^GEw1{leyGLS;2>Q>xlFvs z3c=N7&aW}+Bi}BKp~h0!DvV!+XcuF;isY*zx)P4>&xQzwNR9>W({=Vj&2>V)mi3#e znfhw#w;;m2)eWGjZcOtr) zjQbJ>3JbOd-ersvEx}x~X~IroiFLoES)vKb6^#-W>>W2rxa@S9JyuZaZqOicgWIhz zg{$l{S0XL5u^QT4v$09--z8QySF`Wss^-IAu%a%PtXnbn_l2gVrR_gF8u9D3|7^e5 zuH64@zIu%Rb1UVp*ne(L#WDnYJ1Umy?9Jd>xI>F^UxxJ?u=iWVGLu+h-7l+RH9)zl zie4%+MrS=f?AdSgcwA#Hd5o@SlHuFY>fEU;GNZQc zT_u&Sk-yEKL6Oasgo{B_ujq?WU(4yox%k(&@t8P%VJj1+&Q${2KIFRLGe8xE?al!0 zm!w%gf1(l9n4onD0xfs4z@MQ}gUQp3l-AT4OTBKtR-DcpJ*TmcWdUFF5yF|`k`!~f z!^9jwS3FU;i*>h;6;=tgBbEnQ3jC#(Kq+?>7&oeb2wluU z(DkyqHRsSRxO*)B=YgARyh5UGqiW6ONDUP!RNvzba1z&<^>VDlg>bK_xOP6*$K6~+ zFUjf3P#1u@W`5`P7GIv_)p||eV{&Pn(*lxdAc?rN&0Tt!SMOn;>)V!P&;No3Y}r7d zi{5{~c(qx+|9SEJ<>UGPHp-2j|8MXD$S`kBj`Mh`u5mRbz8=OEC7cF&Csp@YzsAA1 z^j4th*|)|tYPJaGMr9TDc^zV^Jb0OnzImB-8+Xm-qh1F-&#YNGHkTSdLVPMMMs;#| z41-Dp`-7NRJ5wF28~*Vc`p?%|uS5oi2Hw<$`@RDG*TQoS4Nsg3yJ<2mz5)A6*5qz( zM!vTO-LgHu+<^(7cge?0<u_}ug%leCtXK5m-Z-87~1Km|JViqo@IvoO(lv~vp-6Gh`O zVdyQhDT{ljR|6dZ`D;{j9BD}B=&d!}Y8Q{`8a(E5tFW~!1Skmtc-reJwMdP13B^e?Zq%6c(v@!9|$cRd>v!XLh$TW_*NF>NH4MV(xg9%CeF^fj) zAWUv{E!pzlNfdNeAfh9kD`Hh2@r`#3{9kJxe2l^w22YRmkRYZ4LKac4a<*?0B)@zfnKwcr2h~L)R1hw~jZJJ`bpZ z%_bul@-U#H!zLr_ZLHL`kdHch-|v3-?(Mse-+7bZOB(_j_y6JB-8b*wdXwPleazqg z{g?fIDgImk#fz=S_aC=XRwgAmtTf838V)Z{sJH{ue&>uu!46~zD?2u~q8hm# zcVn*RVH$?oMFV4b+g6jAR$`$0Jq<$+KXVZVT|J0*oS&J{;i;s+QvoE;&4aPd^T_1? z1CGTEe$k1#nlt(23vG~NGJGXe#DYn>ww`G$RQQa9Km&#Vj4qz7sfMQ)bw5h7fpj`3 zQnY4c(zHk*mO`=wp<^1xR7f}(r$%X*Q`hb z9pGGeolcSX3LT()`sIgoKuH|S9NL-r#S7?kLld}@@B@<(A$a_g4|QUuJq@2l;D@P9 zs4#?nASj;aB}=I6bb9bZ7Ns5f-q!ZtIz0_w+xyzv{F`p04w_WKz(0>-ic7$vI88wM zV>%)DWd-(a{;`}^?G{jEmCRzg0}{_w9e{vJUobpe?Z6w{ACRTP%1@1$J_$28abF#- zI$#s^>_E1wIvr>CE$HenPqd}5r{Swz;6k?f_M#DsK6N??87%^<;Ur|_g6VtxmtMaY z&;cV+ufN$7FSk3LI1R%C9x{L7SP|L784wNrAZwW2IH5xIbT>RFGx-$rgV=E>oITZ9 zn4hleyCX~qz~Sls@B>dU-$0Z+)#K5Qk}@9<5rWJ*04$-}%rJkX_LLmXOg1ltT@6H0 zWj1`%53J^X%S2`yP_B8MPCy5#gYJR8n5&>h&jl|**brtsg>#;U0cQGAUnHeKKP7?y zi!iB|<`^gsq`OG^%1+VtuF0hk`P2X#(4EdoMwTwctrQr3kzQ65{u@?*1)SvvcO^P2 zpr>v|g#1?Rs{_UDWkMsL<~z$=$1cQ^y~EV?m9lGLc`gzZ9y=YGa6v|NFC2xcfIHMds6jzy!w zh`YURr$c<7O8K4#bkT5ku_jD-kh@Pcj=Rv&Atk}jf+h4LE`VfzEbQ!v{Xv)L;p^{M zr=t!si{Ks@Gu|Ly$e;kpO3=CFY)MBi`buFrG+tN0L2L;DD}WflL9gKP%jq_w*|!A; z?3`>-BUvy@*UV-exs3&{Fxkn>KtH3i_as(2-%c5p`8`wrG|X7%r#YiDw9 zz94BN86JgkFX9W!Qcf^p2EGoGPbeg8LZt$5P9oftBh^}KbV9oY#9T@?U?EFpx?7V> zkO^7FhEgr;>aOIRe8b=hfA95r^=hlZ%H?%dfRGGmXe1Pgy@&)5-ZG~LO9T-xKr!S2z~&mRxpIBA-u z)_S|+{0Iio7 z6a)3j_rbU}`$NT%hFthca&np7(VJ&(c*&r677Ihv?U0p`kg8*JP7_)YRi4exdLPfu|?&!NK4UB%vrq%WUCFIel;p`d>$ z6!cvb^sfg6{To0*zl;LIG~ma0BuJn%6=WgD9Qxj-x7q3Z@-c!Bd`dB`V1IiZwl=pm z*Yy#4x@Aq+m?0UDAnJHDWD%2N8sta(5lhDD0F54sc#Vm4MJ6OkWseVg$(Z(xH5hFn zOu~fx#)UOQsiU6B!Ko%}6qxMh-bb6T)P@>u%TV8QOM%&ActF6Ei9Q!GH<>`)44^g( zvRZ43m#ccJ+ggJ{C~6-Xl^1OqSyyHTz!FKr;s4Lx_4T%G1Mzo#ih~PmO%X|hy{C1^ z;3e5AtgwgJMo{#jp{27;Ocp(olIw3j-0>L8$&h7FeK8*bLq3sr$NNM6$oOberKAj9 zr09llJN=kRRKDLR|H%6uErJ$E2jyJ%sUChwk`(^5*fEMO4_(kltXUldZXLqI>5Ak5 z-o`?Dq=)4Jmd0GO7x=Hf(>h8|p!mdV735gp=_sNnt@+2+JUWQy-r3)Qz$ewpQxz z{Ebkj%v>sF{vX`Xfb&ypB0e1K!uB^Vw%>)i@RaV~@o@>4Tjet3##TrMt?J!IEz#?e zJ&ISj8?!ZQY>;MtuMS}yOc|hu5B9Jzv9W71)(WklmwY7EMsE*}q8}}Ztg8^HK~~Oz z7N1gv%KwT)3$|z)j5#S{ed0Nn+@IUfmZACH!-_(eY7U^CYtg-a$goBiKN3TAB{s@x z6N^g9ObO`Hf4eg1X!sbiqCcwUKofan<0u916P`F*U$o#f#I}2h?^9(+F0r^m8w1s6 zvs!5e9hs~$WlK97p0k&oryG9hMqjpiEu|*kJbp}DhiR!Cx(jg=`Knom$%`_01Kmih zW#Mqu)==x_FGf#)j$_l9S-0IQi-KL}0rXB8^iF%AcPgNF?$CC+M3?*48>?0A^4}N5 z4cEb}J4{KQA^#nvJL7zM1%KtK-ZTz;r>NE2iMf~dvOJXa~U`k5h(bI02dLti4 z`4wkGxtN*sE_Hran%&_&WxyXFFL;K_M;`4K2JzV-OfK6*d=~D3s;0_~rThS{T{>N9 zWi|iuf(%zd+f;I=eY{iU1|uVvAkhtm*$?oR2nO=i;NpEkhPzb<_5lk0Y=9hF@N0=X zjy?rI(O?=f#l7g_R2IxtJGINXr1G0_SYD4!&()SvQvzu#wyS~Xliusn5mbCc$z}#qE>sLD7Lm3vWsqDZ?)%xZf91WEG((C^Hwigf1;kq zlXlQ2prp{iU_66jHh^Epvto9c+!yl?)5RR_$9H$*$-KCo!SoKUr<0pvUQ8!5nEnRi z$tU=unA}_f>VPMHrMT^gLA~X5MV8DEPX>=TTR?<$3AJk0ty&|jO@~~*9(KnP_#g<@j-QE>ed~klb33- @@ -15,7 +17,7 @@ spec: matchLabels: application-crd-id: kubeflow-pipelines descriptor: - version: {{ .Values.global.imageTag | quote }} + version: {{ .Values.global.version | default .Values.global.imageTag | quote }} type: Kubeflow Pipelines description: |- Reusable end-to-end ML workflow diff --git a/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml index 55bc597403..c030af479f 100644 --- a/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml +++ b/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml @@ -6,8 +6,6 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: cache-deployer - component: cache-deployer annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} {{- with .Values.cacheDeployer.serviceAccount.annotations }} diff --git a/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml b/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml index d60bfd942c..d7eba19165 100644 --- a/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml +++ b/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml @@ -16,17 +16,22 @@ data: appName: {{ .Values.pipeline.app.name }} appVersion: {{ .Values.pipeline.app.version }} # Database configuration -{{- $dbHost := include "kubeflow-pipelines.database.host" . -}} -{{- if and .Values.env .Values.env.aws .Values.env.aws.rds .Values.env.aws.rds.enabled -}} - {{- $dbHost = .Values.env.aws.rds.endpoint -}} -{{- else if and .Values.env .Values.env.azure .Values.env.azure.sql .Values.env.azure.sql.enabled -}} - {{- $dbHost = printf "%s.mysql.database.azure.com" .Values.env.azure.sql.serverName -}} -{{- end }} +{{- $dbHost := include "kubeflow-pipelines.database.host" . }} dbHost: {{ $dbHost | quote }} dbPort: {{ include "kubeflow-pipelines.database.port" . | quote }} +{{- if or (eq .Values.externalDatabase.type "postgresql") (eq .Values.externalDatabase.type "postgres") }} + dbType: "postgres" + postgresHost: {{ $dbHost | quote }} + postgresPort: {{ include "kubeflow-pipelines.database.port" . | quote }} +{{- else }} dbType: {{ .Values.externalDatabase.type | default "mysql" | quote }} +{{- if or (eq .Values.env.platform "azure") (eq .Values.env.platform "aws") }} mysqlHost: "mysql" +{{- else }} + mysqlHost: {{ $dbHost | quote }} +{{- end }} mysqlPort: {{ include "kubeflow-pipelines.database.port" . | quote }} +{{- end }} mlmdDb: {{ .Values.database.metadb }} cacheDb: {{ .Values.database.cachedb }} pipelineDb: {{ include "kubeflow-pipelines.database.name" . }} @@ -43,7 +48,9 @@ data: DEFAULT_CACHE_STALENESS: {{ .Values.pipeline.cache.defaultStaleness | quote }} # Connection configuration ConMaxLifeTime: {{ .Values.database.connectionMaxLifetime | quote }} +{{- if not .Values.thirdParty.postgresql.enabled }} LOG_LEVEL: {{ .Values.global.logLevel | quote }} +{{- end }} {{- if eq .Values.objectStore.provider "s3" }} # AWS S3 configuration minioServiceHost: {{ .Values.objectStore.s3.endpoint | default "s3.amazonaws.com" | quote }} diff --git a/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml b/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml index fb7b64afd1..d6f3a54d24 100644 --- a/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml +++ b/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml @@ -1,4 +1,4 @@ -{{- if and (not .Values.externalDatabase.existingSecret) (not .Values.mysql.enabled) (not .Values.postgresql.enabled) }} +{{- if and (not .Values.externalDatabase.existingSecret) (not .Values.mysql.enabled) (not .Values.postgresql.enabled) (not .Values.thirdParty.postgresql.enabled) }} apiVersion: v1 kind: Secret metadata: @@ -8,8 +8,13 @@ metadata: labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} {{- end }} -{{- if or (eq .Values.env.platform "aws") (eq .Values.env.platform "gcp") (eq .Values.env.platform "azure") }} +{{- if ne .Values.env.platform "platform-agnostic" }} type: Opaque +{{- end }} +{{- if or (eq .Values.env.platform "aws") (eq .Values.env.platform "azure") }} +data: + username: {{ .Values.externalDatabase.username | b64enc }} + password: {{ .Values.externalDatabase.password | b64enc }} {{- else }} stringData: username: {{ .Values.externalDatabase.username | quote }} diff --git a/experimental/helm/charts/pipelines/templates/env/aws-patches.yaml b/experimental/helm/charts/pipelines/templates/env/aws-patches.yaml index f1b9351bb2..38964200da 100644 --- a/experimental/helm/charts/pipelines/templates/env/aws-patches.yaml +++ b/experimental/helm/charts/pipelines/templates/env/aws-patches.yaml @@ -38,7 +38,7 @@ metadata: eks.amazonaws.com/role-arn: {{ .Values.env.aws.iam.apiServerRoleArn | quote }} {{- end }} -{{- if .Values.env.aws.rds.enabled }} +{{- if and .Values.env.aws.rds.enabled .Values.env.aws.rds.createSecret }} --- # AWS RDS Configuration Secret apiVersion: v1 diff --git a/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml b/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml index 06d2f4b31f..4459dd91eb 100644 --- a/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml +++ b/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml @@ -39,25 +39,9 @@ metadata: azure.workload.identity/client-id: {{ .Values.env.azure.managedIdentity.clientId | quote }} {{- end }} -{{- if .Values.env.azure.sql.enabled }} ---- -# Azure SQL Configuration Secret -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-azure-sql-secret - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: azure-sql - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -type: Opaque -data: - host: {{ printf "%s.mysql.database.azure.com" .Values.env.azure.sql.serverName | b64enc }} - port: {{ "3306" | b64enc }} - database: {{ .Values.env.azure.sql.database | b64enc }} -{{- end }} + + + {{- if and .Values.env.azure.blob.enabled .Values.env.azure.blob.createMinioGateway }} --- @@ -128,7 +112,9 @@ spec: selector: app: minio application-crd-id: kubeflow-pipelines +{{- end }} +{{- if and .Values.env.azure.blob.enabled .Values.env.azure.blob.createMinioGateway (not .Values.thirdParty.minio.enabled) }} --- # MinIO Artifact Secret apiVersion: v1 @@ -140,7 +126,9 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} +{{- if ne .Values.env.platform "platform-agnostic" }} type: Opaque +{{- end }} stringData: accesskey: {{ .Values.objectStore.azure.accessKey | default "minio" | quote }} secretkey: {{ .Values.objectStore.azure.secretKey | default "minio123" | quote }} diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml index bd8a1f9634..f35780d4f7 100644 --- a/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml +++ b/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml @@ -249,7 +249,9 @@ spec: {{- if .Values.installMode.legacyLabels }} application-crd-id: kubeflow-pipelines {{- end }} +{{- end }} +{{- if and .Values.env.gcp.gcs.enabled (not .Values.thirdParty.minio.enabled) }} --- # MinIO Artifact Secret apiVersion: v1 @@ -261,7 +263,9 @@ metadata: labels: application-crd-id: kubeflow-pipelines {{- end }} +{{- if ne .Values.env.platform "platform-agnostic" }} type: Opaque +{{- end }} data: accesskey: {{ "minio" | b64enc | quote }} secretkey: {{ "minio123" | b64enc | quote }} diff --git a/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml index 2c70621f48..0dc53d8b82 100644 --- a/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml @@ -54,6 +54,23 @@ spec: - --postgres_config_user=$(POSTGRES_USER) - --postgres_config_password=$(POSTGRES_PASSWORD) - --enable_database_upgrade=true + {{- else if .Values.thirdParty.mysqlMetadata.enabled }} + env: [] + envFrom: + - configMapRef: + name: metadata-db-parameters + - secretRef: + name: metadata-db-secrets + - configMapRef: + name: metadata-grpc-configmap + command: ["/bin/metadata_store_server"] + args: + - --grpc_port=$(METADATA_GRPC_SERVICE_PORT) + - --mysql_config_host=metadata-db + - --mysql_config_database=$(MYSQL_DATABASE) + - --mysql_config_port=$(MYSQL_PORT) + - --mysql_config_user=$(MYSQL_USER_NAME) + - --mysql_config_password=$(MYSQL_ROOT_PASSWORD) {{- else }} env: - name: DBCONFIG_USER @@ -107,7 +124,7 @@ spec: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 - {{- if not .Values.thirdParty.postgresql.enabled }} + {{- if and (not .Values.thirdParty.postgresql.enabled) (not .Values.thirdParty.mysqlMetadata.enabled) }} {{- with .Values.metadata.grpc.resources }} resources: {{- toYaml . | nindent 10 }} diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-configmap.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-configmap.yaml new file mode 100644 index 0000000000..6c63ca68f9 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-configmap.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: metadata-db-parameters + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +data: + MYSQL_DATABASE: {{ .Values.thirdParty.mysqlMetadata.database | default "metadb" }} + MYSQL_PORT: "3306" + MYSQL_ALLOW_EMPTY_PASSWORD: "true" +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-deployment.yaml new file mode 100644 index 0000000000..f59a53b045 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-deployment.yaml @@ -0,0 +1,62 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metadata-db + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: db +spec: + selector: + matchLabels: + {{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines + {{- end }} + component: db + replicas: 1 + strategy: + type: Recreate + template: + metadata: + name: db + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} + component: db + sidecar.istio.io/inject: "false" + {{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines + {{- end }} + spec: + containers: + - name: db-container + image: {{ .Values.thirdParty.mysqlMetadata.image.repository }}:{{ .Values.thirdParty.mysqlMetadata.image.tag }} + args: + - --datadir + - /var/lib/mysql/datadir + envFrom: + - configMapRef: + name: metadata-db-parameters + - secretRef: + name: metadata-db-secrets + ports: + - name: dbapi + containerPort: 3306 + readinessProbe: + exec: + command: + - "/bin/bash" + - "-c" + - "mysql -D $$MYSQL_DATABASE -p$$MYSQL_ROOT_PASSWORD -e 'SELECT 1'" + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 1 + volumeMounts: + - name: metadata-mysql + mountPath: /var/lib/mysql + volumes: + - name: metadata-mysql + persistentVolumeClaim: + claimName: metadata-mysql +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-pvc.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-pvc.yaml new file mode 100644 index 0000000000..a47563586f --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-pvc.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: metadata-mysql + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.thirdParty.mysqlMetadata.persistence.size | default "10Gi" }} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-secret.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-secret.yaml new file mode 100644 index 0000000000..3049442d08 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-secret.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: metadata-db-secrets + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +data: + MYSQL_USER_NAME: {{ .Values.thirdParty.mysqlMetadata.username | default "root" | b64enc }} + MYSQL_ROOT_PASSWORD: {{ .Values.thirdParty.mysqlMetadata.password | default "test" | b64enc }} +type: Opaque +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-service.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-service.yaml new file mode 100644 index 0000000000..3c48fe84fb --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-service.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: metadata-db + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: db +spec: + type: ClusterIP + ports: + - port: 3306 + protocol: TCP + name: dbapi + selector: + {{- if .Values.installMode.legacyLabels }} + application-crd-id: kubeflow-pipelines + {{- end }} + component: db +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-configmap.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-configmap.yaml new file mode 100644 index 0000000000..4f11813607 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-configmap.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: metadata-postgres-db-parameters + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +data: + POSTGRES_PORT: "5432" + POSTGRES_DBNAME: "mlmdpostgres" +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-deployment.yaml new file mode 100644 index 0000000000..92b6a390f4 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-deployment.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metadata-postgres-db + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: db +spec: + selector: + matchLabels: + application-crd-id: kubeflow-pipelines + component: db + replicas: 1 + strategy: + type: Recreate + template: + metadata: + name: db + labels: + component: db + sidecar.istio.io/inject: "false" + application-crd-id: kubeflow-pipelines + spec: + containers: + - name: db-container + image: {{ .Values.thirdParty.postgresql.image.repository }}:{{ .Values.thirdParty.postgresql.image.tag }} + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + envFrom: + - configMapRef: + name: metadata-postgres-db-parameters + - secretRef: + name: metadata-postgres-db-secrets + ports: + - name: postgres + containerPort: 5432 + volumeMounts: + - name: metadata-postgres + mountPath: /var/lib/postgresql/data + volumes: + - name: metadata-postgres + persistentVolumeClaim: + claimName: metadata-postgres +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-pvc.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-pvc.yaml new file mode 100644 index 0000000000..5ce8e2fe0c --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-pvc.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: metadata-postgres + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-secret.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-secret.yaml new file mode 100644 index 0000000000..968634b935 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-secret.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: metadata-postgres-db-secrets + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +data: + POSTGRES_USER: {{ "root" | b64enc }} + POSTGRES_PASSWORD: {{ .Values.thirdParty.postgresql.password | default "password" | b64enc }} +type: Opaque +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-service.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-service.yaml new file mode 100644 index 0000000000..dec27721e6 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-service.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: metadata-postgres-db + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + component: db +spec: + type: ClusterIP + ports: + - port: 5432 + protocol: TCP + name: postgres + selector: + component: db + application-crd-id: kubeflow-pipelines +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml index 5dd982c31a..bbb89dfcb0 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml @@ -6,13 +6,10 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: profile-controller + app: kubeflow-pipelines-profile-controller annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} data: - CONTROLLER_PORT: "8080" - VISUALIZATION_SERVER_IMAGE: "{{ .Values.global.imageRegistry }}/{{ .Values.visualization.image.repository }}" - FRONTEND_IMAGE: "{{ .Values.global.imageRegistry }}/{{ .Values.ui.image.repository }}" DISABLE_ISTIO_SIDECAR: {{ .Values.profileController.disableIstioSidecar | quote }} --- @@ -23,7 +20,7 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: profile-controller + app: kubeflow-pipelines-profile-controller annotations: {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} data: diff --git a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml index bf5767f63c..f42901bbc0 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml @@ -15,6 +15,7 @@ rules: - create - get - update +{{- if not .Values.installMode.multiUser.enabled }} - apiGroups: - "" resources: @@ -86,12 +87,14 @@ rules: verbs: - get - list +{{- end }} - apiGroups: - "" resources: - secrets verbs: - get +{{- if not .Values.installMode.multiUser.enabled }} - apiGroups: - argoproj.io resources: @@ -120,3 +123,4 @@ rules: - get - delete {{- end }} +{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml index 8c8dcacbf9..e588219bf6 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml @@ -1,4 +1,4 @@ -{{- if .Values.thirdParty.minio.enabled }} +{{- if and .Values.thirdParty.minio.enabled (ne .Values.env.platform "azure") }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml index 6fc93106ac..a5b184e27c 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml @@ -1,4 +1,4 @@ -{{- if .Values.thirdParty.minio.enabled }} +{{- if and .Values.thirdParty.minio.enabled (ne .Values.env.platform "azure") (not .Values.env.gcp.gcs.enabled) }} apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml index abed4614e4..eac07cfff8 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml @@ -8,8 +8,17 @@ metadata: labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} {{- end }} -{{- if or (eq .Values.env.platform "aws") (eq .Values.env.platform "gcp") (eq .Values.env.platform "azure") }} +{{- if ne .Values.env.platform "platform-agnostic" }} type: Opaque +{{- end }} +{{- if eq .Values.env.platform "aws" }} +data: + accesskey: {{ .Values.objectStore.s3.accessKey | default "YOUR_AWS_ACCESS_ID" | b64enc }} + secretkey: {{ .Values.objectStore.s3.secretKey | default "YOUR_AWS_SECRET_KEY" | b64enc }} +{{- else if eq .Values.env.platform "azure" }} +data: + accesskey: {{ .Values.objectStore.azure.accountName | default "YOUR_AZURE_STORAGE_ACCOUNT" | b64enc }} + secretkey: {{ .Values.objectStore.azure.accountKey | default "YOUR_AZURE_STORAGE_KEY" | b64enc }} {{- else }} stringData: accesskey: "minio" diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml index 56c20a515f..8079d96767 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml @@ -1,4 +1,4 @@ -{{- if .Values.thirdParty.minio.enabled }} +{{- if and .Values.thirdParty.minio.enabled (ne .Values.env.platform "azure") }} apiVersion: v1 kind: Service metadata: @@ -8,7 +8,7 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} spec: ports: - - name: {{ .Values.thirdParty.minio.portName }} + - name: {{ .Values.thirdParty.minio.portName | default "http" }} port: 9000 protocol: TCP targetPort: 9000 diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml index ea6ca75fd1..bd7ef9a03a 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml @@ -4,7 +4,8 @@ kind: Secret metadata: name: postgres-secret-extended namespace: {{ include "kubeflow-pipelines.namespace" . }} -type: Opaque + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} stringData: username: {{ .Values.thirdParty.postgresql.username | default "user" }} password: {{ .Values.thirdParty.postgresql.password | default "password" }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml index 95a99aef40..54757e95ef 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml @@ -4,7 +4,9 @@ kind: Secret metadata: name: postgres-secret namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} type: Opaque data: - root_password: {{ .Values.thirdParty.postgresql.password | b64enc }} + root_password: {{ .Values.thirdParty.postgresql.password | default "password" }} {{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml index 01d57fd809..bf00d18d5a 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml @@ -4,4 +4,6 @@ kind: ServiceAccount metadata: name: postgresql namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + application-crd-id: kubeflow-pipelines {{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml index 1e49306b11..3cadb158f1 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml @@ -1,11 +1,34 @@ {{- if .Values.thirdParty.argo.enabled }} apiVersion: v1 data: +{{- if eq .Values.env.platform "aws" }} + config: | + { + artifactRepository: + { + s3: { + bucket: {{ include "kubeflow-pipelines.objectStore.bucket" . }}, + keyPrefix: artifacts, + endpoint: s3.amazonaws.com, + insecure: true, + accessKeySecret: { + name: mlpipeline-minio-artifact, + key: accesskey + }, + secretKeySecret: { + name: mlpipeline-minio-artifact, + key: secretkey + } + }, + archiveLogs: true + } + } +{{- else }} artifactRepository: | archiveLogs: true s3: endpoint: "minio-service.kubeflow:9000" - bucket: "mlpipeline" + bucket: {{ include "kubeflow-pipelines.objectStore.bucket" . | quote }} # keyFormat is a format pattern to define how artifacts will be organized in a bucket. # It can reference workflow metadata variables such as workflow.namespace, workflow.name, # pod.name. Can also use strftime formating of workflow.creationTimestamp so that workflow @@ -28,6 +51,7 @@ data: key: secretkey executor: | imagePullPolicy: IfNotPresent +{{- end }} kind: ConfigMap metadata: labels: diff --git a/experimental/helm/charts/pipelines/values.yaml b/experimental/helm/charts/pipelines/values.yaml index 9ea0837cce..28923ec7f2 100644 --- a/experimental/helm/charts/pipelines/values.yaml +++ b/experimental/helm/charts/pipelines/values.yaml @@ -20,6 +20,8 @@ global: imageRegistry: ghcr.io/kubeflow # -- Global image tag for all Kubeflow Pipelines components imageTag: 2.14.3 + # -- Application version + version: "" # -- Global image pull policy imagePullPolicy: IfNotPresent # -- Global image pull secrets @@ -608,6 +610,23 @@ thirdParty: # Use built-in minimal PostgreSQL instead of external subchart postgresql: enabled: false + image: + repository: postgres + tag: "13.2" + password: "password" + + # Use built-in MySQL for metadata database (separate from main MySQL) + mysqlMetadata: + enabled: false + image: + repository: mysql + tag: "8.0.3" + database: "metadb" + username: "root" + password: "test" + persistence: + enabled: true + size: 10Gi # Metacontroller configuration metacontroller: @@ -1014,7 +1033,7 @@ profileController: port: 80 # -- Disable Istio sidecar injection - disableIstioSidecar: "true" + disableIstioSidecar: "false" # Additional environment variables additionalEnv: [] diff --git a/tests/helm_kustomize_compare.py b/tests/helm_kustomize_compare.py index 54bf9e203b..e57b3b77ed 100755 --- a/tests/helm_kustomize_compare.py +++ b/tests/helm_kustomize_compare.py @@ -46,7 +46,7 @@ def clean_helm_metadata(obj: Any, component: str = "katib") -> Any: # More restrictive filtering for KServe if not label_key.startswith(('helm.sh/', 'app.kubernetes.io/managed-by')): cleaned_labels[label_key] = label_value - elif component == "pipeline": + elif component in ["pipeline", "pipelines"]: helm_specific_labels = [ 'app.kubernetes.io/managed-by', 'app.kubernetes.io/version', @@ -74,6 +74,9 @@ def clean_helm_metadata(obj: Any, component: str = "katib") -> Any: cleaned_annotations = {} for ann_key, ann_value in meta_value.items(): if not ann_key.startswith(('helm.sh/', 'meta.helm.sh/')): + # For pipelines, ignore GCP icon annotation + if component in ["pipeline", "pipelines"] and ann_key == 'kubernetes-engine.cloud.google.com/icon': + continue cleaned_annotations[ann_key] = ann_value if cleaned_annotations: # Only add if there are remaining annotations cleaned_metadata[meta_key] = cleaned_annotations @@ -154,9 +157,38 @@ def normalize_manifest(manifest: Dict, component: str = "katib") -> Dict: metadata.pop('uid', None) metadata.pop('creationTimestamp', None) metadata.pop('managedFields', None) + + # For pipelines component, remove labels from Argo CRDs + if component in ["pipeline", "pipelines"] and normalized.get('kind') == 'CustomResourceDefinition': + crd_name = normalized.get('metadata', {}).get('name', '') + if 'argoproj.io' in crd_name: + metadata.pop('labels', None) normalized.pop('status', None) + # For pipelines Argo CRDs, normalize schema differences between versions + if component in ["pipeline", "pipelines"] and normalized.get('kind') == 'CustomResourceDefinition': + crd_name = normalized.get('metadata', {}).get('name', '') + if 'argoproj.io' in crd_name and 'spec' in normalized: + def normalize_crd_schema(obj, path=""): + if isinstance(obj, dict): + cleaned = {} + for k, v in obj.items(): + if k.startswith('x-kubernetes-'): + continue + if k == 'default' and 'properties' in path: + continue + if k == 'required' and 'openAPIV3Schema.properties' in path: + continue + cleaned[k] = normalize_crd_schema(v, f"{path}.{k}" if path else k) + return cleaned + elif isinstance(obj, list): + return [normalize_crd_schema(item, path) for item in obj] + else: + return obj + + normalized['spec'] = normalize_crd_schema(normalized['spec']) + def remove_empty_values(obj): if isinstance(obj, dict): return {k: remove_empty_values(v) for k, v in obj.items() @@ -183,7 +215,7 @@ def get_resource_key(manifest: Dict, component: str = "katib") -> str: else: return f"{kind}/{name}" -def deep_diff(obj1: Any, obj2: Any, path: str = "") -> List[str]: +def deep_diff(obj1: Any, obj2: Any, path: str = "", resource_key: str = "") -> List[str]: """Compare two objects and return list of differences.""" differences = [] @@ -198,16 +230,21 @@ def deep_diff(obj1: Any, obj2: Any, path: str = "") -> List[str]: if key not in obj1: differences.append(f"{key_path}: missing in kustomize") elif key not in obj2: + if 'CustomResourceDefinition' in resource_key and 'argoproj.io' in resource_key: + if 'openAPIV3Schema.properties' in path and key == 'properties': + continue + elif '.properties.' in path: + continue differences.append(f"{key_path}: missing in helm") else: - differences.extend(deep_diff(obj1[key], obj2[key], key_path)) + differences.extend(deep_diff(obj1[key], obj2[key], key_path, resource_key)) elif isinstance(obj1, list): if len(obj1) != len(obj2): differences.append(f"{path}: list length mismatch ({len(obj1)} vs {len(obj2)})") else: for i, (item1, item2) in enumerate(zip(obj1, obj2)): - differences.extend(deep_diff(item1, item2, f"{path}[{i}]")) + differences.extend(deep_diff(item1, item2, f"{path}[{i}]", resource_key)) elif obj1 != obj2: differences.append(f"{path}: '{obj1}' != '{obj2}'") @@ -226,8 +263,68 @@ def get_expected_helm_extras(component: str, scenario: str) -> set: if scenario == "base": return {"AuthorizationPolicy/kserve-models-web-app"} return set() - elif component == "pipeline": - return set() # No extra resources in Helm for Kubeflow Pipelines + elif component == "pipelines": + extras = { + # Built-in third-party Argo resources (from thirdParty.argo templates) + 'ConfigMap/workflow-controller-configmap', + 'Deployment/workflow-controller', + 'PriorityClass/workflow-controller', + 'Role/argo-role', + 'RoleBinding/argo-binding', + 'ServiceAccount/argo', + } + + # These get extra resources from the subchart dependency + argo_subchart_scenarios = [ + 'dev', 'aws', 'gcp', 'azure', + 'platform-agnostic-multi-user-emissary', + 'platform-agnostic-multi-user', + 'platform-agnostic-multi-user-legacy', + ] + + if any(sc in scenario for sc in argo_subchart_scenarios): + # Argo Workflows subchart creates these extra resources + extras.update({ + # Argo Workflows subchart RBAC resources + 'ClusterRole/pipeline-argo-workflows-admin', + 'ClusterRole/pipeline-argo-workflows-edit', + 'ClusterRole/pipeline-argo-workflows-view', + 'ClusterRole/pipeline-argo-workflows-server', + 'ClusterRole/pipeline-argo-workflows-server-cluster-template', + 'ClusterRole/pipeline-argo-workflows-workflow-controller', + 'ClusterRole/pipeline-argo-workflows-workflow-controller-cluster-template', + 'ClusterRoleBinding/pipeline-argo-workflows-server', + 'ClusterRoleBinding/pipeline-argo-workflows-server-cluster-template', + 'ClusterRoleBinding/pipeline-argo-workflows-workflow-controller', + 'ClusterRoleBinding/pipeline-argo-workflows-workflow-controller-cluster-template', + 'Role/pipeline-argo-workflows-workflow', + 'RoleBinding/pipeline-argo-workflows-workflow', + # Argo Workflows subchart deployments and services + 'Deployment/pipeline-argo-workflows-server', + 'Deployment/pipeline-argo-workflows-workflow-controller', + 'Service/pipeline-argo-workflows-server', + 'ServiceAccount/pipeline-argo-workflows-server', + 'ServiceAccount/pipeline-argo-workflows-workflow-controller', + 'ConfigMap/pipeline-argo-workflows-workflow-controller-configmap', + # Argo Workflows CRDs from subchart + 'CustomResourceDefinition/clusterworkflowtemplates.argoproj.io', + 'CustomResourceDefinition/cronworkflows.argoproj.io', + 'CustomResourceDefinition/workflowartifactgctasks.argoproj.io', + 'CustomResourceDefinition/workfloweventbindings.argoproj.io', + 'CustomResourceDefinition/workflows.argoproj.io', + 'CustomResourceDefinition/workflowtaskresults.argoproj.io', + 'CustomResourceDefinition/workflowtasksets.argoproj.io', + 'CustomResourceDefinition/workflowtemplates.argoproj.io', + }) + + if 'multi-user' in scenario: + extras.update({ + 'ConfigMap/kubeflow-pipelines-profile-controller-code', + 'ConfigMap/kubeflow-pipelines-profile-controller-env', + 'ConfigMap/pipeline-api-server-config', + }) + + return extras else: return set() @@ -264,11 +361,17 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena if only_in_kustomize: print(f"Resources only in Kustomize: {len(only_in_kustomize)}") + if verbose: + for resource in sorted(only_in_kustomize): + print(f" - {resource}") success = False differences_found.extend(only_in_kustomize) if unexpected_helm_extras: print(f"Unexpected resources only in Helm: {len(unexpected_helm_extras)}") + if verbose: + for resource in sorted(unexpected_helm_extras): + print(f" - {resource}") success = False differences_found.extend(unexpected_helm_extras) @@ -277,7 +380,7 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena kustomize_resource = kustomize_resources[key] helm_resource = helm_resources[key] - differences = deep_diff(kustomize_resource, helm_resource) + differences = deep_diff(kustomize_resource, helm_resource, resource_key=key) if differences: print(f"Differences in {key}: {len(differences)} fields") @@ -312,6 +415,8 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena print(f"ERROR: Unknown component: {component}") print("Supported components: katib, model-registry, kserve-models-web-app, notebook-controller, pipelines") sys.exit(1) + if component == "pipeline": + component = "pipelines" success = compare_manifests(kustomize_file, helm_file, component, scenario, namespace, verbose) sys.exit(0 if success else 1) \ No newline at end of file diff --git a/tests/helm_kustomize_compare.sh b/tests/helm_kustomize_compare.sh index 90ce1b89ae..32a53b08b5 100755 --- a/tests/helm_kustomize_compare.sh +++ b/tests/helm_kustomize_compare.sh @@ -185,14 +185,14 @@ case "$COMPONENT" in ["gcp"]="$CHART_DIR/ci/values-gcp-enhanced.yaml" ["azure"]="$CHART_DIR/ci/values-azure-enhanced.yaml" ["multi-user"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" - ["generic"]="" + ["generic"]="$CHART_DIR/ci/values-generic.yaml" ["dev"]="$CHART_DIR/ci/values-dev-enhanced.yaml" ["plain"]="$CHART_DIR/ci/values-standalone.yaml" ["plain-multi-user"]="$CHART_DIR/ci/values-multi-user.yaml" ["platform-agnostic-emissary"]="$CHART_DIR/ci/values-platform-agnostic-enhanced.yaml" ["platform-agnostic-multi-user"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" ["platform-agnostic-multi-user-emissary"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" - ["platform-agnostic-multi-user-legacy"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" + ["platform-agnostic-multi-user-legacy"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-legacy.yaml" ["platform-agnostic-postgresql"]="$CHART_DIR/ci/values-postgresql.yaml" ) @@ -273,7 +273,7 @@ if [ ! -d "$CHART_DIR" ]; then exit 1 fi -if [[ "$COMPONENT" != "kserve-models-web-app" ]] && [[ "$COMPONENT" != "pipeline" || "$SCENARIO" != "generic" ]] && [ ! -f "$HELM_VALUES_ARG" ]; then +if [[ "$COMPONENT" != "kserve-models-web-app" ]] && [ ! -f "$HELM_VALUES_ARG" ]; then echo "ERROR: Helm values file does not exist: $HELM_VALUES_ARG" exit 1 fi @@ -306,16 +306,10 @@ else --include-crds \ --values "$HELM_VALUES_ARG" > "$HELM_OUTPUT" elif [[ "$COMPONENT" == "pipelines" ]]; then - if [[ "$SCENARIO" == "generic" ]]; then - helm template pipeline . \ - --namespace "$NAMESPACE" \ - --include-crds > "$HELM_OUTPUT" - else - helm template pipeline . \ - --namespace "$NAMESPACE" \ - --include-crds \ - --values "$HELM_VALUES_ARG" > "$HELM_OUTPUT" - fi + helm template pipeline . \ + --namespace "$NAMESPACE" \ + --include-crds \ + --values "$HELM_VALUES_ARG" > "$HELM_OUTPUT" else helm template model-registry . \ --namespace "$NAMESPACE" \ From 54c1b58843f5ef8b11d81166e46cc758ae7c43fc Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sun, 12 Oct 2025 12:54:20 +0530 Subject: [PATCH 14/26] update kserve configmap helm charts Signed-off-by: kunal-511 --- .../helm/charts/kserve-models-web-app/templates/configmap.yaml | 1 + experimental/helm/charts/kserve-models-web-app/values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/experimental/helm/charts/kserve-models-web-app/templates/configmap.yaml b/experimental/helm/charts/kserve-models-web-app/templates/configmap.yaml index fc2a480d80..3fd8b53711 100644 --- a/experimental/helm/charts/kserve-models-web-app/templates/configmap.yaml +++ b/experimental/helm/charts/kserve-models-web-app/templates/configmap.yaml @@ -21,6 +21,7 @@ data: {{- if .Values.kubeflow.enabled }} USERID_HEADER: {{ .Values.kubeflow.config.useridHeader | quote }} APP_PREFIX: {{ .Values.kubeflow.config.appPrefix | quote }} + USERID_PREFIX: {{ .Values.kubeflow.config.useridPrefix | quote }} {{- else }} APP_DISABLE_AUTH: "True" {{- end }} diff --git a/experimental/helm/charts/kserve-models-web-app/values.yaml b/experimental/helm/charts/kserve-models-web-app/values.yaml index 5e3bd6751f..b86b534ea6 100644 --- a/experimental/helm/charts/kserve-models-web-app/values.yaml +++ b/experimental/helm/charts/kserve-models-web-app/values.yaml @@ -242,6 +242,7 @@ kubeflow: disableAuth: false useridHeader: "kubeflow-userid" appPrefix: "/kserve-endpoints" + useridPrefix: "" # Monitoring configuration monitoring: From ef2d1097ad0e62acaa0e0183287e047f8713d355 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sun, 12 Oct 2025 12:58:48 +0530 Subject: [PATCH 15/26] remove charts dependencies Signed-off-by: kunal-511 --- experimental/helm/charts/pipelines/Chart.lock | 15 --------------- .../pipelines/charts/argo-workflows-0.40.14.tgz | Bin 29164 -> 0 bytes .../charts/pipelines/charts/minio-5.0.15.tgz | Bin 21320 -> 0 bytes .../charts/pipelines/charts/mysql-14.0.3.tgz | Bin 69215 -> 0 bytes .../pipelines/charts/postgresql-16.0.6.tgz | Bin 75170 -> 0 bytes 5 files changed, 15 deletions(-) delete mode 100644 experimental/helm/charts/pipelines/Chart.lock delete mode 100644 experimental/helm/charts/pipelines/charts/argo-workflows-0.40.14.tgz delete mode 100644 experimental/helm/charts/pipelines/charts/minio-5.0.15.tgz delete mode 100644 experimental/helm/charts/pipelines/charts/mysql-14.0.3.tgz delete mode 100644 experimental/helm/charts/pipelines/charts/postgresql-16.0.6.tgz diff --git a/experimental/helm/charts/pipelines/Chart.lock b/experimental/helm/charts/pipelines/Chart.lock deleted file mode 100644 index 02e63f6a2c..0000000000 --- a/experimental/helm/charts/pipelines/Chart.lock +++ /dev/null @@ -1,15 +0,0 @@ -dependencies: -- name: argo-workflows - repository: https://argoproj.github.io/argo-helm - version: 0.40.14 -- name: minio - repository: https://charts.min.io/ - version: 5.0.15 -- name: mysql - repository: https://charts.bitnami.com/bitnami - version: 14.0.3 -- name: postgresql - repository: https://charts.bitnami.com/bitnami - version: 16.0.6 -digest: sha256:addbacd4a800daf2fbf006468d352c6749df1a4afaa346d6d44abc62773393bb -generated: "2025-10-09T20:37:41.761712064+05:30" diff --git a/experimental/helm/charts/pipelines/charts/argo-workflows-0.40.14.tgz b/experimental/helm/charts/pipelines/charts/argo-workflows-0.40.14.tgz deleted file mode 100644 index 56b93174d4148ac512c8fa90d2919568f88b0479..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29164 zcmV(~K+nG)iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMZ{ciT3zDEj@ZzXDh8S&h3RT7IV0+1>Xzw%d5+l7+^kSgImV0Cn38PTSqg@ zr8mb>_^*e1hQs0TU~dop9S(=(f48^yxBs=hxBqN=dvE{X+5W$V+k4y3w*Q5O4~RqK zlM0FXzlQg2tK7N2$b;jUNh~Rg#i)l6=8{gZFK6l4qioBc;dn~K2>sbZ2%!PGrg1Pr zI0#4pZx9NI@HtHsgb}((lZ0`JVjK~XV4u9bA)M2IR6z?mjbDB#I+J=C_XC#cal4trSZLPO>TzYwpy!oDBDfbCLT-cesmYgRF zRFZ_$2V&~o*uflndqZz~ulJ=1ef`uvCH{Yd!<2{z767gMe|tE5w*Rcm|M&KG*Zlu6 z9vWfA;Cl#dpuqqgu_$6Ol6YFf|6{#OmyVV(n3U?8AGueY&*j#436ggDWhQDHiv z)0AsA=kp*Rywg}n)fpTIC{3mu2Lv6RA1mys{xm|8r^J?jNk{?(5Q~juNn=y8htM@4 zNwLCVF056L$3#nFJBX$&SEQqlpL#t#vz(}*@WWI{!Z{0xKyrp9B6pI+0U~!)NE%N| zB^aWyBpmxv4?2vIA0X9nH9Y5Jl1rKEbWC_mBmvQ-MA#N56T(UClP&D~M2G=S*@uB^Cu3nn)&|SKaTp_IHUMUwi4Cja$g#-0Va&?l1 zp_=ACk|at(EC~oetDuZ3Fja-+;MrGYnD#f4Gm@)Dy$$rg0=( zdKEni)|7_&3Xh4Vd_kn5AVmU}OhwO!m8=UnWKh86Wj0^P$m^%*c1Hafq5`EtQx;J& z0Z4~%wMJ2dV>%&1ik_Npv*^nRX6+oFoth%T(YfsBeDUN1pAC6l9I*+)sbs>(p(@2m z91uW3^1ri@KPVj6`vo_+<| z)6KS-gmWS!r@qwuRkfcGu7;S!;T+KzVI*ig4IOGhPYT4Nu#+=IL!g#&O*)$jHO*IH zwE!nle?6yhK;x+ZK16U}s-q*l-g+B|{GQSq9Fkayr%+9aMTsoE4-|vuEDQ!ile# z#Q#*3iE?~ATQ-A|byEqV3`GS>5Pm~_a_IXkjpYdJ2TckI@#%yTfo^9+&IngipbIOM zRcRzs}odWy^c(!GFYe*=F2pL;4=G;V9K07?_Q}Zl||0e z80G4`s;5O_k?Zwci1(7j9=gC0Q94G1CBgz-dH>d`r_nJSIei7KRt2|`9?ldV27v%# zFH=2eGKyaDnB5R0%v4u~(2CS?NGe0(nRX6~EsRYI38q{PCk{nEL#kPV{h70B)v9PE z+$Vw%G$tXt-7HX-mU?+z)o&OnX8_L4*ec}J60zORYe88%1tau(isv3>D?mt?ppq#V znMGK`DRRK43e`-jR3L3}cRWgZWsx8xo?hu$D5;hS<~R&VNJWGK>PxJ|*^J#PIW^B` z5cxP()63I1R#S#0;2{(lN*bx9>p;r*&d(tmgJKcVh$_rx1OcnYb`>Qo5D4>(D|k@# zQV9t4zgTHkdXYs`Ql-hDNyr$`jbX!%96mfd1-c|$@$wvdt$d*n^C>|%0(xZ=Xv(6E zVipjfR4cR2u_Uh)-VQei_3!PW`8`g#P``VttSD#ls^pr(3H1czB1AGYCGsf>sgPRS z++yj^G8NcF~m4c={BjvDRnN;V&2H_^DJ}+X57jqf z7N=%rEO^-F&sYE^bpfv1;#&(m%@o?!_(kU}w0f8Wjs zCm96A(n&<6V(I90Dx_73H7D+G^)*XSRYz#cRCB;HE;I1z??&(%4C8Wyei;5R%t5?h zw}?$7iIK0p5N>5ZBv@G^pkXQA0sf6_ur4oO87XOi{S@MHG1_c+uhWSj~s)-rc-?LuKn9|2X=HpQPY2IpSw!gQ#V_pPU;xQIv zlz}iGU?ksU{`U_>6x9D!r)_nsyQ4@6c5c@HGoa)>{^Tpnm>KO9$rv93Z(BZmQid0jV@am-{3Qq?76;;RA_+gz z#v@3M@AnQeuBiUI5|g%ZCxhH=HQ;2jmBF~T9S6PK%tq0CPy-KbAiI-81?pEz zP$K#cK%kPFoq^b00L}QUlr06D4g_80J171^CbQl% zv7-La+p+f5FZ#_0ZJ^zqmYSEBuWD*vUcPbw>{aV0EC8D(P7|kk!h$3Hvc0y?SzL#F zWhLx-`)Vbq^1oC&_PyhRfDH>#SYc-Emb5Hb$_Zya5u#!B{qP^oP*~U{KfT>YJU3gW zuzhs_CI19#ndteFBqT})oojp8V&WLGN)Z4g`3fTI_0fdPKoOt565*CS~JPH0Rp=Jx#TY?WfI{AfVbI z_Nr!2t{H^u+jKV_^;U4-*l#qR_S?20Gv{r2YicPMZh5z_>c~Z3)i$N&xxZBRgaBpO zI|fII?m*pEQ?Ia*I|qd-&iB={?+y4+&CZh!&xw7aS$TwGJk^?@V^dx_`dAK-RHh2{ zN}eAcIgtYjDuB!Pjy+dK+|EdhN?YNIF9QL4j^xO93P&a&*YMw-jWLRSa(X-}yYo{{ zL|JM{h9)$WN`Bc95$V`)8SCMaOmC2A+dyx4&6FjIQG}&GQ`|jo7hx$m9jB6b);;p( z?d8b`olekIUd%Hq^h)(n={Ay~D@DRr3da}`3On#_4W*u&_!uutGa-tbNkD}4_aBO8p32ctBvoPJ4X-t1l z$w0?lWwSS+u;D(Tn)E|9oho&~yYF-&-Uu@Ta$L z%^Rd$3?tMZW8N2%A6`i@dJz z8p^lAtb>-YAOlgTeK{k}zH@1p(S+~-MZ&QJF+$t*H95JV!ZLwtLJ}QApc)Y!Bb!Y=y@7UiL4`WyAj$Tks=x&S_id88{Xo(cQJO~+y1mgSJKutzons{PBd4y1FQ6xkW3_EseFp4aq3yd zWpFLlD1?t&(G|Ec0gMI&^5AUdwB4}E<^+YC>IB%cBK@>_=Zk3)r~PnW9(l~IZNX} z)d9;n#y3amPos#M@Lg+^T^jF{sEsw!WwThZqx0u=OiyVrT6Z~W0AAA4-NO%Bh~)=G z+ND@=O@TnmA`6Ntj`cYy=5!zMr!kccO=N~GCsL{HY7XX(0CIOSvLf~bjCG~ksH(sO z?`>%9g&3V>18L8Y{%s_{$&HftRTqj{y%Cj&z<0!A&Ex&d_nz5r+UmtIlK7gSG(i}J zOe&YJRM2ojO~9vimf~26lL4F9(=Z^pjn`gpe|i-S z`Re1REA*tICo3aL)i#dS+;Ov^9VVi;M# zJVQYHi7KsCaMlp=K{VTo1O2|_6IgA6#&##o2I62O36E%u^)Rpiyh@DwM4xWD zreR0|4;`XE2xr=OKr{}3Or6aRRTFfZF>$L~;3Pmaibz1QB;mYC$Mb=YOYeC=k#T9x| zIw)E0*D*i>Yx3B1!5)-Swo)GRhJ!%lyMLNYl^`uzR2pC39!IgKmta`(nA_}JTA_(i zmm`^(z$&)e9-*Hf+k`;<5_~sgzmr#?0f&l<0Xf0y^)~imh$OSCb6DBQ#rr`0iL=Os zEQrh_v7j)ylN~^nL$%C-K5|F=)R-08FRk=S1q!qo9s2I7HSQrM&_3xOBZWOlL@`ku$M3{sSqg(<~D!p(QybnMIJW;9CLp1e#FolvckELyCpF zvw&zaLfb=cXXxz?y}h9my9zDZ*uVjrQt3r=`=NKQ&<=;8{k_6E7VfgXJAI(t)jB{m zY)irQfsTTJo1IjwHzFh}?29h3g?f7Cii0n=bGwWNnUP$r)>zyma~R5J+gbzd#PyWF zV4VJ;_;{#989Fp+JOF1m%SXk**;o-hdH>?<@Yl_v5^%9NwD2{JO9x$T=j%WrXhjDA z()heEOc`G*pPcOq5Z~nh_njqcwrY;f-l0^|(4Hku2=_@W)u(pA`cTo)*}Hdn?X#@% z2<;E6jp<0o`?U8OAarQ56v||C!Hmfa-%!S_gHJa=Bk)NW=tgBxdO;BxizF?G08Y7xF{9bsn$o?DN;992O}9i46O%cv zofq|)uVCB!-uvcXWaC^}MULvY-Z1}4H_X4xe))MCm#IYY(Y}ri@1uwX#B&xccowWL zJxeTgH5lj$SFrbIV~M={{E=#7!jwj&754Oha&wXmHT{sOZAzx7j_H4BOft?bLf%pK}nCrVG#DR`9yl*$>2rOC(& zqbEF+F-UtF4ReC7bc0t<(Uo>ZT|I@+S|Qk#r&jTtrP(+L!JeW7!d?1SnvMM^i&LY+hjN=-m>O2L<5MM8BPPu$k3diGUl!OF$t9DmA-7r-uBFRcr!`LijaNyWTkr$a4$cFoy z)eg>lJ4N{sw=AyZqI8Zt!f^`WE7rqdoJPfSJ;5Ca^93W>DA?yLaD!V)ci5_uSpz?3Yx$532K+62nU>KBdo5J_8O*w407_c1o9&3#j?Lme72aiD*`_Xs`K``Qf8^iYd<`8%#^;7yX~5X2}vaoGsr1v zXwc**xBlLzkGlNq;_9~YCI<6{r8g;btU8C#@dsad#o=sNdhW`z?ER9xfWgZaQ~es` zL?kR0aq{8l?eR(P0VAzN7GAzml(lK^l3NYdJP$)Pt0xCNJ zK2dr_JaxlvFD~Ae{5c@K)oUu8#JpITeKNwq%+rnsWxoqn9h3^x%wiQp+JmK;#R@5U z;X}z_we}=ft=jT!rJPHp$1!K}8LNl!yPd z1#)+1%g14;6bjD-7HB~4c$n?h=Ftys{KG|I6B*^9XJ$=2G3#$ru>r4FuL2l=c_!Oz z7c%D)6C(YYwS}x{^ov@P2`7TY@~PrqCJ~BFz2)!*AIA_c0kF}O-jEm^QJIN!)S(=D%kx;esy;yjSA`*=W z7c-i;iMa~5F=+F&9&Hp5U+=T(4nP}a+dv~wHryd%G-UBq`=vuRr7_ZOU6Z}$360T= zrCe;<{+vz_(`jO|aCVRl+ryb@pMbzPxl1UY7ef;cLrHiHx!?`+G?QnN$I#dh)4+66 zu*j@JJEX#4rVzMMgq~eH;fI(;I&ek-0vOH!X#L@5_2NjCd{YUbu!$rfl1hLhe8RF! zI5q+sJWP5?Xz03OJ5VK8Ii2W1F7(L$AuI$pGn)<{6!r`OIF5>lJR#oHLvODo*3+Yk zoW>KTPb?IAbWIa5Bq2eZbqibN;l<_UnPG*-E2Ev|s@A_Dqg+J-{-o1KIGyK2raZP+ zWR#}P4$2`AnU6!%e|w%8=~;Xaq|E1JO71L+RVz^Y8lgd$HF2&=lqkIKeW(4l6Yslz z-Fu_2sISW$D>H2Ba~vsa1YLKWOfMkLF^D`UhF*Rz@*7zcR1K;JU*C>@BYt5c%)R!` z@Xi4w)AW2ny}Lo?JuwQbzVJ^=h%+;h;Hfj;BO-BtC9X}8SCH7%Y;cw^MhIs~{B#z6 zZ5Ft8DziQL(=}evzTw==?-pPiDKnUgK7m( zDOL7BIAk*@kf>8f2O7Lh@bB4eQ166J221_IeI$y%UF@PKWa?>aBWBWY)Wcsc(CPWb z;bt!tL`Q|WUJ|WHRkH(TLFidNTF8T6IG_`V--Y}^Vy}Bmzt2|MUV!vucKvni-io@)n8ib%j z?&HJD7f7-UmOTm!8*GM0J8GA$kA=;>lO-=yyy2>nCCDgyFUj0BA2#yH#*ZIS;dU(a z>C+&hF=hR%B+#LTb4_fLx9oV}J>hCNIoyp*I+_3mosLxrR}Y*M-9_!!KNBECN1(o2X@L(n+&uk)H#bFjZTnJQ^uf3c7pp1-j_ z)xtP_saAz37#1Y<`5bmTbwsAf@iNwbR-hD}T|?+UA6>N5cgyM4sqbz_nVzEmOgK~U z{%c_J$0B|9^e4Mn`SBzA-~EptGld}A=K1t#tMnS)BrGsL{&ydJ`qa+|p&0L*ks93G zU`xyufWD39gQ+hD4lnyQ2LN$vB5Nu?D8h7lu9?jieNE=* z+X|B3KFuH*o@F)6F&5-tPt7k=T8h(pzWlZ3qQ^QzDev1va}IlUhQ;84#Bl&;EkcF^ zbZT8BM)`Z%WjxigiQ!zgm#y#G!sRtm9m(?YE`ez#*26#4qJJK;F?_UX;yMy5{sY!^ zMy}aX%)CO;TxhVyTvZ+L#vvPfzS7ZdAtX$x zjOVI+g#8)pgj>qboEJ_aL2a)+%_bZoN+RlWCfG!R>z%}dRBXYhZQ*N(78uxV4~xa7 zD~#jXiOZLZzG2B><@qU4XWi-eJD$gKEs_D{Mr)Q`un$%LhPj4t0`wPuIkazVJ%vnNjaR|gA;lzhMU-SpDO9{RIl32o(6hYRR`62 z-pCpQ1#!ciN_$tlgF7qXZu4w$eA~16%VQ^hdC{E8>cTLJUS00;y3FOBq2yW1>xTY) z-C%Mu`Sr;|<>TH|b9kbR{|G0r&dudHHdBRGnIfE&Z%%Duxb=y>?Vb+wa$Cs!#;abo z?B^^+w^&~ZmaR1R%3&taT$?wr_zT}F=2Ou) zzYkQ`c-qixe9V;p)o zr1LcPh1q}0fY=%OCYo1fsA{q58*__81-lU?-AQ;wFkLh#-Tc70Yev!u@nuDma^Nnb zP6^PbbZ@>b^)e2W@nE|3?bD@<1qV*WV#vO=60M=HYzhVq4HU@tWr2L#>37KnGFXN| zD>5z$4;(Ya=4*)A;k^&V8K=HQQW?P+q-NgY7{EFmfU*K~nFU6#<5q06xHig6p=xt& z6i2gcb@d>}J$5{7m*8oNA{+-wzYzJIx^NyJId?uhHOj8ZGL~1g+MW|$5C=N@glR5M zLt)HO=cvMJSEXc&)j}3q8@9E9{lg|p$ldeB>;(ePM>jj!(E3p@LhtPtQz2mfHQ`&g zGaM3C9-=U0xz1S4GZwR7d;cmwrF;)IXOP%>^Y-%O!jpHhQf299I2;ZS_V(c4;c!^~ zcX)8H|F7-6{b$?Td;15^_Ww29-hVdS`xhGCzv;zKDkSFr8s59Da_9ac&&Q9T_BF)9 zpiiHA$0uj!Cr5{uC&wlQMfr3JGQRiHnU$UbP2xbo=xuxGXCe#dK*^mJ!RYjCONYcM zwc6Ouqh1e^qsh>{Wge9sw)*X-PmtuxFcg6X4BaZd`cE|FBvF+tS(olJ&-yz_!%(%- zM;@TNwMO^iPDP62p67A!h5#SzW%rOb`K6p>;K?jI9#v)$9O(|-1-jP z&}al6pa~6$oG0YRNDYfWBO~KXHm*v2ePdJU`1oi)i82k^>M^8+&NCkkipf}6R4KV#vnNP8g4tC*B z8eOE53B5!8LEe~RUW0P)%XgDJRF|Er)E~Ip4&)WPesY&soykob3jJH$vtn1Q^?0Ho>};lhEm*0&2TNeT^hj>$x!K`frzx>#WYhzb#gll7endCB z-k!JDW2sD1sqBsHRZX$~C1!YM|6mjh557MbPPTV{*uy{UVSjfr`F?M>y}fflCc8Uh zyz~7J+xr+F_`CiO-|zU($ieU#!Nc)hfP?YD4+nM4b!V<0pvNCfk3T?<^KpZ)vvWP* zhhd2N@BcqM_~UT!{|*N~3_f_j{zo4@sg$uDdyfZ`;ot`-WS3JTr#1yH@wACoXCt_Y z`hE5P;>`b^GD#kG!fj@?zxZV9ZhYkvZN~aOlgy&XPN7&hD7u&)fEfuRPt3Mpght9f zYC_(yEzjodgf<#WTQBD;)7F!K`V#d`Wd}^(XHmjp66=@dZ`3!ci~46%wS|+Ul#;{l zgO2Bbw}qIA1)C$SwsL~G9I%OrGRu?ht6_s6oD5vH8r2Y+WNNzalKr=`#Jr*gQc*^- z^d%jqr#y77HV|8JTJ4ecwY5x%8PR(YX zb+KL^8)J`Ek4q@+r6cn!9zc21q@dbVABEDMxw2vKNr$<7{D>AZ$*LQs!YNCQ#18#s z3S+6czWJlS46H@UVGAj)IOI8mj>GQI)BT6Oi8vQGw%QbQgx5F?=WxWp9_`{R;S|ic zJXM=gXUrMYg=25Zd3p81hzkRls@Vb4(I-F|5Nyd2 zxwAiPUTxT?4G@bFxuZgo*k8<)0D569C{x(zs~O{^B0R`jpifYDo*1Xj5))Hb%T7#@A03{ZX866p0`)7&6Qy ztd#t@dKb;;Gu$Zny~7aLp1E8GZt~K}s-rZN)Y+8F&#u8~5T8Y3IP?L=qv+-z21qqd z;_7Cf>y(5y%Q-)_aq*>hUA~^DasMyZ)0!eS2Yv+Gk_+2du>Zfkz4NTN|F^TZ_iTT? z|MwUVIKyx}CFmPJ`k)roQ~2u!hiLR;&JpWEPSr{p3-OzF*bPahf01Ebs{qM5_O zwuV6DXN!5EGs7^7Z!+f?)HiwQjP)^Sgb+4(roJh-q`qm|IWrC_i@P$~V;Tn}mS_)U zY96?f%ouv^Egevz6#b>ESg%q&K*O}`7&=p?UVVVLT>Yn+wI?j1GMmP}(`0`$2bI1;ldbZ3wOz>|gKc!vFKo)9I9xDVAiwS!m+yJG8NY|L^ba?w0xg&hE1{ z|9_09>{Pz)Rjnp8^`VirZ$%I{gpbW+P9Z3JU=y1RkPhoB>r#)o&`3b#XIp4);8kgn zIiK#cz!duH1-4#bW-_%HgFp2pzRsV%V(Grgp2Q}O>DZ0~Qc?Z3x(nnMeKubY5^Wh}sQ zj0laVA#o4u7|94CG27k9Ub_O*S7tvuPz48~!*I(Eq zdjed7oNQ$I>9J>$)}t(Pv_qX#5EB;i#u@cGBc&$4XJc6?)lkDsiP2NrT5Ip@T@q5C z%G2X+h%5!Hq;z&bxE+JC7Tc;j-J5^sY^l;t4PkA42Z2bCvOE*T`lPazUb$X9z*7>- zfsf(NAafCqd+4J<(<@y8umXqeBKwNHYYhe{?X;Va%qKCEUHh9bak<5#x?&s6$;=I6tNzu@y>>)zxL3~qv9rJXY@Pq-QJxkqoSl2CcLH{e^y|V^E9ddNhw;NbDlhK)QNO}??QrNG zx}kN;>c;objuFZWj4DZ#iVfsSW3|?j3Ta7$to_+wo?(u|Q0Xj@E$qB%LZ+rx=N7H# zJI9i|q7jwZ_GbA*QAz6pDJO}87QC92(Tuv!s@<#mrblOmhG@)KuK!&gX#O6B}WFIj_4p^Uv>4bBvurD3S=QyUeI zN0v@tC0Vh}hK5VfiZovs;--PfYUFBHlm7pGKP%R%nfX!~f3U56}p?JQMl)?J5{ zSFFw2Y#&JNtF-__TyV9QK`o1P3CKN>E<>v#U76!N zzq4uKNqZBzw3D@@-`^}Yk*7=V1~I#NvISp-EK`Mo%g~~&H)tcOI20UsmHkN zSdVep#fMa^Wqw^=zRK(7@4B^I2ctq7bQ&{GPB>>=bOY|er4*e56Oz9gxyc0E zEB}cc_SU(1>)bxJ_p_TC<@XB^seoAm8=GxSurgH(!7}#-T;*Pv3#Xp`n$MiRL-;hv zWmKl#)zr47o!A=vSY58Ij#A!A?Ut3x#cfw&dpfr7D3FfhC7_)65-ydiuj~}5Dtdg3 z{gET~t8B@IG{%1Y4n#rUaB2vjC9a=H|9%)JONAof_G^bV<^4MBvUEIzqQ zsH>C|UOz-TTBU*g_@fZhsJI(b_w^xKO$Fhn@r0cfgoXR|AzHC^-NiLc{!KWYI0ISz z_7E*jXebHS(GBNhO78M%l}``jglpPKu}t%v2qo7FDtFztMr>BuFA8aVowWu(9u&OK z!ZgaeV}3qp6EVIyVj;`ezWu%O&K4(<#A>9Aj_=az`}e`#>}1^Www%4?X-V)g3&;t? zHFs1c^9prwl}&?losP|c23tCR^Ptn3u;AyTpE>r)8R3)#7sO|AZsIn4UU||1-Z8`= z!uI8H${{YS&|b~=#s)wB9mA7Wj{o7~uIXPE<$u}T-rgzaf7u?c^S?aG(-IN!2oeGph`i|(U|wc@81QpNhq92IlZnF0#)q~{a>rqFPI;;dPGC_r9Je$`_nA|Xj(;P* zG(o4jsbXI3Ucq$h&_rEYZZB6(?n514Gz!|TtbbfE^s?k$N3ah%P=76={k!W#|Du)@ARn*iMD~q4(t_(5>}S+jcp?-xa(1K`iox-Mi>|ykPZi zw&OtVB*&*QmDyPwcLTogzJ$YS(d>Q=xPcKrg(MdCfo--C>bEUHbFEd)3N&Bp&c&%O zo9NT0QJ4Glmblm#efrcd<6k=H(|Eq8xSo6#MOX>#_o!c^t@Y8bPGLA&xzxupk~w&G!@`%YTv=u~yepjRQAhUA8X)sd^)ooCZ2ji(;e zsDn0L3EVVfV;nXD4cT-sf%xyGpiXSedB4QF7y14Rjc3|iPzFTc2 zlu`bzE>i3(?CRD;zuyCu%ga|Cu#qzS>=-31I0^}l)1(=pgard%7y9h@_?*SRESxY^ z>hWd`1lVEWlwn?JVUT-6gN@R+47iF<`M{vl@=mc465kX7TRC`m{`1=pZw_CdT$~*q zofL{7bfa2?W4NUc2u&yn1G5EEeo8qWss*634imH5aBi{HqWS9N@c88X!^x|Yqs!B` zZ$6wJpS-y|z5MSljBu5yk>8b!sL!WlOr1@%U8bL=ZXTar96ot=Co{HENrm(+|hzSM;4F^%)tztFlM)u+XsC& zX?c0j35uC?D`|I8l{}2ksY)TW&N8Qk6ca6MI-O3jJI#b*T@)m%;)s7xUi zqR4|QpNesug@(Sg)BP)YR^9&*xO4dD;`?7K`~SPo_J`~JpT~Hb_n(W7&P7S!O3_NI z?ijWBNdKl=M{Vq`GzA(nW73@Aq9y3?v0_X1Y~ygE{Lp$p&>I$$9zt@SG>*6#p38Yc3iAq=0g&Gp zRq4NCICzdj9Q%a#Yd5Gvr^JQHWgBPbS)0CA*Xk0GyC{-_wI=!S&pq|Oh{aSgey{ld z-Dlgwa{T|^_WoM`dyL0r9-l`EtYtBVd|n&!Y0wu9b&r&reo^uU6vc@_tcgRu4nG zmsU}%z6i<1ag0lCNi5TJ=@3i(@s)sSzk5>C&GHs7PAIvPBv!vf zHJK?~k#KIb!(#$rG?c|PY`T$ri@IiQ@L73)t_evjljzS#7oT2dccVb7lk9Znkll%&A!a`3a=s1EFoOd;sIEPAPdK;beNWqoW@h<1B`H&g5_jFZea#aOy`4}gr>D1LU)5a zVssGUWS~o8$s+1EmrOW;#XU&l>zLif12{M>Mo98BKP=73?=5jySIry1hnB zp`E~iEF|NQz6IWVb#BtX?X!gZ=Pb_k(`D?yMe=|9Sy}$??ryLBzmM`PkpG`Y6c;Gz zA1Z&H1Sw^q>wKz8T6gT4#Ov+E{-sJ!%3{?4k;~G|$PU)x^fAS$5`9>v;%mU=kwyEL z=6&mD8TsGfQ174#ERz4b&noBtcK6ru-;eTq3HjedN&ieuz>0c%v-QtMmsnhvbb|Uq z744vHGp?*Bz$V!QmyPav5FPY{22G)?E3{|}P5MH!#?Yt=)M*VZh3hqk7BO7a9$E@7 zBehr5A?me=FN;%En?Us%1*&QlwU}2li?VKE-lgN3G-hUe(8sk)!F`sO|Kx_ma;zgD zyZHeY$^YSY#s5FtS?~Wo%JU`Vf02^@+0q~GZMAX5s8;s7MH(f4VN4tR1Gf`0ZjkzJ z^|fzcVZpUMzJC7N&+_trIp6=ky4?ReET8}0*;~v1$9cY_{QsxR{02GSDBGLmd4n|X zB*F6|+h-h(yB?V<{SEYh2%pnLUyXhNRup0rBxl5JM?Jp@7V9+Ip0>8UWQK(mXL{~0 z+$i0OtS+_o%Z$W@hR`h*%97&}I-(wZN5KdMSdxLHSyt~VGN#_4EFg3^B?W8(8wF9q z%c)r12C5L}D5WgvTtE=yo=*8zY3MKveaL3goj`kG`+-dQ2tiW^7@~1?xdq8*slNW?+u6L^M8B$`|JB3 zALaRy`kzZl|7=b05{v6|7>Nd*FfY)k9hT37Rn@_~#NW)d1M1Mj>l`eVNm{-sq-l!^ zA-iK%`G63WB&vDc2F=3sXY_{De_UK0!F3XGV|~=VMdTF1ybVSdJdnPpPN_DE240aH z61JDlm3tZiyfH)0%iWy?WpzK41v4#N3s_%nL4W_C$vwwxVG%R6uh_J4&@_xOV;}?Sn#@G2Xt9bsOi$phN6z~*$p^q+sADxx?r|Z%|)(XH@=;U)m&A&5aa?R zh2&V0>AVdPiPM@Jyz(bII^Fd7Z5IA`#G)i5phk47a7;QBe?x9MaER8C=&-u=!eqR0 ztU9esEW|D2fj&F-5R3Dgv`!vchFs$XB()SUmgt4m4PYBn)K7i*LGc-=;kfTIOL;5K zodzHkgsah9&vOajS2NFxni(iMSeaY=k@9;fd06xKmt?-Cpc25ZhJp|phirUrR*~He zcyCBHzv@G9uZ!cCN1Oz$B-Ld;7B4e3h>ah!!D%dEJH6^s4U~q_jNjruPHHbWb!)eb zwyJz`XKkpJ!A$+eiR!`~#;JeZ&SN@}%>o9~O(8AlxNH7b1^CtQzr{opN)DpjcFV|*O1Hi9_S*V5*cJBqr$qj9jl$`HnSgTNsWM#D?WMdq*xlR|3?<|Y}3GA#rAYL=;B25n2Hur@ z6dPE~M(opto!Dm)TPfnuAs_-RW&YS_J@>5*{AK38b5TE}B$h{nFC>p;NF#^3CcVn@ zHzz$i`<+~4_E}GUYXg6o$?sg0DWz%W;d##K@tWu=&)=Nr?A+^_U(bK#`PZgOp)jrFT$x?)}!fYaRgTu5lwXD#>GV|GeMwgt^ZoX{$zd}F!jJKM zp}_eZCrC9ar}0El9*N~4EZ#4QQYZQqz+2W#rV8*YVloi#O6B}g>G$_ugpYbM;~wGL zZCktxJIe9xePr*VW6uwwJ5>@Mm*KYz4p$^0$mD-yRToN#^HdvgV{;BPU(3Qc>4IfE!2d4`pFa zjLx8b`~4>x$_F@=mL>DPFQAOmJ^-W#Ku4rW64G`O>p(&aiae(L(@9a6kbVvAK)U-; zP00#~gtMt4*bap|W%;fkw7&Ia{d9R&%l|D%=j7iV?tk0atK|P4?he=azaQiI()quE zlGe$;FGySF?+enN_FE02X7V+6!?dOY7v%4j3w(X2$h~m+B6o`P3Zt;TVdN2?739Ay z(pdpmApf_YJ*&w7?Pu%zKOW`zqVhkZq<^>qV8)?I12CVPRRBAV%{oAogiTsN-uPM# zXfOCT5WN+-+Q|ROo#c2B;h0W{SjO6`J`4E&aA&7{|L1ULcg_DF<@xv#aU4%c#s^{E z-vU7a4;%~zy^kLi`&G1dDo|gqwm$M)1}GE79luepfW|^W%AcQDCc5dA)3o^U<67JM zGEd|B*Fhs&(`&Ld#{M;l1HCP?0wY+o{+}I`*Z=mjgZ27Zi@JdTCV?R|Q=#Q>uysW~any%k$r5{ZF{+-0BQq;rid%E8qXUv%OybkMcCG ze>>^z3?L>_5f6>0-t?MigK$hFwpQU!1t{w!zTy%uO~#UIyzS|-{v#aYDG7XKzfGu* zC0ULKEL#8j&-P2}e|NaOvtIv?@ieY~GwDapq%Xk(%<_L-0rFL_0K4V_7Gdf-nF=1E z*0C^E3pP$SAJNz4+FrE&^*;kYOob$Tz*)G2d;q@SRL&Tue}EI!aS~UYxcWkc%-5J31?MazrnvPY7!R|iLHD~} zMi0YM7UOq+&7V`_eH~YKz*`}kwg9gehe9w{_;qSb9|sW5hs-BG zKtix85=(-0YU|}#L&G`xusE0R17wCamkV~ClzLgWdhb+2&1}k@n-2YMmh)L@jl?fA zDv9cwu2r1G1@itLBT%=Pa%#^e8uC^9vogIZ-oRNH67D66EelCv&AaYPv(^2w+NQjO z2|49N{2uC|7$`>RrGoG`Ssr1{oSe_a6w}UUGkW#&)__4Y{|t@SkfZ)o8dp`Ixa(Jn za&XGKoS#C9C6l4b9N2P`n{SD#0b>KFzkJ9Q%$ zg{y*>09~y3=HdL1UrcqfYv&)KK`l$rbtL&qpFeo;X#ph8g43&oOIEOB_o zfj45q}m>2=-Jq8 z{WFiv)G}I6N40HLxRQLcm7xt*?H2Qp< z9+r=?*Pg2Hq}7+?Q_N&Xr;sw!n(cdDFz5@?fRh`ldi{b5$@u&gji{^#Q7!ki$rF^V z1A|NHY&p`E_pYt^*@!pqXB`PAEYRa#w0j$<_4|sQo#%LHZKJY2pv&bo(yirE zyKX`OQQ4%q=hn`GZe5t`V@y)|ypAY-xJUPz+n+HZ4?~ItamLhn`B@r;1M8dj|l zd#Bg{=F>tZkpZd<`v46FS*ZGtZHvx{B-IrL5l+0gyzCZI;US-vxK<;jZx5?*3uK0> zd2e!IJA2InQYI1P3%GLY#3nE~UMlNtygyy#xXB52LN}+dAlV zlU<^VQfbIUL`LYv<>lE$4WzOMzFMoUQkh7!OrbbDCLx|*5TC_?7@_T9p-e(J1!1he z?zmq`d{Z13JN&0_J{-MzcX4@gUP46YWJi6;Q_`=kdU*cx+YfIJU!Po@9Uh(3Rj*IV z4vB~wa;&?CpV>UDsSFveGo>~+&T7$oet2>6;l=sMPxT$F((xDz@`95|(ay6^D>FWc zZ@Tj0oesRXlC7``vTU1}3N0<&urQ6tYdBp~w;*Md6k3Z^Nj>6Q#cxeR;kM9OG%Xs9 z)^V>>!&+KfQh|5$4t-$5RrajDV!qr<-P8k@!Kj;&=DC+q@^7f$+*tEuiSmflV1BT? zo*c7)To}VPFYUhlEb6&rA>m~^%X#@(8pRVxKj_RVMNW5Pl?Iw!ta;GHOP&j!9!nXqmS0Zthh9w8RuO?CtfLFU5Jr}D0R&HndaXYZh+|DVQi zUBPN+Ko;!(9_&6V@BeP^t@FP>&Qq^q+v#@q0aVFvj5`Zf$mqufPQZeLzYIc|6u{id z%6sKOH#D_))k=5L2KsZ>!MMdF(C_sXh>hZ zZ=v}@wE;M(HKjwn+<7f!;v_Lv>|#{)M!6iP$>wp(tl$PTy-HM<3_mRSuC#AObs?!U z^d6oV^2aho)4XMoHCWWRQ%w58vIpXGMq|PYjW-u=c52o3VgaD)%(S}A@{#E+r706c79&_X-MmVi;AcDM{MA{EP3Bc;%#9&fXF0aU@pW#%bqLhD z7?x;s&2GQyrz`*01oBz@zdHw&`0wq#;X3~NQJ#7QtSSDx%+}X&-(N}Gw`JfDAM~9C z3k^)1PFp%odL;+iE3CLZ6l`g&{e*%M&y*p9Rkg2-!Ku@zjQ4G<{uUk2XU6x|F4o{P$)gjOa9G zoIG3`3*`T>a{g<3Z*REX|9zBa1D#aZ1C0##5AF{~Cg>dK>6+ zMgsRTkv0ve$N~K z!MEVQ_QhyCVL&ufDNsyEKmrtygebx1QzDR@ah6VJu-p}V90wkmsNpzJPKwhu6v4PGw8Jv^fQ!1z=LcQ8B_bX>%$SBBeSQfqB1&fFp zH7?LCw5OOW;lR?(WzMS6qw`~dKCn~@8jufq5Qv?}|7I!jaf|{g;0(X1U;${ZZO2Uy z*?5jpp+v;hV4x=6y*>KJ2@VCh(z3+7Q$_O@d%YJh`e37=G?{W75ac)aGaI}A1OWd9 zca&eD2@3e&9vWl5p`o|^gE!oKnvees-w-q= zBt{Xt(TgVQQ09KQQm=6ka3X{raosf(5Fbvw=Jbs@F|$xw$wfpUj8Q>tLlZHs@dKMg zBd0Gv0{qLYLUqSbQ-PSCq9i0(07+^}_hkrOva<|TBOIqV4CiRVS%lsf@1FnlN$D>7 zO*_$yqZ^8$wd{Ig4MM$26CpXl5i&v=agwl6uXlBIHO6ArGuS8-J)EP#HR^X?(_2}4 zTVpJU#%q#>;s4Fa8UV2KuU>nqd^(=4irr zHg)JCA%5>PMvSWgRi6m~cVvPNoD*3TJ_0d+y4ufU)9hlRP=GTNV^Vb-2JzN(= zmb=K}Y&4u66>=fu8Thxd8_|mwyBFMJQxX%7rP+8@nlp4V3Lbo#< z5`pKHv3rxBG__quYPQQ=a1NelBVmqt<)D#!$}8UxHz!sD=3f)ORVt6Hz#gIZ$Y-|# z&(W{hH#nGy-ou^zAw$r^KKj+tB|0sN0;_Hm1(jw_z*W^xPBc3KSKL)JAD9&Qc{W+j>87D_a~I7AT(h+!N!2JZh}&WIq0+~Ft*i9YBhetqK8tr;FTEMya5 zrECf-QDGH$y))f;K}eCgzki#k#l$#z4MOBwQSB^B@Ra1bK*{aK2@d3pD9u380*SYT zRC>?Wj810*?7-RD1j!W#gbS&3+8MhARoBO1MJ7@uq7ymu^bBMiHRn}@IWR*ijOr}y zvbb`NB*Q$#J~=(Met8h3S4P2#+3izwJEQ&#MFhusI<>w@0;gScOT!RhU#VIwMmU~l zEHf`InO=W7ZFN4=pi}~rmkfoFWKLls3K&I{~# zhHBs&20auk_eWh^ye*8DrXpo^N!BuSLa-~S*EXh^wv?;H*!Pus4(GK+&h|7Bf?}(r z(E0PjBUS0*;;n}&1E5e=C2IS;v4J!~if53L%4Nj1Kva?_H^AjB!Pf}a8N0^p`iYV- zfHe{`Nk+ZiV9_cZDLw!Tih<_!G`9whO8n)!@7}(7_21uphawg$k(qNqk&Yt@dlN#h zVyw(Y+ZuJ(dNW3;Am>_;DmAErn8GVItQ8E>96JjHQ5YsyrDz#|yCkGOm8Zv7BXk&Q zvkELmlYCW;EshJo)jYN^88a@b%~JA$s}VxKDE##hb0Q?Cz7%cH7>GdvLH6jkRA`>3 z0q0%?^hD0Ev}{wAc%c;~7=ul(Gyqji`f7x7hmh4!GVqo&$Qg0;@QUa$t0ou0+TnYH zLmFr|L1jc#zXSq7XL#?=phbr$!5l|K5-xhZKhaAvM}H#N?D-QNn`|iRFX3|v%MtzA z`*WZk{a^b};V<~r`_no2{`Q7&uGqDk;R^kU)XcyiSN;AK`V(E~-)cfk4F?F&h*O;t zrAlZPS$0A-HSdQk4W^8#X1+))h-A zdq)zk)G^Hrm3Ha}fHE{s6tl#9ZceG731Api0oytvcf?N>A2)7UPa}6HIex2eGO1v8$T^-Xs)JJK z<@dtz(@19ZL=T?&~{A2B5|wI^PI%X8|isiNfv;j+NfqKDlDMG@D|SnETdAs zBOpca(-@k}Ge4C9+f)SQp|I~^ihWK3r7K~*cvL+DLwm06n!GPECW^Cq*SMGpD;X9IYkd>aZ@LuNr76Zqlfv_;=@rl(3k$0Xsz z$C3p96RgfwMaK@HUJ_XY?xlBC5pWm;0;ZunADJ$Q$LxkUY}R=6khxje3P9L)(f$-i zOFAsVHaEbc1XJM3U;=5}f?NpXL!$T5TDts15aYYuEcUu3D!5&qD-xx8^;^WsJ#K5L zGSJzwx-cUM8YdW*xB}VixQ$8Hc3WHF0xjpBT>+0}fr}QM+geBz;Z-$qCZQM>xv^Z8gIDFX@0f`X&juXo&hoz0(5Ha?Gqb(;-j@7TpEE`!Knt(nYg7cW zdvhEt+N1O586zlKZ-!wWH8objiiK$uSEu$Ub8Wc(6s4fglpSWeY-;~ng{o@5#z`}W z%ll@_*7c|nZgmy(bF>vOU7%E*v)B%a`WMu$Dtmte=kCB`&K;c>{!mpM8&jdY>+SzX zxuyD~$yE~iKkQvwbK^F$e%G%+DGxiYvZT2rn~AIToN_!Kk1KXY$Cu56YidN1kc>4& zFa#)#v$g;I6uJQr1VCLZJLf#a4~fSlx^W}Wm#@3y@2MlWG5U~2Qle2HP;Z)(#GAJ! zkG9{lCaa{sSvi)9A~S}a+`3uw7aDyBXy zJde;y8sTL-6=mh6fepA8$U%Mi&oMkqoZ3k1?hgOGP9BGh_fZi`?Rm9;Q5j&4Smi5Ls%DR zn;^%(U6a#auF15XACVng=<1RhegF3D_hDart7lC9um`P(oaQr@D_MILGk7WuKpjN# zo(rmPcmP@?79mNRE;N?V$ecxtlaV!3qIzPEeur{5xwdIFHhM+VeI*fVC@Xamrz8YxOi$+TwyAvNk`ujPQxOeI%PlksLF0;HW_iE>svJoS!-w!saZvn92OR^90g zCbGrIG$VJVYWQV~V_*$x-IFE}omJseG?4I>k!Zj+h*GT*C$*&)Z@|`5sbs*3i_f2g zU23JWZEbakkU-%UE#LZOBc{GWllKvss5d8vWb!Y;i^)M34>tjCi_z;AmwC?TmX&@x zPB21>6*C^~Grrq&Iiu5huKKK6sQ<5$s!;!@RmH1t5V|BHuK&7I%v^(I#TQD8s)Z@1 zXxwURDI<#dRCPkc$bZa}IhsJ#=}^_FY_V!!=13N8jY{T+afMQEn>+sNih|+>Hjhxz zfxUI99Z;G}g)OujP_&GD0>BA^OzBMZi>WM_GR30_5ijLVWPn3OadlFM31WBx%eTz}Nx zFy|$&)cj(i$;nTze|imE)n5%)^?N7R&D4JlGz52iR;07UfnQ8(Xp^qC-zGck50O?$ z3H@BhTc@dB`+?MOOADIM21+}n85l|7oW8lZ-bBBNy0dHZ{`LFUu1%peU)z#fzZg-{ zo0kG3_^eZD?OGsGCChYRX9MoE*9HpdWB&m2yAUd#T@rHysDpztZ9~FfJ{u>Mx-fDk z7BLYPD$)pzRsF^m^+G3#0sM-RX3LyP$0>^9P21kw;=4TIIj>?)Kq^ViNRcd=4zZag z6$H1;D_$2FNwJ|vkZ1ICsZ_`{rVYF>z?i>al2CY`@B*SFi5>*wMR1I?at^n|zg`8K~ZE*Q`{XZAv)qnki{4o4+`2GOSYg@rDSV^y9PS`E)h5fA} z?Eg2QI}g;gi4;lU0c+pwFiY1&rUv2^$~&|`oTcST*5zy33=uD)-=HCK)Yfy z(~cENFW^OHuM4~%GaJ5ec0zuyX-&zTN|FKVY)FpDAGG2KkR+_gKsnK_WJ>`K#wj(e z2wN;@#u7>M$JmOUot#zbjB<++v^~Cn}mE+BG2G^L}<@#YJMAB-(ibUf#!Mi-; zRZ7us`@r&i%YoW)I?WC7&7QOVjB)Y4(jlb zzo1wonj~cGz7A(|A!>uq1Nx$_mg;1%u{7eIR}x*q9&d1Pq4IX@x)6dx&VZm=mXA4= zhj^o_lVt^gQ%RnKMvY3c7?Klp9w;?%N_BwOOh5eXo=FPLnILpg%5Yitb-=lHsu*d% zBLSvi?XVA>aT_nUGxjLQRmCK=7Vz!oT2BJvT{87+Lu6=GJEc?@h}w)M5+qTt7pDB4 z6&~LhTuCFM_*+(cqXJSw+X^Jb8hqiWafYh(Bcg}#Gy#uzVmw~J0MsLNv^DX7tdO+O z#@PjvO1`0FmU9kktbN_i@pU{tH=69!z3;WclPgh_Bnu4mm|1ZF#F@yq;S}=lM`fzg zn^CD{sp#)D#w2Ytw^UTp0_xryWQxzA3CS_9sv$KC)bu_eptoSup_!JxtZ5$x#0ewvgBsb^I5kN(}yDqi-F?)t|U7yU2) zIqX+7$vzkP;}z%fBg-j9NA`~Z581Ek;#mC33-uKL1{2ifANZT_i;#t*d0F>%H>=Hl zHlN<+t@JW}?QRAgK0V)=9a9V=$!fO32X=~zhH0G7{_?uS8H~&hDpmuhjk6YUX?7$|3bmPQJnp36lKR(IU zXXudVh1I*8<{=s#)TDs$0|pz3wTS|XT}6xnVYm&Yx#?NXjd}FEKD4Tp1#Ip4s*7`BIW!SDuosC&y+xq{rp(qx1~Jk~J@x3BfmsK5c<>vL z6Z-0Kr=pe91sFHh1#R(UuJnIM>LRB?82D>O3!_<@h6C^+4Smqd+u@G^=p_OKbmpV4 zEdc#54eag_`WT7&U?RG`9#d25_AEMYYkq?AuS`wlgxWS*Y!BarXV^`!+iUwC@*bcXs#3i0G#0{}gpx_Sr9| zy7tJn3LyE}eLqMtrG|Ef(Up!F@wAw;LWlfxFcu|EBNKSnVyDEXFCLj$#lfh#^#DGq z2K#s4ZXfmwHSiJ)M2;3eY{^ea(2i6Jo(5No4cpe)5@;4n4Izdm zg^Eqq0+xgcGMF5aNt!cS$O!-{CqnRv%@Sfz+d}XrZOw+P3xcd@p8JC^vC>1EU?iMn z4~uJ}t8EKyO+wrWVFj6Tc~5RjI5a=-RY5XVQIz-_R|9yctQ%bESD(hmZZ!48>E3Ah zc1mUP7PChbbfqPr$Uru4&QH8zf2b2RCu5$GW8Iim`?(n6}+w;3MzG>)}{ z^kjTX>ZSoHn-)*(!6g{bcIt$-x$V;rrvkofl!G$@J}4lg-XizvHC7N-Wc1tcUM_M7 z*#u9kQLHJ%3>w7Cl-@HCA(pffkp1S{BW5}xqMm|*uR&UaO=8sgR5yf4on}m8k9|kS z86Z}gK9#g`vGO!yazj|uc0G7dbZ@qeB-$AlQ!_tG$g9*44in#BcVO%k zUF3!207>IxWkpgw4g=DAMi^=8S$y2j6Ile|A(~NJi*@-tUs%!FG2L?^k8=jOmV6$U zNm=3-D_4{j}{3z4R|ek*JqR057kxHDFO85!c8HAUZ$%9O$YUkYY|>>hZh%R4e` z$KC2tu{AJT9UeondfZmd%z@u>NIqj?rOD~HB!wswe0>5U^0^rRti6WQrIRk+ZS`r? zk4}5%s>ukr+I0Bl;+nM6NAU8d>2rd>HO*&(VjKC>;4uyTO>pO@NO0qj z$rG_Dx971Q?~70Enu|XVm+G?J7xHOtcbDWgkVkhG9Jr$;v8Q!5lwqC?sb@cUFBRvRoxA6xXn0yK0!9&qg%ptemc1j+tYgNbTvm58tpe?C%BH}8N7;xQ!}5gpy- zp{2Vuke3As#@Rd_AQ*11mjMRGiSG&u^r-9y2)sRS;lCv2Y65&wP=f)CLse4Y&-sHfuLlL5=viIwoxfe`G)UU1My4y(i$td~=w2lSL{Hz{yBy9%gf_6}NeZfo}FiLSn=k!~kaeTyF9w3L~a4IV6sxV1Y zDGpUzlPaG9EAWQAeqiN4X~kxb(QIZ9tOTuWsa`Q|7;}<;y15yf#q!AuQByAl)TX(| zGgb^STP(QRj090}F3IiHh4GER7#Ji1laJkl0d8QJifn~rHBkj*3e(iJElLMh>av%J zeu9kPt(gn<35b2&R!xxmyjmR%oJmivz8tWgjcPZ`by3}bS{=I|9SeYWWKS@F}gr7EacM%sXXU% z2FcqSoZ2f^kb7QN;=t!n3tF8QGd>QG!hIJj2s)1d0duBwfD9@+=&fN=mpk7Vo+J)0 zxhVgMFweH|2mxXrSk*uJ_kZ99*KeUr0FY3u9tP?bd`)yw@+zqw(cszI5YL38wbWi1 zL+eWJ4&S|SvznX_FAK=;0+?+?1)B%dlrC}XyDyNsTQg~R3=@oxzToqCkevyih7kQ0 z#4J}$5(s2AbV(PJKTze`Ceabpv1}-^f*eEZpL0Hihu~0*^O>ii47)9tgq)pRo3$UNxdNx=HeLyp`k9?1Ua>)A)Cnf> zcC!1Ybp;+<@Cx$+a=gAfB=b}p64GF=q84p}sn~nHH$!+n(HYrK zu0Bk(CBhbIswfoD8)6psq*BV($|P8ZR_?u-@?wbrlt~U*)<+uuq0S-A4G67hMRjN` z7UxS!2D#k9;xc@AkM-Q&OiZihz`)Gw#(g%@73fCV1+_>dStUYGLS`>4$T$RbZkt?W z@K?-r*Xt2NOaU@mn+xl08CFL`d*{rb%?@eGlo=Nrz9EzO5m~+)z8=2O3CfTJi<#108uRIS0%Q?173F95TjExjeu;?^11kCsJAa)L?bRiplgI zr$KjKowc|r@D{LsNMLR=)T-8c?V*a1l`w0lf5m5jKH=)MB*zdSy-aSUpc!e~E<`i= zHWxqEGGtgXP1=0CQ;UbvhMKlKI>9e;WnYRKcVPSI3Xu1d%oxS8olC}Rkw21$g0Bit zVQf^;b))uA)D}0{$tPaO(WKohj6<4`0cJT-6FX_;>6!F;4PdQFt=KlyV>wM;q_mXq zOEa+}bO{Ygo(+_cH0njLkPTg^YvHIW4x<4D(4EIj!!MENcddVklBQpmZ>CfxZ@xMO zGg7|j#Q=@F_u`(f29l4ovu&UQ>&16w23Rrwsvmwe55MY4w_gqN5?}u66fn_Q|BlG6 z*B7)K^=xL}-BV${dHwgF-t|03%i3jEGW}p)vy6_^`VO#_TKQgwuVpPcnTQ85s_J3@ z);!dK?Utg0GMkQiI)CAI+tH?c9F-jZGsxMDrjKb(N8dHJU!W;^Eg}CY8y>U(s1Doc z5~JQ4iT=sMFxtgH0L?+#5FPbeLemj({?&5|1-g3-#R**t(G+=ylDVTuCBy$-@GQyy zVM+Cx(p9@}T+f0v>U9(X+mCWm%lp9s4x#_+GTVRs{PpwK&tE_P?|%L_00960yJsEZ H0M-Hkh8XRO diff --git a/experimental/helm/charts/pipelines/charts/minio-5.0.15.tgz b/experimental/helm/charts/pipelines/charts/minio-5.0.15.tgz deleted file mode 100644 index ee5e93736690e6abbc2b275c0cce90b011afcbd0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21320 zcmV)lK%c)KiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYcd)qd$FuwnWpJG<_+1Nd%WIMNA?Yif;`L3=%%!gJ;Jl$EPn}aS_jmxbA1^B)Fz?{aX%$5Ri6*5TtD@1dfe|B z2q8*xsi+8Mgr$nGlnSg)XM`m~i0RFLBP5L(KuR>i&schVx&_B+6qpyLEsLS9?XaIP zHO)u;h|dOQEt!%xZWJH&xBEMfg5P_dyB>q{4M}n;KNSO5y#62VJbL)3zWy~!xLg09 z;aOXQGr@0IOo1Tn14|)GNJN2;4b{V#3%DX#HoSs_jA$Z*wKZH8r#Dmx7Skd0dO^?x zr8vlZBo$M+q6kJ!DU9hF4Q6s701Z z5P6!ibfN*;f`?dc=<4+o9+3nAUr+@x9g{p!z-DAZx6D76WU>V$jm@vKJW0-Y!lHQ} zW;~|4m0Z$T_mHJAi-@ASZ@NjWFbn_$&A4QWi}?_K%gMa&n1f{z0E$eOuvvhtXnF|8 z;}fpV1eG*ZK~P|JK_fwx9KwrVf;9*&3+v(cP}dJ7mgv1#kBfP`nnhQzq5m@YudP9p zFq*1jC?Y}iK#&6dD37kG+U)CbnMKDdj7+p>SRREB{}w*jT|6vnYnBr9L~Q}<(>_Q= z(wHPXrN9NmOe(=fxuS8P3l04@u(k$VO{oBxXBijzm0^!S1chD+$_(dz95XEiNn#g= zp@}HTV+yaBnnFyogwJPspt#VeC_2s)NmW3Kx=*p{5Z=7210`w7712N55AuFlzW{$Z z&sao~WRCM%LK%1Nh;eHB&4q zMck?@rrwX{dw11mJhzQ6sBa3j0g)`G5fNoIY3dymzl&POsiH#H(F+99d^Vy2`2^}s zaPa{4&yKzJ*Vf>`kc6b-hDEfG6*bE+uj%V1zyN!(e|CJvg&IQdZ`<43y;Yk3M(|gX z3OL}Y8~i%EE=m~VX1at0fh(%8@6wn?fSw|aIT(9Dz@;iGa{w7 zJaExOn(_QV=O`}?OVq!Zgb7Wl9^BZl!;~wq4DHa9F@{bnBWkT6?1Yhp2}|?W1D;Vc zl5$`HKTWxGhxu3F{5R z)~D;e{$$#7y@6VL?eB~BD$`l7*L-j5;k+ja=q1c}idumk0eHod1V$8$tcZ0dt;#TF z3Dw$<9fJZhO*iGMqEW43%(Q-?bt4q6sESA}CutgMfl5=|W{lE4N0A(HA*jrF8l$kI z5mHM(aMNhUyO`laCTOJLYiL>q8)D4(IEbiF@{FkI5PAcuB1?$)de95Z!q(qsQMJS@ z8rXG9)YPiuXFtL#o+q&$n3)zjG_JOI#bc2wPv?H4aXS(SX>Vg#0A=~rYxF5 zp@*VK(dyb1Tf<*47JP=|emsVRjumLOGv5MMEN_&P8VJsm9>+Itp#P&WgbTFb-8%~< zrel`U7(vW(sdTqcxd}_m_!uE(x}wZz#Kz1}LpSKasL1D>yBUHLBhUC4aPdkYN|9)4 zRBQrzaR=th85Jj}zwT73nc*!NU||pmnk6hEatJ$*D>dlrjHGeFn5>y3gN$>nmF^g= z@#%%;k0=mIX&b0SYRO{h(y2zE_2>g<4;U6{l-5kJ8=`3c+Op3qA0;g6M?&Fhk>(|8 zPnz#&TI$2O&nnwpE(0Yf9n6Tv+^|uVNKZtp?S{ra3CnlZFbV)Pje83PELsOz%$FGI z_XhnR=zQpR#f*^9;}*jK6^f0~DwHilR_o3RoIj=N6&KebO(QW!`H3ujO--4I!;Ap6iL@Jh;U@@OW?0kzqDR`dQFbZ5FiC=$}8v8v5Arbv>Y#)1`dQuq<(9O ze9Kq~5z0F&m8Dtw2)2TNW<|y7SBdPG4fO*%6PjRTjZYt8TaZy9nN&25s2!Ix_eOzH z>ER$DY*tm$+O?xxQV~va7SlliSB2K_Fe+KuTxC^jJ1GF)8a37=uJe5@v~n~kjZzY~ zAgPBD5jAzr>2Em`RQet4*PE-c-m@PM3f^moC@xzb(XrN0OlJy_JmPs8>yB7qh!~hL zM({?2Xi40HvUOqu>WNKstee@>BLCsP`nss$jLopuR;kz9vNFxupc3MFN@gsAOz;~f z8AtBzplKTyF)+>;E6+C31z;LVb8TOzywMm+C1AEMpv!MTH02x#%Y#<7Ohq4HW0LJ^ z;l2OQi!JL+5n7r%IN$wnw5317U*I&O>4hTE^(KBaCEP$7Ri2Nva-h&Ej{ES6`Lcz`?P6SGNqa$d&IX0q}I11H#gOb=O z4XEVUzfB3-Q*?c3-h&-MQlFAEHWr_Sl_mLvrIuC4JiV_VC5qk9WbWKphz6phr+On{ z*Aym3&X470-%Xb7L&=3+XIxTGYqJJCP9L>nL}SC{c&?;V+MJ7a<0&WcHzXlxq?I&N z8$E)WU`>5J?+8G%XUnjQ`UmEZK8e)AZa?oU!WGPUKjhQkmF zp3r_369*cygs3qWvzD$K2(9({ky*Owgr%=r3#vqhB*N8i$t-(Oyy zU93{~)s!ikFsVYb-x3OvPUsM}`}p5L6KIAje-Hpnxl}mq0Ct}>q_4>=OQ_vD0j&~Q zie1x%j0$H60Tm-xAykPpg*>op#;^{nMcuwk)j4;|#kbR3d^_*I_$(^NcMIWeAvmXT z|FbWMwKdR&zT08al0V}y5UI^0&G`e=s24kEJsF3zC~0DRMab**s|$R?1=5`2iL*bk zTVy%cO z8mPmIJOv`r;L>1o=RKPUo@a1!dU-S~U>z7xT3h02GB3cfahsK~hek%NZky2~@^oJs zx9N_N3Rc4Ne6S*~bUhU=hXnHHfjP_$u|&5X`) zqj%@A)|J=R;3<=+>1KLRnA!013_3E_YM&I?V8dg9;e zKVE5d{Vo0<-`T~xAJ16&HfCg!a;aD(f6<`lJjd-tV_5<52iQVK*Fls^#b@5uFfra% z9L_S7!BsL_@@s%t*fpJ*U3L4@N@$Xq!&%l>_LDAq+a2;!!-=nEC-|GF^$v^_O&R{ zoK0B^#dlD%SRZq?fr`f41k-{3rUb)@U- zcT|~APkDs2*WRJ7>gBgumu1f|6tff;5xq73^zyHhc?XTxxIlxC~_<jS!TP|KX#c@=GfTEjn0>oz%)+roiPOHvgeMTf@A z_uNkT7Vf#2&EPb43&=X#q}>g%JD0k?seM>ig@~ZiyHh{WEWa23V?9Iu{UB2H#kxKe?tLwO(8g{9q@O+m+I8_RANM^NlYu2q_Ju8ND|bGfJCHjZZVazC^2 z*=$(4AQmj(5Oj1>4+vXvVQbQL%i8|^M_bg+&rG7Ih_PF~zU56sqDU29fwgi%x4ft! zv(YMSAg$~$2X%d~;uXIriYMu-+-eGyx%LLcYWl@!{cvW}3hHKo7hNGatCTw8fP z!O{h_Y(*_!Qf=t{q!~&gIUeh<18s?`Ml#=}vz9nDTDf~@Ug{OSINYB)Ik6*IT zed^V_t+BR}p8DJNxoVZQmGCUHw$I(x8FDV`Y~DAd`Um7;&JJ3Mb=ktDKNHpfH<0ZP zrTK12*QnU$)lO0T%vW6`Rd%zg_-@ty7b#Ea+nB#f^^CmDabn(@IEr7Yqzt-f2|pJT zss*+MglkjV7+RdoT}=};Tf`(`O^u8_pQT0on2QEdO62q{)^a$M6)ZT=M(&K?Sk2*@ z&YkuUbBz0ZMbc_Jl&D-((J+Qbp+> zmHj6ds*bF8%xGeQW8)ZTdcy?AO))P*ThN5wXnw4?7soG>xd$ zwhi)?A11~hTQtX}FP?vMa&-Cf>G8qQ$;HtSdQVwIQ%O-$@sTp70fybU=4C!gsp=an zdOomk&QJe&adbX}9v_blNnK8})09REHcpQZ z4>m(hCPXoe0X4^JMc*7zS!ByKLWdm+AtSTmXbuf|MpG7tR7WjD{b7nO89np7P&ahVh^iYGC{(xi3A(|>p1jQS+@vaPZG0zT^y0(H¬kHJ+X z+_Ze}g%-AbM$4{exLC5U1!;H_6v!PhAL1Slyo!OXq z3?@0UBxbckd%Ek8Ns(hD`!{(!q3ZpPn(6j&|6!2vSPtR>s10Urw$qAd{m7j;?{HBB zjyAk6S}bVw)2S8i2u&>ea6E?l#=5%iB0&|0``DIdMCk9iuQ}rVBG5cfi{m2ZT&!#7 zzSVi}7osVbH14~&L~E_;PMIeC$Z4NJclV)z@O=&8{uc7|Y6aY(!4Om2iN;18TGpQI z7a25qRyYxDUsEMmB#knKFrV?1X<^@`O00p51j#5+ts83nK+iCqwZCDp3HV0NRH}Qn z5v*?sJJ1?mJwcd zTkgue9>Rldd1c4dy{dswY({ymY7opEUYc)O+p}P5?~O1%x;!{6xWLj=CZmLpDyK}O zk^`!uxIe=qnujKk!*M;c;h2_=ek;!Vfv#hif-Py)&5WpNjrR!}r+7TB`W|`s0Trt8 z!m;$Ot1=cDus(Nt@;D!+sfA5`mtQ`4&_s#{am#X#Q$%PSA z{uuz=zO(Fe7`%A{jz+Cy>APGpJ?O)`cYWR9IYnEme^Qtm@80$0P1LXDeYxZ=48imQ z+r@8FF4Y#A+34k@34i4*Y&i@9fSo>=W2$g=die5W|Jl*r`i53fB9(yFW3&LlgeCy}i8>VnJ|DNenYCD3B4)m6=fsQBPQT!eD8*phe34`_e~l5ClKj zeVdfOs!oYL)XP)XOffw}-H|?gi{#@?lT;LQerS6G?WmfmZ`7ArWcjn3%|q*X_Vl=- z41~9~%x96=`uZ(r(KUt~+1asF^X%Z|_oo+^FP9Q@h#lUuy|2=kwMJH=vX+L*$Yj%0 zozN;r)!+5tEuJ=c{1C!0rV)?n=J5By$DNaJ#p%CG3h8D(dNLmL^$aJn4VAVz2m;L{ zX&USL%lM^#+UdjjBuPsb!tt3_S23U+zN?*oYAFPx`7o$V(oM87Um{tyw{&!Z;Dlq^ zE28|fug7OFBXdyXngW4{=UNpsxx{i+MEe%jR{u6k;2+MI43E#gzGDP#;jOf$R+{>c z!w5dz8m|(-$6Dt>u%94UMU(kf5lo;hd{je5Hla|(+P^sb0e#Lkp3$yGD&B)uI^DS* z91J!*#mifpNzgHen;>3fjw5JTjXau3p1KxL|2S;lwXnIq#0gDNwb+#ev66EV}w^FVXF<&>KuNI0k}Y zmo#>kQ9p2Mkz+c3rXW;;4qhIdo_u@!-OIz{^Sz$Ow`(PP!Lx(C-XNC((VA(Mg{9`8 zAcS@|AdH!S^*0ru?|Oz|B$Ap2yP)(cOm)q1p-cYEB(=nCMPia3hBBcv3-NDAIL|b% zdPSJ>lFil5!#lMmsKVXYgg3#xi-Yfvw86L&+}pprJbHF^d9k-0+rBA9 zdjbCmJDb5h)AgTw-B1uJ1h)hkLB8ujmyJiSFVX8r&09 z;W zo8|G%^MfBWb?yZB4xSzEt#6pgmx>hgK3F$};Ojw5Zw6_eB(VGSU-T?k^wb@MM7z$qNh5n2oNMj)K8Yw;SVKz`cJAHb5@YCMnvdrQ+8wIO6y!#fh9UqZx1&@zza zc9`zlV)y%+v`eO8l8+d%F}C3uge96d@0sZoGYAEQ;h2kv*5}p1>Za0rn#GrBeN#gR zVooK7-{IJ8fxKo}rV+>aMr`%c8Z#Vwo1x63Vm8JkaTO>w&0{^B%{#EX(Fk30+O7s- zvWnt479YZG7 z&kb^MV~&I7(E^$T)w;!eAa&T8pN$|4bzyDNUbjs>P1+GL0hh}$>1wv6DdexVy@f-w zN=pkGT0*WiT-C1hI(arDRXsAT5ud{EavYG_qee0x-BNzNq zL%elvY#FyTw}2!nO?T15MJ_XH@?hSkb!n<~l8OGGoyU%8^jgbPyWI6;+Iieph85xt z@Wu}`jVKBMs$A40PyIM-*L|~lT9g?#s1-vI){Uq_)@}8xl&A9< z&kZ}W-y6e$a`jTo3$4&s@-SRf<5@EWR3e>cLTlVIn$j6XZHt5*Y3o?@kMwH*K4Pq%A-y%EOyCpt1yTx21q>JlTCa(r=5{?2!OxCa!&yK-8TB4 zrqSEE0{}jjKLGLgbS?n|@7c}xyRe+`SF3Lz0Ejt8GZoW#)$|TrApkoVi#-Ik^8 z3H36L0$XX!>>tiy@b?v62D(nY^Y!k>Jf#IOSPE=2%>Y_F2xc##@E{m@((oWyF}j)u zp>leJrDJX#3Vu^|wK^hPs2H=w8*sU|QW+H;4Dv$1I`x&&|O9 z3>F^~FQT#e_W0?M`L!FE@Y_;FG1oA)*rDaBaw?)bqP3189vdtz*7CY*q5rGo>1HoL z()x4DteCnUan&pyJ52Yn+vHBc$gG-)(_>P6NZdV66|G5Cl{dk=yySp6)L zB-E8n_rcRP*IE3!a6_Sr+5)6PJ6fZ)dRS^ggY59V)L&Tq`a?9pCHkK|g8X3$;IjRn z$Bp>ECy$;yy3_wY$8)Fuwch4(O_4d_DHSa^HCs1|P8=IGClh8g! zf(r@g(8E)!Hf=K|b|jIaLiPiDxS;a35;{GE`&+%Oe{6ld^=9i`VAL0*xcqSR(@Uee z=)W$Xo$d7oYL*R(`~fdX8ow-(1-KRbQq8j7owj?Y?fy}<-Q6!i+r2Q0&8fQ$J=ZSA zTPeD@)#EC9>TabiyWW~QZliWdKpRuNj=~*SU&voJSJ#CtO0cn><9>7Pt-k%6z5l~|YoitA37($n(NQ?2+={&T_erz?Rv$hgW%}9SeKX#j^ zFwETw&<)_4e)mpK@XI!{yg;e5YKQIqdE#G}*#8>E4?YdJ%>I9{(}@4v*?D|t|9_6> z&i?;{+y8fV|DE0cN4ERFh+QsCZ=FUziuqex{GO#>LSJm=e`@m{Z!RwP@XD7hyKQ&Y z{0CX{#!2$Nro5qsWp=zrwB1HLg1V0ep*7!^4;#E~TD{uZwWQg!D4%lv|GR-DgONP9`v9Gzd)5aA@ciiT-hbH!F4w3&>OgloN>tyZD(x~M5{bt zCKVJmczt2hkm>4o>+*|+e zUegl&BGu1ozH_5=dVO^X#9H*uW=(I@G?R4cy+4)G(_?QrKwZ%co=aP{!II)8_B%L9rQorlmnlQ8v-Wr)2aymwT?*H&Oruy5`o_0MRb zc7KI(_2=-iLJifaYGMK^zFcv=neIZcyoBYwSJ=kb++pl zHi^r**jhJd-mv0L*&dk5b;b194B-uiNZShU-oZONIEjMw>pJr9-fel@6Ux8+xA@fG zyfd?`p|UnwN61YE4e(3w<=Unl{9k??&TpQ@{D0@+lZW;F--l1O@AiK`%k$>V;48Rc zvmt7CW0p`g&*Cw4zH(_ zy3fUkhE8C<3A+m|8+KD2wPqh49)Hrzdgb}`YqN1VE(TIH8E#ncWW>GlZ8^d%D4SQA zV5u5I?>}VtAF@}2Yw%bZx~0VGuTC2ECSNZpmxq8za8U!r$}+k3I*9xyGTM#r$NJOFcpFQZk?7hdDJ2%!DURjW3f@gx^n#!1K1#{+`yn@!I=wubuOoauA z8huMTyP@KM&oaFddtdME^dE-XTkwya{*!Q9_n2g4#1f_$m3_U4Eu5m`{>0&ML65#x zG{q~!2RAz-s>n`*+}oReVw>K1{N8QbezSDazy2#GQC=N||No=6+*^r`SDW|p-rdeg z_lm4KP@$N#7RxSTAM+G&XnUMIA4u57vEyD9sGQ+7xd6*E2RHu3U8 zebGd7eT!A3W_oA03&oe7>%VKVW}hytG3JBqyPiinuld*liq_K2lK%)9!7_t))< zi7Z>K`jU$R_B)2GynFZNjfsO;dG%+AJsk4rnhH3ckqL!!I$=_Yxw|3_vxax_XQ%M#;WL`#m1L!{+|Pgv9s_7lms z>^>U;%5iT{vUHM|%ZIJSn2>+Xfoi<*)+k`6U6R_#FUQJTa7Fk=`-jFr>XP+$!xl=G*PN9*yO&Qd za7PshN{L)*n?IsLb&|eH$?~)=UK*^eN638_oFh?x{|qC<3Om=JDuU^ znGgL)s718}yCI622&`&#^QC`H=bwg%%FYh={7X|>>uK>yg_eO`BetNrYqA`w_iwR; zf!i;u(ewfu@W-L*_DZRbv1zT7BGy+WgT;|@b)j!Xu!~!b6Zb)r7 z7vs)j=s(4+7p=@`x6a-+8yhi;6!ePm>$P1Z(=$5vLZ!NAX08Xj+tB-g&U?^vxElV^O2y%VQF7Nq zFBl~k+tf8u#rH3Ev>sMQ%i+6Z#1`%U)c5~PB-o5({W+O^q#eLz@t+SK)Z;%M?L2vM zxBvG!p3*Thgqxks%3)&ry zqNLqU8#ZpSas{STFhwS{5*Ic~^WD3^b%>=*F_N%<(ILDI8w-2G_-EGpmlmV`H$+lY zRD0d`=v_T zLc0K(DeW*N#rt=o3}EZRzYNf5^kqR#hEhQ&69c#oUi^4qdZ#~?hez;hYkq)_+Q%Xn zb<0dux?23uS=t-aryul~C{E%IOTgoe()|1q?H zCG!8l?!!j>=YxlL^8d3uZyLuo{B0X&@)xn6TGXG|P!G?OZmTWPV#^3pBa6?tvC?ae zU-(x!7IiE^I-gVr!rC0hI=)`hrc`uqPobh=^dS4WT?;D5;@?5%ajX{aYUvm}(*>x+ z$T$yclb^Q;mPNoxnsPBm9}pWFPkwphgjc5$&suXzdHaE#6I%Qbc|uq1 zMg>E28S1dWRxeOl{?rAs!Vm1Paj%D-zpNz!$L!p$ue5br1FBoToM&_hXM&E|>&nLw ziLPlH4=XRho%X7~?q&?tM>Dk{G+qOw`D~=^15_V9{@5}Oz9(WeY)@9e&1-ZuEPUE> zYoUCD7jYL3a9CeTpzTh}-&;QZ(Vs5;&)ui|IrKmMcU}M6z0?0b%YzFY54HJPmA~Cn zZjtox*j&&2i=O#b?~&Cy-p6<36v=-FeX3pjJLPqCb!U0~Tj)a@T)>96>s5oQ<{5?F zfw^M)_^gMUbsKb+srjNRo7TUqP*xVz%VtxqmVk*|B@3$f-KK;!Ds?$BZ0x|Y4nO52 z{)QwZji~7PUFmKbD;mY#t8CFN4wCh{s#hzuZ6d6WK&9IlGvhV}OO?Ye;c{EhN%2Lt zRasha#Uq{!;qu_DD$zIa`0la+KH)Ly37VR^N~X4qBPLXO?7Ky%x-vDzp`rsRA|_Nd zAmQRaFyLB1ELP-Q`_$f+e|%!qSO(S48Lt z|E)8-XXN!oE+$K`aHY!mn331d(;@G4;>(mIEB`hLxhzSL&LSsf$G=1vP^2*e_ zluBKF+mN@)0B*C=^pjLDyakRctfiKvlJU5{Mmp9>S+XS1zt$mFd?P;j*W1^Ma}m8d zNx7hSMi29c%6wKqC$cYH>NxD+N?T{P0!nndSth4s_t9gS&stkBQ6Q@~P>mW{)I?)+ z+NPqxK^D(Pm23DB`ah?=6{pE@ls1F}zMR@jY9=un)YRp7?;yPBiv5Q$BylXLlrXGYzWZm# z*6KBh^U$%|S-NT`8&+Erw%!q>er<5Q0~E48PG*u3L};_^`j2-Qz@)PG^85B`Wd$Yi zX`0M27Mr%VLtkgy#zD8X;rH9UZSID&u*Npl*Y-{3qjs3~Lrq4s)~xrtMQlP-X5jo( z&xcB%$Uu%NYk6%;yoY_rD_mB+o6~~hHRr)-q~HHTV8G1K3z4%pjoTb z1lpTy^u}|x)yJ{T>E@n-)wMgM%4NA6vgZ)LMGLJiFa6N;roB5RLE`!8>E%l=YqWcA zTwrr^*Rx3OdrrsAuM4z(|1@g9ZKo8>xd#^&z6|ZN{fmo#o}M3mVW_(h?GIeGrlU65 z>CwrLFTXv0deqj{uC;4b^!}}uaf#xLD#4;c=0D@9mb~_$%wnX^&QG5mU4DP`{Nm;Q z^ULpFUjB5}g}DLuKKOAKM@@aS4V~k|qm#?y%b#AJo*kVWAG$Gpe*Ux_DQ|6IBW4_* zi{zHgJUu=-xqNxtN}nyQB#fr&__lp9jhM`)ZNue@Gc#oNJkJ5PcTq-!EG<yE}c1}N|v-c4QHDp^B{9=4x{&4iu`{Mlan$AB6?~JJF zD(elAKHIA~sw(c#GjV!RrvOIulhamQf`=@^e0N<8z}Ys0q;<7;)Nb_>vs(B8J%O}}u%n^oyy%42$AcQ>j*1@3KUWbt1Y zI>{O)Y%`^aXTUyQfFc=>S;~}0Fz(HJqgH{hctS>^tP?g;>xz}GCxmcg*0|v+Pyms9<{)yv&#P4T^#Y!ryl?FG0%OuXKDQJ zlShsG&pVIq;(tEJQ(KH|m4jbPa=R@$;w5i$wu{~JP*+=i$p42V=Vj_xcRqjQM1ROAY^UC2Jiho`VYgN=yF)>>poOq` z7N{!VMw2W9O;T8;9Xgu*l=qZuYme|KT z0p>k*{-5zUW>Vysw0M-q6Z+u?ftK+9?I(@%zfT@KxV!)Fvpmh<>%(IDzcJJANdZ^7 zqR#K19b3*W``;4@J)=Ch6!?R)Cd|g+;cAu2&77^p{6%a;Q%fr?A_mF*_F}}b^#$iN z+D5MJsK&=QjqwpVSabbL75>UcAJYN2^zSQfu? zNri0Y^ecjqt2fxTrOfkJ@JJKQKYa9Fc>)}n-g+tfH9xn{u5l2kH>Js}FQW-lPigCgUX8!XJ4y5hSL~(qT*!s;JHO*c$@Alz9|U8q;`dM8{mv5Z_{4KjUe; z+fcXjKYY}5@WJhThmJ=sJ_UCBb6v$`tuzhrdCzt=nlfYAb%tr#Nyv`+tL)nP2bcua zOij6S4yWn2geAG4A7_8tHy3ZY0blE8rTTkrfEVQZi8tC^JD+63y@QOp4rBT5;@cd! z>VkWlgI5b-taJdY3qs={)-B=^R1{`JO_y!v7m565*{VhTKS@~80=dCAXOQ3Dr`b!`#31Khy4>ls6Mnu)Lfg%V0M#7}pY)Jjh&32fY z#@nutx_y7LqycHY&81(tN}>LE*JV^;wFWo4$83q%Gqr;S$aH+uz*}(bh*?|m!Ol_< z1Ec4osf7JMGaaG4iGH6cd9(2Ln6isqx5$LE=ys7heoM{!MHUYC%9qU4NBLl14r+Ie zv<^pYFh1uw(}pPZbCBbRq)Z(wIE-o-{)}8Z<=PvImcyvC=y>sQR9B#RAhmwRhFse- zy(`InSS0_UQwLbwN~A{H$GTc{JlBn3*8$yyBS~ND2}k?1TB}G!{-Qj} zy9K>4>cKBwF(M|?F9Y^0)u7C_|93e0ac}*<{@y(_RV3gqe*ygl){lOyHrBp{+<|Em zkrrU}UN1QV=uQ*VIDi-4yt;ByI|z2#bqt`n*iC>?Yxwl?lg2IDAXLw{4ycOE5xGHy z?RII1_3HJnt0{|hD5hMcz-m$iZXrpa^YpW6Uca72@a~u#vIQQ?(xl5v7?YPlm6Uev<7W3)a0fkS_r8R^6)I7SYjrZZu3t|0E{Mq63N%d+? z7+t)%9f41vQ)?vd`0^cJ{=NCKBlwTOm23O8o$R@KUDxO0(G~1IC8J*2&~3T(Y5;!+ zmfb+?lkx2;$+U6rCAfTK7GA~Yj*;In^52t@H)#K3aP#WTlx-u9YPK!*IW_Hzir4;A z>)gv*dmrt4MX0}%9^S<6BXRath1`-*mrlM6nfGVyUCOydsCUQJ@3{K!tDZM$|5mL1 zcRi2Li2uEbKI+;3Kdo^7>&fGK{O`l84f-4O!rszKHWtc%WwYq_8kcmn6zb|j6|LUFd)F6q&q%bA zlFHxRZCC2lT3#k+>iploD&k}Dn5E}`w|5))U$%Ge`2Xj4T8?oxBLX|FdBl8HD>3Jm zaMLfGJ)66PPBZdbP6O9Jmx~%})gtKiH{9}aNWX>AWB}alC%s$U(DyM9$`iu{5c6tq4 zpsQplb$sLwO8Wd`nhBUi;d?ICArr{k zeW|4}Nhii?31LQF*Bo;Tk0;Gox@z&#ZQY7Q&f)AMEB|NrI7W3#(}x8JaI;w2esKk{^YPhMNaTO`!n;G%L} zyS&BC|MCN`{Fk@7>%URD!_aKG2`s?=oGK#-Mh0IexIjf{aY31v#kFo4u$h~Vs<}6s}*&=QR>Zju@jg5h|dOfZejtwr}+*(e=4UQJ#C_TY0?- zLLG446DvWHK^bb3^>D=1mQ4*txpE#aN6(KkSn)>T5ha3(OMXq$i`p_+Iaj3MGpeRE zFM~aCIVcUZ4da5tx1{jaE29W(3LC802!_h8^~wofYmorAO#=??o~hcV`EnxhxL6p5 zw-MlHf{)(I?H5`f(4GALH{HVFNFWEMB+lKI88JRb zyRxNL`=YAOW!_a7`+LsV2aCKrMP~t4pI;yNV|%6={vSmY9p{Ps2=_j?XR-fh_raql zb^q_9$2)iNU!UdaI{y(fsRSG4ipKT;2&`wECkH%FD>pjomlAVDD_2e%ZH;n*W(kW( zt!VLfxCgU5@!Rmg*ut@3H&mWcfzBllcJ)K0f%o6r*0qasWIllMr7dNWjh%Cj9p(Zy z)?L~1)28>ffUsGhnUCttR8AMz&u$_d{vE|DH+I0fcWk4hf>**Jqam(vrj&MArK`*AF&lvLW_y2^g}2~X)~*(_dpQ}*NU zbJnfQw(|p?RO-0h^}-gpSoKW8zhmS{O((mRmG`0U*s^}(YJ+7S*!HTcI9>BBCW>At zK@^?L&D=C2Ey}5Xj^+E+YA)a+enwI5`g1W6K_^n&JnoGURMC&A2$Eachk zTt6Rr1>E90H*S|#{`NO?6h*(Z+c_E+u5=h?Y?5+85A>2fn9}H4=CcYqk$ve-$5q^Q z8=27h#(MwK4p9FaA}NaNUgbDlZxfI6$SK)<^jPMz))q`O?&=Mg@-vbxZDM&ck*-@y zy60o*B_)k!{cB3<%?rjc?zwt{vyoV}Pqo94Ej3#%{%dVjQKR)ETkN3akW9juCm2ugaQ0TrT`Wg9^3+V0iHkTD-sV=y?-P>%z?_f&O z32n)~(t)dAe&JGDMr5Be0e)K!Wlw4BBb#e*!Rh%yCoZ ze5veRmI+|0ufvGeUk!38uyo9$YbrvTDlx}h+rFHdWF^+KTMY>u?txy;75nquJG_#c z;N?WlJY`k0RF|M>e4NJgwJt;AxTw-7IHD7l7Dd~@K~<|C4hFv5y{ZcBE4;BHeDemb z-dr^S=n@z0n|SxG-|xS912P|ZIPRE&SMRQj2XhH#dH?e(}}iS21?WA)gVJ zVnb{U8xvKx+;=uDwx!+dcVyc`yjHQm-uz)5)D2i~-k23pt_=t;Dw$xYQ0^vs2t#5_ zFc@}QJ`jeHxdykpboHKGBQk5_OJ6HN_QgbAq2}c0(DL&bb*KEXjHPq|)HeEE&~MwQ zx0P*K0@Z4@Iz`H%*V?8GF*>$hrk;Oi-S?PuRqqKUh1st5j=3pT)0R zuJ5n*dG)S-p*?JW?Vsn?MA{xlQVxX@#AadY9+2KZ&9Z@?BPyinO>bqCsvF`ekw4J6 z{IOgcm0Rkn`IWl5<<*w$qgm5(o3*-9I(7D@b|W_|@#46r5acU{$}fQIc-QS(XubCv zmSSI&{O}7y z-KC89z-4PXdT@I3?eTXnk4}Dk`R(!3qqeSgtzBzx_HVU}mu@1~cR<^aK07~sc69mu z(esO!`_C`Ge|h=SSr_I8-233iSsX3PKWET6K0G?PJih$t<>}ec$?>5Z)92?;+mZ6t z7B*tW@wrHD+04`9qm#>*$F20)(h4TnIKFLPOd}UZ2j@qZZNudovS!GN^j^BXi-NG1 zmukJ*?PILA)$r?Hb*q4ALB30eSTaTJ%eI4Pm(AY(@w1mF`_DR-M-u_UW^IG9N;7BY zN8cX*pIbC^Cg_;GzFkun2d8I87ppZQBc9QQWM8&+pFKNTkN~u{aAz}`t~ynoihWJz zTd;mZ5`TA$rZ-IRv<;#!$gewBiIFT>Vj!=#XApg_%$BR=u&msXixQ~3Td(OsG&{{3 zYM#;4G4wZ~Co$nquMJR9YvYj2ibz}qn-t!|u34{v+n@onz^eg7MoD=2>I zi;zF%1<1w|sJwlYouwYXTe=2!{^q*|TY8uAgiq!dnV=;0(#!aL?I`lzJzwpTW21}J zZp>NXqR^G~&einEB_`l4ZZ7R;plhRIfi+lJ9F6UVb{lZ{21ljvD%%_N4UH}z{!N)f z#qH@-PnEQIZ~QGB6y)n&YoG&07jDpcd&;3PR7b=6cmkB5E6G#=#n-bb=fMHZrsXX>9bbKX3NoG)?BfQ>>{`f{Y4CSW5dr|M24Fh2nw+YjD74 zGoHea2Nw`CA%p&esR91qpda){{}KcIzk4yA4D|opPkECLNVJbZ_>l<4a|y?XM>6Q!tY<-=#gq(8Il+Go*5LG?Cr9TO2n)ts zfL1_7N)?rX`Sg3wcl|Nz|NLnG@Yzv+7Jrg9mc)NQdbInnw*Ge>KfH_o{w&WLJY(tc zDXxVa&BpI(G6M_}2!a=XesPhI==zrp438cT22%g*&sf@L-2C`2ZO!k=o5^N-sZDtC z=NAc!Xew!0g3Ts0sjJ2EOM37Wb>dIh$NIHiJyFqa?*@7`8*j1}-Z2=cH z1QY}vQ$f>6OC+T+L!|ofkC7OBjsLSP+VEsRlBOh8 z9_cS2Ic_Tsl%Uj6oz|Z=K}`t`6CaOR#7L6Ny#Z<;AE^XSC&?THHJ-&-LM}}w9fh=E ztp_g?JkBFC*ECD`d`43x`*1-iyg0-+WrcrgveX?cyJlB$%p+<0c3reI$TO(~B{SP! zt>h0_DiukR(6koLFr0Q^f4Ol9^;q<$YL@($pyNF=ew${ti#TA2d6Z+0QvJIRZ-ZcM z4USV)bSVffi&=&d=SnI;GU?crJ@v;^cu@|I>DqAM^#d7DIiN49BY>#N>88_hz;sFU z5uGzZG4ZTqT7CziT?`2`YuuX_8~Fn6A|x93gAmSs)KhfBB;zR+1>Lh0Q9)Ld%r$1! zh2Dpaf6|g3EcXBCB*>Tv8mTan?CZ@SM9$)&c4ORMaI@3j`5V$;LNuy+K1Nh3Z8TEB z(4fO!AGki)p~g-Nn(-jGy1E(>ISr=n6$U(+uquZhPMCBOLRZmH_1q%uxQVa=#=iAd zwCV1!)|jO{2*dLDwrMvEB~``t?wO#@%x^h&<&CIn-yq!~*L9lWmRv$FP7nOIvY^+s&Z-S&|B_~WOiNG}Lno&iCG>j!8 zDU7H`V-V-UkPi-yX{>v+VMFn`9<4kx(vi(b0*cQ8-yuaPikn=JIT^(a!EOR+KUx-n||BqQzUbhl!!>_Zg4rUU@CiCzyCWaL==e zAaRXKf8sd~L@udW4vIUmWC+78GL`~?L>oU=qv*rFl=)1UY2IUGctP5!US{cpBFwQ?z~HKgOKuI($nSn8Jf?PYsPCpL8@J zy+N}po;hM3TZPAvc62n&Y}v(<0$eI%wn-N28(- z$8O08qNYfg6s}y(_^U05nM9cFT^r38OObAI&=};e;j7+$TBNIYBf|6|6jFf+w~vim zi@+I=(JGXZO|)LFI1o7DF*PO(272hvknnV(|3#{@wM3q3yUVBoMB&zNAO2Z^Ea;5i z&=|&o&wyZ&!q^d%NSBxqlbX=1c9FHr6 z1m+`HQc46R@!WQAtKcq7b*K?LeK;`Z6f}PICWGCDpAAM2`cNu(cA>34>Hd|0duKl$ zU`$K<60;Xwm_r3YaLK`t45a1*WahYmo&{t;TY+o%1#yLNUE$>#){vXodlY1CO$3$Z za3o8UqGEa&+e#66Ee9mlNb9xa(j5(A8YM)~&>|2TEZ0U<&wM!Nd58i1n_B_ej#R-_ z>No8c#uf5-C31Sz4}xzien+kNKZ3vLe;ym=Im@L2l1L6J7pbP?YJWzk5bB{1bib{{ zJ*4TpOtj=}6ls~S=+H2SS1d_@reiK5>JR!?BuT!4jB8122)@kOgcK3zaf?*q0DA?> zyxb{O{Sj59UmfJ?@a?_%xyRCppi=(a8|ojD8>0TXH~hKhb?4{ayLUZ9L}(pXhry2R z7AQfqJ#YFpqL8o|Q-*K+s)rJdy1#92Z`-k-aX}#yG-A??!B*Sk;VmggHs|!7nLXol z(Jci$P3C}Q%($Vf)FbD#*ceRgm^rbj(TRAP(g-C!QhDEq8}r?vA2XRHWL~OBJXcw+ zfNSm{bKHF!=ZRaH{TlDF8~$jyQ4;HFQ*u-F%2kbp9s36H+VPlLr z9}zJk|7t5y*c?Tv-}2X<+Rf^mu;9UF@QA8w)0NIQy41oxt2g)U)w|WIrH#2JG2MdI zgtZz?UuSr$izAVu@wd>|Qmg;-Kd$Ng=dbm58^G}v4k{YT4G|1k@LM>fG7>C9l>`9Z z!nwW92i^v6hh09%&M%D*-ix=vTex!bbp@5D^UGVnb3@vC?AsS}D(yE1&`s}E17P!u z1?aBv>sJ8=`$W#biY!IOlFN^YaUTE^p$ z#Tu%bZyLkh6PjK&3D`j04ZMsvg%P9jY!lr(sT(@tcuURZUCe+~B10*Y(7Cjbx#$Vs z%ZQo|6dz=Q-4I3l*L2?g9rq{tkx(DTa&%EmBdf3*FOv?6udQm%y}fF$S`|0XPt~MK zHDw~!Mx2_%!M=1mI$C;ol8&SCVGK&SWy8cYKvP<4-%``IUk-CfNGhdEL0qs3Vt|tuc)B7w;|{R6+5??P2f7^uTnVJ-$DY5-`(xEEL@JHwmfcXoDbl3Oq|@bGAB-8Scx`fgtPb|Eu$?nxk6J(a zfzG3ZlWR*ETdgzU9JSW$w6t8AM-y#eeK}W_Ib79L;V`v6?hdU5WxE}O5RbH0Swk^| zmDks{3ni~O9=mbU#9ZjT$QpQ_#AdiqAJyHc&hW>@xryARwH5UESG8Ynv=s)dP?Wf4 zW|8vE@Dg8wf}K#~JG&}fM7AHzH8Qjjr&X+bLBK^2SVq{WwAWIlq%8Tt^Fr~nr`MtB{y|)%s%2Qf-or?+W^2Yeuo}IJG zR)@c@k?Ywqe!?&aLb!5`7?iGTu(5+dFVhr%bAE zu+}P-c1)%3s4_ts_dha;6&SSBm~~p*lEyrsUax-9z!N6XxGvv2B}LPO=`_lkV$2r+ z^d_bPygaOfVh_VWsB{{&slEuT8~h4Thuj1MH{uFaYh&s56s>E0C^;hxLogAQ@&`KK zEB@LlJ{IQ$_lm#Dk3L&7FLzqO?Ur8+lyf@Xs~Xw$61u*#`^s-#gCCiiO%opWdIF0d z8Xg`P@k$nW>XO9|SF-4*U!~gRvYuHuW^>xun$mL@bhKXuX7NB*xK{jFTJvHQV%{|1 zD5+?|2VL^__7!bh{?^nH6?(RTSzs?dyFF>w0DLHw%bSQqk?JIL|9cy)``>kE_c9`A ziq4PnTNl;)1v&}8S*Sw${^uG3yB~QC6iqE!2DSOA)$3jqu8_Yjyow*RG#2CZ2^NY6 zJx4cKZKdE88Qxx5E_FnMwKaI2veuBE+Xg(C-7an5Esb@sMa$}jc(fv3jV*AfHH!MN zJ!YFu78~;ZrYE0eJjJaaA|;QQ**|n1c;nClIQ;h P00960T@~jA0L%dZqm{OC diff --git a/experimental/helm/charts/pipelines/charts/mysql-14.0.3.tgz b/experimental/helm/charts/pipelines/charts/mysql-14.0.3.tgz deleted file mode 100644 index eb620aa50b5533466526048c7a1649582ce4de89..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69215 zcmYhCV~{36v$n^!ZQHhO+qP|cc5K@_wr6&1?bx>YoIUS%-iWV$bw+1pcSm$p-B)EL zVJsBTe;VMb_u2)Q`!^L=|1%obL$qmC2H`I|&(|}&xn3LT?UK!(`D~ll=?}L|atx`} zO0Cf<8kv(gii5tv@BSv-ro^T>R|6wM<3~^-;AdfsXJc7B&FPj=8X{mal;~+O3lkPa z0FFnXXT#j=>|@@oK@SoyFK_p5?|Lg>C2`%q>G7-kbj~BccO8|tyL9-VlESQ*Jdt%AjfhDgM0L=Amf4^LQ1twH*0jlsu zl5fc$c}v4qP64Csyo`ObfK#uSN5PzrJ4fG(vUBy^H%}4pnmIuGE=0ZbiC=m)Iy3DwFxwazii`p`{Rw?z9vkY6Ho) zjA;2}JbGaH_GMBu3E~p*0Y`z3uzGuNj4_i~=bA5}F%?Iw2NOGhf#OB}9Lx=0HKcgI zeeMc&cU{2^d7U7pa82?gPj?w)%dB4*tspX}l$SX$Vc#3t`)u!QZoap?UU^&M#B(ql z;lUHL$;v(0vx>c?7{ZAdkqUL%Gb2ct?(6#i7t%w8+VwHthfswAhQW?lzePbv!}Hrm zq}k4*LYdHR3oX)PJW-;hKJbN(>7XB#<3sHDc*pl~LexsMW*97?0y`?FZHV$GlEn{-yYlSvayi*GdY9CjD z?Uh?08v9JMR3P+++M*e_tyBqq)g z5mY1{Ga4ZiP+|+5*?rcpK(4U(2njl)0HNBb6g876rXNbhk&T)_L7(=fh1*x-6uCns zhoCstU1V$%5n+SAmvDMTsPUrOL(a6^Z=nP9L zreHX83+bSzqwxEv)3U}UPh&KeEDB+ocHJ-vG%GU}s35yi!n;Cf&Im0?xI{>RA>b77 z2>9*?$mYr^X*?Cn^+X5z1OJ7T0<(;sCKAe(9SV zqiVIlyc@F#bV#g2-GcBcoS`6<-BGIoXrEfEn0tGr9{a$ys>Z*i?Zf)k;rcQG{Z zc%I^bqF|F}v9}FQ?jqjY;(8*tK4patJ9u|ZqE6#O{gVH%_Z_qkB@4ABoA6RG8XPc* zQpv9PhEI|X2q|bucOtNyzvTWU4`OEq)GHM|yr&f@3{p885-oOwy){DAq|m5MKUcm! z&z3(c!iZdM3gd}yHJ?@Fnkx(Z+mobed_LZdLwvH(gA~hHfKaj)h{*84RRhV&Z;1_2 zI3PsfU}T7=j6$OE&9brnq_L2aISEvR+*C&5fO|Qbl1(f_0?L)ZC0HA3t0jscg#QI7 zUj)qvT=wwjog`a+wg?r2-x`j{QAWPq8El!ZlDjX8^vqnr0PrjbV7xgE{GJ2gcFUL+ zGY{j0sR$55pGijCqiw(j$q30OQx|qZ=2&UykTC@W`J>0uQZpvy?9;rLc33;N>C7Yj zSr7<_6}>`dt0>2A2NByzzl!5&eB)vC;Nhij>Rx)scnitaT_=_3>9BF4YmZM52bUA`Qi3-<)uc^dQVYUmP1C|9co8^x-K^_}WMX3}_`ZNEoly-J6iIv)_A8g|s ztg9unK4xQYz z*xsm1-hqX* zr&dLYo(~EweI&BLa)4WiG2MD+C)^wIsL%dlR^{&*vUDIAGznwkgBT^gBQCe|f%u)` zkg0&Yk@uH-1Qvt_uq=_I%9Y1U{S0}xl3@ZhcjxtTn&R^=eHRAFe+ zM--Z->o*v2*@Ntfkm?c9rW$V2K?ShB77VFbx)!{_gCOeFHThD1=kst@>NW^d=*?(h z3cJlh=n%bi);tw|;8jQ$xC=?5Nkq)+7;N^ughTDK!&I+6&Y4Vv=pkiCn25$1QgxCI z5351N9@q&MDj|w2=o2ur)^v6e_Ky}O93$-@F?NdP0-t8Oe6Ldfp@>k85fl_Q?cgK~ z%D%bDbFSNxa~sN> z2t|YIzM!mkQ$KJP{Y95n4qF5?v~#VDvjcwU!a#P--&?tKlMEtI&EbGLiYK zHdijZ_N?n|txFFS3A{RRBmC*CYIn_KSf#TD^<0osBxH#wu5C7Xb2B8@v{3li`!e^R zQFQ`#*1va+S-~b!W5|6#)^-LK+@ane1<>9r5yfesEo^+|>Ctv2N=bM#$7~jwBtD^! zeRDNZfsxM}ddcn*0_Gv{t79)R%E}u1VrphfzQ^!b_V1kFyKSw3ej(~G#B!J74BFS~-%%i+M(bJnsL1E5g>phRlAOY*&$SZ9 z&Z%pg@S&fwfB&+BT|r!eTYS@R3#To%=Y%Q|`+;g!7Ajt$LV~6@YX!Z;=4nb&Cv%V2 z{(DphUJTW>Yr5!-FG&^qB<}ZvxmVO92oAr;;~MwDaHpG8ae$$%g%sYyoaLU*1Ak%u zs#4c+NR^k&a_W3sjqw@D^x;+Pc|yuf_6W{yvzsnEe1R%w{P-A0*6%nS=r?-)J2dfqugd`Jdgruf1VTN<~pP8 zrqyungkqjE2Xs685{NLinm-FlvT({YtxP>aE-;E|+p9UYK7^9CtT~kg@=SRT8$I~k zdW<$UH`1=NVQG4y8fu@d@{ zTeV%A6o)srR`Vd02sAK#>txvckaxt0VM~5V3x0Cp}lXca<71zyeqq z)coh4YB)5&-A;ys{KZ|Tz2#{tyT|-1Ksc;CI;|b_BGD)j=^Y_Cp3Lqb_`v;~uim0W7#GxjL6qFiKHiM75X=*W+Qo{kW3@0MiuFS4K|pz| z9B3D3HukX)6IzI-Wj&{r^PlU&s>3I=R{!22D`y1wlIVqE8_5Y(X{|_tJs?sJpF@tN z+tR}TSirk&C7S(LGwX+Ly-JSqetvnQ(HUYUPZIHo zkNylc^Mz&b4h2WtvYF6UyG=-lOG1+viu$lCoJvg%a9HMuAs)dWS|q}N0r*I*g$*kf zK-8^(OLq{+HPy?JVNfg(<^3Y_cZB@jFywuM?je`>4oUmyRK@`=`|EDhZosFf9hZuM zPGjD<$>VZyocy!+IEDX{MS+3C&*iQm0Gn%T+|zM4-=B~F>^Z>y^Tw+G8&N-7*B(*a zeIijXzlUe+VDv?n{`(Tc4il&h;m=81H8#rs_X@*I?yPS=k3qL{V_j zB593^5&XZQ-z}82KRJ<5>2OU7vnQ1tfb5aaBigViyil%{e`d8y7qx<$n#%1T#|Q{b zKdD46XGW;n5|GQRK-tQicF&HEs&q{`xn8V=I{b>I6%cUKQH_461ATd5hnKKQL*E7L zST%R}${hb7Le*aDL@~n4`N@qd>`(QrZ~MOFT7pI3gc*e7q)DLd;cVC+ELfNwb~l2b zJf7Sjykt*L@pTlTGA~vkBG3p&h6m#uY}DkN@+$jbhW{02DflZndJ)InAT>#F>DbeX zUI$rcgF2M9+sP?-|XOQn-LP6BR7Sf+E8@VGEtX$g*sc9Q0Q$GM69Z zA_{c8@p;bOcCxYxXOtey7V(#yebL*cNC*@ZK)UWx*@oj513iMIhhDp$?1LBv-5I?VQnp09ZPFS_TuQPEbB&O*$ zpB0D0eS2Kto+~$W4w@a4we5C3s`3uO8g!0(5`nn(uKGBs0B;A=7S4x>Unp{3HphqcT>sLf;|)F>hggB09R@#l+rPp$d2P_~s{Ov9CJ68UDIgf4h+b$6Je5(@wx zUr)#^KOf;?k+(I*Q@#_p=n+UVK2t+PW5OS_xCTMDOMUU6K)|O zxQJoT4$j+-s~H}j5CD#s-1TpkSF7+au`c6prTorYTs{bg4^Jp=x&jlC4_lREDb)VfHaYYUm~e){UjnCpbGu^6DX%1!zeqsx^UmsqJwO)WOcpIHy?M<$wK4_K z%c}G?zjDUmyvER`%o0B9nd9E55-yQ$C3cgf*%q9=aHgcpM}N6q64b z(k@M;w;;K!l{%E^`G@hI`*JwJJd_{YZ!4?Drtv|Pr``5P?@v&=I-vyf z%^p~8wTu-=VFpgj0k$5KCouEzD2C4A`d7GRa|acCL75Ktd09ypkNlB2(ws_nKO0k9 zV@Y#?Cvc@JvV5PGfmD6yBoFlriIHufB$YW>tE2m0trg_O0c(+`WQzh044`+x0dRn@#}}1SqRfT&?Dz&Mvgj`99-OW*=5wEr6I{L$Y&MYiaQ*u zpS))XG;lgLcbG2_by#p3r}CgQL8Y|##jcUQsxoL>_|hW5rf zfflQd0uP`b z9)@~-y1uW^K!+hDqY^oQt1Dq3klzK<*Vml9ts0>7Ju$GddB!l=vU6RD$^pA3)B~M@ zkt+QG5Q0S>%aVP*jFzMc(-mFfO8Z)+&K_x&Bd9@%$Qa4QT)zvkp}j#LduLfHPzM=u zg9%Yd9bSgAL z1IB;1XN&bHI9vVeB@w#-Y5xuJ?LB`TbhLT^-y-=YT)5lAS|3AMgw#1&}p z$A5Yja+mIOZGAvwidd9H>%HSm zD8Fl}wfotFXRuH`wtyiE>+76Pc70;=+}*sXN>Ai?Xw9Y4+f{`RFNaVx3C2Avvcm(} z*Sbfabe`rxstAI3@+5R+LLXHs1ae3-MgiOzM6g6ZPIQ#+O8Neovq$h(FulIGH*?0T zMjxn%yUMqdir8oMI(~qxabg2SKZ*Thxl{UULuGF(@Ru)OGfkM=^Afsa&DHWk(T3XA zi%n!nTPxK@{#l<%s%XF?t!#j8v&sJ##Xa2lEGnUbYqpOf+e#d)F=0`|b}rdw3|aSP z3|F`SD8gazaapYZ39E)=iyYZcZcpj6=OKzBGO}xLN%01GhRWlt>n@^b+yK!#ym^Fc z)vS)+@p;5f6W-_&v!n=`RY(#3ReUL$3BB0@f`)-7y$64V66ZR%fSEKOC__eqDZ)Mx zjAglYwJo-G)LGqIKi=VX(d(K6u+6=vZ+41lg1>lGgfw&F2X)+B@BD#2F7Da^e7;TH zm!9cBwL(Rs&vS!f$*E2cB`@d~IO|H^1gV48?Pc?1ifNWbwP11DDzV0of_U^(#$20D zp(L7IF$W0h;KOrg3Z#XlYqMe?x+P3v$c9MLOU8hc<>Yko@B3tgCkV3)-tmof+>Q=jfz-`sp(XWv1^|e&I~ic`BQHHz^n&TOyp$+b5+^*TevmQXO@+l+BOdZK%fD3&=0lVN zhT=oKUPd14Z70)c$I=rZv&}1mM=a-DSQhv(JeO70V*OO_)R%{H&TQG}q4fe69ldIo zQR%`&tjS?6ByO=@5-i#g6VuoRm^~Z^a(o?p)qT>941HRXc0^Pk`3W zRmmRXgnKtjHNWNl744)i_gjusrx^?|rW0JGvoK%*QTd&Sq}DRDTKl&Gsw!MB-+a}n zVeY*=zFh#u$1%;c`x6~3UhLi3#3eHbAtD};50{abjM9V2A@$(y8M$GdB&U70SQBQQ z@u{=i5y!47&|{a&qSd-zBM@UR9zi-L&-QQoSL*SJqWmt}I9CiZz`x?FNP${-*I~ge zrSCjSv?Vv(2Ga-+@2IKFU)6v)Ph^Xl!1zE=NWo5mO}Wp6HZQN;9j!M8ctDkYz@5b3 zO-WQhc8q)3%!MW>;OmAl=LR4v|wR% z%(C~0EQt9EuRiP3_u+hK=pxTOL-xPb38HHWuUV0(3OF{EkSTF z9c}(z*$cLpJcA>*?>(3^=70?rqQTVgL_hRl(w~_zxX5Cli>j0-dd}RYVC$Ztnlyfy zyNkMTwS%#wnwae}CuC#oNy2`V`;*E%fJ%m+b96IuhAq{ltE#-M+Obi%iahv1}{DY39x&Tf0p*yv1 z(z{6_Cn9~TP1hlNauwo7Cj&X=B4xm;*EbNGrxFk>`}MsG@{EJfF7o;MrtkfA->Lha z^Vi7xyc}x`M$8-$Pdg0=FLUk3?6_{y;&g^zk1FcOQoHTn!%>pI!>7c8Oyw#2Ib5WQ zp?~s+hrDFdZIbfTk2ow#Yy@GN-Lz>~4C#E*Pn(sSlmrY4C25FGmpn7xeS?g`F*o4m)p5C)OslIzt(zGTE-(wsWflm}S4 zEI&1w75Lfn9)O9_Jz+#{Fm}|!%m#%kJQVmpWZco>F!J)BT~d6MpEQ_CJ8XI<5g>Fu z=J#e)RYPZ#UDa`7BHUhet`r zDw?%@i_AQDQ45?7CZHEkJn_-j>DO!P^q+aZBw;pKaTvso7N>PL*xKkym4_lQF5vqU6oCaZ(zAn zw{*^}HE<=HpZ%&fIF*{|cliU3S!v+PiOV>hoxeuNoYdK1(CGfgrCTMb^i^nq8U^4AJO$MLw1$>z#eh$RS8@rEe=}AN=Cu1UkIy(eA zLtLK{5X+CIP6K6$JrqJ66{11f7N%)@PnU=Z2xYBIZ&XRGXBMUI&g=q@dkwsCvHsnm z?2_#$R-?XgWa*WD9_nC1++Z{WJZI6+CO^&Ty8La|X3hrS|K!RfR)$ zel4VDENj*sBQ8x36(OMwcvjnd7NsZ4Ku#J`g;7{9S_?c?Y|qukZE7iC;$N;9gPj_)_yj^ zR6jPqWuRzLQK2{-PN*r8R(z~KY}CsB_PLN0`5-lrpP)ml#L&B`@N;i(Ei(B3^+sgX zYzfQDwJIWWse53hKf5f>6@fdD8+BHlcA}n2o(&AcT7hDuBnT~13S&)oaixWpSoJ`~ z(Mq5G^S52?lnK#8_L1l}RVAWqcXJ$s#yM?mVW<-tC6R-|2gJx+rlndx$5`J=((2w| z$=r@`V0hn4NI&}bLt_KA#d&YL!$sfCH6NgFx3Qrw=Jbk}DB&rb`gM10+tG9E+rmFw zU)kVWh@0?(JIeD7qXgC*W#pO!HY&V9w*a=}{lmZVFW^BL<~U&FV$*VF;eUQ2^z z^|^|Gl#wuPw|zOW!!iK1n5<|ItUP{^80jsNy=5_5tEj()e%bSK-!rKhz-Xvgz1~J z|HO+TA*5?2<_{6v-9b+Pc^plpWT!2Y3AAz0lf2N450v4Vb)$}Ubh-I!g~|j^rA$LP z=(1uiknK!91l#FdVE}n5%d%(++}g3*#SY%+EhP{?&SLWUHsW=D!G$Yy#!0Ho?4Qg^ z(>8LYRzAx(_;Asj#aZLsvv;a|h)maqqYC`v#vZV0>08EW|Kr>?S9=0Ai-K#vO~xMm zh)%_C8Y7Hlo@xKC_GG^sNn?kK+rC!jac*8;Y#b(uM}(cYrG_wbQT2@)RrT20CFgLy zy414F8|0&%H2D$37hzJI-M?d|*N?imuBk{3XeJ$C7s3ItFqYuche+`g+j|Kpq5^Fd z!DM=~%u8;23M+Hzv$R+i_GwMbT8&xLcCtQm31Pm4z zH;WNgzL|3~OyjDC)fU2aqq$u$BBTw}3;aD5(f+e=rr?!0EAG71lt6vzjdz#1r+A;yPjF=&`aliR)~QJNlsRUV`?oQ5OB1DzFP^jU71DF%`Utl?|zP z>V;f$4dbb9l^JGYPAxK3Yg*M9BPX?06ceU5K;JbB!{Zygylf7il?8%|w)66~1NQBY zH#8spiz`L7GRiR@b_I4S1`(8FJJtGNiZ?3SmwvGaY1-aq_SI}QQ{(C<3`qjm8q=0d zW{=ny*2X*TM+|D++1|VgHfvY;z}@=6T1QS-5!~CywsN?9B%?v~Dm!#+lb+eoaemET z%|bkHj0JR`a`+__$19}lp?FLr^GT#l9?70fOb@Q8VWKXkev*LpHCJR|YO$YO@onG> zMn8({j&1QMWqlU%wR_)Ptdx^3vv2D{X3_!JqLn$C5sk53BmFLqMZjpVGI36my%&lM zp>u3j)XV&RHsj``=K^uVUCD9?~bTF$Qw$H^TD$!Hi0>B7%;z0*&k| zFk!j1peBj)pRMv$6nzs>3|+Y}NvN-Q9wNo|dHmZ9#TQvZ;pLK@A9ADkp6#f zZ0~egWa>(|(!>OSx2`j2zvW)}-w9JKF1ES&~mCb8PufaH{H7d@s8S z<@n)WG$38$mshiU-FubVX$ovh+g5MtF2lOjt?QHj>U6i)u*WFLX0jG;7Q*(3*kKe$07$tM$(oVH}vFi-!4R48O-ImdG&%bsnBQ^k%S);#=5> zzphrxDp6@1=`Au>Oq9JqQ8!2%P{6uIHprUS6fyisE@=#9EE{7yXr-aL2R3yP&$dn9 z>Vt8mNqf&OlxG{?qOR0a<)%uj4V`3YB!I9hyTFkiO9fF&`Rj=Dsf$Qbvj0$Vjv$;B zE^~ZQ#8NOkEi$rpRAHSf;IqHxo$d8J>=H~)St6`Ts_J0=`y#y^)V(tWf%#qxpoTfO zlkx~KJ_An)kj#Yx_kM4svhB>3YkpV03+Fe_5o6rQT*6+ll7cPltNQcTAT$#O!P4m? zsDJ|572{-aKT1ZLa#?ls>}t?lYBa^nIS_tP3gv63n4)LIaq`FGiRT6LlbB%~ z4-^ILd0DlV6#W9ayA``by-xU9 z)kt!$WD?hJVKQmbX=8`CodMN9-K(Xkhv#|;y;DGqV%kT_w<;nzdn0=^?a}eP4+yW8 zp^Pr_Pm7R;#CIGS_y z!j^z$&3z;jG#SC^m;$=Rb$O8Y9Y;|43-=hF`JQ7?Uyf0NF=r>izA+y>QzQa1MbiU} zwzK^a5!~j^%bC4gJW~clA0AtKDeE%G30j=Hhym*K{WDhlFHPJ^27!PvJEMzq_(_Xi z2t=bZrriB&Ny>`vFMITp@_5u)#*wYAEp+EzzC%ColXS9N!>CU8PP;=N?=kOq?rF8Z zcg3ONnI&+Yi3E=EwQ}s4x?-L)Os^dc{93@Sp)7*6kvfUopb`TBg8OEW6DE*XNXDn) zL6s+T=2MN)GYiWKaQ7O(B+YN67g2AG0lYIH7U2CWy20AjQ1K1K_&T!zX(TQKoangX z*_5g0KV3zozxuvCkWUm8GGInyHEO_5ej=W+^U=wSeJ7ryqIQg7hHZA;=7~|_`V`ab z$&y7VIf*`_VzuMdwc1{)djb2q2im}{w7Jh~dH0Uu&Md^N19ake2jA+I!Ov+PtEaih zm*xKKe6J>ztNdIS8ZD!m`&QMye2Sp9U1NB3y=Y_p^LvY+|E9>H8+%B>xmo6CiXDQw zptg|#(VB{NKU7HezM@Z~b@lp-}`00?z`S%l zX@sMqS)RzMW*_B$%km^x7BQl5--rAueGM!;<&yuBL2})xFeGVUz5)7ovBJ!Gf6}2V z$7dL$y`v{t$!AWsXLJNx>3GU4#RhAE`MryY04y%|Fw#wIY8BGC9w?6amrgu0zOy(! zmvNMsQpBAf!Gm91l9^E*d$0q00fa9^&JR7T=uZ!9Pr3m@DPSOXEQL~W`Hs6j z32BdLc;Eihz>wS`F2L z(#G#JmYwnCm=PlNu6LUERmnD`R#nUR=5;_+rEaOZCBDCX~*dZ%^0KcbQ=NYyY` z)k=`4QHKuA5OW}!m3V=NTr&tmK2T&W8UVX6_nl{9J8Npw&72Jxst zgz%%JvUK$Y;MGmyfjotAN-Xymx9d%N3{Zj;U>dcTe!mF%z?Jyp?<}=6L;>hUXNvZi zEgs4EDx<#GW0<}e9SZbgyC}C+K41Jg{b`ey!Jz61{Tpb@8XHqa_7e*|$-9TGZ{al2 zYAYtEu%4>3>RoE@!xf1{A4A^r^a2R?7E%O`1-o5upx^t2@784@w)i3Qd3kB4Y1|Nm zUE6&p)@1!eKe@iwvgkB>8bnfOzB?;*FgJNG={*KrD?f!!pCW?sk@K*bkw5FQ8jEo^ zFaN$1>DzpehO2gvP{_Ig9i_pYajPBt)9^|b(_7Zt2s`lTM{59|c~mlfK;lU=xF5Fj zO=1VDvw}!M)ZA`Z}IkD)F{v955p&x?J+XWf+PS!t3P`)-%ux_Eu{V%P)e8kDBJ%)PG}b9CcZ!tl+!_;-cYV z4AGr!aKrVnu3}>1d*}8C`IkD&5X?_|I#dtnS8sHQr?Vbc$icI2?C*-I`NlAdP>#kq zj-O3|_QOCl&VX#NjK-9NBW4=&Ny!=~{V&y?9C7sjV25bmQbiR&hy4d)XPFmqhzQo~ z?|llF;h6bX9b_*$VJd(F(V+LQTiC$K=dBTv;@uB!>hYE6KE#`9+WkY06QK4hM;&lbia*7 zG9?W)K`aFG5QycTIb$c{(fxDAvrOjX6fBM#dO0tYg|yroq!IO8J)@6Q#-qetsQ{0L zR7ba_h$cYY68;0aV#)L6LBj%meQdpxK4q%Gnqjg*WWNMwr-s2%mqL{4ztwLv+x^Yf zt-$u#z5X(CK;k}w;Rk2?LdUb}vdj3c>u2$a=Zn2-v-QSijeTU-<%3p_ch?M*(I!YIW&Y3>AJ?l&ZV z==K8YQ4f~?4R`muHL35qjlea;;F`yOo{rZ)-40lK{@)xbfwr%^?jwf(Sjj0sEQ`fp zqdKrt0oj2%Fo6TV@cCO$HzuiSzV-E zp&F*&SF}XZA-!TBA$(8x4^j-$d;+xB63)MjiT7mSkZ<%6&YK0`A5VdP5=y3CXBup~ zF(8xgevOnAf)C`m?4>Q3Cyn4eIG>XPUr=cg587I4R-vkZA74+1H@GP0+vcLKZpFLRB_+I$GC!X@pv#ZwTh}V@=Y& zOE$=(l#4vyVN(sCPt@<*z?Z{G`0(RYC9QU*kX5o-&eS9 z&=Y9P0x8t(UzT>KDAzJx}xo!R`_r zAUj_0w@h!s_vlWl{M~pV*kD&X@Xn*?&!Ls}J3BPXI|fr&{!#3z<6S%M$u3ocA&~nm zXwNC+-P5<>>GdDWb-SPiZ@40fpwVRbn_0b7fkqCcCCwD-c*w(Hc&kk zDX)!#%m}I0x5e?)S!CkpcDSJs}GFQ{UrD#2ar2dk!)P=6+%Wd>+V+(Js>; zy1kkI(mu4quX_z3m>9*23 zT;PSy97Xgv{1%DEr01%O%)|J%Ue4u=qQd9_M?v= z^t;y%M)VE`m%yMg!joisuW)hFFJ+m-arvVP zJFu@?X93!M#KY9UI}t`4I&l;1KnL>#@b)%irHzk7EHV_l~~B+mBq&!(=N*+JlDu4cy_@8$lJ$WBP6oY1KVbADoR_ z_8%kWa;A?^h&AoKcSv-P+a@YlW_sqk`8FDk-I3(2kzDee$d=RGT!)>_#@j1kKI8t+ z@b>t-aRmQwnns52tgoiO2P!~rw{j9tv!BD z3OpgA+17^HL#7@K*AP{Hflk~i{ww)nAsudcGd@Y2&qLWixE;^Z?9Kw9?>VA~L1q{r zk+6KIC&f4!4lm|TQ;-z<9(AiiA1I_2^~>s)d;D@;bmoKgPyQ2)DR#0L0qL8tJPiQ9 zKlT40f50(Cx;ypng8re{!ShUsK9yyj!o7k@BTNV*j(Z0NzvF{Ax7_C|$pHKcE;OQ{ zT(S!w^;m!lYy!kEu;|2az&h;zjG1#1^DkB;0POj^AH;e-_xF_KdPj1CMv_Z$aVKeA zz|SOE@W}?B$|rAHy%G;v_NeU}9-`2v7=a4Uqdi(ff$?7%H<03f5)UHw?~hd~@A_<LbX+>ImvT6-)4UMmFHrDZlNqNSvzMaczb<$l0-da%>SxwH*|6*_9MyJe~s=A-2 z8>)un&Tz1Qhi+kUvldJXBDpW2ev~__bEI$EqER|sC0rfTe5)B!1!^$qYs9xIw%xUpA+#~ zejtlI9nc!y!(nt|V^E%pyf+EDFE|4AtFvowKClu^`*jPVYDpZ*d=H`uK=)R9s*yTb z#ExdD5^b2H^7#8Qac`4IU2<;a1gNQahrPbmjWBGIguLQ1==?+qe_21g`HV&kYOR2tmOWWu-BWPf$zTZLMOwPLRk{vW z<+pqA!RHyi48BI29b@a;tAF1Kns0NXOn0pgpOmfn<7e9UTG5$Sov-5Pi)_&OfGIMj z&+jQzRkMYlBVwa}%ov`?>wOcXf~k-FAtNraX+EFp!T_G0`|nIP=bZaH%)-j?a^Q=v zSnNN;V1b7{?bkl%av7Ubi8A@3<|~Uz5RcBse7pt!bIl1~D@c`SIuoKaYy*_OiA<}5 zF@Cbpq)xX2dUFKg-Yrqj!%4c&23}QAwQ$=&LsGI&u)t?JE>F(g1Z?yL=(Pn4wofA6 zi}B#n)`MaA@`f9ZjGXgTxL7lJMI1=k??8x!%M$kyySpuzs$rY!Wsn_H$0wQqomPAJ zHBnRh`tlvRSkAJ$`njyQ{1CLUAx--~lh|Hvb1{&CHRUf3zb1UYkv6xa{7d|HWtV(8 zht4pvje1jOkyNnD3PtroUiQp)L|Z-^;+E%H)6eM%E~3xs#_vZ zXLm*~XbROo{-W+NYGyH=H;kl@KAIHgSavb4 z$sRMCS6BOGldg0)6`1@)NC?<~Z|^>Uj9Z0{P?8g2IFn68gDW;bx2-P*f1t_c-^4;q zwul{xJIl(22!jAhrE2bEKL+gG9}AZBX4-iJu#SivLOTqG>WE?5{5)u2ksq zoj<$;|Cb*C#zN(e4i0H&5(i<`_QV6hFK6B#0xh~lNBD7Y*-VEaaBes)4;s+wtS@W% zX$9b*TptE%h+o8LI#HKm-010h*4D=?qu(B@NVzjC>UyTuhb)f*yRZ*PztM!BIgjWe zDHqP+2+Sm^jyY%JhXGS@#JAI6RQYkkA9Cxjg)@bX3*8tqAs%ZFrgVApWZ`j5RA~fN zrQZ|AySSB9yg&30YrVP$YKUA6#*FBQTpadUmfovItHpq;LGLS?vp`dQEBKUbff$zj zR10v#13qzfHQ$jl_&1a39%^|aIgr@zll3(;R|@KcS;- zm$-(&a54bjN-1@DAJb;t%TLTa6@_r zB6f-50o)dOeP2%#9T5x&U-J@U7dYdWh+?6GYYG)LKUqNgHY4XfKiWK9UNndsaVxIf z=!P5nvHSyoU{!Q4vr7_diY`K9UnX;6-L9^C-#m67dG4Z?ln#g?ug)s{AXXJt52ib` zW%a;~tx{D&QoKtT!~0^$q(2Txp5!f!t^CBZOAu@75~rZ{PR-sZX8%0kqVp$b6eZ5B zP<4wa;sd|m?}<>@4Eh(aqGe)nJ~hKpGF*Gd?Mw|*DtEMdCsXf0)=YFo&U z*K3eEVSha67+#;K6Kb=1)BmfM+LFVsvUUGXOAUvKW^nE9`5yprK#sp8{|wqnlQHr- zpD>x;ECC5w*)gmWgpoF0LU0> zg}RC^Q-w!MQCZ$b36JZfuh3JWw2=3N-`4A`Uxi0UV_Dut0gvmYxTrBM&|Y+zG9De( zWqBI~Jg%1pL(f7rMwh9^qtRiOx3T^@&O&`gm$C79qnrVpG*yAHub=P<27et)q2Fc^cXpp?Fyvn;!rV5WMU2D7W8r!u%=F0XS`+Dys_3}3A(X=kN zqu+R4uXN4hX~i*uxm0B>Tpm}}uPJEoAQ`2YYYXG#mQ)sOf5ccuN(Rj#jOc~RF^p_A z{%Ft(0=X4{_h?YNvfV7lqq%EqN%fWS*n5PRc&&`bH40rU4avAzRDQX+W5q%|Ho;&W z9@o$7VrBm51Q|9v%{FegZh+!iJuB-oE*{Y#83^M@hTfdnS_Pf53m?*tkLA=QhD?L& zf?Bjc?iDV`D3z(l<5VUArO&Cxpl;)oimfFA)pK3GXQW)F9*=0?h-cYiL|%(CF1E&q zyr%0?mz1f;Pn=~;QdF5t1rhP`ONUKiBjx?X9YDd4fmhF!T|uL}azv+{mj zz+;mQyL!J~FGSv!KQ`I0>-Ot)!sN>Pbpek}Htf3ndd={-8a*oW>+3i|huVhw^&#I^ z-E-=Scgh+;NKFTcH$}^j_PF3Ywz&oP?G$fZiy!Do!^(aD1w5L&ZWjQ^{U6b_$J9WN zv#W7{m+7jj$1NOdix2zVp63^~KhWo<=QA3Qe~Hg2L&YKxe7SXG-C|EUDbQrg{>aHl z<~O`&G2OE6r6v^!)x^X#hBm)!^1iC)ws@qZ(U;y5JlgcAYX~n>S#Vj_*W`LO~9+flS zZSk1OP!8aaMIf$=0*|h=MmdmHKA$PzaU4f+vLvTe&L6GPd2A68rtjNK5;!DhJb5(F zZRN+XL0T^Iv-iza#tM&=T*iY%Oqq$#c@wf6FM=q|yb$k00g-18!lZTyB0fMAZF2jm z-Vm?ALqkq<{5hZ7(qj|%l+wCh4_6fDSs9PHF#K*7HnrojUYH8cCOD($xghcMKaJ^- zI{>YwB4U|cGL+pE&%o=+lF=Ecs!WVl}p%cXJbRlbJ7FlHW%dbtE>9$c1m|!&0mtQ*N2w*6R_?j zBLYVW?Zz^kXd$J~1C^ghJ!x%h^!t72Wg3Ta!!9Q#^LC;b-+yih!L50-Y}~(<75*&0Z!}PIZ8ykNRZ{keui$BLR5lI%! z(%Cs^QrRry;P`0>EEz4&NI1pR3AD!*;P+YP@_1Hd#1<@)+y00c7>qGhhl!j<5P5n8 z7kxU=UthWM15a%Wp_%@oZOrZ0OAWle+u=a%P7qgj_NAg0>QvhA*b4@o>#z>YOlrk4 z_7jTN^HdgTb0<8Kf;(M63^_jswKd3;@e}0@GjX`VLdWdM#D2ikygfVK7jFrH=qt#G z3sRMJS^6!62){x06M(=pz$0nA$A&Mp!!Dz~91Mu+pOSYY9wy{hucR%QZdd^xTDxV! z;p-^)s1d+-UJ^KS9dTyEP95krP-QvFlS8-gd>0lNOu@XIBQh0Rr|;8kNBrlpGHtXO z5Dd13eZ%=K5AA#wgjpNt#q%e?>0$Ogx|=79W9U#Y6yBv5gvjLpw+CT{gT(^jyaki- z=Y!X;C`O34TfWRDP`NJI*E1Jvaz3iIW!RxE1GI=;jRVS^NlRRF5*o@4*g|`#TJFrf zN%(bT-*#pHL45PgSup+P8%7GwohBunP5*<1@WmSs0j=9~x)zvi%)uqjl6Zz2>=;-F z(8b2aj{$VKv>gK9nPDdND;&KqK?o#-D<5Q;B&gYV;kCKs$@OcJ($=X`32o2WbkU+wRcFewmIEJd7N1t});c zyU#m2ogHMP0ZJNwfmuJFO{R$KR9&#s|9$6qpLb>?3-qbxtv?UbQj=tWz?PN&UD5Zpq^-mUdc}0y=P6j_w0_l_pEXE_Bwl}dzZ*6 zW4xmvBVTB#Gf=_@;7^M4(&43!f1J(|SaCpn2n-Yg&gKi86XlEkD2&g&u%kIhut0tavOmtw1(86L6~eUAeYW%ciyfqxlMIM8lDG|F zSOOVvG8VW>!cX54%&x{V5(a9)?UMrN$8=*veEa3GckKPC)B0eX*$Jb*~WX=45=+@yV)eMjF|FekG z1I0)VXCa1j#OHqjnoLocB^x8!W%}k2T8v@Yg}B(%=axR01{Ojg9#I^%i-mx9WT0vB zz(Hue@|4tSmeSFi9_XT=@e}L9uF}FAc+_fS$d?!jkUoD1@~q^W!DKSah~0tY7W(NX zu|Q-59&BVVO+hY)){WUiF%Nkn$fQ6&I3;YOI(~J0&^xBtwP$LZ7aFB%284?fkTa9P zILH7Rx<9=Xn=mnv?}c$BH#aV3(D5wPy}(Er2Xea&L^*(Az9f@f4Mgmi;{8iOwpEN? z3G}jhCMK;^lh*hdGF15q_6?}qw;4>Im5CW_GazVN>Q9TAZX~Z16)LeAl_7Z9Qu&L# zu$8ktrcv^FKsHU6|9Ba*UJ8xEPPTo#FOh7vyjj{}cTe&Re&FjaEav;$))NI!`fYBg zP0CK&YwHf^_t1fDj%1>}kQQDp0a$Boce!x!y^(!dtZf9*Ae{LW@qCH$m-vv#NsNvR zz@0$%Nr2d)kBjAukXF5tKfh;j2(X&Azi&Tp?~uUpMCIqBH*F0V2GCaRZJQfV8I$@( z?-mVR@lRI=Qror?pby}0L@t9p4QHbu>Wi_5$Fx8%)C@DNp?3)&VSHTQb1QOQJH-bE z^Z$YyGVhx>xcr`S6YB>OLOz2#)iY>c ze!wgOsnCdB+NcfupslVrP#Ka~&VahU7$)&V^mkaoq9j68xF*eo3jH(E7?&-?I+mbg zKw@8;(opJ{%qAiY{w9eNVxIx~h`9_CDWIIcO|> zD|XQHy{S@9XoiqY<5mmOU)u;ww;u(~dT zc$Q)msl_3FUm`Cpjlt$g<`>Nc6gXmMMp#86_k17fVlr%$gr$f3LP}x{kTcs+@s8Lq z7)AP~SL{$C`A%N4oup*cQmj%*Kw5va1SjC&!y)VwdBiKxcZys2ER+e$2DqDZ@9EOn zzS?s-g<>QsnSG`36{+J)<@7`a*jS+GWEha`BiFT9$YwMQ=ymgkh1h;xoT`P@-C{O?IuDG}No^72Vi%YNpp_G$@_S>jgx8kg z3HpX^`#yT(NZ`25k($#HzslS`1l6u$60voHf=2F)j;q3Qf4$O?wAI#HO$JEajwz3CL#Zb62`@!~Ga{?le-Rn{|P*Sxm?&`Byj zPCRrT`beYxn1(>&A00v`{mz_3s7i_mbuCR(A5ln!ZlV)n%kDc&=P>wWZd&Iy`#6d| zHJPvPLP?WUg^HTYAT8WSrf4j}Gm8$5>CiGgSxtL{wKy4h5y*YMWr(5;(lV4bC82zIzfB@0!_PIqx(dIX!DD07Dy~3EII#Jme_uN z3XSIZKAt0e2!rzkm1b(HrW;v2g>}|dML#M_=@TroO&lK4jV{*FV6IKD_VKkkCNSFc zEF&>miN5D3vY@%+1JdQqPBC~yPt?yg|7@!c^C`f#wpg}cwOzo1=V&jOcGSUWIWfcc zt@AuHt|0f^Wg`MgOemt~_HdB?Q%olk4V2XRgEzRi@=zjlvmH;X82L@X=#Z_G2E7M#Vh$Hc z8Sp_&i}&?{MTnj_hq2_h=ILYKmJ$)uj1Gn}9S>snR(UAUsir=<{lJPnsHqLkA^;g^ z8!%P)EP^?~JE>2^PTwCI)=pt(q)rkOV`Rip9V*V$kXz<>E#JZQ(JzQfT2MX=yH=>s$b;E>Ko%0xl}O1@jG>o_hV&KP!1+RG>8-Ev0;xUzmqmcppk+R#AAF$ z*kvfVHkAX2K`xb2BLL2q%5aDT>Luo!P2=kL?W-KU>MXbM)&Q~)9ZS$bUF_?KCnarY z`Y;>T)6_BqrJapm6hpHP6-tM2v`Z+or#WS0s&;QbZIO%a|>9LCdm5{$-~__qQ4`R_>8FT@W??D;Sc9V2s#P5o|@SrVZ` zzXOeO&_VL(qD2mcIaup~A9!>Yw6UR@v*p->DIO;2j)9kGQ4Hr1!i?SJ2( zx%TEn3A`f(Df{}!Pbi@BeF-oLffrGjsv=1RqO{0FXA zSgu!BR~?VK*a0aV1U)K}c3+(w9>3`wx1pPV*x2}mNzp7q&^MGz6J-MPu22K;Mv3Bp z7-O3RxS8Pb^e_W@mm5AZ+2DMZIq1=SLc!fCfRbY*HV=B@q_-)4IOv`9S{uKfoc;0b zFK6P{gLm%^-khBr_r%+G;_&U8qm#3fw{PIrkK*9XpT*B7Z;o0*0&*no4ww&*JjH?u zv5d}!K4{U6>tl80OErKIMX(S@QjE|?IUkyBE9mzdJj7dwb9Ko{QZVJI}v^Kfe18%I!Yi+1ve{*tv(kl%5%I ziR5=X*vH*P<(7Tl;Ya9y@3fQm_dF3fv(t>L#hmK^B+GaBA;lmEnV^+yCN>xx~+lgb0Oy&vc=Qf1NObX zCA|#U#)LRJK7Dt5cyM-nv`-VVpmo%Cq~{~@xD@K14$}6;<|e#@7LPll4*yle%s#(H zt#C5e6@Qj<{)|U9R6v8aYl;uQ!Rv~ysdm%yD`?4N@VCfYwV+Q~_sPY*jHm_PY`8kn znm_Mao56;<~Ii#^ohp(9t4MM~{cIH6Y7a?ngN6yr_j9f{P8U@^S6`ao0_ zXV-rHl7UnpVwF=>aeyzCt=9db7*cA?6ZsiCrWr)YC(8fD{^0pS1~b@5IZ54_L@ zLmfD<;(rzS*df6>K0U_eqb!|yCgv?rxZ28)&W92i>*zj)5^i_IF)k}lpFcX7EH%F` z{{D|r_f(*_>X{B~yys4FqlNxZ()A;bCpn-C8i{q$Ye8`qNs6!u-pS#<&^94eT$Szwv3}E5 z2mVPHdsb*4_YTzsy}zLtz}YN8=N?6Int`%5iG8VUe>wk0fO(mb0@ipjIBps@r;pEsSSDhcvK5^)Kdl*DD!2lgATQj6L7au`X}I zZIOmjP8YRFx%v-O!v2+8!`iy3Afg!7xwBcQV~e^iL%a5HF%o|8bz)NzniiFj7A=fP zA($oeMrqfgX!l-e@j>mx`@*pR|7JH?C6O1eRdxVXSmTQTbv?!gh<6GiUgm3;#NS)F zZ!nHVosgR3`P-##se6WPIQMq-aH;Y9YG0=V)}6hTRkTX8>J_!+!OQd6$bbpZ2Nhq0 z@yO1krB)ph8dXCoO^kwJjQ;di&(*%`egxlO{QKhfo#!2NNIeg{sJ*+_PIgg$AbeZ; zf$EQ!D;$g@{_Ifng%O01FY?)M9MNgC9WzXT((aRK0!(v;8AjUtO0T|rCiaDWxiB~R zc4{I!Q{6dz?JtPVC0%nXRew;3$T^;;MKh|dQ{pM&GNQwFv$17fdO-tYt-U>s{k9oq z7YD)PLTB(xbZ6G{)Kyv_+n^OMpWZ|XoR;)IQt3!zK<`x~que={?tY-yAE~xpI|>d( z1fVsBaRMM_3CJ)%9Q6LEt;E0;PM9i3%A?g7ICO{TtF^@`3U_?zd38Cjwj2iqbiuIw z6G%q2%kPW4IJUwNT{q|gSH|N+ch@W(Fy?-M+K8%Rjxk2JBTn)QLyWhfTHHbI?pHxN zH;SNtqWq=lj0wAuJIBXSn}Vj=pi*hP+vCU=uRT&G_9Gqs{OeefO&NWNSSG>&ci54w z;{?3pd;ovFvyoAI?C`Ifs2r2q<9#CN^0-Vdab2)msPZ_@W4+BFKl>Pb)BDZI+yt6e zOplHOuNVe?`brYgswyH`zH)7lk8jxwt*A7pF`Y1v{9bc3|kV)C$fZ1c{S;%o}I zvRY{??a(Yjw_E$>MVu@ggTY$$IC59u6_HrcL^VD&EZ#~mTUZJF&`M9BLsR*vQ58Uk zZ+1vDc7_W%*Zv_c3+uPcYYUs^t8$E_0_G+6njV&*NEGF$aXN2yd@ z2GudTjf0SCq`}l^7!*GuUQ=QUCm{o$cpBp{$|r0Y0*OUNmq8LoWZ=`OV65flz^-F{ z%pu%v;Or^RVJ3`wLD9VZ%FdVLUOM84veE#MjNR>dI#n7DoFODm^uI$7Jx*bh@7tIm zF(|nzT8-Ene3d*@F*G9=VITHI_Y(D%t`hIMIi2ZZGg^d|#d7{2tW{bE(_O~6HC1#D zEgq!_s*12e0WCFDYfCKjxrIm4+RhrNFah%Q`9`!_{R*zO!0C(v(&%(7X>){A+oT=h z=rni>Z$7$R`8dsGd{l=kMp3YU(h1_FTmJ=xqIo@ypw}*EH;l~ ziaiGfG8%O@Nx5=>yVfk{8)r#Z;VqJWf&xp0B;WYjcZ zW<%S9&R+Gpz22*W(vf#G3)7-q4yvFEMSFg+q`My<7xrQf1mLMj3C=(!8K11lv;Ye~ zA%h~}(%P?qo3qmtBB#(GFz%;V?7n%s+v7K zxY206n~&8KhX?iy&}Ip<;)rYg^1SNpuwoF!NzLzIz9CQ?8^R z&Fu??H}&2ki+$ZS^(mh^4ac(psz0H4NghKmc%Ju@j-D&7H66h|&;fR2@EAJlMo-&P z$RiYVUQ6`e4AKcQPaU<#Jky3*y15h`Nn2E=4Lh_qlF2r2uz3?~cs7_;k{`aomKj9h zNWR=nY~eLGU}p6MlQ<;KB?u^oYVM*wE8i2uhDMgox#Em0jzXrTzCD`0>gM(>diXPD zX*w=+>jGZ3gq_7yeYn$(0CgRNBaAm4PsGm{WM_V9mG=Iy0FC!BWE|V2*zx;s($~?q zNBWQZZxh(7y!wH|P~PF+=zhf)Y2CbWs{!q(;S!Z%8s!g2ZPrIM3HxGi=lS>fJ3|%w z;$81Rz1P@-!n@z@jMZCfeEZ@VzOOhsGIwn|0)q->a1E6@!XY;gDGw2yWW?CHw&X~^ zY^ku3mIy)d#D6=Tjybzy9;=F0Hm1+)%vrqN26K^67b%U&6xm9*(G>!{oK-3*Qht?o z%0_?FCw;I3Zhjo6*+Cdkc=A#1wFHm_e!`px_>Jo}IV@7A=V=?htNa;V&-dFZQ&BsM>99p+ zgElshwo%HbKijIUt{#)sx7rEit(NBRPg-IiefpEY_>e{#H@s2S0@k*>=%bKk{O;4o zX%c7gAPzr%no@ce(!Jf2bZ&nUo?X)Uz~Se=DP<==S3p0!OIw5qKypaZsvNvhDf-{?oB>EY$mzyH1M zo<~&LQmzdr*hKTlrufG{o)T0%n{FeYYom8dc&TKlPiV>svX5Gj_z3IoBkH@b@+MO% zgRImKl9%0hnstjOr$X=qeylEm;iX+J!G(c-UjF^>W+2v7GJCMn7C_gj4wDSlqKHPA z$z8Eig5VIXQ}g30jegQ~Q#?A(s)m$JriwnW3>C&5hY;YF#Mks4s?^{oQ%Ih8pFS$t z_~X>0rgy)6&G?56xed!5V;x?8!)OeAqN;`6^E*(Ciih#2O=&wTXDA+}x&r*o(!XA#H<^JtV<{f7n0}SC`h}Q0G;*H~;S`!hH-qr07Q`oO z@3N}x=PVY)`e!@eS}d1e>{K@HCSd7g^QjanD_*Kz3VP`Zl3KX8!UQY!?L|@)N={+y z`Rsgyxn}T?(}LH1qVnviegwG-j@xM-I(8DYA3JAF<$ysnvmIltyERZEv>jxPCCZEz zN~9|e6CJxQ19{a|k|N6}Dz#+SYR<@nZU<|nYAiNn-#<>wR_-fE?OXsOPN9TrQ8VJX zDx~dc9S)DMtU`SZEo8?R;81?mAM-_`6=3!ipm~8Pg+mpHM<0`mOuZ z_Hd&GiORgOG2{%rc5xCevtT)*>}J*wTO}OqQ@AL1&E=@rK#_3nV+i=)QeW|9p`!)k zjWIWrN;2A=0acp%V?NpXhwZqu4FK+Ox-`C$7$H zqiI8rtcL3greEWQy^?EbE_9Ta!IJf>vR;%pUaNXpbH0j-Wn^6Cm7K#yCM_Nr6N{=- zyXhd{p1v;fjhX1Wz4GQ*Yr;ZorlC7k+w&|A)yYaK&aA)hce+tP%V=uz_D#R7F4fYf zPA^Gukt7vqXf4Wbs9-~Ff2(n0qiw0AV zf_NH-@o3(gVv1AD%LURn5Ie3e{v!C3u)mR?9?DyV<#n@RwRI=jDK&+LKHZe^Vm~w4 z@$rW0npiEf9&3hFzdB=|Pjzr4%H&t|jy6Di2lVC99YA?za-jcPD;W)RDMZ)A^s7x~ zOiQDx=yg8zZvReWKTmNz;DFUt)@EHuf`-{vFuUm4N)G15FKr+r%*P&A+Z3imy>>ip z@xZ;<{e`ODa_A~b`H&JO=be+}9dF2MB`fN0|Ad+vs87>84TjFfVAox(RXWuGUH=l+-AEqee#H9__P}m)s>=vz3a!?Z zom!ydC-Jn@#vP6eC>K*toF*}H6t)FmbBAQeU)-vyixCR>0c7K8th}kA!BA0PA>R?o zAz0>vbuHF$LwETpMM&hyH4JYPR+f$8ONDz2Y#qVmN5R5LBBA|TaKeGTZXUdPwW-*z zGex?bvYNbO72=46eYQbVqt=c#s9!QB!9J!^lc`E1M7hLPm9AWsj)f>pe>hqyH>tEl3m6MjiQm`z$D#I!3d z@k#x=$A9PqocT!;+Q3S^)gdTbm0zsr^uoO>+X@S>3ztVOjxFO%&*x!$UX|&&tM#L_ z`_XixlTE`XuXbM;&ELf{EM!7Qjg7$!rZ^p?CCc|I8F%NrCPfJ!p8kR+*a;8`Ec-1W z9FA!!-bQ2eB4s>qV|a2R+Ay&;+!6BvSle{AFZS}^+~&S`vAg#=(671A#ahqGwRU&* zo~vp$@^Y(H?k&S&w-%FdVD62EcOiD>?=|eUxj?baNjSPstjhGJa%y$`Q!G+cx15g+ zs?(+HUU(!!Z=M@f=pCjf_2$~X;-IA|^r>U4VQa^&0OGzC&oT}1jDOw=&g$@&E}qpvHdsEZ18ul~RtMa02~_~z z5QVpoX%Q*UU+VWmTt{~QO4dH>+w#h*t)SIlFI_#WforgGRs+~@)vN}z;fg5%#nTw; z-Ab!v_b!%;Dqg!@iWUO$WPW9DljVQ_SB9&wJDbIk=L#*utv%)BvhN%=Q|?4zM_i#7 zF61TM=-E7@)Co1WD|f+$TH66)U_XYK*2{&OU*ly0L{J|ULoBPs8WwZH(TdGtuKqUk&;IU+No-sk=Z?6;9q8s zE}fm9VqzC37+4(|KBdb*IPnV}tW&@k8GdpAcKZre)0XY5>4hWVoMyQvI%&D!#PkaI z$xx-6@mo$-bM`3t5Bt?0tU4<5%w~JDs@f*Jy&4CDu}+4X3V%*FBnir^ggKho!m8on zRT0Pa6HvuHljo(0!H_8gy z#I44(%qS-EYA3qg(k5iHV$(b2y0B~#7St_nTc@R|H(gTRCw%9G!e37fsVAMw1xo2O zZrnZeHth{O?BV-jf6nFDy9}T=PX@4_62>Ru!jaiD?nT{JOeV1{992ys>oEJ-gTeg z)RH585q+kI1r(YjBpQLW!kHfHK-0~FLFQ$0@{*mAp=0xXU%bJD%2r3MPfNRk`}hxD zh~5z8{uDk^X7+qN&^@xRtRgzBM8iWbVV1~5f#63JzLWfBjqfrCEDylr*2p`EU_%ZY^B zx+Ct`C@(wpRVAP1F15K!?Z6QjHHeeb+|hIS$f5KCaF(HJV|55{0?BrFI{d$#?%wmd#pBd&L`(~fW@0PS*ZPv7 z*(8GOD+!>_Hr4F{Ew~o8MLqcc*?ZURHgaT5w14v{vT65}RP7{1U8~jGGh3EbE=}9A zMwF|%cXcf(MW&=|lT217sY_LP*1W;_KkJ;e<}uFm>=!t%a3TUgCV&L;LZ&F$#q7OZ zR3wtkVIKkbEG?QLqXDv=<&`~_T>7Z*~Z#hPJ?)v z5~FnFj=AQ^vf=FX?hq*}GP=lS=J-}pE&q{a`)8*6Ft39+n$mvA zc{S}{u~UK6X|$Z(feFiKI+7)&e0JWyNUhra3#+}Lf5F2k@MXKrO@@wVqm{7sIl7+l z8ceEkQNj6EYN&r0jP{ZUpy)h0oDQAQ6eFFZt-gm>Fs^*_-6ysr@t87^Pm$2XwLE1H zEmx9fDMWg_yAPu@Ch0rxt~x~+?U~}UfKon93nL`*1oiQ}L%`w}A&B#HTaR#nayxoBRlwk5Y3nrL+QsD%X*Do9jPj12yOww5D;?|~pGBzC%N2F|! z)+-&7$l(hpo9+^j7XC=scPRm%EBam9f?2R0Q^u3cO7o*VN&{O7?{QvwjEIjZF}awJ zmbxDb^)aRDsRI|vELIA-Fzv}Dc;V9nFI0weM)*Pt^X3B>=2yIgFeJTk31Rq@APmdR zm8S|~_yE+;2xBP8iy46omDTx?pbQK8GdTsz12ZfVg0@0xf-_93pApb7RrEg}NW-+z zvw#|=Eq)5HhH6_(czp`J`Ot=vn*LeB_2q*bYQt`UH`KbT5a95$?XwJZNZbnt2~&8l z7y>bs#ZMdrQ8JX|!XU=E4Ke^B&dCIt+d#%7mrHjFhNzKU0Ed`Ev#7paK*ZcuD3FN8 zhU9`GR%}e&u!z~}O~8oi%JDmgMpU_>035Ndo~sQWF{Qp3An{>_eP)P6X&JPDB&r0@ z1(PV5i_JiZdR;R^C8jlf=3t2uYt;gms46s{4Pc@?!+yt*iHg!%A86uSo0x^bCZ-fV zS>QxvY;2Hm#n6eSu_#kR4e*J%<<|h9m|yqUQRb6-!iBC%9rMb~M8) z=2jTz;3}3G$tAMlVD7#F34(N&odJ}v_%eT)JVbRo_CC1_xg0U$3 zdCi2fXe*#DoJCvtb096+iku5;(N^f(Xp6Rjmw1a$Zw}3jxM(e+Hq1q94fCKbS__>I zchOqx{K$*eqL9Zp+S_+&8gV9ptyeN#8Vn1IT#;yFh zleH+!g~Vvs^H#!Q)ajTVjj@=L0z5{Yl-Up&i^<5vWK4<3pfaWu2}T5|<5e zqR69TEof{C6=*Af~oZ>a+@@`4QMoT465VJ8|-M&cF zM#DhVkK0I2Vsyee3vy#c5}yWk;|xSU9(tpujMawU80Tq*HUvi_G4*0N8r0@WD2^$K z^WZqzdZz%%Q8Jgzisfjl*@otr%D~EajuN$5M8}klEVfcbXvb_lS#ZbP>I_hbxniKO1bweDB3RRnQ{S%ufX`Qrc11ff%Xmfy!b=sx(wZjZ~|w3pY}usWNh; ziK^<@k@lY9=#hD3&WIl=6`3GNYEU8tH<}Z6vjRh&yxQB>&>$PhB%uydAr2d*Cql2# z(M1Saj}_f!P?U0QIv%C`O`RgleOgQdI{Z)xm%&t}p3x?QwXzHXCd=5@yGJZ_==_c+ z2FM$nP*BIPmKQ)8l?H!b`6IkS3B{xr-xv}S-kYUV`W4TS`EIX5tRhy8T7w31DGE0V zOiLYkNSb@%W3p&?3YuoR&LQxao+zQF1j;7RcM`(zzd|HCY!0jsWq6b>S+@~qFN}g; zn5d2}ydE(j{!m#D-w1~L;o4^xyhAav|AoTH(Ns#kwxNlmaH8{B7WR;ej;44xwa?Z) zR?w6OiA_7k|Q}`+Kq`d zRXdEf)j&J81HMC~V_II~FzovSazg>Ffgi5WSu#Ix0HACUc?u5^!Dta2u)STZUSOBs`1Bs_4O%ZQY;dtqo; z$xzn!@kGnx=OxjE-`Z&K%T}U0s&(GM74C)^{ta&u?GSi7k)Vh!CgagNn(0smLkTi9 zv6&E7bRjvktvehsI({TSqMyh}&5ZQ{|B_5$BvN4`eL~9xO7l9gqblr3Uw)%-HhLJLkBg zC+G4gSD2t-Mm;9<7{UvxV)5APX@1z2>W&ec!cgOJs>aV!Yr}C&E#m5%rFQzNtk}n1 z90S#T4t(hRE4;u3bj)8fw8Q@%UZ{&`=5xmT%TTPoaAsAaY_XsteGD%z-{Pe+S{V5Y zb`4GFkAqAmkG@<&HgZCyMvyo+cqdr!Y23ogsTvc954vwja`u3nWRnFlny8o~T6f`6U|kPK#$DF+&Ek ztjYY5n3cx*3qvO+-_+KmOuC<34och@GN=btOolc~Nn8(nib>iPz^EMX|E3-mUs+mM zy^ea3R4P=QBC04?u?1Tv*xTwj`Y=Jw5j zJd({fD;1^jWHNNva?VOg7HsA%_Yaf5-~x z;=?FQ&M~veKY`UohpjUx2u1L9WE<~uAPe#)1Qn4Zc2-<(9tOy+R6R8S+Lfzkc3`_I z{l%uYd0b4B{Zd-I8czr6E`Rfq+ykUbfRbLS-91h7 z?DfI7dmrAOo*f=_Pxp@Y56+JE-W?FN)c22j(|-|6Qp|fu$SW zD7+e&PY>TG6y69dLlA~^0r?x#jwAL*aX`K$7@}v@M6Bh`poQRhe8$CTS5Yt8JjQcL z;tt@9kLj8vQp0a64B1Df2mocNE6u-dC(`a^MZuL7M~!&^GH zfu(`%Kl$-h_PQ}mq;VK9^wHGrA$Dp!b`6~l^03VE#>?SNY9ZBGl9Un4eAbWhRCSC= zBDQgkD>VwwqttetXBu{%XqEOu!YsjbmIX`l71Rfd=#N}7?m8z2Zw|YsC;y-w)=%2l z@q*m3G~QH_BT6b+?Kp%jN2pbN{Jwj5`kq(a4r4dI_6LJ1G>Bq36&UQ*O2ovwDqS+B zT?vN6>4f=DA`B3^sY=S(YN?j@gDV8OC|fU290`I@|79!X*uq;i1#QpAp@g8?@Ml_eOnm~2CNRj|z^XoUj&7=vF%Zo-@V zM2pVy3H6x65heQ=EQz0E@jSb{5DK_iyGt$jM9K97mbGZCp_EV{Ck#whYsSS8O9sEZKX zTunS=MW}iSf0<|_w5vgn0+9SJq5zDu<(AMMM6qRch66!zj%uKnSwbOTag1F-xS<}? z@~>8e3%Qo9^4hMJ$d_oX(8Vm=Dkai#tx;oJm9zODX)V@#0*%T~WlNOzUPSJ~!G?9Tj9F9c4Y*Fq@IKSa5=ahy-*x`d zS=*GC0Fx$~Ty9r^pYoK~1hJY0@ljBtXAgauLO=|%;|Kkh{y%@>>lO=A*PN%PlVo!0lX({CC3?9->Fu|w!YelNO;(OZ+w%9*1+ z-4|dk0cfUYJ^@TyJxe+VuY{y0>}HNjD@bphSF56%y{)d!8Oa!s_Dx$WiJz;d7~HPo zf9 zO~Q}UoK#nX@WLAueNNEFvnkMOa)w9>L$AfIeqvb#2|4L}=t`xfy`y};sUUrCI4+xR zD7sXi8+5lo$t=riIQ%7wO?-e(K^u z!&rBgWMe!tJEdO=a6P!Bm{sY@gj z9(4t2gHhkV3&IlB(}0#wIS6|n3(`sVQqJf+4W=$aoSa$sAs%=)zE!BOuXMPb{dnzr z(c}WQ{yCo#&dy6iZA)x#VsT*0A76q}Urr(U<^8!;)p~rVt!0GEM~HAK6phd0+$2M7 za#~{i@|m8IWZSYl8L@c2mDxL1h#ywxW^u;ImFDbq#jP}#K1b9_bAfZktTY!ncf?9_ zv5yn4va`;-1gb;{GY12+1>P%K>}LpAS?A=KIV7c}fZ765TFRd%9Hphm`GQee3Y|X` zrKRA<3PicFfQcf_5aE9NE*#-mW4^frG$B8XC98=mP**e~7cWhRqtr}IJXJCo_8Ib7 zSNZCk2tA!57ovp=l`(Mw!b4Xj6unOgN>1)_BV|}A`+7Ki35FZv$PG^B=UWA17>LOGwEGVLmEz+_RJXYDa}cjL(@MMrdPu} zmFSG?5caYB9)Xyb^1I|*Oa#o7CMV2H^5vE(kn&cNBFHxH0!>9L)A6sM2)%65sbKSy^YW$BS(OrBq8!{BN^b_)czGZo zpw)!wf9pkZR7n1n9SE~SOzBhRoLA%+q5<>@a(u<^D~mG+=ht7);r~8;a-K6XTJo1q zpYRKXIfE8|rP44RG84L1AWy8Q0tqVbAAUy<^-G%>cU$}%d61apX*!OJ5Fb7ZF6UHeRF3A(sj1hcQ*gvtkp?`?U~|T(Lbyqjh(!7Mc-Lt z)*SKQT8VtP=U3qU=5sdiO%R~u{nKtE3%Hye!sx2aem-lyz~t!MfoYke2z!?XxPULe z(u8VyQSUmKKoZ!bCvTNY5v+Nz_MA%xh39F>{%DFN|JY2EH(xkSDKU5D`t*XgZ#ua)wM*(3wOxCV z0S)yWV^Gl*@>|GP^ghvthh4bnA$8)?X!EBeZEz_OWgZ&-h3(s4^J7Z*>in8^0}6t;Z-P+L*h4>h?k{`pf2_HG`RUiri|x%%O(|&}C6;=d zGNPeQk%bS{F=M=_KxC>ZHRh9}VR#6t-4BOOXU)Z}j@LuY+8d0oJ?LauI(rVQ8;`v=(^C3Uqg+`XCD}onG6QmpL~n(1SF#B1{iZQ zo149V7a+rCd6|9vFbc3Qn{R?_{#xZ;%0_cBxN-SSyay{Sz^S+SlU;&SNBZR=Tmv&E z-T8Sf!zmXm7h?XoUT#<}#mlAm%U_Bm>GvRuaRdMV)6+R(VD|rgJcn=L6^n$7_WwmZeSeJea@3>C%K~Ye`M)<*kdCky?6%tie!}gv5!6cLqh^BlZ=%VO|B>KEivM=0{Rq)P9- zQ6CBz`B%(Vy)usc%ivBRr2lcX} z{5u)K@Bax&zxN^vx^?*a04fAs5f#x!3p5h1&CEal@Q$LAiQeq zf46t^=HTu7H?3j+$w(zO|I>?3JzI#aV z87>3FSlK(?+yCysU2m;5zHWc|)M#Mw!6iAG2|R|$AO)tA-y!J08?mk@DOUWSb5kLD zfYU^lb=ketsJZGu&AejTALSH)8Q-ydNg%$%IG%#wfkM1-@V(GvpIhpcqcP})Zg?rXgJl!prD9kCMw2udA4!!A85WyrRQ)v# zt(}Yq+{=k?M3d&Stzk|D1&zhp@MbgofqtsU0I@%~?80h;3hJ+)8m{X$q*SHYY9Eae zmw2-47xNYf?P#)OsxO_X1Z>~jgpe5by6L_q~{8L(qI0hJ;VW4gZz7gyHJU zLL7-T_?GWd=9zWg*?3|xS*?{ub5qu;ZOR`A(cX5LHwSN zHVT}&!Dvc#kx_%TD#^LwltQua3#19O2;TV=SDm=!6U72$&AaTDL_TSgn2+1vY^-nZ zG#X!jEpVU-`)z{oWO$u(z~Nu8(RgFw!bfuq3+)3GfBki`zY@Z!A9z=z5SFUmyRe^L zA%X!oO$xg9uq1z!78JnmB%?FLoV4#_*Zm>Gjn8ugF)NTTdaivDzdkJ!ufg0x*G=Fz z+I4mtT-)&20$fV)8s)CmG*Po8d=0V}G4*P#mO%RQ>TJC68Rn0OZHR~pcrD1k1ExY9 zg1Q8kHH>VESD7h=>!vrQ$}QxdJ~{5dutthIE-!H|@|#o1gl_DKOa4B^5JDs)0b|tn zFQ!)tsWPSbaz-jAf%9)DVVw8hI;Y%4@~bksu9XjvV^tmjUH zF;(#cS~PhSu_VYA0}+!V*Csrl9qHSdy0OvV7Z+oq!tY&&0&P-N{iq27zEai{ppaJC z+FJ2VuO)$Ka)teMh_`g%`>ym&R3!s!#w~W?O0vYHb{zR)hkCz)&01<(Hw8P(wk|oP zT}1oR%5qxBwXeoO!kH567qa#hmkVBh7>w`)jjnH|;Z3Ds{lMMFcj?*zynXkN?*Dju zcJluH>DlpKxBJ8Ulh-ewugILm@%U$&pO8@Ke^UQ)#Z=7U(7`J~XwrY#{FRXHSo>Wh z(G`y8pPC>K(MfE3({cN&Fli_V?Fg?CLm$uuW8p_e?d`Ojlj#VSd=y4>LL5U%?v|Cc)Gr^hI>W zOj6Lr$h^jpG`u9NK(@jqg3@u;TaMbH!uMF9VXPo9OB)wpe>Leb#;5M};-!S+PU+%| zJ8qg9*=$r64$!0xy%SQ;FQpp@Uc6K<3wbV5Q7rz9P!wF&J533!&AmIxBc( z$ee~y`ws5#6wMpZ7V!u+{X+9TlG8Gr5Gs-@;IvFJP((srQKm!jJbOv#%*pL&7EMJN zG4j}JX{}ZiSv`UcEx;jy6$+79O&HlHuwY>kom}l|l|N7Fdzqi2PXy>Lw z2Hyh`xy2xc$kdjcxEw2mm``5HY#lNuEaCu3v8*u7pr9eC3h~(=y(9qZ zbDhMLt*fv{B<9eIgQ7ISX+E?C9$Z8^VZSBa!g(&a6Ta?IN@T4KRE zh{ZjZ)9=2_Swx0@gho_l4Z+>1qij!SB}m?zww%?v7HXw=`NxfhrI`gn%RNn?AakYL))cN^Y1p+~s?uuore$=bZ06V}Q^MXQffU}eT-wWm`i(`* zMFqp*!MB?qhJo7TZ~U54~XuG zNXoebWz3_>PE(`Ez&%5&(B{3Ox^-u>4x~(7vKXG=EsS`Tx6}5Walj<}2k1RB6fY0w z<2U^9*woy@iLWM}W!?#nI6RJ3wt^RR&oQ~KvNOwZ8Y~(mhaVN2rgS+K!78hq0`q)P zI0<{!!i zBl3viidLxd5=oSwFOUpk_4zpL^L;_Biga46aSvkDoFzPb`<92UcZ&UudDkPgX@svdz|M$-N*2dEQvxsML=RYf@j6kNv3OSKOj%p?%gB_>m zjlGLt5FnBd$UWu_lVst>Abv;`_Fo5aG$ribSJOVM*Gn5yeKw}g>IlWEHX!p2gVA1c z-GEAv-&hnRb4JVl-jdzB`!Gr`Iw<~L_KNuWt!B<6Qmy5;8m_x#7b_PzeVkK_3ifr9maqqDK0uK(+uwe2PT(?Xul{`^PsuN2%WJ6Ptd z*f^CZ(R9>vwm0Z!Fzik*FM~U$={A?kd8H$xL{L1rv!jZXJ>f&7#6Y5+;vB({p*?eY z0X=bj6i!Z{#rJ#teuQDc;})|9+uI(XSQyZFvE<)cuuRFiLkpI%I=m{mxmS? zPfx>HK6#H_|6?%ULvK8{6{tY}>um1m`rq2-R%faIE#g^>{%6<6m|bUI@R|U$FzSIU z)jMyzw9@>}tTbY*hrlo+Xzp+B{O-ew8j%Jx$frusRMaF%ZK`j>kE>F^3TK0c;Ylhf7a~6v&dr1CyKb(#xtPI#W zG2Ayq_Q-XCGo;KqzCFp^Hd%YemO-ELZMNcK&?9-YeA=w;* z$m!j%nj6x~x}1mQdWz>r zW@N1Nd&*@6>O<+WLmiYSHIyWqgO_QCN8E^E|L++ePuuT_a)Oq%2byum4hX=wX^(W zz@occ-&8$%ySZl^)hC5gQ=Rj95)S-GU#q2;bu~b)r%w7o84D>=AE-F7fPm!lO8;cr z;$(`*s6gfE0mXU}41EJR`WM?~O)p>$Vq#Ed`%~z6l@jH(&G%i&ItLAta2yW8tNZR4 z5z9dFMqEHppcQtrB&m=2Y&1w4$oW|@6H7F!rpIama|2P~gQ-GRpnKj(tz;P-DtT3g zqrh+#z``V!7!WDPzfGUPmZdl!HPoPTI1xSX=|0!yUEdiAy+*DXW59eDiZ72R%=_z zJgF)`(bV+GYwC8Z(IL%UKo7skGxGBa5vtS(_GHZ*da}Krd~OjJP70MHtZ#fVJQFT1o2rvh|bupQiUXTE`)Xc z6>A-7xuN^UjvLwXIBhBt%5KA!orx3pLo4JmkX|qtx_7u;%=9&rPE@3!7}{{6iD@^j zRyW0%uhi!(N+NL<9Zydo6_-YgHHjh74pyW)PRlGKVJ$Fym1M4^j6x|~VXBmy8pDq; zMvgmnnkw_?0d{-W;qAz2<~x7sLzu+Wc5FD#hH>XX*&$1_h^R{p|DvaH+#w2|@#Na+ zz;Ev^`HfEdgdgfOTk*&n$JZeUH~IA-j3S7yD+SjqE>sl0II6!zc?SYh3j7qVi>i6;X&LQ~;nEuCGVmxNtBy&x06 zLKg84`(WE!mMOy6TDeg?)!ztm8sV2R8q~vYNbEOE_M)H`n~(!!2d1`KWZ&V zzSY$Xt+II<_@h0)gf?FFhF9CIN$} z!jwhl$!F44vz$$7;=uvAsL zbp^id?_s1|-Bl$^`u^L)gQL^4{k^kq58ql?KFhCi1$i_ZG*nH}QKwJQ7lyf=e2d}c z%k!a#6_QVV65iBcmxRoO&OsGITNJ;fG#l1VZOW{bi00Yk14wDil*93| z17K>+w^2CU)jo*mo`3W2lbE>blQ^LkMPD-H{pjCw1zsJ=isSS28embO7YG4DLok7eD6gWT5I%35;c_6dF)zGmQPAGZU0?% zAbR3rkCC~oi01S401%?bGO26$m}gbCoop^FvXPvU$_LpH zfZ=B(?5-xt?rPx&1!hjkQkQ#S4$NH1smxm-W6PA;0^5x@%!?|x@kf3fA7czzeK#k4 zs?88!sI`{;v_h@9D{5%aN;nKmPW}V6LA-iBWL4gnW7IQU+!)Vu*i9BA5xej}) zpY7lymhLlediS#V{8zg=wd;P;&sXq)q4@Df$GvoNGn0CWB{)C*WI3iZ(DQr+6bwg1 z;a~JX^Z8J+NEb@6sDUqf0g!TQ-Y5KbqupLe?OO8RAT( zc3R{EoKZ08Uv!z1-@dSQm_J+TrDegzv@UlS%pt!Y_CETN3rrK;kHcUzaakp;D#kP! z@C^lcP7~Dc1#O_jGc|%&Gjl5}h##iSHE<$G!F_R0)!HOVP>qowOPc|i*ye}fYkni; z^AhtxgMuAvGniYCP?&MH7x{$s;0^MI&2Gyt)1U|P>#16_dDv{Vp^qzmuer*c_F;|n zw-Nnx^J+Bu@#b5=zL%Dx2Q-8!)y!iL(^_MQL$7zW91WE{R!1r&ly)VDDkW!C$10Ke z3A$Akkyfv3#{5UJ^_<1_T|~c(CVun)r?TaaHb2)h%`eD%?rAuYJ*=a@c@JH*dCqQ} ztM8a|cFQHU$(b!O<$Qg1@OSj;L5C!s{iU`$W_`tG`I(sIj_a~F7@Qhmd0!5~o;N@k zg58acwe9pfNTUen&`X-xI7ksjmdmb3xajS{cnfbQ7)M$!*ni7zzVnXxGRqo1_T#jS zBxi?d*OSTEI9caBZxYqbc08llsfDmb6g)DSf4MweU2Cc*+NR=iw+(wm{cbbZ6ACX#2TqVt06`Q2dGh3VL zF?d;xT~(W;kptBjHU##TJ%k7^0!SU}%cNUw|AWt%8wrb2x|KXD zGbQI`C>C-S8@$E-UZRV$p4r{JPUhQhR48|y^{wqZ((_gVdiOd&u%Z$IT{1>JfKXRT zuPoRNvnc=WdO<27G6r5SH1(!-&XnYxuoh50b;Nu4DxFhf)4`sdgjU+jL@_f5Oe=R! zU!z}R1a$*ld@v@r`ozZA$ zMN+DTHY4~j@3V!(Qc_1MBl2Z%P#~!XyQUadip<7GwZ2%oB+ag}hs!K7RujFp3Z>gY zWsmqX)T|q6fP*7(#!zsQ9{gfJC^Iieywp|?;nkAWNGi>UqLqGjnpcDH!W*Q?*x$L! zQ?WZi21BqYO41Ctxu*Imv6ib#IO*qOWBlhmdi)1&O>pCzUQN-Ti{n3UZfWs#2 zTg&(li+GStA3pv=&T;Bo5g&PZGPt`EkU>Fqry2UY4dam7ShH$~7#4A3`jLY;&PL7? zj@)QCCJ}8TJYU=POi!FdeN=*}8;fiv(IT6?E2NN-IrOAeWL0)0f@CHvqh1mwNLNZF z7NcVBA4lON?1h6}=XC#=ztfMP>WYe3ee!)K%lOQXzWztyz`uYFfL-;X*8k4-+LpHd z??9I2`oD;0G3&o-bCN>?@R%dX*XY9QiFcnw7^f3p%2$jjD@BN+!ZM+u@&iHH^oK89 zeb6+urybg9dhst<4`?lN9Ytm2Y}H6tkqR+0Q224jCg2(ZMBH^46HO=AVHEsAq_#eO zgMA38QueIIOYW0dS+eo}sBeJm0M^@jmVg5Jzlpy!`MDSwQS~JEvi}M%Xh8FJeWIBnt*ZY(fpn2a+eE7SL5s;qs{l%Y7oLP5 zBHolCFPl4Kk1t0?+>5-ie;N#Z1Y1wf7v$Gv2(pwjBURVoD2(`=PO?fb%TQ!2V@7(| z>l2^E0Eld<#LTBT)ymQo%pdB*QdMMXX7tbvjAn0IvJ7>z(Lqr;PR^d(S@t{+7IL)G zPAE6{sL@JVK?YhozRey2cb2QNIDMHMR(Dfk!(Wfi{Ry3&+Zoh$Mz^w?&sSi`eG1$59E)Ey;lBC3>(FT)(2YcdQhP3!Vaz~D)pS_bzga5j zgU(^-X3l8Z7fY;3cF6~7%*LCvN2lcHr{yyY!mCy`{3XA$eugstW!WGh?pjOT@Ll^} zA2>+y{<9WPVyh=V!RQi(kWd(L1jXdxQWj0&R?Gr4nYauGwyoUGa(mb4P2SH#EnxMY z;N}0MPH%wsvhR7_{g%`H7>p64PbUEh)pK!u}&A5>6WZZ47uK9g|TyDlD;l7m4z)i_4ZN=76bAkr)Y@ix*5 zi?ym>VyitzGK3^kPtFUL@P4DpX5zItGmk|@b=Rwbq?%}?rB%*;=Q z3+)M`NzS2O)VmHQes2Q1E2k;l=`;yFqX;ux1Fay*(%cUx&O=Xp?`Z;9K?yo(Pf#istGE!GsE=K*R!&Z_+w{X2-SeO25QLSH{jlt|Si!y-iH2iD8r zV^o7bETwc4*)_ulrq*=?L%IpVX+HV`duE7|Ry3m@xN#%@wyt(rczf2ph}uo%<4lSI z7sQyK6Hk_8$IpZlt1_XTH)q5<^aM%FeHD|6R$1rbvoqVow%UpP8Dyaxo}pt)LEem?x~{JT+K{y!y%3 z6NDfyPh_2e?+vvsFm7VG+85B=Ma8o#S^olByohLGrRtuit;Iox%7Ch-<`6AWf~|@v z3gS(aRl2y6$~B#mK(ppJCZdp>dO*#Xs|v_pdetou?GF`!qfB)VI-h2Oc+M-PGT3uY zVGj=bTv&Id4$lDo&I*ZHjL|9)QfY3^fWlgJi&GCUYXwu8TeyL?7HcVSz7xP%5o4?G zXbn@otOT8p5sEf|b@D)wF|qpa=h%ZDbL30z?G1xobjdDvmCpMF;%llu8covWUV7~)~pk^!jX*k0II#!l%c4veg zhOLTO_Z;PNa$>jpGS5e!mnKrPKw3#&QQ&yS!p3?*l02Q6KtmCqtmui@NyQiD)TzC^ zFjoS3VtEN}A(!A7Y~uT|_w&?m2p%?QU&LbwksN@g=rm8SeMfM$>gS3t>R~3OQ1!MvS`-&aotAMi9Xf`81trg}(dBlY$-ztSLxV^vY{KWJ zAARXhk^YC+Ezkm-^o#o+qXM8CAbTJ{NtDx@@WH{f-|C z{;S8)nHPco+CiO9$A8^i;(ssXSuFmO8T;F=$S9D``5>+Z(9U1FdZNb_Q~LRlp-^X{ z=1D=ImWE@|n_7Uy!bh%r*i!Bv!hp}M{;rwmqxMzV6#t>Nzz_T>jQ^nT|2tco+v`jD zzldk?`+xHqU<&{-f4Bz;qkkFs;d77t5KRX@mwqBbg9K08WcTngffU^od79qb_|b** z@yefMyMdUAs*`qNvo={O%+}WR$JsyW@}FPsZ7(qX-%e*^JrnBTvlY;e89@_LVpAMm->K zwM2%auKQoh!L-f z2sD=OW(WV0h08SBWjfl^~oGjdcMdp2+f+2@i}F);NjO5^|%Be{sZh!9(!lIe9Aml`ww0f?q9eJ z7Gda(=dcD9$A8ZF|81`={r?v7Jc9k7aaNasM5||C>bHFX$+2oZB2JPNy1P-)E{CQr z2^qa;LsM&JiEOmttI_*vu-TVgMDjKwPx^R)G0;4kh{oO0s-7|SKMDURoBSX5=a2g) zo&x#5zO}iQvHx!_?f(mT7VH0$Xv4r=vz_rW$64Dc?{V0F9mLUej1&LWw0~vugI=C8 zR6S+*EabPa*5|O@Fc|IKc)lsGEPnrooMqYmDW7WH{b9*ZeDk+l|DVGCP36ufzkQqg$?^Y6 zqD#!X5iHvO;cqSe_ty4u|F@9m$@zahV1T!!FfZXUmQ4C5<~vrAt|#n7mK?CEzFbEv zR?B#L0sU}&6iyJE_I|J5kD%4)?1FGdbGSuVLfYi|KolP0j>CAaQ+yLqcR5tuEzfMr z2#E5>v9|TUj&ncp6z%^uw{`qCc(*+NTgbDR{hw|9W;Wn`!E0jQHOppiY0>?HcXSAA z(CUq#vA?CcNZ+&YU zN|$TY+`UcK0xQs;Lb!(;e;x$|XGa=Iec;6tm-RcfWE9@w$uS() z+adx<@`e{Dq7apl1WLT1HCYmn0GkAL(|JBlXe0FO?e7 zr7B{H^em~Zg=N_v3+{3yQoX~~&9zQfod!xh_08T%IPfEVDVAQ=L%du|eYC@WLy8y) zx=VbNfQHg6(m$E^DwzT@noU_+KuewkL*GD({>4a1Ci~2U#Gsw_r_kLh70PS#%|<2T z%=J&gaX1LC?z>|QDG54s45~n&gu}BYsf*cc6j(e=W??gdYuq z5zXH>k=OH&eV8~{UpVT^Su$a5tN&d`Zsd|#Yq8UpE2Fu&-=cYzOLH@+VZg3(7g ztQqNnCLL2S>j9z@mC_cq|H_^Pe@6< zh=&g(X|AzyKVq%01J%R9wl9KF`@)N_rT1QRl?fZ``aSKkcEl`&adzTK zKb%g;lNEmYksjCs|)Zi+V!zifVNWiV55(X8{sP8Bxb5VhLg=&ZI~RL+gJ$@ki)pk`+9n4Xi+S zoR(NdqGrjcE@Lfa6iVF+)1=(A7=DEDaon-fR2iqKrR!aXw#W(=q%5z^6#F=@n92)PPGJwe zWEIx^00GU>fS)5M6)x7&*%Wq3#PNvdDpABeA{4rb1`ALga<{v?$5e&Ah=U=;{@@4~)OaYz^7x%f=0h?iNZDO{vx zyNMN5Kh1g#3+{JP`s?g;THkBvaVn=~QkAkFV0StQF8$tpZ{U9yh95QRlW+AjL91$> z#`vQf^WysI@bqZ!-Qn5mgI6EkXlo6T#n0NJ5m3Yf|2oY}4@R^}kZr2)?E4qfs{-a+ zHOp0N2q+~MgKafs(a@EU?`Pj0zCExuNnF!z8a94B);wXNVR-LFr6QQxBsG z$v08{Z4?f7wGXtI0J(qj?@#>8>{q8kw2EwAIX1oA582k)H67NtgQLAyZx7D8-M7VT zc8{kvpv9)WJsG4s)_nGnH^_1>-M<6A?eFdT(M0#?$dbPQ_VD27^lX1GpQUmt$fMcN zbG#`AI8!AqS^?P~x(6p#VvPkp+{~R#D>s{W!7V!Eu2vbn40wDB!a1L z=8z6~PGa8FFO>(QoA13SLuYLcZKC{=TDe^> z&$3O4X6?Vr4nR*_(=u|JHR{qltp_rc_Rr5;ykO>$mu(}P6pL#hr=xNsrK`dz@n!=S zKFW$Z!yRH{zs7WOZo~XGl66-LU3WF`fdWviWT?j*H3y)&>wsG%_{;V{VF`WUE<;gvU6%a65xm%2)))4*_Y z$R>~Z^pJhiPY~H#{S*-&v0|Tj)4P|==f4`v0lV%e{TL4)=>H#oblgkFoQqU1%Q58a`2MbGnSt~uoQ!`??fa$$-`_Y?%$WtFr_nW8-nd*Zc-%j(9TX1`6P?&023g(qv}t zUgQ(#i8sg_GW$Woyk@EIrA6q$tW!!Q^BAeLmJIz#6!eTTDd}aFL@L7NAAPZYGaBbVsN2lv=~wwQTnvr9qVdh_*F9$2 z8P~BZo^MB-*$q?9*JlTRN3V~AlO+2~Z6nP3icRt}G07d*Wp7#@1IT?j2z%ZD;Uso9 zHrBS&@4~6*CEI7axBf+IK${^~WZgBa3fhU%^<;vo4-ImJ@uRcL8LpXoz%ODIWyxu3 zR@kTK9oc18MSSdsWBEnttnH)gWH-cHP&Wp`W*H;27Rok zxliCyQXKUCo>%{Mq@>)LE+R2L$YRCSyPLP}m3B8a?IP^bmrljL>0%L~UHU3#z~HOsF`VCaRv*+Iv@EC)NG(XI`?TuIrkG{*$BHD^^BE^9WqD^GlU0n?XRIn~yc z$-&0r_Szr4_6P8(pMTVtSGQD%=v{o(g&q97#j-M_!4|B2ljaiuxJeWp9iZ1S4RZ+y z))E-ZqG6?Qzg2j{3P`Jx@G*s?mDs96l&7@Vx?(-mK$#IioA=g2qCKf&lvy=o%v&I- z2fM)-7n{t6PPHl-J0#7lvW3g+OxAM?Z5c~9xysJ>XQ)>%7cX4}k@fsygeMFuOgD2`KQ2xwDnl^Dv^C0b4K zAN2Sipf9;hqpBhB6MqWgKOv5{8vmoSy|uH9|Fnu~4=Bg~tua>@shoXe7K8c0E@FE`+c z7zTC|C%=gF!j7pB5v)Ngqad8aTqFD38xZr5szG|gim2}E)}LbhzxbvH|7jXK$=vSj z;Hl%WnpUD!F>}Uw>BuzJG#0ayH)Clf#<-cB%{857XMLx&2LJ1%cN5Md12b_yIPd`~A#<7jRIZ%)+lxS&9rs(f2B9KU?DN0W|9uNzpsx}EEdlD_K z=e2tLG&G}rveSsxAPhfF$Fe6)gaWCMyhna9h#$d~*QriBYrHGI5cquYO*d zfdHsIj|8&!)kXsQWg_8k{(Xi5n+-%+mt~uWX<_C-JsS836~V0`Y>Kavdc2eYOBo=^ z0E3~Ktrd!R&_4g>Tx@T`HZ2<;EZv03vm3X~K?J4Ou@AtnKwr|?PHW?t2L@eIPybecUbgX4Sv_sukh&yE*n&v;7 zO(xp=EMOYzy+yZwhNh2eZ#Fv`*U4rWUtQiPB%Gl~&Z00D?sdO(HS+V*58scypQpYQ zMm#fJ({QAk@Gz@oD#Oc}$N3qndvDl1FLJka6=rsh%|;P#M#!`BQZCItgRs-bRr{$x z4bigj3RnWRhKgSaiNyqdPqMYDxO|w@b=`-N3~qeWyMU62{gz_lNr?&0Iv0#}uu-fn zVxT#DN78(6a(~yRZU06Er^$JI3v*(+xFE?AQkl$2$HLEFh@H^7g%o!mWHONXY*g@Z zd#++%iofJaB_zwd4KpKnHbFa$0wq#3_1|v0jahg96#x65V)(bsO!9JRtnXO2z)ROZ(3vo?m}`Q0q_oYv(2yqBNa2m%+eC5%_Y5 zecrnUd6624`u-((Vc1MVuCmE5w}Zm;l4HW3O1dQI)+R0w?S2Y^HahQ02rX0tTuGGZ zzYTq2^2>1NZ)I9+`F5s60`r{s(EB3_r3h;x$PR}fUK#cRsWglmvMu764YnSm&k3>v z-^0>hC#1tB7lEiEt6yMAqWPu05@c(T6Z4X<6gPJ>Qu3XZ%P^u(cy}qiCv1qB$a1aP z8f*=hIB(M}1<&0HuL&&2Ii$i){Hq|w!!eGcOui$zEQx?2EOA`e#<84MZfRlzQQ6cy z>IEILTCqpoKr_X_2i_p~gKXD$%i0(El{ZNS7u?Zv`OZLI&>sK z6$6U84M1lRKrxfh@dHR0v5jcbFsz@~xyP#H=L8Kbjckx0J4GoWmT()yIRmehb(E~l z1_yLwwMk-@A*s3ALar1nY#50Xxy%+?HNvZ>S-LPQFG#{Txd04fGP$N#Ofxv<02#_< zWn0%;t`i$3mi?ATWSZNudj7W2w9XW@Rd) zd-fAxE2(1(7;B}}2F|oY$zj2quC9{_*%2D!K-MwxRdsvUBq!&VRe|hNs#BvTtaI4R zl$oH?$fUL6%7t-qPd9zX`%=Y-w>K zbvgYX#(l{2-@^+;rJ;<+$qBG>GNQ`t61nxBoV|;*sUxZ?KZmA7ru#36&TjFvs@}-V zBx!%`oNCmQ=IP%ic5Bob*-?8RR9XrjIT+kKU$--bDf&ir+MZ|Ubj8Lh{P3Vnm3AG1 z47Z8{LXjaVSF%-X$;CZ=#v5Qx?;5s9E$0Ux&q|o$F^wWTDdMG(F6_m)*SPlBo+pH9 zo|J3oi*mNWO3Jm|G;G!^6AAL=%%O~Moke0)EMks(J(LrE<+Q!5(y zrAWO}>p^9xbboavGjHd+S)*Q%ST%N-4I!S6V3Ye(O25mhgowA?Oz$S`0Cg@XawttJ>3k{hI$nb)zYa#+X-%XSL6?3UzP zFB*}*T9l;L4>xAXsalw4DZyr*H%|Zka5#o8c0j^7qEx7E5~5xVf50c0JTc^dK}$;y z6JC13z-m;Zi!Q*g!ZlFz-*aA0dLP*M1tnrQok%a6*)lNUzd{($=08VGDQPlA(c3|! z)lb|0Ir@I@?T3S|1OGXEO`%zL|2ZO$B$4GeYGSS=c--?}{rsOU^woL!;}3=RKejg4 z^!p!Mn_El#hlM;$lX%ekA2~Nb3zw*pYaep0%QT&cVK~Tcd-7Lwpr0C*?X#+7uuzeSZ|w8N5JW`|9gfxjFLb z(^u8AV296`ttH(L)_d1dWZuoT8M4gVF}0MP(}T5@KXX^x&}wGNR8M!$U3y`M*HT0I zE}t3Pr^bS29GXScm>sS~)mk)!K*tT}Hh=Ruc&9hG3M1GS4KvUOs#=8#(>ty&ef015 zL^dJ*5$=yGHhE?St(Y!?1X2loeuP(n43UKu0Lp-W`^vuzBfml!Qm!TX0e|%UALw+q zVf1m#5LN5h2p0H%cQ)a-y8nY;OaJeMJd4@?iTTO7D+@u)6v`v{v$BOlyl2j%X`nQ3 z>`|1oAX9Rhm0!v>?(Q-rnyzYn(VMx?ahEm@`&|LfO6`VxaEDI~&r{AJz6FNHlQag0m2u26j>Pg6aCmpVw;Mm29-T!&wlFnL8 z;i(dvuEG6Gj$*|?Ka4Iy*<=>y0O7SpVu-e3fOu3R%+pxeVENx+|rbKFOu& zAt=o6GaHQ=M#Aq5yZSjzb{xPJieU*e%uK;=&U6&~>}c^nl3R?t9PNZs0kNKhDkp0a zK|Ez8L+!Zjo0cKf)<2BZAkX&FAt21s)n0nxTA}o?@<%W-NEm63Uq<1u+$`sU@##); zg+RkBQocXZM4ySq*`~Tir6q1}4dj&wRG3ei68wafEhW^YQ@LC-wux1vHYKMK#cdK; zSF^P(KlglX9M9S`*;ltpql)r9_K>CY%{FLd^Dk=HW?k!*TIXNvc`wT@EjikAn(tBx zw_H+9P(C=NcZqIq5YYJ-Ll^rzloJCBs~yYN{-f@@lm>MilWp$|J)7_Vnm}d0(ntUP zg<(2Wqa_)y=M3#s?7(F%Ge$@;@?80@IS%_)ie}@u8T%!Vkca4+qEK)7veKZdaWYh{qZ2g2vIAunE#zlcK&ay?=0v4A|Bbc-#}#i ziGLMF_q#CrFC#AoZIqwh1g0eZLR zybUfQboTxZ?_?08(XF_vD4dSb!J4t4GYGGOo^t`KxbKe~?g-BAaKtSj=JYy<*`MP2 zf1gAOnkMktM=*)~!DXv)9pYIJS`y;jws0s%J-HF|kRb7;e?OS~JiH(Ld2S>J3HH%_ILk)H3<{W#1ih_Bw=*dx}#T-KMvzy zLd#m%3lKw=V?w!|R-dq<@N}HZI4(7zN~#x@i8vhi4Z1{_)PL`=dz48sBF~5wn}GDp zfhnKDmy`ax>oj3d2BGsq7!CSOM#S4N{76@nLb|yMavtr00#AW%EsdkFKcxlezd5Hc zm_L61?iO!1oqD5Rrm(ChshxeO5Jo2*XldzegTC)a<0y#z)fXI^V>tAqUf>NJzaLNm zAp_`1#*~%E2K7Jx_y76-{@?%0IUJ%D(1We_Q8)?EyW+zbN+SNa#yvxfZIMgiR5co} zVEjgS#mL#4Ucn4@*1wrtzi`&q*0)|HsR;?lzX=C7KI#L`czOZt6Qt1L^yQ@=Ari)v zoW%%22nYuTfHC&_FVG`=8U>Sk_cH83r}o92uPN?HlIxZ~iW<9^+-JCBF-PB;1UG)# zjgUNUUGUYNIupjo?fW-v@zMo#fQHGBU7=_Y8NSOZ2U;RG9Z5Ixz5czM7+-^5{Jtv> z{3_@V(0EjpXix~?$Q#8pPP+|I1m3`(6EQ8PK=nQ0#Kt^au_JKgE`tcVdJ+tM?7XCJ zLQw98Sgn|QF;+AV$J2oqNw{+yF%2qIM(6^^5@%NABgH^QJROsf8tWZF3)AsOt#DRtFAXRY{dHpF}eU1EK2<(8RlD%WXZuHS1CZSg; zJVHI^p)bXBFnN*b9xD6VyYWRw(Pg*D$HD76qv`Mhh>-Y6^Z?%5jT5r51ypT(4N1Zg zP?@sVfBvuko6-57|LgxjQmIYUW|)JJieX9s|M-J&JVZ0rcp8DU^JD1dE~0&9CJD43 z%DxR95V{}Z7j$7yT>8w6)(bA5@LM=Qs-QYP%uqRX-iqJ4YQf}dZLPJ|UO1h#);7om zKDN#e#Y+JH=n zAb-z?Imrb$@#DA9sB}g-XgMT(pL!d!xghX{f-hf)*#Q;6QUHRME?DE>_hBwLMrnW) z!P1h5`-CTw9)~3H5C#JocQWyMAECbJ((7T~{qGOhq=jirtIx_!;5pwP{Js6RgYTiW z-B<75#q3^XDx3wqAaohS390G@L!>qsB8Igu-~?zg!?cYFiwD>i8AG&%wHP*5kYyMW zt6bph8Ao0ZWj7NN$&AFBfsdg=oRmP(6%~&3O~>vebg}FCmJPWU<^+zx5IuAIuu294 zXw6Lk!%Wm!1T5}dVMFC(38{!RZD*1`ZqsF#xc$w>`u5H$TaFI7z@?2@L)uw#i0_!N zl1O1m&Vo2fG9T2QH$qmY6#_bzPe)oCm^J~Qu**o5$|z5(Ae?(2@wTYAqef&MM?Qsf zgHC|i3|#uR#Fx+MwAMHN46jHv+h~2$TKhB0<3*q}00YD{<}fIDG{!D2B;QS@6If5VJ6^UsNc8SYUhrjvZw`CRCYH-TNS^Z$*xXua`_EGLDpkqF_6`bdEm;|Z=%KR1V zPbDpY>*3A{(=M0`N&@OC9kxrANocvNcv0cJb$=uIw>p-;aGXt zpfo2z{W4`0cw?3)nM*g~+DG#Oogvyc9c`jp8-j>fDM^+=QAPxI*-;3pHlazSjX_Nq z3!Ake^NvU_3P**FDccZH*kd%L@4;?;1e}`Jl(G#;3wwlyBpaQamXvNtTG+xH5>J4j z%jqC@Ov*MyE^N^a;a*eq%n?yorAE}$km$ne6^gC04LNzWw@){R=!~#Jduowd{rbf3 z{>keZhNGT_jP)ysrnZ!A22ina`Ni+v z@u4WJLM!TN$o1H(Z;pHyM!_#=7#ldpVc*%CPQtk74T?7mH8kX=&W3!CC{DdhGm3|& zj>#U6TZ!gsX-CqKI8lF((J`1!Yc{>+(}3$(*;(Z=S`@9rl5GeM4VgX35_yxV^Bb*= zKUZ!?Et9>^(F*rvuy}2dvJKfYW}A6f9)+U?M*4$kYih6bvA?< zR`6cSB%o&8SGKG?xH094JaZc5sfdcko(H$@Jdvk{cB+$7X6RNBzaRUf_&Q*A-*h|; zmYTjlVMC0EUoZ=YC>N9LHF45npYr0sUWB!8na{c;r?7Y?{`eAb$d(NoAz0u{t9LKDZCd&ncp z!4yId1Ui|I{4_EEa|gK!Zu}AYo{Vsg;Tu*Y&Sp=9AUM2q&N;R<@F-*Pb77Zf@Z}AWx;LZP@p?=NO6}lU1>y|px%OdgE&Hmo>ey}OysLr{b4p{b z80RuypAylGV(JRmXkG<)HCOj8V$kY7$f=&+C$}h`jXP9a+++8$Kn}4#A-5B3^*M`v z*%G3Y-2gt+NJ1wfSj23aVbWFRaGfBYuUnI_1@*O((50(|hlX5mF&fGVu7!)NhY>yY z>dMHZg-fFaSE$S(IX>>WHZ<92wm9q2A6Sm7~1FO=p5@iKOTx~ zKdn5YG!ab`;WCY!(YZ2A&H&m!tda_IL02aJH!~ZA0n2h6R>tUWQ>ad50J^ z1ZK^wkHoNCdD+SI8Mg7FFLI$*bbo;2_nfu>^)I)m1lWM3l3ypRDrsdkLa6$e)% zo{opWVCp!9(Plj_dO^;fbgJ~yA|J&Odz#R|7z=o~gk`uWMFO^7k!li_1=6TqSzszn z;W7;y^_)@7mt4u$OeElNww3xIYiTo3AVK1w>cdfN#AUV-upRlmv=9asHhv6MN6m$b z@;SnyD{0Ox6cAI8%2aaEt&24tdIc@F+||Wr550CNypm!qpBsA>Ie>2E;U0+_E7@HR zkb1~S=LtX%q<9a@N(3W*u9h>mVYqN~)aHo0o*EhZB21;Q(PFxL=jW+6KnCsODH%01 z_vmZ!%}F4^tHn1by^t+UqY`sRy2x#xLh023Y!O!< z)0{57cG)#;mPUjs6jDY{Y&_&R5e|3x>(J9W*|02PxogkU+U8iOQJ5_Cg#=9=G-Jsq z<0X!gq;!K?-W~+QAc_9Y?kIeFe!(8ufX zOgyst=kg_f$=uJckc_-h*i0CUTaq5x3jG-x3?BM4%a6Ze7cEo!jhlzibJ-^110}YE6+rMF%S7{Gu|)s zF^$MaygR9NTrwg&5IUr(gWS1Bgk9l-e@SQfthzntF^x!iQqF8m=Y|nUqS1#4zA%Xb z{|0yTgx&xzt_U%NJpTxTHo`TJHOnx?&V`y5z_=wJ?8HO26PW2J+GpUx(8{-(iF@ zPhmmBi^NQKNk)v(WNexS(a7n2=FD(7wqAmM23MQlwfARf5a`VF^C_Y}L^iRr= ziBE09_u(+69`LD+B4tKQksVsJZ8;4OMmOJkOi2FbyqX3B^!B(3VDm$mg*RRl5C$1p zRG7KcV_TjD?17W1MHQ0q#1%Wi=6RKZGR@~!p(}qv)Al^^*`On#h?#NelpCR4Jh2Dk z+WcakqCJD?75Rn7x3IY(+$*vUOwyH+c0J-u_FS0G83W)>2jkVC z{8~iw+1^MF@#Hi#8;t(aq}rcM7yK$Xx^kKsfinksb>EV?W$H#Cyi z4;VTl!kCKGH8ekKB!QP5wNuD6V(OM`^Ly8R?_)e2p7NEA4(9kf1U^d^JYHc_EQaj1 z#McOFH1NMp&skOyDCHMj$!3P<`gkr9_GM3{7qFEU$fekC0({w7!L7lw&Nmx7+mhoi zBcO>;kx<3Z8&mv?@+`m?!S9o+6f=NvJ<7uPgf||Gek!AmJ<3}C^c09kdlaFQ-eQVd zTo-2&L|o!xV=ShqpDaQ{#74%`)(f9qzp8!ZRh9R@DGLqX-|4GwOIv+qodnWMFSy^i z{8K&YVdg{1Pqt){P_rQ)j+`EP;}BGGa)<4O1QN?Jv{V1xYjoscDvPAXw45&5TtxTM z!uHe3Gd?VIDZyVY8nNMYFbT%Wm?zfBs@wqPYgBPJHRZbD9L-&|Vl_<0L*0J=>RfaMd zZh8Nm|k~>7_)$bG;s2qlU@_iH6V|6OU zJzQ}&H^J){*LMw6*H2~Qadj?gP^skDrO8#`WOKmxj<69x$)G^X%c6l_SdjKPVuOP_ z8)5BK_B9Bs-UYtK@FoHAbqrgZp%+V6eX+r592gN^21B;6P+16(FuH<2m2hbMHlIkI z1})$>+&~5mLb}fgIsH2iHcU`f67-`Wg~ z2{*}%@=S?=qz>~l96rmSJ#(Z{a}F<$!pSi@E{-Pfi8Y;5L?MW9I!I@3>4fk{Xx{4= z_R|mCL4l=rq!YWA+zHr^jRrqgq$9&3mU?p?!hEOwe?s8I`92&>hrW9F_yX}6Vm4k) zw~0O)X*EX_wW-f`Rw_F2z5Wk)@bI1nAP#;htgH)ZD%9}Jn*e4Z2+X2#kI`_)d{@Ol_LY)Ee2!dp z7OcYt{h2ZxtY4owQpai?0O_#@i0EfzN|dTicvf}IO4*izbQQGnNOA(yoHvxl4)1yC zAN~7x-WbE6?doOm_en%FRwh@e&_c*A78owM;KyKOBZAS-R><7WW>Lu!QlB|8vClxN z3cVc@9rNF?pG~?F`;r=rgiTRW{lLLcaNN*`g#u3@@(|8NQskw@nwIZm?;3{~X)3@= z?NGoicZJ6NDZdvMv6D)WY%zykV%2ixAb{^$Z(zU@F}lxEN#5ptMejk+Ge>gcy)t_k zH;3PzV=EN&Yx%5f2;YyrpJ5h=+SAd1V$$x^p4$iHS4E+@=6h{Tz9z(Ta|3UBa?P$`i+Kf^sf6;7m6ng=mU`Ovy54x?%-9!mb zh~ck|wu#-BO6Y8Z@o8->YkoRXtxtkb7t*deE)TbDGJBM%uT!DE4zJG?B)rgmlniV= zRva2<8XHg%ZfRx4%WI%oB=#t_+%XU*lvxDU1iBrGGDM}t>lk`eNq#+yZhrtLFiU~! z#^gJz4AjeJ^M%%z@QOJ8D=q0)S=-nPXUuNidHufgmAQiFVxpg65j+kX_BX6LE$5V3 z{pj5^irCpL2Bd5xvkulPjNK;{^YZx$-j(l729%kfak_4L4NiD5ro7;|q4Tde93gox zo4>YD;K;pQm@}lG{QFPhC z`?oDFm46^-9!~qM&muIkIG%l`nd$xT-!P`zvGBu|LuXQ2sljw{S=C6+ofUDZlt(glXA6%(d!~VKlHB%`HMUNh4V%+Z-G_X z_9C(yFjb$$`ja=i%)UeLZZa@1^8$gc`hdNl1Rg)n&~&rWQpmdWer`YGsrhwH&gKQw zV1=hDH!%nNG28wMW&3ODSX%DCh9m=*-M5$YXxw;PPFLF*%k^1(gRaIeE*hzBY- z^0(f-V$c?`4%2aqIgjllv-bUnLvawcUWzYB6D3GE3AyW#n;N+0EWiS_l-&0`y%Rup zGZ-iMeKs%n%y~iOF^9s@SI{Xay1Wtk3}RTt@>Mb&ozElAsBsVWdQlK2A&L^Ke-1bN9n<2*UbSD@Px-CT!c6qecHhpnTUpS%;G{@_? zP(Egl-N_X8eb5`Ri9Z!U3>3MxJf(>iSZ|-#t(V;6MD!3rB#Tuea8m z6-?0NpPH+GZ6J>iL4M$LnbIL{5Hm64-}U@4i7A8{9t3QwFP1o7M62-+UO;$(jyD8M zJiZreGEe1ru+^YWg+9H#y={3!Vk?ZU+5?s(ZofU;KRD_hxIiZKW# z8Hz&c-Qoe7cNO_;A%dFGSvJ6{P$GtGqmNe1;9@#ax<}9i?UWPX`Z02vdtK+S+jL&- zbq~8Q8b2JKe)s;vsq@3$$;sZ)>ES`wd4J;Uzdw3?czXE$2>$xk**p4&^S8sJ*DoAD zV5Y~rF@YZ8fDK6nX*IeW7fMu~9A=R5IOsu(MpsibAzq>5FCABmfjJoDK3(N%(B*J0 zxiigaHU98O&+tC}c_7wpwy$bSd)Qj)SnI5%4FZ#n8(%8vMSM;5?MKsE(mB@#CP^%D5fz3nbSRf{dbr1t$UapDDOIZ$9wzVkpv)7ng(A+rlLU0gjp^Nh5f_! z{%eo`;eg$D1g#q_xhNYU{lCW_gR!unG84$NXASx9T@LZdbh}5JM4$0vn*XFj2UjCd zrE008X#Q`nbu#mRV`Dl07x7@s@Chg~%nO|jI@?)XpqoCeWGp&4&PnFjBVq-?M!-kU zut9j?5D*(J;E|6&M>jrm+)m%)BKXW1!A6?4dn>TaUIuq!gYie$V7`ZaCT%{(j3it` zQGbQyvGuxphKEsjqO;Fpv46kc1r;EQ8?CEg(x!hI`Hj}aFHxKR6)&!@+W3$7Grk$M z(}ECxayllpDQU2-|Zc}Ie7d2O>5YHY?)=6|LFl&58PBc@ zSF zI+Tn^DDs$Ue275??V64O41QQSh|TD!gLZsgwac$tFgMOM*fw5_br;7nr`&ZfA;_U{I~epTi@E=ZLZS!0&G@} z4v*gKIwxuC(*Qgxc+i*D0~vPl4W{|@X*ID(jKT@OcFN+!Fq{dHu91(;#Jh#G@$zoM zpr7sSm|3Y(q$&BfR;%%6G?Y1yY)+s(6rqy3oN^Z_cas2d zA6yw;RK4wzgQ!_!O{|t;S-sNapw7FzB>rI^@QjY?Pri@wPTD)rym1c;Uzu-f@>#Sz z&EDdqG1&9gUV86_L%AoJBV6?y`K{wnp?JVe1=3*$lu1NfqGCY*ZT=~4|0&-6Q%p09 zd&tSWk6gbhl?Zc_CS5W&wJNk`Bxl8@9(sxP3gi;~!%WoiI+R|1$-Kc*@&!-ozoA)b zPUPGF{NMk_=^pHVI5|B1hm(km_nAN|v2*Q#DniOm%g8j-9l=I5!o>LsA=>Zhos?W-JG{1BNIsK+1m z(w&GgrFU%X!m?Z32=RfTdWQAn^F#&Z+?mFg{)#RVhGdpH6X>84j^BH&RGi%?C4h|7*YuF(SRrPiXccP*DEzE&=g z()x*;A0=fu9ZDyA_`rnmL4E`16j$|V)NzV)<%30!0gG_Btd|aq)_46^y|ke*M$Idx z7P}R(p&Bu6O_^r``3~b3%vny3$xPE<7zMXlgoiO{?g@w&+D=)49f>wznF@@9;ditR z=Q&7IXIXiv2or5C1WG0;nK0pTMA$Z%!D039eh@CC)(uG+p87F0nCaVt3RK)_o2b9zev z5ltIls46zpvanHBfGuG}swBP7wFHRxLRn7Sc6zQ^7QT5Q%Oi*uieQEtt36J@Be)hy zz%6kiXcHnPoTq%#A5GDKWlvrcPk87+xFk+SyrfQ(Y2ZsBfBVu)Hv}$rPGjw~qdgZb zyI&YI68)Y5a$hFJ!Sz#tCAeDNL6fL#B3XfiQnM`wmn#@#gCY93fyJO_k))OI{2KW; z@NT+KsL@ldETywJ`T-?KT>-r#7y${|q{nTzG^DYczcv0gV#Jr~AYY~d=_Tp)@Hz@di zYUS7Fsnh@JxK;}&fNSo54o(hFZ2j*+_i&^CUB`1*^uG;b=CQN=RWdI5ol)7#$0|p; z_(NIfRUL=z=zZ)^8qFSUFKTn%xvJ({I3)H9Qi@b60NoDFGbD)Rfca-7hBD3|lrcAV z2IVy4@Chia_=|1*A^&SK9JC|7dupxszyHk!HOc?C1VLRyFQ8|~m`{hXX)`FJ?H)w? zfxmDDx8hggw(tzgGq76{=9b=-n4q|garLA&!Kl>|y|vXbnZSaA50MlA2+@Ho9T|dy z+@h>{qE1+$!^wCFq5ec0kUB2yV_;K8)V7_nu*MoP5gD`(qkb6+!)&%smrTgH+yzXP zsQvzo$0UShhrb%F3@m9;?h?ZeHYI2QhN9;_>zz?E0%d72t}WdN6n)fWahhwBLa9hZ ztwl*{AMjoP76|*vX%jnixG1N zJ~9jL|26WTPJpvY`R`!QmjCuoHuwK)dG3JxCk4QT(keYzURos|H?tE#sdY^gFL6~y zi2t}vbcFC5@k@+hCIFZIPz-9&J3wxXlG{rj?R_3?3)Aj&%-DWfB!iA{iO1o+A0&)h z?C$gbqW62*Rv>Lj{75Fp^LwTC#gJ&fa*+hRGJe5F#t3!@K~$&hGwK-(~E- z4lL~5(H{HMmJRaZ+07kU&HZ*i*xddfMQiY{)-dZ8O@ZFGdf7iZUIgfalOvDNh^ofOoljHEjS(r}y*>pH3)ZM-PFmmPqyL)sHo;wq6r2nR5 z9q$qlkgD!)UcGwp>NMg?VT{r7@bmH^YY}ZKVxALKM*8Kv z9}P;di|wCtwk@@G*m&{YOhV8`@i55-ULE(Ws?{3#k4N%d#Q5{SYURJ9y^~{0{@Xh| z*x$&1>v-;f{D&Ykj*4D!3EzLVDa&-Q&rj4JTS~2tUe6WlTd5= zCN)%IJHjuFSf~-CngjSV;#L@oxzcf6;8LXW-EAGEqg4H4R8!R!WVYS3? z)?gPmXAlfzVBdB1;Vn`bH@nG;^PkavC1h+RW>fX#Se}x0=p0LFKzWN_XDQEw4y>~f zjxzZL7Y=ydJvfm{_1G4Ufb99cJ8vS47U~wB7Z4D}G#9uSUg5efXMc$_SyHX?zHE|BgG;Fw~!>mA3NX9ns zAjMJC(np*R5%*AmDccxu{K}r1^PkM!xX=l(=KR;)KXA@}CkK0*^WQq2J8=Fp^#8%6 zpg#6&WER!;igRWStz^|!S6c2Q;b<}TVa(fW$Htcr$|99^9x9_75pT}?<%kRmo43)L zA)Rigzu;-cC}J8&>;2mec>K2Ywna}(mNjX0WfTVjpoK(<&Hl2kr-uJ?zo!KifHnT# zdwU1A|M%X}{wDs*TAn+=|0(+?++U|Ch#}sp$~1u2oc-()f~CrgG2)cC6LG5+?qYbs z&X5q>0X2yt{mOVOB;MG!p9pHl|LE;RdE{1jmFsOp7U6%hrg_F0?kHPrp<;GVPL15X zRQk!PokRhqJy5sx*1^Au4{h3<^ih=!8VxB2R=vfgHKzbK+M0?Z2WuyHc%Tvq(#c9X_#pi)wOs7Xp zgpS4Vzm`?jo;d=k0he0iV*4;G`|F`5}2!j_s%Jv z;LUZwEvRT74!9mj6Vx_`)-Fj8Vy)nMulw#!_#0T7`ImT;C2c@ORB`0J(jVLEV|z&x zgqXgNjJV(s4@SPjWK2B+J-uoDGJT781g2+=E9y0A$MMebE2#b90gNxSSrd+`@ygK}m)URLhRVv|#1Gi^&Z<*H(>s##GGaIksS|JFO=JxF+u#DNE zX#h#pg8S5*XBFsQjDIbA3Vz*OkjwotFf_y6We&mS`D?i3gZH~kz$sFG@rRz3zo|Z! zREaj%Qdd9Q!DRLWi0ViQ`uO2BlhkXaeBmK|nZ6(Mn`2MjfbX>T!ETujS$7bk`FwZ( z;Nqy;PY=FI_P#nk?j5BUd*>Iu!~OGvi<5(+-pN6C@8tYsZ*Nxs7+>a}JwNPro*y1Y zYDn1y@BTXoJQ)4+@7sUd>-_eoFWcL1WAW<`^k)aZKkocr()rih&QD*8Z~8_1i#F8w zvQfFMe-d072M0+fa6OGf4ir=}dOya0TB>UN9cB;Ee`X{4=Sv)Kwe0UXJ5`GIpISMw zc~<_^=>IU^?uzw)2Zx9Iw*K#UZ)5+rmglbM|6X4e`He{LY$dkf0misMpt4LGE^=-b z@CW)*U+1Y}{O%j*fM>;OkWGr_k2hr>Gp&_`YT-;;3c~KHAe<-nmPaHJd_kni(8<4D zNa(QC;mtE6E(+YU8k558{NbRl;wfJ{k=+}6B_)$c){LX)+Q(9q7MUGo_AZ`udQ$ zfvQ(QdG6L%7A3E^tQtQne-If^3@HX-6gUNC@Y|70a&;C7V|UCb0qao&Qsco%WU4H^ ze(mfv=<~!oZ-qM*Q6kNk42;NrzE=@YQ?xl1F0(Ys%1)FI`Vm_J$ik9AiJ36YdHf-n zbtb))Z{LZ08tVRbBI3VAy+M)<%cX$*?Xn&B0b)_JBAo&<>}TG{EyFLJ%}DCB>5diRwwcxA zDr3q9x+J$qA~Sg?q^!hq=)0iW=jmnAyA3zOj+mF>M)<@i6(rbSY{ymlG|zq5N`l)4 z@9fk~oeev+NvC0Jzt6?F+qfUH?}qG)J6}s?BN)2Y*A<@X9oyzmR9#-j39W0XDy9OVhz_-N;;5GwAI655HtnD_+UZdXbyD{6_uY|W~3hAV%YW1i=)u< ztso7R?`tJ!&GdAEDc!aR%Z-FwNh0<_r^q!v_@}sHgU}Hg8K1||F+X=Uoby|v&Xl}hQIa5x))tH#N8;wz>mvvhcsa`cg;T=7jhaN&qV4Jada#i_pkq&3A!x^Ek zR&#uab7ciK8j_#L2?~dR5?B_^Q;LblWG3foaBV=*t8#hBqH2zVtwvsotyVGM@K ziWE+w4;vgQ9;ze79LKI150d0|M&)heI$?2CJlOl{XwP$!Gdp&VZGslVd~~fVgUSdl zWK`17kRe=fhBNdY7DUzjUN&97Uw`UdS`I^5TSDO}s7Y``nU2!;V;&axJZ0m{^1|E* z2%Z#Wl6Q!>R{|U1Ex?xg3=*mDqK?hGmHu;W(Chqv_(l3E{eSlNj*lGsZ}xL@|Fe$g zuH66p5fzt{R7N!5cHb2`F-e&`jDt{xq4P9b&}ZU?!>0^0%P^sZt7gz8etL0{y~pSR z>Zg4L@seCPSWR`JZ;Tha>h|Hq7&c(G-(#pR^!FBF)@2i7+$zpE$-yhaCNPl#J}@?1 zATu<=m!S?(`cO+{6FW(W7&9dvGcp83Et+&!Ddtxe+At+ai8i?;Ycl{Swni zGcn>k8-Tci5MZqE(*`ocq)5|UNOQ4kCWZY>j-L06ICZ9j-r{IH@?5f09t@71dDu>V zlkzRM1lMV{hgldcL1CcNY}i*fPch~rxwnCtlAmS5XXURf?Hj?7Am}n?P^i*3%19uN zwH1%j0SXRYLr1fvx{TO^Afsujcz8THAfX7-bk^Fuz%4|V(uoux|QAM=CeuoSA+iPf@ap+IF5 ziR=YfTl|hU_ItB6n_F$64>4h`XW{Exw>L z98^AEQ3q27X-*0-^Dkngmx6Bla5(MfLxnir44-QDBxC)J5G6QB{Y$MhlM#5pG?A=S?npOJx_%LNMOMS=K z5$zcu*Mvas@gT)&&zGQN37* z$WMvQ;ely&!4OK?YzZjX(J*mj5{Sp?K>$w!#do!BHb~egG%G42VYwj|)04N|C=d6aZS z2mkQU*kw6Dx?>od2Cr6#c7-=3UT!JKc+@^AY0F*AdaXB2Vs-;|k@|xyDjOF2Jt@y* zxL6Ed6&5rg2~WftoFf`tY;j>vShjYs+cyhDfb))_vUff`Pa$L)Xe1 z%kyqPkKu_n+Li%E+Za!jL|~>J(Ss35A1OcMA%)%tDDt2gM2*rPqVwa!e%iCL0?C6H zUi9g#ZKOhcs^$0N!u zJjY(gf~|^DlpzzBoL>^AWn1O@*P%S7O7f5f`A8-$33Q9V?~CW?7_FJ$pn2_Fz6I{c zY!_cWcaCv7H~RMjV;RUT%h4bVkC)@wY7cArV{y!LD2dUh8sIiWb1$N8*jOJA##c$y z1;a7;^ZV&AEwUba5pDCZy;i9YI##bAt&Cc+M%l|UXX^@A0awUZ{LcOHVDKDsLk}1) zd;23n5ymER+X0d_ z9FsGoEe8NYAfTTcrR*3o1G9A0id*{$mKXzy0&sZGmdy|DK#2+y37N``wNG&sv_lV*ewg^fdYo zsO;(2QDM9KmMYq#?Ef>xit$j$RCzvAjb#6*|!nmu!Yo>JwmIcze#A zvc708nNF@^$C|=uQk>`O+Y%+M`bp8dVke*;I5{){h!nS5Vz9m}Qm~e(ZwIVOpn}zg zR0x1$CRG<`K_D*3rGjG>2&GG3_FGQA9hcu#ol; z$CH*4N8s zJ?$CFF;AF7UvmwcS=O|R-9`>#hIv=gihr&f&Vp*C0p@9Puh1;W0aak!$PtF>RsRi) zFGJm;_@?Z1mNjMwb@B+Hgnce6-0|gR3}KOVMt7uH7FM9%l4e%wTw(prFtb|BzL&GA z!LON7$|Zd>M&H*ek!H~U&|P>Y9Z-$_=kVm%_WwHUZt_2^=ecwGAE6Fu7}76Ajs!UY zc~HM_1iWj0_q0%WKGIuvxxH0{+Vcp%A(__QuX zlN3Je3PXx&aX?X-G-#|`?e${{$G@~yl(kFJ$SX}H2=a5YXx6)EEb*+)6+#t7Yi2S& z8aFfpBVUl&F2WaswQI+@4W2VIuU>H>MVe#rIsx?=lc!rP8hZ=UUCUB!sc<824D~8% zRrN(P(Ge=`AS?&5l^I3YaGvmpo^LDz0HX8)^{ujXq-oTho;KkymMjd!CDa*h(pOGY%@`KgSosNwzd6B zl2s37wz;2A(Hd@NK~<}6_>L=ErAm&^_e?kky%w*wQpMbAS05cj&79k-%r~9pmrogV zY8Lyid~}(w!2avt=)l?k_YV#?`~NzgU;X|ssdhbAtkBuh^QGJlG=O`as~^$q{k~!q zgP397=e=SLw6f?Gt7!I~?E1AAaVD15a}kG2?d#tZ>E(Z~V7)cJjUx+cj@!7E-h5ju zWNkO!tDTy~ufOG#t^WG^Q*ZCJ&kXY4I4>ucMGAW7*+jq^`S19!>)ijcDjWH49namC z|N3bUQba#+G#ko#4Sz+`{k1QoVk-d~FJW2xCi;JDgaBAx*i{z75o)@)|!qOjQs zp!t-9C3BJIN8OVwK%jOfHTJB*{ZM-SItfh(EPk23Z4Rem&kXi4Yr{)EI*=jGNHOyr zASxPK@C-pG>wX*~B*djO$)0fvKqhzESg9mSxZ4us*v2DT3HQ)Ne^4Gl^&A8*iQT1H zLAMcIWBR8Yo!%A9EcI0-M?undb|HU{E8ry5nQ1;&LR+}kRO}tk^0*sC^o*R&ZndVV zCG$Jl+jxGK=k=Pq$K(0Z?qDoF=O{8!xe#MBr~@L5YAjdM718t8c~$b{uZP z$dB7m%ihoJ4)`jVH<*zPZNLRflR%(*x?)uz$Ga~=)Z~91B;K<9vEx7P z9c}#o*Yn%~{SRl3f1aMF#VCa=NhHt6Lcv{YKF#|&WF$Os}3~HEAh7`)9X}l zJ*N-3hGl3v5p6iOx@{X@L}w(JfV>)4VZ4&@IDV&wA7}ZlNNO97FH5;o>na@#Bo;z-Mlf}4=4 zH4J5(_e0#|!H;Rw&qog@(M>YKL~MgRA4fUEOT-%!Y)wSY0}C=H?hU2)uY$8=P^M7^ zav`gG8x68|>EM<-wPzx8-FcBF??_VPawjVJ)UjXbq!&NIc}69E^1 zbW>!Ll!}{@=>n5eSz&p}oR=KJoD9=R0xtUkbz)}1uJk?(FN2*%xR#@BjEnCuKE33N z6cVJFYrbVIcHa{JfSDEknhXbCekrUdDi~>7cDX2%pPgB^F&I6-zEn;%E7!T}67YZa zQzQQ;7i<}4lPq223{WlqGs*w>z>@#Fha3Cv^*nb*{^yx2AbDth0YNkR>0?fNw|p&LHJ@bCYFfYDV4fj|#X1@shYQKbE+Q?La; zW25R%2kh6g%TcagfAc==O#yzc+v1f{;F!Yr4fn-w-jBfy5R^Ugy|6XsjFlw5<4*xW z48HR|h6p`5!~C-mdCiA;O5PY@qxfgDCFsYN*X-&l%Tg5Gq(Bp=QP(59sd>BtVPcVa zyrKuYmCWOnI@ZZpTFs{sS~wH3+Uv9!3WRs=(_B`}-U1V+<_s1)M9#w1vXU9Jw6l<( z4?2~75|vy-u0aFz%M3ueRZcInw90G5HOfE>_mg~_5Aw@fka-)Uo>bBkK;I#j8>82# zDr9Z?l2MmBnw!$#RbEcGxL7kxWc2JLCCO|TezVPFcCup?Ew<0fjPnuOniR>^9^{z( zTqNJo1B!N8Fu(W^E9$@0!$GECHDZ4jwZ4}bgY=S6NrWKbkosf4)pk#u4Q7)dNaVqyz&q<^>V!Lgfqf{*iZ=RY?8MC&7FxmuQN;csXG~3Y_4z@sw%)hgnP)NRw zr(d~QXl71M!>7m_oU@=;F%kJR6I(S$*$8t?j=+~|g!zq)U4a<;%Pb9w3fgv2b(`a! z1XZQe2-BEpQgXT@;)K1eZLzIlhlJE~3CV_Ni|?nnw~Bh+-OuS*-s#vBpbDs^XYAv^ z41H&AEJ?dm+Ou#E{+*;&bz$-WJ+)uiEj!~iUJy>^J^s(K!_X3KS5`Ep*uQA8&Rtwa zTfc^Nok^ECF{rqZ4zkxWjG)ZPX3QOM867-csnX^1~ zB@tNaw3VOw%4+j0@u|80Wg-{cV=QI>TIc_9;>3U5JK4m4U(a)A?tlL%@{L62N$(x7 z8rE(h&7Dd}iZD(l8CZegZkRC0h{@>Vm(L<$c~GDLO~l{~Z=_=Jua4kzhl zTEqyJ6#6>RQ$U=Z&PINgZX?PJQ$$0KDHX$j z(w;GSm5-#Qsvz=+TxPUKKhixFl2FSAv(dzCt*yUZ4Dw`hd}x1{-Qi5R9ew}|dql0E z?1TOV46I&SOgh|#6I@|LI;dpC+5_g!GtRShDH8EK zCxNX>TBw_w!!4JB5|DD<*FDvvN9ynMnH`+D!d>ouqO#?lp zOla=L^iQDal)oO~H zXQ0WT>?FM*7!(phQH~~a%W5_Izht}FfJxQ!N&$8LANvP~w*P1M;CQqDuj9EZ`+sEf zHegBp6g=Z0Y9Av<#>$4UIlW-pRe6U8h^Fz2Iu~&|tB;y3j zz}bljA8(}W@u#H3%yQEL>gTdHTE=&E8?_5p@_RVIv;CNeKeF*BVyD&ncg`8PL{=Pq zeXBIDd}ALbY^P!W=f0BojtL5*DCdEnM8y(q%{8K<8*o${=GUoe4$Q)LFN=ISmI*si zQ0wQY>$%B`cTql_NSWk3f6t{h?(zs*GE8gdx*MH6dD46yxjy5}L;w-tdg_(-{NdxZ zD>YZ@H-bRvzLS13W+RimfT~G@B_|e1XnWfUjmd-AwaqTunuyw__5k)LIPCiavzl;g zLPQ81Y_MtP#_DK^zpPf1=_H3nXIY)n;+-a2W9;D-r$5$`UI`6|kjXZhj4g`98J9b$ zJ#&wue8|>^{knBg{t-H4l7BHBJuZ1h82(#)ppKwI_ep4g{d}H{rti5cb7>CpnG(Q9 zJYQUtpa0A_;0{@VtN^$q$fX2HLfO)nV)uF4OQt23!XKr|?Ff!EAaIoPjoSp<9A$lc zUpj?+U=|wqI00XNLSry}1NF|#Y7AAdvw-UiFjTdMrAtZkHRkW2EgbHH^GBEDBB%@@ zOL#mXU4y*d&&ndbOo~24mA*+PjI10*3Wus_Z|5GRbQ#of|C18#J$Nv67*Y^K7>F#gj|8+cfMgCXfKj@5- z0^*Sn^`21eRpe_UWC{o8H#9JXs36QB#1B)1|FfCe#KxFF?1Z&Y> zGvGv>m$fj8f}kdU_f=`1ndbrb=lr}OkWjJ$G`*Y_0jF2wqvjyV(kln6pD<~Yar36s zrps4=#`-H4hfKM)Hw8^sST$^H6Pu(L(?N5*(Hg6UPU^UuL8esb>s*|#zJB}!&anBk z=%q~|EzXl(6`1_{baIu;&{@j@_sOfLO@Ztc{ptzM*p9{IC)}Tw1RmR=MA(joK(hL> zLN%=H#F>?oVyVdd6HE`BR|nC{ydOQDPV%ys4B(cr5%5>z#0&Kn5yyNX!NQWFGH0_9 zAV7ob*X#YoM*Rlv-+J?o`FiPsbFGu%o|Bo{eM3A>p*RgPm-W$Bnk&8>^$Sx`KO^r)9xcQph;Ni8 zeYiW~mLWkHqK_Zd6%4ivs(BiRotDi;(eK#6#4Qd$z`6zVqO`aG6L7%DMLq>PsI(|@ zkP^OljhQ9+Rgky=K+S)`1gZS;VHpjxQ8t{8e6#Vz7Y$D-oO_o9R^#%UV5ar&?d!ko zb$VjsAZv&mGYJBM9HTAHyCB%xokKWRbC9%*i0%qQU8GO}~g&i6){_-3%pPFs@iY zR^bo8ghsvS_yB)q!?WoHggtL{LWjI3MGE2y4*-r}0?d-jCL(0z`K<<>yJyu6Km_$P zHf@ZS-2s0mV<7yu+(JHJhBMeKT3@!J)<0VF4=PcJyO<6Jx6vPoNKgyi{9WnKKZ{wGE<3Tw#OUGsuRSFhZWd=DRIW_4NBQD2tZR4&0{NZnZPdfj4`?mk~ZRe-aaJ8sfOWE9XE*){bK;Qd+;!uR$Etz%eO+`V`CDD~ zl4ATNzY(@)ntHj*D$jbKbM&w&#)Hog4AeTh5Q$X`2OdTn=V*?~CD0~2J+|Z7?p+^R z%0=Dc zVQyr3R8em|NM&qo0PMYccN;g7C_aDdQ{YEsuBEwZl9FY6hG(=~-9S~LP$(1%g+fdipHjqr4SUBkNO^Yw z=i#3=`|0=l{nv+w@^8Q2&;LEx-#_@%{^6Vc>%;z=*RNjvslWg7u>bl`pub@{NNl+eWhoIno8kfALA@ElGN8-U-o006)P5uBp| zAk1Sk6o&GlE;IlE@$V4rV$vI99>F>8nR0{u11~^h3?py% zwoqZv@9w|q_H7)8hov}vZKOFXIoN;G?YmI_wgo_leH1all%p8>Gvpm~`&%%^KOxF6 zi3Z^QU<<}^`cpyK3Q&xq07X7VY_MhM^2vNoqAdU@iV4G9@}Ky4(Cbk&#f;O1x8E&L z>5K8owgBL8Dm%{u`i+-Dyf*|KbU$=IYz2t<6vtdX8mdN){`(y4=$_WfXpk)KxMdU%CnGgyn3Nakx5c36qQ6LKe9C0+Im@m5E==khJ z5Eh3RMtl!o4pcnICy#3 zjic$-9a=mP8p!^$Br5x{<)sYT@_>3rQ5<4l@^8xnewZ*J;&X^24snF2B@N$E0t27S z_rO`?cfnB@f@?gTaR#mtL-Za6-7NqT8fFcfVm?d8vRy`kwQv*!U-hX(*I2v9DMCF# zWXZtTN=0k`b+_Mrz4f^@+4EK6rabLk*sOw z;I)|d@DwpV0Bi;iUcDZ?noQpO_2rwt?Z1L(((k{1GY(!J9K6{_=wQ6x_y6{{{+l<4 z{{CzKaQu3L`tXha3XNZ*S1(_^?E9$y*R3gv5QQ8C1JF6>9~^r7eQ*Eu&HlmQpg-vM zyRY6H{_WM9!~X04?L6zH+K)B z=ZGW9#M@zrAVYf~A{+regBUtIKrdLf~3!Hbd}#*}w!$zLvU|0VoY{NGVjv0P0S=>RTS3Zi}$|=L-5o z9bTUvAKjdu3}koJV8;>T2nMDhB}Fv%S@DszjFT(A0#R9kE=>3={XK+?2_qk*UqnNN zP<2d^4uHWNM}m9`c8tg}a9axKcDEe?0-6v5AUaPXg}_f3Cv$~}6W~(>IVz$wRokM9 zE8Ex-VZ3D9(p6)zZRx8Tbf!#Obm#+S97j_n@d_!K0Q1-kI;}9IAZH6=pCYN*V91J; zxR7smyX^>42!BKi)|H}T`2HhWq}1J^1;EiJh^)bOyW0g+pV+(0|e@+wa2T=6lg$a)kPOPK*}J#tB~ zUS#bw=mE%!wN+e^Q1z~yQ)!X9juvJR_EA^5t*jSzhv_mvlyQ2js&cB)wp3GPsf^8( zTaC7+wAAQ0Ci56_Y z%;BB1{pM-QLIfT#pJh}7iWsNZ=P2;*55Va7cmODxlY0bc5*@L34ADLH^(`eyES-)M z>wAb4Wrmr?-ozBZFeDEsNKK`)6V?@y!6Y4OeKin+h=3g!$Ke8C(+Z9RMFMXmG=laJ ziZPf{zPl&*?<24iBPubAs#*5{7(T!S+uZ{JW^f#$0PF-rSlGZ+$O`YK{j_C9Q?VZu zt&TB&EIk9RF~4P5dw0ngX|Ff9GIqQrg4#16YFS04EN4K3?no<-I29b>L?9JvFkECS z1m&pcO3S%BNy1PXPaMrf?;x-v*AvFgD)~i`TzN%(x083~QuUgX~(Bxs| zQ0WdoA%a+&VaHk$)EfcGDLODUyCLymXhB3DIE5#2cz`3Zno!mcr%-$JnP`U^KWdEF z?Abv=T%nLju^>sCB3WjEmJ1218Ilj@aEw9*j)DM$>MJK=a9AT&$U4%_XxS*M;sAX7 zRDgMuZB$9X*2Yy^=oScr;xrNi1q2vQBf>cL!JGsLXjx$#ax_UohPZ%|mAW7Sau<^B zyDo~<0IYz&Bgl;@86&;q9iwc}3MC`Wa5A5Z@1}qjwgAb&D+jJdGXN79GE_&90d}na za*_$JAjT(VC!(zJ*>oW)5A?1Y3lWOzkU~0T;7I(l5;{@c;KI=aN0=`hWn^1PReI?= z<(jn1sug{vi2y~DY%^t+jE!g0C-WXH+56~mIAuLcL?y&-x}N`so&W~m`th+0|AvqFX97#jrg`hv{ ze9XWLq2U>Su?JAdWFutpv&kjlLy8!R_zMNj88LV1Y}Li27Z9KId=mL6=JJak=sgui z(A$nF@&uDn;HgJuyk~tm05(pA@+wxl&*1_v(^l;#6Ej=h!=+tjrjO?dIr5AI^c4Kl4{#obW)g{4 z@#)27?!1xeGfopEx_3bm@F0aU#COuY!VuC{0FH!Bs7yx^fsjmD=4BfvQ!}eoqV+84xI; zXBMMRic2g~^9Z-h8aq=xYIQEBdK0H1Rw>|!F+7a`Oc&R(=c}BQU2w%`h(2J3_T-9( zBLICLvCIS+heA0SM<8aho@;B^1Pr8@6-iyui`oTe(?|w);7PfsP3WkYl6xGW)FE8q zhRBiB+OKNB=3E&07B1&IU9n}gEn3Q(nvQGN<&pzw`9%AixCG`gK;rp~y z6`pDl$s=2hhb5@W+rBApakpv%U@z}A-;;*^Z52mu(@}|ayG8LB%K@6iU_!}U4N^r~ z7u+a!i6C<>_w!)Lh_SSpWa=GSfZH&DajIKy1v}4Aj)sDJdO6WmWx8V=1@GfF^yG%6UU;9AdZaK*X6)cPKWh=B`|MBMSSOP9w)zSR4Vf^`WK41*Cs(LJUll3vGqNU<0- z*2LjJx2z~b1odZ_BcCS}+01sH@7yvDqX32^LbrfWaBDU0R(fIV@&k|+7pSf+?P-iu z6@9q9?J80Wi0nNZufSrVb*&QXMVBoie@00}5_U}HbIhe3YCV{@<1xK z8N5dh#Wp~2NjMsSW9f7fI&dndRWVb+NDvAGgT;)$VNoEJhTR#Thud%89ebm{UmpK(eRX;DZsZ+bU0j^qq$4fg1q09_lZjBV z4AS2fBweOZSGC~+M;C}OI2D>) zeUVgz=pKb~#XEx(Wgc#6la?^)97kwK(FA`0H)j7ph_!@;BLrgga>wlM;ECK@VTkXR z!Mnk8^luU&kbW_yt$-*-IOA|0GqvUmu-rPcBqej|y3*i0cY8K_m9f=?!sPj!sfPj^ z2LMq@sKSu3MMJWhX7(_o{aBc0G()5*ne&Is3Of2<6wTs+RPls{;7luKwK*ppH4sb^ zF;VS-SUC`d$?`(iz%SBRp&W7zcqN#5 z#*4+_$>ox~7nOp@VSfQGj)LXk;wWeVH(jEv06u=Glp574a7_p|OhZ$%S%HMzln!Xg zZd?$0MZc&ljYlOY;-UB}jMT z)1l)4Er9lCXpW5URW6>{t$`>yYgEQZt`-_hjbY3{fMSY#$Wc(jIUX{jMJpL+l>sA^ z`hhvoD|>KrJ_0E6=_1z6*b?;yWjl#VwWWcS)U4NIdoECFNdHg+#{7`7Z~cfLOdYp( zou{gI%p=KF{ey>k1zEG@X(j@to8MiZs}U;$Dzs?GC|6uruAlcp#ht%0qj7hM=cBKVJ5)MK<$7bhWtT1MCFh_Gj7jo|Y zZjpxHsI4}+UQA8+0PIM=l?uvP=x)uH5qj~(b`=_M+Y8WrkIm+CJm0=Pytu8~6;LQW z#g#?M=N@J(L9BPU|GKVrMy35#*M0f=)uG@fo6S|2wYwZoALnf?G>`d0h|9fvEhiPz zsJy>Nak>y)P;m2S$Qb@NI>%r|Et46>R-F}cXiO%zT%hNlq<6pe^-JP|Wbwo~c=s=_8THKHS_ z$B%J56Z-1s8Il5T9g|Rs{smLWWI3R!qR2-KP^fCmpz1)iTzHY~+7{82NQ@xJZb{D1 z4of_eGss=V0UOy3eYNF3G?_U z19?keaJyZ{vanjpnx86^pA{w?9E`djz4nz!bN&q%Oy2ynE#p?e%7cgGI*~r99>gRlpZg<+)3%zMt5>I~^!{esrXs?rNP3N@m%59JW8Be|07|M2J zt&o$$whpx@5IO+Ac_!#Wx|pTSC+$9(XpsXzdh60RvYdQUEHhuEqB37FxjNbZ+d+5# z^$8q?PH3DO5(cXv(|{kf1SRTM087rQ+17wYqc|>@CEkl+Z?LC=i-8 z?JUAP7&q#Jbz+QEFNTN#y;YH|cEF*cTm@FyAhV-0 zbjnJOnB?GQr2Y&;VYFvqeLA=!G}EDR4p~%i3L^&9EEtlhjXn>!nG}^NhJh0-0Seh5 z6Vp)r(m*g&22NQ6LCsh+i2&q4#yWv9A?EyuvD}sIuBxQJ0hlk|3!LxOn=YG~JVf%X z`1}9$#yIM+ne!C+GXgp%1TZ3O6dX;v9p{vdMg%^7YQ=_Z;N*r-%SNi9b5lIyd|Qt1 zwuIOebL-C2R<{1rhF-8E3=T1OVaxhvC}No7dt?Uk(CCZ<|C}B&)|bl*`+gBB7m6Divc|ph9uPhV*u-V}`=8L{`UWLMQ^kcsUIcQ?ywv z(yTrT{HW9*dCp|bZLd5E*lw3q90dp^x??Fyt5iXeR7V?B^X;}$6UZ}*IuaY`Re?Dd z^@61jLeynC>WvY|`mAK~TmG$$nC+m@*1YgkPyCso9bG zyDVpsMpeyQxsl;;Q(B%5KjpS7;kCH+?;pfM>Sl(3{E|iNsPJ=*Q~UHC9BqKh-P{|g zo$2zxWvU4YuVO<20ILz{h_i6lErTdLjqZPfl<6~jvOkqsSr&Ds&b|dwbV`i*hy>bI zX8A{LEY@iQ<*oSs>S`<1e=^`K*W0#$SOnx)YTyh-k^D}9^Ac3J#8O7IfG;Lr48s!?!o>*r zBnp^ZG-H{cCxkMUkbEj5m+ZPNpvQ<}5*WBc`D-nJ9M2I+xPf@1ey#;_0&$p7bTgxf z%}5x4@6`jq($w4V*;iA8&G!M=S5IB$ z4Zth;G~fOKc&*H$TmuGRzqHw@xR(m7`*t;n&0$zhWZlYb;5Q2G@*rD??eaid3hwfN zTZ(Q8V0k{qe!7(KzIt-m)e5>)a_x7k$*lu@IjOa)Hp6a|*yX^rkk;h@wv^Q6fVPy< z0#Idbr_U-)?)CNSeua$gzgkL2m92E{%>2jNb?ZEo``cM|AeFpAS!5*2wvUV8w9}yq z^|h4kutDT`o>C6A-g`NBuJzuaaTJz zE<}G?ZtE6l(|hBPj9mw*;z7@(!C}4kDH&ZJhui1-ud+GS;qw4=1aHb$fmvJfy-|=E z*}MQxWMB@r`b(bdKOVjlLaXDVb`WFgEL)B#(jylV80ckOE)PKpH1!J@LI#;vz>kvw zILLl0oCEM`|KI|r8|jkfe65$oTKoNj!wX#CojtSyVDeG{otSD9YeAD)Og=&%c=@1- z<-}np#VS}HhBn`(tr?8YDRw-A(G=+vCPE10j`4zVGzU|ebj+6}Od^w((Ak3dJXB(J zCm0i^r=6_xIjWSrrGZYMUp|kQK~Lg1L~|5z7|K?#Jn6~_ARAC_ZdG#xsDIB+Qe7Ss z>$rcuT9wgS!ruWlJ~8EK?z(t z6te6JI4(&fkCHf8Ua$%*y7Y2*Vv6r^h^8|7zX3~DcFDFN7+gi+LQwa8Hr$(ARa$R3 z+|uwE8p2PR57mt{&Dg7IE{ETTvD&@Ch;;)>Vi}SM$OPDL^!}1~A1}*YhJ0bx4Ji?- zw96MUYHpmaBxZSeUNdzDmm{gtm+sI`cQ(^;m1;j=4TStdEw>ifOwjD*TDdq_)v9{T z24ZvFtAWHg(YmJs(nfq;1rn9t-_Pvh#D##uyG?mmbzmkN`eK~2)GDX=oA znK`U@LILbpXTde_U7HY%q=~6#!{Y)S;@L)MI2#_<;n!&~FJLUxhm|BVNY8Lu2@z|t zg;~zbQt7qxW?J9#ZL-8EmDpUq%nCE^ytyq>Nb4Grf~iAfldrMmvw$v#GM^B%3hzYa zp498+a;^cWOX+p8o3AMC4P7Wrlv2!v!N`QGkcb(i!R?Z+0_M+9kc2pzdgxw-)byat z6DG%UL;}>?miE5fC#lsx66;VRPlFCRa>E*H($!T|k-3&~qoSx3Bf}2XnzC(*js-+H-r+Lx? z$E`><*iY4sE%N)F%q;Kr8#+K9{I?guIdXzh?@F`K*EB-s9j$MNE@Yb0lKo|E(LIrc z$az3DWxx37z;2l7!m79gBPFJJb+Z#tGCfZdgRbaCEewr@fvt}!R?Z81^@?y$M~7AQ zWg)$qjK9GAh3t-WO4fBMrz-(?ghJ$VLcwJ!1hw=1>KtvOa%?4ID61r&lQ7LDb7Osf z0(lUNB`*6uBO)c(HbuRtTbS>G$Sg}zGFxx7!En2 z5#$J@ulMAYT%f}^p?9675(dlscUW|#r@Peh0nU?<<2XcKiYZ%QIkZ)_fFnA`k(AT7 z6#8iBY&EA_&gCXqgixTeKWQikOa%~m)xnZOt0+pnZHqXMOk6Rkqr0-TNgP0qMw~*9 zri+3hVoWQ0A;olIX)v?_N=Ycx+dKKR3A!!}6pY-?qICv!RbKie;sS>d3=3~>TOssx z>u?s3=(c8H*-F=T!<0|Pep0ESrDI50Y1=2f-%sC0`ign6eJkn8T=j0oJ{ezc>sa`XLS20hR zJI5J=ax655VV2{c-t;syY;OY>OM#04@zN6@1s0AIwiuHS)>da4hWjEQFZRHTwC;<4 z8JszUY3qk104JEyL`p6AE(xZn`M6Kzd_$ASv(hWt8&1KveEto_U3~?g+CJ*(i%9^i zB2H#DuW+UdaC00TsiO}e0v9;K^JK2Gsr6z+eH3vxmC@MJo|6$G+6hOIJk4unzjAlF z!~s9Ni(rNYerSVb4j1ZDfB=CctKN04OSs*aNs`Be&#YrAxpw7GNf$`EmA7fgWGrcR z+}NBmRSdV>-!|xUN$3Ksk9HrDU=$<2TKky-%c(kDE1MmRLszN_Gx4-HIm=wo`l}af zkl6}Cz0g{zYL#pk%Ccd}wqwM^vPvH5fQEFv^`3cs_016W`n9-Q(u7#~fWe11R z0;t#y17s&pD+pw65^XLK%6Hw%2W^(F+7+i$nE+DymW-cR@>VfGP)TLDh13(2%V7<} z(58)5lcm5fL7W^_XOW`nAM!Snf5}PC+|i-l&_tj05hOc;TXqK02I_cgtjh$gsXMl3 z<@Al{wUVMix`ub(srS;uq&@k?_o@sfLU~Oni8zXal`x!sh(+7va=Z$SbfyVh+1b*g0e)A6#7GaIkkfVoU_u!?1dy5&*` zm3r$0mZaqepd(1COmAd2&g7*GA)c3}NgV!ktoNB+HDuMROQmGtk`cc{$Y;`{7enpw z(z&>el8~iP0AQ9cP6nG)rsBxob?gXoinBE1dkdmjGEPZm$yli{qB3(?)QiQS4tF*{ z^>t7<0s*?WGuEh+Y{J+c{v-#3+3e2S+iFNF0$TEzaYEq~Da%%L-AFyq0Rt5oNTdKe z5+&4ofF?LX!5*-ID*5^k`_X?|b+j+G<8{iabvk?CVTS#gx#69GU?KX%SGl5u?y_8~ zXA_V_nhLOE9gp1=Q<6p%C1JP+B2qvno%{f^Onu7GBe^OxhrY~l+Y~#ZT>y5b@c|Ii zI%T`F<-7 zeyg(4*&et?F!-5bj;cH8ckbknmwexS03uhDZTd){1hUnQMB}t!TRRc`(ZiLNpups5)zDL7x68BW%VdKpYf!9_V6g`3T3Hrr;I5r&u?F_IoXFVXl*=pecuAiN1K$EROcN`wPkAn45IIhGdTA zB&Qi2t6zVmX&oyX^qkxAi^}cj^wOFM9-CloAj@MDiVdWCY(%kozQ>jfdQSSdjL>dA z^J6>kn@#`N4(HlAAeTV1ULwdP5UrmLvK_Jwq=anQs>jO<*^YomP7c|Qh_y3BwjGAg zX(GRvG?DI}z~{*o>CjM-Ffv=weEBIO3+>rV-pDd;w9On@An9|Yk8~H!OXiTQZGA-+ zNmsbr9b0h)EfwjST#I=dv2{kt+9p-!l+5E>A+cl==DO691@x5}6CGCmaWhO7`>QU^ zWR7Xii6%>S+rQvclWy!!bH2&?hCOx0$tLP^X(w~Ue_4qqo7z&$K3UqFFE{_BJG(U{ zq0AXj&uJ)2(ohyercEx&O)++3*(jY>v`k6)IMT6lQp#Kto-i zCa83`X}{Vmm99)Ko2t?^>%N*~l}@7@GFE0yY?rgL4t8bY%5o&X=hT(ONY+QoUsuo7t11!?1HGOJs> z$y}M0Sk_CJS4L)Og7)@?F%=5?xUewWEJ%Xzr@44RqFUL}X-@>G5qi8NQE`}1Yf z{0{neNo)sF$t_M9>#R!Pxju+2hdagh7=|DuQ|!YK1j#%GJ|)pFWNf%-@?LgRP&)2t zj+|Z4NRQqHm?n_Ih$F;O7>PO%IHho&!NCkfgaZgz1YL<%w#@oX{3tOg za*zej6H0|7Sa}G_Q54E5mu>#(Ka|LRQAZAkR9FEgV!4VVlm)$3(o$YS8{s>pniGBu3nk=CdrV36 zUu4`@*9 zP&KXgBz{{L<&xP#wi)8m=bwKg{oBy9b8ys86B|h_vI>e#q!(EQ?}{l#Rz|f-nvs=} zt(t0N6@)9N8(FD!n@%~h3K5Tzc4QSoR!lvz+A!Tn`jMvGBjswmBL7Hv29k;#B$Zi6 zs`8Mu$VAdE7fE?GlCpdxRT)WYa+1{crOJRVx~XJ}GOsxioY8%wqWx31VQ-A19-F1V zJwN^0iKp+yk_(0*@H{_6Fan;3C!X|oG2p3P>U#{mSQx8v!~`a+DAT|k?F!7Bp6YSEA20{Wc3tMdS?^l-Epvd|9;MZNtF5Hm@y*Q` z9@&XnCPR>9pt=))G^+M2=9eJt&ej%N#a3w6l1FT_(6+)cTgTGV-CIn)S&=Yw9a`I^ z46R>meFbSltB6h$ht?HnNF6Gf^OVU$Q#qMaF3 zZ)m#9Aaj83+fVYjL-mU+d#GFYpsa41LA3N3Or65q=usFeT%aT)n%WU-BuSUcdmGfE z9D>_nnH&2SKxR~pwJ8gl$!bH1v-v+%ky6e+RTWX{)~DiyPv5G|(U3&wb~hu#-a8)t zbPVPR;}(y>97c&euB8(<=b{0|B#DY0Q5u2JqSZ#EGarm>Dpvc{cL>ED8N_{0usokf z?sXUh13^MnXU$MJ*ZTlEZXT1U3(gU|*RbTtH`M}NPC`)msHI&X)QYp0U^ULAahcY& z*zp(Giy~FtTdj8bH^0}v>D_+4XWC}BKi?Il8}}_b!ri#H_wDl-N_OeX_vzbof89QP zL&uv2I~(?GT54In0(IWftqrzng&_+XlK*5OHd)w>juf(RIZ;0nZci--TJur|{ES)z z`&j53V|t1rmJ95WP`<|z0z8oyX$dg*nf|>o7DpaXS(!|v8XH_zJ&R~K(uXI1O_1Qj zJ)&u%0Z!CLV}Pinb&MlvJnqO8Bv!OmNc=26!LCi&<3b-HRA+8eB#kB?^`hx<(yhW}%9w9|Ve$8*&rBzue z+EKV(W{Q?{DU$3Bb-jf@M0AeT!8dnveM^)+%Oz6CN({tnMT`Yv0Mzb)B#k`COWdMB zZ_DU`ALqj|cG#}c0WrZ57r!_GFEVeG-X?efNTf0Xi6ye2u!!E-GXpupdU5cidak>Ei&@<@(T%W!@8{J(0{r%<9#p%CtU7!JZvDIx7 z5-^ZjqKyf|QXSifhgYMsn=6gFN0`TEC=8dO!GP6Nusa-A62alLHDpnC2hZmTSD^}U z%H*L4YpuH5s^tNmipjh!UR_)t4tRiKeFG0(y&gc=?RL9h7$V3JhzJ)lkF621r_TED zJ!Qh08($~K>RyIyO`+b zMmh$-cmRxyDL)aM4#56_7oahQk+*-~(U)pLthync-eW5j&DrFVaOn`p99bm_6-GcF zgcgH1O=qJUo_aM5Bh2b5T~(Q7mX?F`6l!-f9zoF_os%b~_#R5nlwxA9)YGuN4XloH z&STFP*ShMDXrT&RLN}n)u9_y8$#wo0b7@aeaK8>0Tn*i+7p|eqS9*s}U8|li@~ODU z#~prEca5ks9Lo7Y9wIwJ4@^iHsAMaF5O!trH*D8$rOT~|sKwwIh2){63){k~lz}I@ z5@i+2f&9mck#P;fwcuAo;cO6t2~Ti-Y!*-1es$Ti8mx^MWi(%l(a#>}{b=h+x)q&m z71v!z;qLu93!^LP)*cqtLYi&b2l1U+LT77=W0e+7nMLYqM}43#U2UI!h!Mpq7;1Mb zpYBQ**QLcYY8{r8Cvba$lpOv+aor&e8HF0^D>8E1f$B6`t3czn*H;@_HAb6MefNgD zICI~*N#)X{aYDbj7kHFh1(AxEn-Z-Ol`!7q5wu93)sq40E*8Te<3$zPQOwvvQ)*H% z)2XYV0)!bzT_0AQpqg7HQr4X$VJHhbS$wyE&L)^k&z7UR_GT0rM$F{3C)%G@V4W&R zB7FkZX>Ybwv%L+D)p=t(KbSCRB|ESt|{avxbcSJOH+F^0YoRz2dcMI|8^;>&kttm{l|`tFj^Q1C=p zC1Tuf2`B?gWJ~TT5qMOBCVfum`n#j!EM-%AUB1W=!XJx2_G1d{6ZeC6!Wx(07 zbjzgKdX-ZY(Q&@S)DU1EgqWo)R-|Z;lXFF(4nAIH`JfiFGG%odn5myx9TT(I^vlnB zXHr-|>It+XlJ$-z`1$Di^6c{Mp!iN5QHd$?&7#jbmVdwBJ=hmUs2IO=&&~%aN{ZL7 z($}K1V-^?t+MaMO5U1E@xhZF^p1N97Kg_mcIiwswVfxNRqZ2068Bm_hx;0KI1Ffwq z%F?^T4B@FpSm%ezV`|}jGRKQO)1cP)%Y0BS0US=-0bAPb=6Uv^#?=h4wxT>)lDK?|~P` zSC^Nk$2Tu_!A-8RMag@H|7}&<1*hkuQ_u&emnVV(@%H_AA?DWaudgmDfB>2Jh)Q3F zq+J-BI08E_j4j=Nu?Ny0`{IXT^^4t(?U8X=2C{VCZjN#Q5+?I_cg~N#J3a5@-v)^J z6vxW#*15D66NcuFiz91jC8oDW@7|s^U>f0n7cdD+QQ6wLJ2i#JMO8{?y+|w^yj$g4iY*H&Pa=i=Nk4)h+F zYca@pUE={~&T$l#+{to;IN%zbf7h;V*SK>Ue;MR1zy|CidrWHF&Ny8Lvkj~ouYDOG z>(0$~RJ}4V#j$U7JuFb01{{{Y`AWkK zi*0!hF02SH9IbGfO*#oVm(A2Ghja_E)$KEReNcye-8)q_lg-{7ee~odspKK@aO^>K zRfFBkx~p08Ez4gEVD`?`Zbv?Q|CH=G3HxIb2nJA+M1e=iILq_-lU6O}S^&eRjdH|p zbesJB`NrE(JL#QJg}Z4o0pmnQ5qSY-GH0e_gpo-U1@Z)uO_Uw%qs1=Ugpe_IK83OE z_AeGucbRoQegs{`)NcLQ`1DCDDvR(deKnXFiWelpT;1DPIU;qLhJmR|<>L@@c&^P% zDs4p3W-&P#igh;pBDv0Kvbr?56-1@m-ZC-1`j(5%Dp0r+SeL4bg4aleL!=;_wBRX< z=CNj|l1cmSVF=F57s&O&YQvE%i-8i5pnTunn*Mx35n=Wgm$?qkOoj4xl@sD8G6GcE zAuz_>d9Yn3xrWI0%~F(_<%{{N&daOTNwpzM$qdIEqYo}wzAo3p*v;K32&F!E7)VAW29RKx0=0mUgo zt^rlj3ER$=s;f&ti$vuW&7w$>EOTC7tMAI2&C*+BWfY<`e3(f(Vf2cP+k#YbDz*Z> zdVFmWabZ7cK?;thh~@cE0Nb9T8HothsqRgjll{y}@v3*qv!PZt46M3KDb!!cSk@{8 zFsrlQI=YvOwE@qr3sd2^P%RdJA!8k}9N80{O`XA#g(RC~^=bL*Bt&M5N+p<31qEGA zIr#SgBYScl&|CR?DkFfbCdP7rVkA=lfUZ&ggo5s|7#Q^-H`VHDTdjzVbu5X-CdGnE zQtvdbYT9*6<)>YkXHm|8S$Sg3p{<)(a}+%0#G0dODJ`8NbE8^1N0$nl&LMOqK&^A( z9Rf^x>Qu%AP9fG=GMw9TRTgh)6oSO@Ij$PbeMflld>gAVBK3d{5p=z*87 z`+e}AZmy|`m>%}$_M@=u)?J3~5tZh>Pcg?n4DI&?b33i)#4uTHlT!QfBWR=^eEQ@B z(lQXj=~URQuqZj^8Tmi z&mZZNseqpH=E)ll=CJz%&9&x8LvQ{_gkRynOkm{lho?*N6Q#uV20T zQ-A-}{=vbYKz~E{Yd;C&kp8K^Hm-c*zLL+!EdV;-s6OuuKt~uQgI-VczNaOD&}mP7 z-R|s(;(QS!QC4|8)U)i`zs>-BlsctjCx%O(TJ;&HIGW1(>N)12d|~M}2S!?QLg>7s zHY?L-mWmcIj&eTgTPri2Fvo5!t$C{qgim{#?u5_E=(XDG(tDKf8H%{k!Olvip6C*` z$>++b(tMDCN6veTs4*cSf*Bf@GQCs!&yE@A|Lg!5%yA^_80uEaGf*a21rVfZc(wIV z0j#Tzo<3lP_GH>Tj&x?TERSDE{5$QYmj^?NL|9BpAAib9q^O6Qs4BMPWGg3MrbWi) zT!lX;mN^Omg;7A}V88E)^)K|rS_+2o3`WTuQS1Xn5Wwka%;nmh24%0*-fTSS0{|(w zx)$0&;kkNx?PQLL=)8!c;J!Q=BsXL6ztdx|83oHVTl9jopf=O&#!fA1vZ%9qbM+3F z1R`^0&L9JJP8w3UxVN^88pp>*$3>0>s{LQksIYYSyA4T-ZA`Zo7s(*GGL*OdMfxP4 z0o1K*mGgXtg8)Uj1_02JQYTlQaL zR#+D)wy`d%Oaq4hy`qp8u2~!jP_5WTQ4~lUq{BiSqTMLZWGH&(Oa(%Kc>l_RgNQGg?krg@E)QXsEa zO+kF=Q%oT#(mE5(Dm_=llYEcLd|}6FKyIyJP{_&ZCNedtn6n(L!h*QY=U;7_U5qFC zi5ZrulXUV)1448-QVWrHTeKzq|J2TtYx)1O$hOO?o6}L3f8d+(F*W%A-n`l0FZ%yp zKKuWk;&c4N(e(|uJi0g?fR7*bCS_NyYd(G2Qg44cy&j!iUD^eH5@U}6K1}NUJ5Xr zMuc(fUyvZ%!Tj`T>m~&x7TIHjq6`p@KvogVMNM!NhWhv!n8ACbYpA;k0vUH{QecVi zDNJ&Y=mW)^qe$MfJlNXGuFC?F+T3msk3;QT*Y*Y239!%2lu*?)gSn%RLwCS8yWlqv z;V2N?I}~jO0MaSjC|Uuq7U$MmeYKTI@GWy|jTZoYKz_n;G?gs5OUB6OA($erN_bwj z?<)jFZ{T*bJ>XBDfENN7$6Yt1&%~sBo3AFxIWi{+R6yZ^B;Wz6X&^x3Wa^M+Qzjm^ zq>5wkA7Ye%tM9?h52s*wb@HFU^FWWpJdJQs(-!MlN)<(%F66A#Wixe$+8DCg*49}B z2o029#TaSGT-mK-fXx|64OcfpC)G(WXp>5RNje%z(n0MSd)wRItd^_Eckh>&K#Z^6Sw@2K$LyJtZVZvt`>W^r_ zy3)04DEJXAgsmaH4nL_8vBL%7bm0OhknX-UPr(lQ)nQP|mCo+2JD@zn6#>EICm4Jy zblC&ez)nVwy3WRG zod6o4%J%M>j@J$vr=>!gto&uEzbI9)>FL}+YFHTv=8UQe_f4>QSKWr8?vd%qi(~{}BbBx-omSDWnHX5B4tM1! z8CFlJwbWXhTIDIv6Kgt76TY%+T9Vl`00g=lB4;o;yV!ot3?iUOq|l5`x_Ntj zI(mO}a&dO~et0w*{d{$O^6fV}_N1tjt+twg>Lk&mywY;$5%3EmkyuN9+xgfHAcx(L zA1i9-HDU*JKYjYt0l$GUWa#xF@Pb{3`<3kY_|ajw08-7DFPI}@@@mBpJuhxTlswimn7K3CgR*AqMycBc>r6MGR%mnbs}+l|yreQtH#KRJSK-brxVoncQ<->Y z*e*b{j0f}d=@`$WbKrTLFMx-|FC^{0Z(m%ZdqiKfG*DGKy_#;6?@bSSD7yc)Gklu~ zPC=GB;Gf#9;CW^x;&~uuzlOl`W`xyEkLB|g@ZNzoq7J;kFuMYNx;d)gRukeDw8m{3 z{Fn&`@ZzOr{8T7X6Zz|tgh%M(udMH)Y!Lq^xV*YK9e}fmTtvcbU74-P)L|pBhT8-4 zEZ}wqxvXQBXq^=oh8=*dg`v8GQV@VBl{N?aH5}-B&g7=EFe<^>39ycLnSN=JPwdmD zPaq&jSTw-dP^pLGJT_` zyvwVT)Awh?+(akup>8=m+{p`bG-v$5>~PGImI3}NIW+G^lvg~q`gE&vz$3W5oG0COBA+`5R*1`N)II^ZeI z4c<-n8h(b{9L19(M!7)23&-xfZ|hcjSX#bcIO;fl0ug#phn+*6Kov*87e=nUQ8=dP9+QN* z@r-H<^8qq*aXa8MN|LFX5#_D$+UD(ov&bAAWM~hNFp!sHOG4y!BaGxjL*kxFi)r0P zuOUmQ4=fpOO-nO-(J&r|pnW3uLXh|6dya@qUIv zAyi!+hZ~47sE_}6`Rdg{{`}YL{{E}y_>ZUfeEit^%hoZ87Zgut9DGL!416-*180%1 zA|$TG3kI%{P|hgmZjFW~|KpuwA4Lp#X90@1ka#)(N5iAzA5Oi4Zhz}9J+)KQ`wO_o z^MOp?HNhbgrpdQ+!E=9x1|Tc8m7YL1Q?W2C7U0(ehH^jw7{}`1e<1;}9LqpOX|b)J z5#Th5jYtY6Mq!{X$B}ufkUs5%8Q2*wr1sRic|zfe zRr_vNHBx8;l|rD|Lh`chf_B?(o;3klr>6hh^1yW7K$K1WO|eHXMtsHdQ8Jm}51`{^ z1j_l_M&9hj^sE^<|9#CK7CPFKTY{Q7?91!y^6gKOuunXY0r=pOS#3@0XsYnTLZBo42=WP;mvZ`s$o}h^ld)MMa7Ub z&T)3ANqgqyA|cmdVGdLiBUi!P22UlR7KUT{-m#V%UfNT4XwkwHDV!UYK|EHMZ8Cn< zU5PY`gB9;vPG|~mRViu4pgYv@#^!}*gLJMzRUSvDOe)-!s3TcoHel1_|sZ5J*RmD#8;mKp{*o;$!cFcR|MzbP`JX)vrs;VSfKX>K`0mKYhvtfh%; zFf`%7%6Devh}LOsn#PMNHqwW2(jzQOp9u24&LcrPK7k=(K5CN3hT}q2Z>yly9y;26 zEV9&*p^`Z4+EY#y$DEwg8ejX#Cdq(i-~Fz_VG4!;jMKBKHV@>BF^+=wg`Q4d#SenD zwjz*&y0r?hd|J-c7LBVhdpEw-dd$62AbZhrpG{(Ke9K#Pe=aNWU3>0Z8%#xL6@_3T~+~W_uIae z!`MI2_er7>Y9tJ8fp&J|k*Zax=iFkNCste1jre84C2~L`og<_~$7M0U`b(!3kX7U@ z6EvBmN@q22a=tDjN)m`80mm0r-m6t-;3ci9mg|vpUx*?vmo?YqI9YmnMQ0J_C*!UL zkYh!mrG@e-zNKc>8aOSd)tDl|0#Da}wE1*1d=^lGwoJ|yUX8<5Kw7f*QF)si>o?~k zbt9Qt-iuk}{N!k8XRv5suUzW$kQo<2(W<_~QEbTqT^egBgY;@*m5j>-(6^aTvulZuRi$VSewBNb2IVLw1bPvo(x`cWBXETx|-N zLmv9@nm4mgmBjW@U??%xUsm$Xl19 zm@v!rZ}3|QI2PLTtm;4cJ6p!3hDV-@wRj7e!(FBdTS&*-P4^LM8s z@YB)x*~!t(+12IXpOKiE(<;Boj%uORf6^_}7q3{iAxJ+WIGX@Dc_cIh>S6?$;6UcZ z0)af`6e0Bt5I&Q)G0P)S>a4P;yPULC_hQ8|)^*kAHwv6m2@Q8^CSG}Bf*Rj_M&ux6 z!5***$_e!EBmus>h)f=u7!y7Nx2E{5B=pv*a|@^ zQrF)d9b3yM^OzH|q!7pvR^_)~u~~gOP6|4PzJ5{c!K~6AunI~}R#1%#%}aBANXNrS z9=m@a^sc5u>h8LZNr?SLFD8M-xMsr>@T{+2qr(K+K9QD}l6x(@w~v zPmw&;2t&4+n3x%uTK2p1#=>M5{Qn;52YtDIb zc^rbBO!FL>9@EPv98Fc9THLG9LabT~<@w&Na|eI!BfU#~O3#0fp?{aeJ)e^37c%ZH z;C!uP;0^izU%lEt%$@&!bMWf*^ZDn2Rrl%B7L2jJUtj>n zko&XV{r=V+j)DO=rX>0=GTxda4g<*HU`t&GK3F>0lNV2eWFEsX+;VOdFVC5id84+J zp8H1MA6^pInC0f5BPtKU0V_8@6ThAFV48DgNNT{DOGyCLm-*X4cwL}(=c@;FJyBW!I7Q zrj%#KZDNB^S1wrhgtxtU0XSDLI56w)cd%S;EJnT%X6DaOkc3JQmCRMN-jsA2R8f}W zIr=w=bXoE1e@Uc;y`e@y^3#k~Vr?p21^!|vSE6#+514U67w346 zc^#ng8VVXqWLnQj5-tO=x>g(3$~3Mxnw}sSgg8PY=DAXaI1BdhxN^<3_}L{X!pAfH6DQ=A6A2M5>vHSGM%!g0T&G#-b``Vb!P|sIm3^cP)j_jzRu?M@;VYS4)b3cE zzI3Ol?1Q5Ff%711-tg_a(dl*W<;T_;|3U5}du))V#Vdhp=j|)0nY(16qCpisSw(xo zkU2Q=Ice!0}dxA@W zH!F>FNM`UFT*4*bYL!aKi;digS6Cv{-ifz=m?uIdNx2Xm&hi6!mAbXLE8;KO=ae!$ZZEFn@v5FeEg`AymVrwnil`O1pU**LR6>`PgV&&smEPWuyV;o|>&>YCYw@>e;kCvpdJ#$uK zO`1hEUoCy1TF=Sf;XG`H(nz3=y^Jb7I4P_lMGSG!iAaPx4dmrJCw<+GC6u%$PnK^3p$7Hfm2z2~na4EAf=_|>1f{oe@j z2SV>u1lbz9!1epT{ezcpa{Ipr{Z}ua?|*-i&(rMx+MzDFJ>V*@tdc!t5ZLpF+Ty*m zUpsrjB`G+47(;27V#IQQvzmI!d(=JS@-UaQOu_xJfnAN#9pVk|;96?DxU;&U4by4_ z4UH+;KK-bY4+ZCswL71|Zofe<4NSegA^Ttsxj#E^mH?|<<lS@?GW^=- z>rH>^=KuKKUpw~ePtEiH%Y*$l`TIW(`){7-|EKtTsq=q+<>-XNwpoa&h*8@zNaMqX zWlO8)XtA#%TFlLEuu`m7gA^IBm+gzP?UyY8Hb#mS6aD3e!L7LBULwS;;9Y3v7%J;2 zAg@d7vRTi()hD3Z>{R3c=C&do8cWPdbzSrY36wP?PsGTJK^w4I^$aRc5?d2im-%~T zd5V5b5X&qq4Z79t4E}dP7ZtN`>cV~qS$1Z00_Y|Keu>V{oC}rye#UYUz+w;7#TT>eMQY-H`c~&AzeM-l_ zHg9`r2CMbdK$HKs|LS$#|0`ZUkN+q5Jk9#ADg9C*r>NRRD_y;%nV`y7Y#U0*DlMVq z)^J~_w^p_8D|6MFZv9xEV9d>Cr~RjCpNh8w@Z0K3mwI&VTTw}WNxnVRi0S*;7;e4+ zCHUI6;(2EM9e--)|6In~RSkiL`G5b_{(f%$-+%RH|9Sp@iqDrc|Ci*$-H08qkp$My z2Ei9$5RiLBDGrdCBfZT{lI6LTYf1e2r^Z_S@1~Ajc@}8W|A+erIsN~#fAGBidy>!7 z@BfwO_bpEkUy%cN6O+Ok`GLQlso<;A@M|5csE{mcOUtI<;P;`x*ZkD#e|_!rn&y9_ z{(o7t{|{fjde;9>@p*dvZ|*YB2Y{I8O6y(dyZ6^u^cwE7j{j10{1(u6tDCNoT1B!r zcdV*hBrK-?;nnxrpH}*xtrrN`wEy$^Ft7jn2X79a_5V|Ro<{#y>HaG1K2k{m{_wiJ z!Ih(%Um6-$9rtEB!DL=2BqMr^aI`w4OU9#p?(A!P5u*V(4im-^JsWNTzz`Dxd^DMe zrH318mT!2jZ0Q`Huj3nBs&x9Z7E!K}vn^$XTyV?h!17$9N5hty-2sXRycFPYV%2L^nAr~zS*a=)uc-n@GD z|2@g)Y3F}4Sij!WeZSW;eP89K(BmEGTW41JT2At%U0!mG?<_Ls_?$z1t%frUL-KI? zfg>8hP*qhVxkmy&rk$DH0cI807OIClfFNQr+C5C6>9$JyC6M{?fJ+aUtYH$$T~_QcG!*vK&OUQYIKxcBG?XCiuW9v_)n^ zTf5a*8r5disSf~f)aHoma@}9K6e!7{Qi36Mn7Nn=7J5lK4~@5M7p`V_XZ0seBE_c> z-4@!=71)+!6_RYFS%_O61oT5JrN6^zf;v@wTh3oCR+poTj3T!Usd2GFOH4TaH+N{U z2fmTxMNF9M(UkH@fVJ!$m9SJHSuR)Ua;zGXppvrHdVEbj)e-auS`b zHUu798yhsssN-kn{{}v_>;E`d|MHhM{{Q`#`SV||`_J*8PxARZ>wh6w`o_iF7C_Si zKdxBW>y!N?IKhl2u`txXOM)rlkFn-|;?w+ws;~GM|5xu`s1V{gj*jjj4ux^9>f28o z+q}M-LF?=XSlc;HZTR6`1k?IErAB)Jjg|?way5l6j^EOoxc95*{c;>I>?`uVd~PGA ze*ULu6CuEj@xKRo|NqNZFQ51SpXBrT;(ud`&LYfD8bW{z#WorRtR?(85cmt4b~hIY zTokeASm4~JR{!gL{yB*-Cv@edV1xcYJS@b2?eF)We_B2wk_@&RNT$YZPljXhkV@EckbJ;|c60{PdwhAafTeHD{g#Tv4rcYYyb$utGV z{Hlw=DzE9dd)?)b6f<`3Z*HkpL6Jebgsg3Lv6eM*-NjladMFBFf+Ozi`jrOHwd-&k zaYXO!eKhl=O{W)YS8eMx^_5RyjBfB8k%VWJoyTnms(`a1RTRnI#?jQ;L9rgLIjmb? zt%wbMy>`>odMb8BrKXT`v)SbHn%8N6 zoTWg${m&sslO$wltweye_W%C=%U6em`yURU?f<9wJc0ec_VxZvdqTFDR%{H7bdk~X zhR_#b%ojFq zzDg>}b<$Ur2w6!6_>vU!7TGJN&oof{8S?K~GVeM0uhdou_U|(AJ91|PeAB(ra?zOSlUsJPJ`T z!n>j%=-i=2tI~I9kr}8h%B!MPEo?hf1YqMY%L?01H#1}$JewQooP>zxXGH7aW{9`3 zY?Ahvgk-uH#S}qd$1qMIc5a)mEV8L~RZ&n8Cb2>*V~UBqG&_WhiFq-rQFv7zj93kz zEe*_{p&$tnb*np%OL4mu3#T$#_cWq&97zFsOQDa3h+=u2IEezgx9f_P($^Aj>%MAn zmwNsl^-@6;8XIyf9drvVj5)}uY%=7hkD3WuW&8;vL!T%w<)Zc8t#md#uB>o2JTCB4 ztS;?H2m4BQC$>%92R{_OCzFAs#dddH2$coFPFZYlb{|tAx`NH->asfvRaxG|V{>wl z*;|tK^TO(M%_f7w*|q`jNxYC(-P?EEOG~-V2ruH0RSK8crzZEXcr#c9jBcD3FF&1C zF{ILay4=#)^<8ytv8;0SUF4~Oi6QM}W^T13qM^xhK+kjVkWR)m#;6m8NVP zJ_;IP6u2iFJ`a``)id^V8O=iahp ztlsf?T&Qe7Q}`e?5uRWla>VK}^zuq81C%wlExbAwyG|Ib0G?{t^3@dhMoocVA^j5^1GqOsHJW>fF#dlhB#D9!_znE?^J)YDK0rEUz@vJEde@Emq4k(A1O8+h zpzWxxePQYKf(=qN!n|wn=FJ<>3($R!&E_kjvzM_&MC+KspJmQzAy&(`qHd;nN>N*Z z@@fUW@*1TQ_7iq^>7td)ah(^YdhMB_b9S*CZP#XsRv@M=0eR+s)|F$!Y^i8-UD74k z*E6LWOk>@SV%@ZX4d^QYRE)D_)wAb`;cT`melH& zP0>D?BAXPb&C{I6@Wgx<>xPXoT`oj6z;Ov@!=yT}*I_h@?#peA@6K*6k1o#MpPYX8 z?rl!nOJdaBH;6`%E_BXF(3!xHp^oFNRr)4C<7C>(kMHpT8eo zUEjRFe0TBP>2)P>SwM7E3e)!Ze!4n;cX9gu z8EQCs?bTQ-VQ2^$dmuFU9*23Kc(J$N506HppRcY@-hY2~ep<;+laQs#-eD>hTZ(GV7&-sxk@k2?L>R6d=Kv0Aw)O;gs8uSwZdvShhIXa#pz5W8#< z`7=r)lCWbkpJNU>WHKr2>J+#-`upYa57$?hSMNsekFPE+&Ti^QRXf~mhtTD_3x@*T z$D^gtN6CDYQJ@tGTT;apNIO1%H@Z2!et$F^zP~)WSdJqwPQRBq>D!qhwhuHIm-sJ) z-aL~p>zi_Rd3JO1-TR~Kx6O<#L%PyrQ2{U;6F|L&U6fi>HCGH3qtol3TJ*j| zCsX7!suTlR2v<9}ag7XB(tZ5HX$wuI==Nvm|7Y)AyBo)iMbUiLufTCn?${GccE9)W zJ-zqU$eteG*lr!m-N`; zRfVc@CRcOlQ8gV4s?Xua4({r#f|2dK^im*Ig7os_m(#y`0+pd5{* z+w;aZHMggG=8$u0+Ig>n|Gks;W#`xH_SIE2#VTJ%jvx!h*H8#n_M$BW!-HgjeWFM5 z%jwxw4fU)Y%MeEbm5$~Y{bnVV&I?c^Le)IBl3dlZSl7M8kHdO6q&TCyI_oKlcXDaC z)OF{y`|HW&(u0*G5wV`)VjUch5vx2qR1Bwca(Pv+oT&G%E7>O%J^lIQzgOv~`s7|% zjsP*yY5#&FL`n{VRbXf@J+^iQ82mk&0v0Vg+{?42$9s9UveNYhXeS{SbLq;bk4o_O zI)OpiVk@E}Del+l=qftue7d+e>0F(Dx|o0DPW$ROd14pq?)bE~RFB7)2lG3f(L$;^!()ie7aZwXVMPVFy45iko$~| ze7rF}wm%=AUdds5j@D^@HGNsW_v=pks=dr8I#9r| zuFp?;z4pfw-%X*jubyLajyQ)Sw8&2E{o1}fIhF{1_Nje*eRlfuW&85fqw9<;_QsH* zu**>c ztWfR&ug>}GLI*Y_P^lbW>9Aej>|*M#dDE4{)W?P8$5`HO<+9B@EF`f8&L$B-+$?ph z1+F)4cd4Y;z`Yi>h^JzeZ>@3*=?Tb05_cIL6z?TCZ`S0gIosnJ{2kqKR`><4Jd@qr z4Uw{*!H(7M?D>-l^eqeimtjfC-Ct07|5v|h1~U&=u*K^(<={X(y2hL#7*0X+vzyW- zbFZs#Fj5z%!hzfuAF4%TXBhe}W^J^T+5-pRJdzJ8dNrlnkr7gmJ9G=RZWhH8Yjnmd&$2*itaU|DE<1+_;#Be#<^q$^OfHYAsX74^3VArHr@Cke zEcL(TR&OEaerGPFZXaIM&bsk0F<1{jg9nUPp@q&*B$^%p!4kB@1inM;kZoj_Wr#sg zO9bN4uLI~uCl;X(>n01U+I_w_a@>R2^KzZ6V$}Ord0y9&jb=fgPks!1;;qmOr#7O5mDT`QAM>oowpCNfoBLlVm znIJZ8#!wWa=NfrN%ZIkrDV#*U#Ed~QLf{9v!}q}t7DbLec+XXjQ<4B}+WjFzGeYHa z-EmsI^{;>F?v}S;S4LquZeO*pd!IgEc22H8U!E3N!|c_`R2zBp$1oLU{)_WyLPPW+ z#9Yoq-JYzDWNOlaDewIq(3VMM)wq|8g@~K8QUs`VE}3DN+CnfXqnYdRFv%0=$M7(z>2U@7K$>R% z(LGGcdS1#hAcD9@gBv7@#2{8^Qh`uDvQZx7k7FyZ>vIk3{5@>trF}28)N(C9mMpYB z!9VQsKb9r1w%0Xa!1u8PmiPWtk>F{VUp85-=y`a_sjkbpzB|qmYto|n!-{b(WnP`F zys&w7rilkPuU7Q9pdMkqvMBW+#UTpd1H=YfXmp6Uh#-OC5C= zTElgjzr9W0OD|q=oaYNK*#buKMGfK}%P8Vm-<_(F{IK+Tu5Ksyvym^kjD_JXzlQUy z=L=cRmt4KlF?cTBJR7Y?VuIyKhltE*OQ?sxVYrfS`mLa2&04?wkZXK783L3!*A}g0 z-Xif_ad_T_vbxIE=bwOQSUvvL-nZ?k$=Xc9+~bn7WK0gUk%*#RX-;P$0~{tYJK z5$Kk6+^`#2zz_yP-T|a=G z9+yR!O!bq~2mLcayr_`mf->uIQ|s7;e=Ch@;p0?>w3*(mW}mGed}(hh?Y6>z43t_y z1UXMf<8)g)Ca^P#=m199NBVb(KK3 zc=+zk8};wu;bHOLSMT2*{^8}D_lNJ^9KL_|_U#`IUmd=A`Th^!aNUzFD+w2n{o!!s zzKWCEkRs9a;HO51##4qzV*!3<6ovtv?1NJhv_LzGz@>b_!6j0`EyGr$*FFAg^9%=w zaMV1NSdWJou_Ms#wmZL`G+(t28$TWV``?X5L7vc(L{EFgG+e2{S3$>bgO)|f^mHmK z^B4w(_+6q1L5^A%_Hmxv)$vpPP6u#>EX>Z5#w@E|?>ILTWv#OrMw&ZHzOgh3#&PQ= z86ZZGK)i+NK|m)lC5VV4U?$keOEC>ltI@IkMd>DWg_Qgt+p{8*Z1)GZ#?glw0zN@( zgbIgiNkKt`c}e_;TfE>h=Fc)<5hKmZV>?kG^9+mIsK6#We=O~1CAw|k`GmT zx?2hc2*^W@{8PZ_M*fRnFgERD0w@a+(_aR3Vw!8o1Cbu%1r?X_QQHnawYxb!J++}? zV(YwP`OF+=xr~zw$VNzL=bxFT6%xlZG@EdZD4aD~y4_1rwO$%B)MWxiFt((moWM9U zS`S&NqW|;w{|anBEpzz)yTdnc^ZftiyLYc&ZTbIWluhIRj@t4$^8Q&?Kg%PlSnZ5b zNi-qJ#rQh$IQ5gOVBS-?oRDQ;J6)tFqBy-y_ za6#WxjNa&q!Ql`GCHG&7?UdHYBYu@ed-3^{1mrGOjGQW30dfh+2yyHGUiH3ql2w&d za`^w3Q}TW1*@+-8ouGQ*Ju~(7cV4p7ZrMDf)ayBk({B9?s@h29YmYgma=G0$j&NQx z%&d#b5mr^@h)YJ+9u6_VVrm7x+{4RC<-V?I#YcZ>=k*o>VZnwa`DjNeyA{Ug7!gUVnDxJv^>>c-)qaFBRv1IzeKL63&tct*io=r~f~E`>JsMfB)w7%kBC9G0Fp< z|Cex+%8>6;BA?@(;VKg*lwd&_CZkqB8KR259;7OUZmy9zng&B)l_p`Hg29GgEKxMf znFuM@NGKK*t&RUwbRI3ri`Jkh`C-<RN^B4(80HM8_~Ho` zuRrQ3V}BA=UHtH#Nd;12V($Nyg*zA5nkw=dtk+w%X% zD38bgcW0vIz!<7fIar>Xc--PiEKl?%k7RnH;&Cfr2x^Au$^fy_a~|Bj93TNx&QC)d zHAm|~*U0>zsC8iz8Dq@wEsoF#g-2irBaRw{n|V#}W9s()qkW>UFr+_AePPqL)?qmN zoY8-vK%h`Q8B;DUkhr7lCL4i#c-rmcA9PQTvj;38ZT^`f5u9Z ze}+l&t@?cu#}S$!B4G5<5f99#=Lr|;b3JUP72^%-6FI@;2)s;P+$Qkb5qS6Z?d!J< z0ES$(x31>w4ckjfg}T8upg<%HfgPqdtD@rTU=VX}&OQT1^y?CaUusEKa=AbFeI;Yq_UmxqIZCO7q#akiWp=r5&Y7lZZYKxWwF5K5SQK7`#m$N3}>eEn(l)i)E07cM6EEe6l$_4gZNLXZWUQW)=}qs` zfr6ZF#{R0azn9m{9pCeePT?BA4c5nrD@K&Y`}Rk;^|Ik?lB=@e0{$Tq#uO8g)AN>k zPx!4mCIYcrS7~aW_^kYF?J7m?yafvwjL{XIAeuN?RdSDg22|0`lA~fsEjcD5M=j<& zT+?0pnqDiMgWjz?HU0RqY=Sed9%h=eBY3dMjwzw6sNQrQ`e0UR8@cgRXucJ>P+U5@ zD>?TBzlo~B8u{M@u;xfSr2p^B!^2la`Tynncdxeb{~x71LI0m0G+X}65z9#%dGR~p zh&dwnA__qlvn#ZMPW1MzHt;D)_Di@f=VtyWe#@0TZkW@u*`2&vv1h*M(1Zwkcu*X8 zUYbdFqm<_KsMDP$aR>$K2?hlk8E@$Pv(mKPb#2r~F5PM)SMd|Gf`u7sAy?c=Bo)lb zkLF98=Fd5o5Y%VZAeXsiyi~ z$vWLtH*&xaEt`6<^)uwC3&j}hqH%@j!-v=BAxD-YlrJH5|;=%k*)V~eO=KTM5;MsU9HNaaIS?h z<)R&7$WiV9oIYGFx_4z)6-)LX{s;Oz9CCM-6g-_9Piq=~+KK1jQ>WqFc+#CFcW3P& zO%?535N3G<#Wb$l@utl#LmUa!nC2Nf@*o7m9Y3~IodZ!(6@edQ>`uE6Hhw}wuk*gLC^@XB5!pb zu5lV}S8Ja|Od~p)_G0#Q>iFpd4W(`G*24wc6AmZK359hdBz;N1mfv27_}!9jk!qe3Y-LB0tvE zI_-95U@Xseu6u)soX`fYLzRwh*_A7*E_0`!?d|1TsQgf+EGdUxN#=d~Asex#k+_f#b&u7k)D^4EFTtqkm0gb2405QQGFLWiZ*Bu3B`%qPK=Yg~#DfpLU z)~aZu6{nw<`_9*GxHN}%zDkzbd#+F=zpefqi; zt2TTDutK+c^JY=Jf1r}AhLrp$RO)lg5P(hqS&-gT(z#bL2LV*UT76b0tvr_~ejmbE zHNHxT+6SU?ppb<5t0_%zFu>%1j|)$mC@4I+FWedlVsIcP@qya12S0%W8jFJg76eZ4 zfn#CR?~9ek6B>edhle#=TSUBrnE102*z;4-JN8_jt$A@inwqFMz|9b|LdWWK-i2a( zq()zO%aehACnoVsW!p4^cZyEoAr7EGyuOEnB1y{{%BJhvULA<@DkDP_9+kzEQFX~j znrgM1Arx>++P9>A-Gr!C?kz#!+*q6j&_BU05VR*4CL^$0gF}_r2hQ~?m>_U&WOH! z_+GQcjh@L6lW^fSL31MMNG!(SC1_6IH~aU$vODI>%&jRaHYGz+0@QHF8F%hwixU|0 zF%>(x#}Vb?PeYoJa0lFjzkOfw0l+uNMjSMm*~!hzxx~bqHk38~9VmtG#;<+h@Prvz zrE2RnFTwlw@4-QcZV&i)vSfGeiFmf7RpjRH!m2zFZ1Gvt@uu&CbW6;knuafZw5e?S z5tqL7z)L5%PPJ2gWKFxv8SUrFz4J%3#9({}Pz3&WJ#ZEdPK8$2AXNEC&t{~v4Wv`> z8AxR-729c1I-Y`TmUz1hQAXbVMo>fAH$^{CG1&zjW~vm&>vWhY^syF)8IWlXb3vDi zAYVjNvAAAhbuGm2QiWLnNQmKxP%dzAPQwbCV3iFknO_SXnKl=iic#fMz)H8b{(+Zv z^T;b84-{(!zdE6;3-+q9HIH|;gH=F|$ZfgE@$>1`Mf?2p`uODM&mRk1L5)bKOhAl4 zHZ_sNz>dn~wv*HOYHAdsK{A@NvU^ogr0kfwpL$mxFHd^^`|P^=>GJCO;`8~>Czq9j z%UqwUBHa1+_vO>s=kt^63Bldg7;@SBxfW+U6$Za~c+hl}Uk?eUtegQ7C zi?Q|aygcciop#z+r=Kpa&p#bkV8RZM;BA3e);3j333C|iT;cs}?&Llwd2~nlczeK^ z<+^r4*+e#JGgN}a6Uk=)(mDIwyE?hNZg;!a7wz-v1ZZ>zq3 z{nWdv?5ASl=9*ghZN&QSd=5hOH=R`eoSEbhS$P4j&U*9CM<*BUpU+ONi(i(W6{m>r zfE_}2N)?}eJKaN7>WuE{tashLJo)A1^163&`Q_yDx^vq7_2hEt!Nv>?5$hp#i&zK8 zW5oCZheFLiW|Gi2_)jP1T906jY)0QNccx)F7hW659Ygd55 zU)LV6Xwl(bo-IAz%d?f4yF74*Sj-#g*{6?+40(|}C|hhrbo49bqK?;AbkzBDadFa- zJg0g$&GglA@>eX@-SKH}sUD9p59W6|BTtyxXPBTaLqq({bC_qR7blA3oc`5+j7~3$ zDPpRIeT66JH%feAdv$t#@_#;EEPyj<2WuE_JW|MgMn^vSzmM(D$ER0v*q)M_)*Ms?;C za{C3c8DW2In)X~Lo*MUhGe?zQwA)kXeEv83w)u=&e^$u%g<76SsC^}6(MKx%Kt z)wk_OA{Lsx9I!r(tT@{8#w!c}}}I&!)=m!H(&FXV0Iwpl?}tjtp~3 z+}}Z^3%q{Q3}znAarOSkqhX;3dDmLDTu4r^ly^Ka(FvS2oyjvhrm+}g``p4hk$if*sxu6RAS4!98>qF@?C=vPW_ zcu}q}@7B2K3;nI)M)_T@aQHM`<-AYHu&PTK$0}UrL-boSQ`v3VY*#V$JC_||^Znvu zwnqImXs?_YsOCPY$H)sYOsglb)NNb!zgD=+>^y9-4|v-|^UhBsnjQhc612nwzQfI* z?e(kq8s0AsQGHmwey*ogl>7fGPef9hCtV{zK3fm)xe;VrUHoQHuI8dQgl5vXKb^}# z^M>|2ychK3F2ftWv&sMMn7@LZreSuierQ&V#S8<4&mC?)5gp>$^DUvhhAux5cHMzDrTC&P~=^4^@XuDN}kNvw3Ozt69uz z?5HIv?`)~IqR#LOt>l`N5vBE*QuEaBF|4RHJ+6U@Kfdj>tmmZ^St5vgG`K;cNG&RJ zC|2TS?IJ&p@u#lOHLUaZF!_}Bz0^|6wftDJ(E7Ocu*?5gdhFU>*MI?(y)IBF z5*n?Olkyck$n?)QxdJXR;p`G{ zZ|ff=uUXDj=MoB*qp;E<4+?_%ghN^`>!`phz7o$k>QppszjZ&5| zeaNbeh(f^LXiAaD>TcmHh=!2Y{M3U2wT}MPAj=xE#(=pTy1N@mQFy-pk`d}nwcZf> zNg%|@@p(NQNl**}i{Z1He-5?VY#u~P`~-$mTWY_R*d&J~2_y`-cuSM;vU(Mn{Xgaz z!j&+$7%%>soT&<}(9-Cu8h7j{oSPb^%c5N-6*Lx^)eAVlM<+NKtCH#l`1`D+m2+jM z_yp;;!E%EJiU#rsmz$_AnGAz|Mvhxs1B0q@QA!$-QcR}>s;vEhy!DE{@z7_j_ zv!>NoUfIZ^)fQVQn*NJUkO&K#E}oIS%>!3Bd$GA@QhGkzq+_a0c`#^39+C zIBbKw`y@(GsK;g)!K4;doVqCk_zF;i;8e<-%+wv)1j% zJn1?5LK;_R4zxHVB+8-p%Am#gZ_L4ctIY|p1q>1j0l2eKHg%NB2|lsd8V2Li_WZ9J znKwLei$=}ywLpcUC3C2(To@tf8Vu}}0n0*+7_dv&Q@bjkU0W(#YLP?NrFG;qL^R+B ztpe{Bt#W@CL(AH@^-T5RctCZOzfC6jcbD(KYk$!%U(eUCBdssr*n8i^Z_E8hLGm?6 zt;OuLSliMX+}i!(F;DtP_lZ-1U+K4WM;+sz_m;u;)j3+*659F3HYc(Dw^lf!Xe4F( zb-qjO1*FMf)A69_?`2@*QrZYe5j-?)rYh??HZ7ld6^&O6yn<;{$|LFZwS>G+&3Jcc z$*B9wlpW{)K6$m5{MZ(*Bocp3t9R-A2(B7W$y~E({h@bX(ma}@)bWH~WltlR`!>xl zxV~0L&5NgonBJBz{YE zrhJA0ci&CY@%7nX_$hz23HgVK)v!&KYLlMz`JW#KgB`8iNJ{~8JMDy&c=Bl2Z!^t% zQMaS70;rAx(=p86MNCNgGNH>%!o%bK7s5LK!YCU#_H zJ3_Z(?qwnHM}zBRk+KB#XY7*f^N3U>ev}XKI{fXqdhFk?SFGPbA13{(-I!stYtUaA z+Vsxy0@^ArM$A`}nVecJvGwQasvnTkb^Ga+4sMl)-{1;j3Q`v+;K#S$zZ`cu8PC6W zx&$}>1J9L1zrSmK+tWh{FtduFp`oAlB|Tvo<4jB@1R6a`#L^D*oJ}fof0HWvBIY4{ z1%cI$*#zetRS-gOkn*8Ue1f1mr1d}LOQM*wc{ zzKKc&!2W%kD71}MmUhlp;OH%EW;R-iz8K;t$wZ1APJ9 zUMog|#mGJj$X8?xRds#7jy5A1_GDw3$G-VpgFq@I-3Y|oj;@~G+8s=a*x%>r)G^E`fxwlX!q$F(Ux`T=qAcFIo545Wr&uH(6 zQg|(ICvW~u@q^btdQTXAv7y?Oie(}@meWvJaX|WX^NpRDoC6%6m(+dy8_N z-pukG{6x~^)k0IiM?>g~lu{TD1YIeFvf8HL1FKKOG8Wv@f>? zDLfU|^CR_78h>j|2sa2xKE`qW0p^31(d$B7M**cr@y!_IbHzZlWz!prM)8;P9(!=R9$TICSBQ) zAQYM&U_O&*@lX57;q8RXJ)Nu!cAzbi2^SELq_#@@FX>x2AoW|=sVH^YtpHRLF=`Dl zzjz(F$6xU;ejCvSVdqw#{PnJ%mr+6-k1=pL=3>;2%g( zJAPaD&DGH7j(FwlceOuK3!=FNd#}OA%|+4o%E!vhw&T-xf-2F*!NG9{o1e7}PXu9Y{N5-KFqgSY4^qLrI;>*u!5qq3n@HdY=#(pD$XaaMD6tX z0Mxr9ht+cd;KS;>%+jco3nB|1tv^MOaHubLwR zHKicqSW=O0>(?+@pqrreJ6(!BO=%M}=9S}XtHT)3V>qL85o~{wk)(Fic+1cuqu!U% zCO2aicMEF|da?jC;pYYz(z*_6%u#-3;RpjmdO7II(hnIjwdtlT(fXI`k{^{%(oEke zoJ(sIVUG-t_aN&Nc)H@~><1M=*Z1W5%EjzvwR+7*(CvOPxR2`1QtRqwp)$w`FmU|$ zqR)hVNg<>zD`WO;v88@<_KAO}12W|#1T^m|{8oo15`S*9Uhmt0U&Pzkm!IYq zsZ^ED(oiqiq^S<-vq$A=;RSzH|9}aQT1ZBPaEfD@jZ1q(o&as&qFqS()-%IUyp2=| zU#2uXWt0Bm_xtER{eJPG{}xp925nb?8H9lj2J3oXC*qZH!Z2k6V?}V-g}Ku@fc%SR zhBS#ufb@zZiHalX2C`_Oq#*SXSO^2>7o#zbb;3(wasWwUUE)f22~73xxG{qHflyk& zWlJwLnj$D=RT%+VBq}&6RmGAu;Y*!wFx#RRD4%Ug5OXdO4C_JMGE5BkQ-I*7l-(^p zzsh$j@^CTuoA5`)#gKXmVgRa z@gfSC156irl}R{RjiZ-Bnpwyv1b}Qd(u5W0{c-a67k5^cz(i(HsF$o*N_-rJ2-PX1 z%#Zjp#I*mVk5S6gj~FIgntbrDZK}K&T5!f3NEmENYVFR@=39YTPqk96gjcN6=+BG48lb#6A*RNxX2fhpEFJR&d{9e=oAm5Ba!k{7C8eNgATEoNDIAW6%92F981F_a z_N_3Vz*T7{ex?U{*OWY^+i z=|Y$@t37C?_S>3p+<1$5J&|#yG*Xw$?8Tfc!Dvq7%8AW95k8&f#U_N?Hh(ouCB&Aa z&bmDagNaT-t!c$qSKK%^?PCq;f|#)ZnORvV@41fY`{TXH|EG(S_x!&_{;qfP0PNP? zfM9sKS=3j@ERbLonin!ZLAS%MF&KNtvqwCbm65b)JiI`+dmy^6$TmH$?l%$=O z!>ZpTtX#GovPmSs_9G{hjRWOf*952j6m%!tLcj|(jhJA}@%JXF_$4lBX!`0SaTX`K zpzugcON~8ugHcOI6Y}LhEx`Zt&ke@C{ErYc?}Y$EB5Pg&>vB`EWXJ#?Y%}jQN*%Hj#XtRR3?j2H9ATf8zv0(>gwG{o9;YV(UQ5x~a92JFNIG(OZoawQX6 zMWjV#l_v#KPd_#P{JLXuS5pIj+Jgy+M7k#sZQWhXsA~!hCyO`A>GJDX0qdFPccbr1 zMhDB`WRdxCp|=Y_5}O3iVxhDVWr7@l&7o_v%8{|u%SFN7-%PgRWl zA;Fe2sj|AqZ@CdrIzXyr8YYe>hDU*pPgNM~OdQIwtHuVS!mvoZW+^cHk3*e@lMvRl zzO?mlMZOAm>K74Dt^!)2xcflTfGi~3aAW>*Z-&QWH038qQpiRWMopF&oE9_-dQFxT z3ZIH&mW5-iVJa~U4u?42=y@kCnka5Ksv{jyB}b)V8T4A}fs5tbAce$^o*8Qta7$As z{gnhSHAa&K7M|KH)gL>1un9qVTNpzbxoIjDRf_7Oj?OgX+jSWKm>4}6#FvBZflidA zbB?~0M4*Wlp`v2~$=3m+yeo6LXhF)wb7WS!9A z*lW=j%=K`t%CR*vT(39Tbvp$byVs9U2y3pU`6muX9qxjd|GpRD-{+tv7*nm506TIm zE3?C-nE6vYL=#q5c?onKx6$?vH`yhrI7DlegMkj(Top)jzNAJ&a&eyA&-`5`B@533 zHNL-FDqjgjO>q*73#U?1BHPB%0&k-7)wCY4c1lP)eaSJEVG|kg&T%#S*NwdjJsPiD z@Z$=9nRTkgXtgt%j>qbEHx)Wq{ChkPn5-8GC2@JK0#;saZkH<_Ym*?qgC{kXIYerhkqt3}?t;8viOY!flan50N(_OjCMiPP zk@pxBYO>%l)J1$9<&2f^ze6L7B(STfx^peNATvfs`kG5*lR?#{EEmV(l#4%5$=Af zZ;ZgglE(Vd3e|6SzhB?$J~vv+d+YP5#7RH3G$WjT#=&BdV^Un`RS%=b<0$LT&7^Wg z-C>=w%MA(bsiL&29D~NO>o`H^vOt}iXs(d$uOR$m7f~pA0x~Hg6Bnr!gp5($DVnXb zS_8ba=je?hI}!tuS;+jsl-58LY;Pd`^WtV0d9P35{k-?9sm)V)N+uVR9SP znB-r(d^?~dB2=OObngCCezz4t_>}8!iFG=Fq$ora$$h4%h`$u`U?N@Dzw^kJQIFXm ze0ggQAmRb}YCtFR&e@u?`htH{ybOtqKSV09MdwMK>)j$1Wc{{Ugcc30orerBtXmTL zWxAk4l+~P3kgi%RSEfIT)HN(ar>{l*1BnV2J6hrnWLVGk#LP$f@;VgUC+3n2U+km; zKp6!JO+sXt0Il+^Mn$luly$ILaW$NI4Jvn?c^(U?C<(6-Dy4L-YTue#C!wYV3x#7# zNETaV$2&L|R<##ex;Fd){qK!-%rzB{4j6_gNU8Cuh!R3{OU}6&E2%`gs8~INy0qzp z<%!X&U#_~;zt>8U=e9!>Wc7d=I>5?V;wJng?5&yu#wO~?!SlNm(t*R{Hli`KHmzzB zK@@Rzj?LI7EL_QLOSMxq1l%rp0yJvD7|H3tHm?QX*qx(fdmB z(&JjzWdN3!R6vjk#T`O0skmPaCnJT3-p%7u9_eze^76clQNLpIsx_otFL=0F7qQcm zi2m-P>`Q2OVJ{t=;=dnlHK`ibJrIKC0eb}&-J`HFNJb~K=DamB)Ma!1Zl_T=Z>nX2 zHK2mMXDZB+Z?Bi6C{7YQTEj%zl6@(F8X6+;LM?zZtxow%D7(rSWqoGy^WX6sg{X8m zN0o{K`M)j{$wmG>LnHtE-xADG zbc+!(YFADvbKxy{i&ae_k6eS!m{o{sH%QubKbE%JW_kUy0bPN&26fhrTm#AuuXooY zlXTykn)Lu(OwHLVy5@U|VacjIPQ zJL=~!g-@1@PC&>6jUq7lI8x3jU?_*WB30NDPU}VaNl@79<13)srwbYfPqY?&MrbT< z`Scy=nw-zJkr#!$Irrton4b6tZf$WTRSUKy-0#r}tE>LCY+eq`EoN0C_K4(~rbE?~ z;2o=2q+m8#g`fz8ZQI(Ks8Ex$ZuiU@t+eD9N=qd|t{UEC2Hn{D)4WKnD!j=4A9m5; zJ(1t>k?~S4DyP}eO_R@TSZHd!w10Z0~_ya_xxI1f1z65PS0MexgBDVEdYXRzGqT^GNCV$`0y2tZ8QfL~- zlGo=5bMC_xLDXb0$cRfBEzMSNsg1gQ{+0QyN_C9m*Oh!jK-X6U4i37;5f6)1c{QaQ zrw$vy`q_5$W27*=)5Kk_WX!>$nWgf0L|y8_F9=T6Ev!=jyP|_#(hYwI8-I=X9;r)I zs4%*E8&Qz#;05gfllum!09;w8VsD?34?p;R$;s43x@ zgNN4tR3Sn{dK#=X z(L^?-jZeB_CJh_+9m*>Z#ysQR-zg$>ZkB4(eYHctg!V}=`p6^)LWj#@NvWXbvoe!D zfbRRe`?g6{@1HWrX$uH*%uS=m&BZ$775R(mI$-%I2agju-zRRP5swnt5kG(qbRyr< zg!7?7^$*w(>((!UWK2S>B&|!z5KDmbZujkJTIFAZ1W66Zrqk?fvuK}aVVOqqVV4k` zl?2e})7&+>Wh<)i^|DeaJC(KHFAwGbx*mQhz7 zgq=C9^>XR0N84Qay zG=u(fp;|S5YONROu7xA4=uQ;2^{M;v_;a%67-OtcZ~P?5Q*qXMKCxyLV@%g6ur$n* ztN<0+HmEQ2XbrWjt#U6N|JwcJ*H{HLB0e)mXzd;*=jKpldQ^|H50uBtQs@;Zf=o6y z>`s%#nVGpSaq8XEu+c?r<7FE_RKY;-c5J(nNm)*)-g&iZC39$+E7c2XX_JJ(-%5^xn)mK3H9scE{!T1N#tZ{Di6*-31YHu_z$Nr?I_X zL^m3WR3e=x!`M!$T(gss7H?Zll>0z2;Xp%vKR2eIs2`2&`4wY7UtnX99`np^SM4w= zAU{8Yy3Gi^Q;i4Z78P*Jo8~%us*9u29B1AM?TyFv6-4t>W9yuM9-#@4Kc2(;$CbJ3 zLi*ICTW8zYIm$4x%<+v(w*ILcWKgdJnQ?%dVTQ`-*kblm-dK!-_T64uJDYc7-4KV% zW>xDdY{(oe_z2s8l7phtKeHzFvW%;;vyd>%EslHIsifoH{ON!#BeKR5SLZI(cE>Kz z4+McrY6Ox!0#7zw|wH`TEfta1gNJZ28oh8}Pk> zGxJRbD@KF|HTY4PG{7cyv|f`rDNG@ehIeq4y;47kM&lQU3&+x0u*$42?j9@5nqG@M zJS3TdVsA_~xA>|@|5_?B5uSxu`((6bU!Pj0pVfAea7iIHc>tr7+aVJbiSE7y29^f_ zy6=}E;nA8Q==oza6KrfQ@vnGfs{H*73 z07s`q^glQB2MUM`1FmxouC0UWDzbnS zs~znaQX6cNkXji}50r5AE)X=?H@8oOhFZI{=+Gs9;=$3{{gD@o-ZK;6m3T^nXzOP- z+Fzq8`wt>S=RON&hvb%4s)T!3oRpNYQubX0))=GHrb}QG8Wrs%@$;R8-Vw1LQ&PY1Lm#h;f#}0BhTWH%o5PN?BrSLM zp>RFd@3R*_fw^SsuQvpkon|B3(mtGo;YbUoNg-8!OQ9OQvnRaNqS(1~J$z{s*qX?R zhVB)csVersJVRyPL5gXBUGtX-s4oGMcUE(0 zW1H~>XMaVJE2)L;?zLiPo6;f+3Xf()kX=-PxO z=BNHB`1dPM3fDU^MP|oVhkgCCBVUifQ%CYeUb+qlnH5(4%&C>9cZUf?6=&g~qDN-{ z1-y8SpqNrVJO;aptB0*#FHMX=z;`>wS=(Z>RvW0e!W=58bkKIZ9&g7twW2Qv z<5wzN8$Dv0jXyCgOq_#uNh*}k@qEq5Oc3EOkI4A0j#+G{N6@~AKu*p}H5|`F)~I>w3&_J+wCMSp1*%N$&Tnqr z_|~-pnOdCMH(5Lbfc~=DwP=7RWi8F=`ZY?!NOSLsiN0NoMzitHQDe@I*xAgsjEaWzL zh0cuf+}5?JN2KMns*JoTVM%d>9yjxhX$@<=E3M_VZiz~~)53?j7iB?88AQ*?2v=KL zrFl+&ad&Syrw!S))Er1m39;|@@X_iHMeF5Gwzyd4&Re!#9O!;NYdz(SuV=35=h?m1#sY?sV|Hnvx*OYNd^#y#PZ)fdZ9PRpu$vKktM7ZwMsti)Wr1wi zn}7NXt;MUKz+rGX8V4DbyF%b(&FYWj5B%-J2|!wD4GA5`mN*^#D;Ac(rqq#9nuJy} zeEFEsN^$JWiR2%0EyEibXoxeSThfOD>o1%{$tR$RrRap2{nd>;>!paWlR0+yunSBIUpmqxW(erQfM6hpC@dAf!%(ut}E3Jg1 z4Yxs6udj}cRGDO%R7QeS<;iHt`iJ}wYFqJ~iLSqNBis*{w(9rmlW8#b$vpP6m1UL2 zJbhF|z~V}==nh24NVF5?ZCPy%(YY5Tj=}*6oNayflmz^aWO5A+voR*aNN{ziMQL~t z*pD0xE$>d7h{Cb7Ql?fMVO2pijIuqax5kFIm7Z#;1B%-7j$-ErzdCmr%wkwp0@QIx zq-mQ-HqUoo{8P#g5C!YV-Z}!c$x{&Ll?Xj`GkmrFVeD4o3_=vL#$WMLAKokR!yM$& z3#^RoN%B_vHrW&6Dg`EJ&J3ikF@xR=Hy*kXfe{CCgG;iAyLl&HE_01 zdh8&3R4uNJfn^i3T#NKyC&tXX4Zq!IbR09mmAmHbZ`ZuzabKDozVwx2F~|QYbjN+# zdD=Jj_G;Rv)hOcGc4~ zEZMe*tjO&0Z5GI}TIg(b3#hqJJWm88_s@|BpLV3Jv5S1U|KW1Rqp=C{C#PSlj@f7g zv_bz{*n`A0ypj|sE}y7Z?6O-EJk3NYp6<^WtWIG4b)scC6Y;i!q$nO@u5poZSTluE zF=KBV$Z&sb>hgWlF_{PRP>=tzTv0V$$^~BxC(pB?rEk~-%7TbyXZR_fo2a*&fp zI^RB}`Of|l=~NnrsE=-`M1z*I)^iT}bNC}@zsr$m8MRT967d>)Ew@|2AcR@#^N&r| z;R&l&vKQLMs{8nZG?F58IqN{q?%b+`_xKU|Us3g3Pdi)?58P#K{s zzh}InL&jXHDkTuN?H*_x5}J&(W=6|x_gs+Qv9z-(i*(M`1#(gjJnT2gR?4LKm<+Qc zTx~Y%;`Cz~00)FvE3}@^&G2{d@hX|!4@5Z&p^<(yvPcDE#U{*;s@Xr`(<2ytf1n7g z?UyQ>_2IymG`{<8IBv=TTI)^idn3^nwMlgdLRWp#v{HQ&!8ax^q<)&Uyrx2!Nx>?5 zNkg)Z@p?Taf=pvmO-J1?tNy-89Vm03x>P3{VyQjs)MCn-zTpptOzw$e|CKy_>U8QY z`NEs}@eVsvgD6YYh49KX6Lm)hhHV3I$VQTcUQu@`fwMlSGhhM~#{(NsBblfj63|ix z3s8!J-JhW=cEwW^{Ii;LzZU&&P(oBUz^BH!kyPb$%J&g(mrZi7tDhl2Op&>8?bnw9 z!#GANvsvx2mz@wiRW4gWVt9bJFd)u-{# z#j1^r|H=7QNN+!H}Phhk>lr0V0!G*2>e$fx%zB?Dudjv573FyNc$W<&+-(tqf&vNl4{&m06=u^mQ+s z-!i%P^{mdM-L@M@*(0+9Ke(E&(d%5X1mT4n%V_~diaOTCNnOivKv$#6)3Rk7!99#T z?>l3Bh5j}>SHHq8P!31cY*Cddmp>MJjkD{6p03N5!}YEm3iKEOKZ|M9^-m4#O4(in zWsEPp)W~qad4e;XfZpxiF0G8E7JV&mkkUo>sjiNn+MK?s#(KXvD8WKCKB~rE0@V0n zsetts4dG|@W0>D9u)`J4mZ?uos2VG~{v9mIiBCkPw=*a z&wssvuUE9C&hX>|yM>Qvu@a}^>znT`o-!8K!e;?z(dBs-0DYD`lM&jbk(`@Sf zNT+=%<^4dSF#l+>Tfi3&?`IN64co&dVi|R^H1B|Z7mR#@CTTPdG{nTN9ZQn%4?3jp zZrPrsMSCQ-D4^y3HP(_jrlHs77#jzoXv!)$JXuziR-PLI3?Q1Kkf4H8ev|B+@^cemL-$`g{f6 zbL^$+Iyoy(IFm8|epZaPjggHl_VU+zeArQAr_gh;_013^D2F3W`Scl=PDc3JDIm7% z;eBuC8dA3Lq+7|*$d&8yZ@LXS_u1p32Dr~7#vCVcI3XWWr&m^F&$gAeOSCTPfKRjR zL@zyzZ?@2e8{M!81pR35etheRak|tbo!6GP{O*AHWyC36th{)-nuWez@29vqt5a6U z;Izhsw-)gM0BeixA|S;X574h&mO5=@gGt+zZfSt0!QEHwm%FajYSkA*)6hyYFk5$q z0o)_!{)2IuRlS)p`4nL{GOi?x<6RvpHim9!LIIICv2qy3L|FF*IH=qO=L#~}-YNnF zh+oDNvT>Cv^|>u}w@5^M`jtq~ikv_fJD{n{=KWxgE#Y^iY-KrC#Qn>NdpV-T1^ zJ*jzdmiKoWy{vxOprg#fdt?Amks8fKjIP>ApEdnczTg^F*+dU z{HbfrmdD zQU_*(Yl_Gx5dfRrO|F5*D9{kYTXIrGJ=m%^kiFdNE*f`r$_ksk(-g`J_iAIYkzgA* z^)|Xg06l@{$3$^qdAcu_-EiLoi!E1AWC@RKIBN+wtTaM)0lYVGyDXU~`OdHZ*i5ao zMo});Xj4t?pZ@$xTP`mKKhuPND@D&yCB$dwj7@MM62jNS>gRvx;Quab4a#l5{`V1{ z%Qu*xxWU{oXj5KgmFlVO(OEeXz3xl3h^3we;pt6f#^D zJtX!kf8|fk!Q|+G zM!+ARRB3eDlX$CR;9%i%tkJv85nyutXeFJ{^oE=9R$e)1%c4z6bM3e>^G9{t%A+xYdJu zK4$ULeiP*C8_PPMkUq%CFc!0qc3>`YG2>Cw45W3L*WDV}r5}zxZ@#}#uRhcbaEO`j zuQdDcKVPB~`j`Z8=g+wOSNH=N--Mg*ej!B9bm3S(3OI4ksXb}}UX{}w-^}rUQ^u43 zeNuwU&iE!wzh`eyM`xe^4m@m{f%QT2un?2u{BJ4E@42ndfoa>u@HNvnHs3M-3P*Wu zbI;gNs$2@c-mzFeO@J~RN`cv!2(e=#$C^u9inW0r#%%`H3%Hh0{*!|$@#U47N^z>IT0gWV!Q1C4vXf=EGGhK#T z6n(r5fbCq@@wfS8iUbGtEN89-fe>?1boOdM!o?i2xr$4>#!Y zp(`F_55IA6)TLZ2%1Z|na++}$RI4#KcmKdBksmN2aR3uxEIJ2EUGU5Q2meaFTWj7==mhg9yEjwvwzuH) z!Djc1A?L`lz$R*#pcCG9ThZ&hG+*+H0K7&?s7LYjlr$Rbpx?;R2T=1csA~=Imp+{8 zee&*s#wuK5Ag^j_ehCHY6xD=#g2wKFNb3Lw^+}6yC;qO?J)ffd-O2cfxu5VdYqixo zY%?wQ_xSVh@~)$&d2fg^E4yi-k=Lzfink$o+sm(xJJ|mGlKD*sgkYHe!}192j#_hl zuYXFCorY3LJfd_Af!aa`eY#6Js%Tml`#LcRgO)+DXa**_m&AS|C|Mi%dMLD6OXIi% zTv;(OC3LmpEF^-j;RLp~KxfR7V=)^B=3GOfd8TeL5S4==bN-EV0J&vVEK*4pbj4$m z4F&aR!V$r`SH8(h;NH&l;E&m&rtiOQ-td-k0`G2!*#%iT<(=3NjQDNd)L!SRsAVGr ziD8`~u|C>JT98F+)D9v8gXtutt+wO9U(FF>$nLrw~Gmc#tV8lzJdZXP?CiO@Ozge!bAJl2g+x%eD( zYS0(Xdzh}C?_CTd`JK|P!srOX{SrDU8L}jNp=4F>mve9SF%eC(Wl3$d35KBpjmfsNeRIj(P%>T5cH@PqFq8`voa{`b{Vd%xm zQDFl+b`WDPL63xW3__P>FB}sB2@*-+3EoF_ulV^zgP>l@6i;%3Z8XMkExMusR+T;# zPhA)eVA*Dfll8uf*~5xe&5($E>r-s~$Rr1W4NMY=mGP6&UxTjTv<)I;qUl)kHl1`0 z+K1#1T7^KD>=0tZDY*Wmx1h5uZHL!%?k0f=gTuE!fQdEV7@&ye8myj!2fnUiK|aBv zK~f8CR&Xk*H#Gqr)qf7kINj^KD%J#l0N;LjJjsk$A$kzqjK#kAG|dbfPozr)a*KC_)2S4d+c^*MsXZWa7_P2$)411Jhx}uQLy=b zj2JF9JS6D0@m$*WJwvD$`u4>ipg?~87IN>nVsYy;TX$mqi%)oJ2~t(!)u)sv&wrL9 z_%sZ;S~)vA1LL&a=*(jciJ<8k-ssWNb0-?vCL6RT|ky9kRk=t6q?l(lng9h14Pq3SraJk7)vKjBN-o` z!QXS-D|h|X5V9kLeu~)T>x9(v*6RRP4hZap2#2B2g=PeNnk`;m_gsPtKmesz?(}f7 zI>OeYC?Y933M(!dcL{SFr6$Z$edm759L**QEhVpKH_h}>-CW1`^qL6h=MLU)01R^l{v7}wU307sk20jME+%=gYJf1Poo5sS1i+3 zg6OT+rQUD^4ZQ33qM?5pfpPTsb2oa8c~Ul1_qWz;F<#06tVEpW_#?=0w0Yh(Lz)(& zr14FT>XtS#{xcT?jGhjb)RQ;`dm)5O-9s1#&y+P>%r~Iq*fEz{ ztzYK@Zz}}spnAmBl%{JsFipO(c3rgL_Kavk@JOb2p(eCOByeq4RyTHCI#CA#3ICB` z=WjJ(goUK%?|n0?e=wWjme;J*KQGOpJb>~&t(FaRpp3D^9mHZc7|q_2fE-6E`6a82 zJAAm!f^?N}*Yono@yE-3czT$Zq=d;Udn^eoiaOQ>-`jO;KMKO+3W*AogMh+b=_}N6 zLBKJ`yWuAHFDCfCT=))N{8pHbg==>3&7ubJV-(-g(xzl50i_PAGF23lBZneWL0J?! zPi2pe%5l?jb2hmt!C|(}3tE7|xVXP}?6wDv94a0zA+ncm! zbqEL=iwHm(j)h+>dF`yi|L6La_h@Fl(Y2^?{F*WsQ?C*=Y)z4;{Ro$xnT;-|Z(-p` z#wmDem^lXhoDYl-?`md_eFMo|DcF7ZHK4~=5Xe14k_9=^e8$eC>$r%~kT|V#6mGQ} z^C5FiM9}_x)`j+1cQ>y?FCMd?t+|;6s0@o`9lrO{JH8?8ZtN)W6iTN>%=2R5oh&ay+)EwYBZmaQi0i(sxATks(?Vq zAauW7x^y~|k(55aQQg%$a}?9EYYPQRBt8I!hPcYt??MZc&Vyh!9x+!9&|RaenY^#n zzwO$G$C@=}=(2nL4GUV!u6WBGthTEyEwWla4MR7nz^YX=4D#e526|W+W~tFg2n);c zZf+fa{#0SwHas7lok92sgp-JgoudqU&~82-^PAe0-D9^nQ0x9MFyQ+Rf5==l&7l9N zVnE=(_RPN{T8&A_bmyWzTcd)URoVq;$Fi=x*n%E|r0rn&%Eep24=A}v)^=pO*0H?N zQ*_fR**53TR4_pC7dlJi(lRCa1=7;GPfScV;RfR__x_N?Up90`A#Zrhxu93>@*3i} z58m3sNiqUKe7(NXnk3;AG4qO{U9dHi1;>phClDzZ=)hfD6RVF1ohCu0>RWKL>jdtx z6eLW;zx9zkfS#ISp!G-|deFopNzQPT7A)}Y>O1gELZx^fK!v=~-~NntJOx<}#krxj zrULuyt}nj1<)isriC`FS%FD zvuO2_j~rQ&GAjtZHtn&9P2g-&&9Ly#@b(+hAYJK1u6a?*vPH}5i6At8bh={ok**ki z}oFuO>hNW?zI)NSu6TzJwgGdf5>fH)arfMpYKUYx0Ss+?f{EPA0 zvRG|jIjk2)?G&2#Rm!XX$<1S4JhXp44O;r+=B?M6ROLAN^xa5`V?5ChY}>Ew;|IK> zZJn>W?bIh{!K0j?@vj#lRw&?r+%^81sYki);P590- z%ijrMryA+rS0^6|D+9JezB(*K&eAw~=&Zh)AWKhh1@gzYV#3M^IuoJ3yc?YYkc>b| zk>TqPb|FuNLiUyE;R_y3_DTsp-}+`T{%4W-WOoKLUEb`!Cad3mrCqbQ7aVXi4oJkN#HzXV4(3-MBoq;K_*@h8UDjTg4gR}jXWZ3C#%AS32j zR7quC^w3gW0*&6~7Rq{9{jgHcx6!rt_{jzOs3=7ufi*2Cs7MXuejy^|AG&0R5I~pR zV8cOPvh&B(eCYpIyZ>3|;TljAyL3kNYu0YN=*~7;qpS% zs(RUdNyD$cxgVugDWL2)eY8kCa7Fh#4l0uGsn~J989`P9psT3tR1{%!7Y#V>n`6 zfvOybzi1`xdD$=bp!?utc{xd3Xl44ZQF;*@I^p9^;)0qWtzm?neV8~)-UMixrtt*s z6KJ|GtP$H936@WZqDqMxV>R5yo2azXwC{lD{{@RcbidwyuaR3>6y1OODe9WZ9+-t2 z4D*r2-m}gj%p~DtJ~%*_w=$lbE7&~A04ZZl4p`yltP`)v0~?oTDDTm5CX`5GYT z?2|RPrIJ)CRh3GmsxO3$tCee@O0?n}Sn7H!eG;0NW-U>SF|z#zA9I=PhK$Je#@~zjiwI0wr@#Tx@NQsUU!b%%XQ;WGd_wI}d1MmU zv7yGDNjBqTVKXQAN@f5az-rw>+}-zO%TeY2!QOIj9LuQOKiF%D#`~B2zC9{iMiVOn ze@|j1!1Dw~=GiwIew|oE_ly8*OM7$15=lLB;B%6Dlz8X;FH2pMHKZMz1AHD;A_gA< zIhwK*DtDJ8%nX%o9P$T?&*e~Qg4nlpDri=QfY^BW+KIq}{rtv13+6?#%mw$%n})Z~>=dB9?mhf9<7xlmev6ge`;4d8mvM)_Yj^ys!f)|6UE*!~hJrY`GFDHB~fYb1^D4gbbT) z$?{H?CBl%j$X-(cUPxlotW@gU4;aVk{+9Lh9&KQ`N=GT|W= z%bs?Q^G1IKRJPpU36FWmN3)|?{k+E`nFzw7c?rK1Ds>~!t-xufeI%7$lp|5$e^qP^+q0aWx~Z-_l^qX zrAm8LCR9vVgnaKiL410!_?hDcn9t3po`}*wXR0^gDU1~KVH$?_%#F1@D$_V12|Y># zN$6SQWgt3(I?2Yem&DFS|7zN#`@%PLL zW{C?4g-Y`2$B48><^{YbpW2{uLS~rTk^~geyaNx6BNMWM)BY*2Aq;pjUbxC63R(=G z2nuu(C$nogY%Lkg`+x^WG4-!5!405$E3RwG681{%?o*pAsu8$$bTR3Ak}(xP;DZau zaeL8>_@3=*r?lslMO311H~3{`=|;1w zZneero)=^|WIhZ9p8!XJ6p;y)G4b!K7w}~zc!tUY9yDU}0uMLgUj&`)Q7N@=ek+@A z2KLctbnzw1M<@G}>7jXLG z`Sw-|32spt@qiw*L#J^mK;)Hf!5?5*;>OXS(yt@pCg4X2Nth2=8{h&y@JQd^jwuO3 zCDje+d&nQ%vOFr6@JVm!i*C36Jul0Posg5@8xoSprve=X=INF1I0;~2YJ-n{1#Flh z4@D4@aQe;x`bAxLuTYk!N*9$2c6k|<`v(sekC#FvH3#owtrgY;y>ft?Kw^H#&?QLc zW#2#0uD)V=|4rvk$(5l+FK_FpQYX4UB+;n$77`DRN|UF?#7hJP8w3zDyIBfl5?tE6 zz4>2bS>nbqr%FLk8){)ie~agDR7KS(hklB=q`;zPbMO6?LCd33QYjgac3U_BU%-*B zT=f2Rjz9+JI`|}l0fny8l)6e~>h2*^*gxpryfS1J>lO-T0hK1zB`Il|nVx>{xBC*W$*7P_CN%QfX5+{ucrBU& zk`Qp9nrMs%KkePrWN)@CaS!gI(qseP3kjRdQ_*euU=rmSgcYug`|pD)OWcDysKgi= z2P7FUx^X-nQ;112M&_vftKgi4A>Mi`AT$8-Knz6}aJn&INkk@W!?pCxN2N>$ixFv2 zDbqo~1U^!kDGGJ4nsAL!^^bdBE=QFz;ew3lf+ILw;ATG3m5pbAqfrA{r$O<(ESE#2 zzJh(vgZZwO8kM{HzE;&PBl13Du2L{$(;+3n&w?fNW8_mEylDYdvOn6nU%6CysH37k zSae>%>DS+}8gGF4RN2mES+!Tb9U^T9657CwEv)#!Tv`&A=qk`e%1{)4A<(s8rHc8`u_@eoc}XMU5z-jm7domP?>A z(G-LyokOaAU_7ez70eHgIr$kWBWy#P|G=io)SA(e^R(x_l=jiDTA}iMOrs;87;IkH zeVZe0K*watxX_kg#Xp7a0CW&RqrzKM9??W6X9-gOuDf1d!091P2rI`;yaFoU(2ypG z%Xz5OR`oazjUQ`Nhr&RE`clYQt~8^vv$C=R-w^*Sjhz>AosO|lT(#RbRQiI&59xAe zZn=P;P?+AcqPA4;$_1_i$h|7}bjF&64sh4Ewg1+VjTP;RKZbn7d=f&CPGT!|l;$)b zl0sL|Q5q5v&hE_}Pi<)u?{~bo>+K63{l*7bjD@9MXe!~S7S;gCMMm~Y>n5jh`{e>o z|7Sp0xJ&?~J5cJ@J7E+0-*uOkbHL15Zwu}wr*ZqG$(DJ?OWk_=Jn~Z^Xyni4dFB+r z%vtXWxLcXnO&82l7xMFaw&Zo|JtQoo!T;g|`8|^f7qfS4!sf@doVVWf;BIeE4+2X>#nl|Bjaq1I`e^=@kXr7>{H@>kg3K8eYIg)CuIdV%1% z+I9i&Ob!mW4kR1~6jaUlfw1wc-~-a#ckjCQ%gDCm`P=vTBt8&)$ihNM&KCRIG=^BW z{?Ze_YPo>Z!!%0RgqnE%_hZ@X*LyM{(Nccsdc8HA!m=A!YNw0zzVhz6g8O@8z@iN~ z?yh%xes{eak4}Pg5|c25Ue6CHiJ;eG!ydl-l+f#ioCLioBfXf21f${fggUPDdcUVU zp*<3YJr>X?VM9iRY7y~XOr$*LB7j~`ZIstf{yg~Z&5!R7PTuSve%{FY>^)h6#ahR2{{N>ejpE`{3g2%l-}~Qt^W=56Hf@L^#k4Ol7^@ z;7t#)JfQWy?tI6F&D#sJQjXbsS{~4PUw2aG>Uuwb%*+qy{##HT5<%(|`)wvw($66S z0mS#Y2++@k1CJ3^(wa6D!b)c@uA`V=G6!s2&1TJ7gCtTbMPqzoe&mZ%D zwI~nBo@!#-E%P2c^GDV$iUdv%>LPKR0+Cjn-}7j94{^QQ1uX>~K~C@~3uqy%RMVob zVh|sUpK?lsKV}K_lT^^xG9ghwLLSl6yBS(4FAwN_iCu3iVp}{kLHUhq%rNR__l(lG zi|L@IoMD1iJ~gpxB?PI^TwqMYiHVN`JU05h2k$7E8d#VbREKFb^$-FX;!7oU-X9Z@ ze1RA{4{*I(#w>+OgvdN@salbT!N1Gp0j_t;z^Y)@yAWIT!I!20t9QK2-QVq^zd~hy zyAfa?>JD#;0(-~H-2J^wS}nES58Y<&mIrkHw(H$85Cp!Pg?`$GwVUn&?8 zAMv)KZyWE&+Rx?Z?ZWkg*3Vmn>wic~{{Z#1X?|utHUXDqL&WC`*fkm8e^04e@F^8$ zxzVf1=aFOq6)4gNEJB0&D)mO5K89;EO~`!`+GcnB0RDjZIlk@G29oz`U$(HLUx0pO zM8a?e5>2Z!^F}l@j%_5*Z!~pamzBEv@&M@p-H$oz-S*1lGV6VBmm15xU2lw=Y(hvM zP%&Ycl}%oE4CTLG1Z7?3#!kWecyL_L$WWZSvi7G6Oj#TQ6Sj5aM~B2L~Y~l0w823YfPh zleOzQFL$!8uawlWNU%VMB~QrcRxnwe_nedJT+GI47`BN5*B-M9;=>6SbpfaQ!w)<` zzq6?PWOor}kEk!`g2UI6n3WpjP0#!0%Uo@|0kg}w0p3*)z&Z2EoB$Wpxx63XBFwhZ zyRO;SZk0;4xflsclYw(C&O%OtJ7PMTYawdD>~f)9{(4bebh%c{)&^0z0E~>{gu^Nw z86If4fYZ(9AYqrmY&l#>O{B5E1ZuAzv*o}b7sZete;Mpv=5)K<6W$A``S~w^;JI&j zjivD;`cfZU!MJ$0EQz^&Z)s!R-Ll-}qtTl{QZZ#d{YGEDEij#3z#jU*n-c`mx&e@1 z-@W}5Q^9l)(Hm8{@*VEZWY@<;lAiX~Z&U}Y%@3<73gbgKDnKZ|?~3`2IdSVbyi_=Irq>_3!HTKF(j1{47CL~Qb-tiZQc+AfzMmxdOHWE}d;|ow( z6|4(f+gys^ulIq*eCnF-)N}#6`L&m?6Z4Xq1l*;Sx}lQm#X1w|fcOh`huhZd_(L92 zJesM0d}H^zIm`8;_}RQwO1PnUAa00#^8^Zh-aY)V|KYox>NlMbHWt*!4Z|Y%AstW=(S%Bv z_PwpXI$DPWZIx+6ri6v&iUdEu$UYa=1V#UQnh560D+&Mdgzu6na5JS{BGxP3B;GAm z}JSy{>>spNHD@xP6?53yiKO zcnCC0cVK~Bc?x%=v~xr*ViM0sMtyIa(CrN!vWU8Fp60x3pkBvZzHGS!Cea1HSbVFSWZLlbF<$c3~abVe+q?9^^87 z@-gv7U1^tkVF^tBWm8&QJ0}0K$s(?kPyV&jFkHeG^qAnFwp=Go{JP>@)GMruJzEEfjJX&-<%oV4bLcq0^OYq4eCRxN1 zM#48VB(o#x^C+k}jBD3IF5vV<3;SX-CSxiXULn`x(DrhG z)gB}YO+n3kxzl11lZr9$G?|ae3pm!*9xy7eOp`;x!c@@Xv7mCy!+M5!0pIFskl1<; z4wn~j`tnMclqp73TZG9YUF~6GvPI^|0w&dIge@n$Gud5i`=EsPC1iIAlWm0eJlS1B z?!7IS;FBdxwh`X7viqT+(3DHa?h+>32=98?{jgAZb((A=yqjeAL&D`HWOoUZZG?A| z?0#UFTvm3=goq@KmpFFkvfFL@pd|ODtZD^JwmNptmE8s8-s5r!iBrI2t7CV)?0zUH zH07$8Y<28zklhapl~?7Ht&ZJ|vil+7Qekqj8q|!*R>$r}+5ONkxg1He=$Eli7BLx-36(MN=Yh}>3$Ru>Tt@||t42GQcE2A|{TMU@aW_H_mdoY& zq{SqQ5-O%7oX;osO)Ve^oR675&L}TWu8TxTgIfwybKT|YnAGz(_Q!>OK|1lI#w7|E zY^~H@t?=I8a&=5%!kLBoq!XY=H!$JeNu;PTl{5g!%R3(5Zx1L~SwsUE%v{EKm&(;8 zPJL3Jaxc#GStJt@`4ntRi`)BMu8v7T5fLmJ&AZ1r)V20_G-M;K`NVNJ!!Si;5{Fd6 zfDXB!xd^{!!&6`d2*TlrVXn`eX^a4_%)u zVX{rA#)~qJ0*a=%hm;Mtt?cet;Y*1>cBHPh*QiZ++UrT0z!{yr#!Hu9*0X>9>lBC> zr5H#g;cz;p;Y2IwVR-7nw_E`Igj&#gr*nFG>hnnQkapa9Fq!p)+2_A@)Sz^KBu&Pi z4ZC$fq&(*$c&%d*#pX|}&V9DCu~7(pw$T*6>-SI%qtp2ZeBcS)fq(K8B1$ppiA)DF zVM&_ks9AAHd^+Y~K!vPCRwGeRi4k@R=PVfm0YUMmMT^)OPX&Ch=If7%Nb(SK3Pc_y zt%xie0nArO;Y0X_B@vmhbp5q z27Ib-+*BJ$6RjIK_adhjlc#u7mqrQL#Xt%uu8)HI5BTu$`0WmSM1+i)G>FjAK|?;H z!8$PfJK_n%TuL?w5qCN-8V*4z)yzlh?NhuYq*8(}XrM2vGf#_i>B43AqhE%Bq(X5)1O~UHIPjXE&;=xR>AkI~sr9Q)ZIH4s>9Gh%Fkz8eO_H#wLI)2eupwk|QoYVU;MudEiHKMGo{qF#2sg8#84va& zE0iQ%2Ob8PZXE|QCR2qu{Lv0^9So)6&{>i}MyTgJdYV8&&M45~kot)SZ`24>-A43d zUy_GYjb0{YNCq3PM1kxBT-Ft3GSs?B%I;%Cn^Qq%(49c>+HesQqg zhSm~6>BfYxDEp8v!-fNEtrq7fA& z+!*uoUcxur<~>EPJ?x;Tx7nJ9Gy5?f2@>F(R@6i}^j~5_#wt0MAgODXS#05 z0j2^y!V6Aws!@D>l1@OfKPb9YAnS`j5q?5t0y1TZVrN=#)`C}}azcs8AmF%Zq>_y2 zItwY?yf1xDS7zj$vP>yPsHui@}ksx$d6CI0NYGKMEJ?dN%d$2#$ z+60r?&R=&m2D=Fj&o#Ku1RRg3^m6)oUZJn4k}qoisLAU-KF~xhmkDpq2ftB2KdVl- zRLsg(0?F(lkO_$bA_D2b2ekq54xdRNAgN;h2{lk~6;ETqCPd6M1HxYv6&_O559H}O z45-ajZz8HEW6A{hTnH*-t^?*vij*Kd*hj63gpy}LC=ydkR)ITdqFWX}#*OZQdXa|{y)K5}D9aL{OqT1cfc9fMExkzjM7v3i|?*2>f zlad5#=VP;9_;AR4MnaT>LT?J0^QZIkb1!GA9v35ISsVSA8>!_w-Z+_rYbY`BCz|z? zPh#sjCh^=%;>C4M;>Ds#yeLefuP1>j8xGE$;z0?ldgP)Z`uTMP= zgmrz;mkCMeFb&bS(eNFkVUeK6qj0tcV+aVd&B3quYkl1}Bq0MLv2Fpql8W+g zaSj2OgMMOCsXjWWa$yPZgfpNtQd+B0G-v>66c~r16Otz56PS?LLqmQ3w~}&7Dk#RO z&NZ=Os8eb%I%K0%P&9}r@oCpla)|8~BLVnCs09s;B}b-%UQV7tp4$R(YJydM z=B&n|x%rjxnGf~j%u%8#L06cVq2~C?2IaQK5tkvnEmT&JN*LUaSP~3&#Yh&>p@09G zgl`xUF2lc%wDMH>p@7rE7emb1&uTN7Rn|Xb`)Co7+o|bhn%s+Xc~3s=X+ELET4}qQ zf8`b4u?b5`yBCKyx5}{}|3GO>LN=w+`UcIb5`=Wl=RKZAr5=tH2UV)5@s}~!a;7pG z56B^%@G1S4g^EASW!FDS3Mc>dU;kCwXi5X1B+w|-sNI!EN~U}>TVnU36dKGlS;?X* zZ${n}Y;Bp16z55RZ0q&HHmmPpy~Imu!BkmJBRrgC;R%+*v6%DYgv4@*=@mgbGb_O@ z&!=f9S%tcB=_R*TE?@uJe#Hwdt38$h>+T}=gj4De&j>W3DA9++Pq?6gp(mrREHyw` zQgWQZxd8RR>iqg0@~`DYwQm7(hE&S7mg#{woSLL^?~wf{Ur+SUC;F6e(%EO<(PM;k6G(ps@M$xhHILAspv zWNjzESI{|8*7GD~1LB{hv04djGcc;2b;A*96rq_sni($%Nr9LnbEnhy;Nw7X6>U{; zF9RZt5_7pQw0M*4vF4}3Ji)<1?-Z=J`V|=*d^|e-?(pr=|9*FJuzPg$^T)$Cub-@* zr330GA!r8$`c@&dm{JQ(6r^ewN8hci&{oI8MQ22TI9D2Qg z`aGa(bz}KyuQZH!>O(J7Cq?fpOYG(`zF|G|UgzDd3m($f_MA}~8cXOz{<8TmZ$cA- z<5Uy-++EYA=e{xe(=uucXPAg5=+rW%CWt(o^m;y#J_+b+^m8~_Gj|ONdqvdSX_?y{ z2e9-=97}K!=b*S|1xlS(K01ZO zew>L_ZP<9fJMVV)Ne%HkjcP<&u7ih!O^N`1+H>0KWH<>Ky?#={NQUiy^3=JgLThxvet|#(DMR4G6^GY55<-T!lji!hv) z6+*`3PXt3glA1=8vebD3b`0tRlr}jJWcyuSt)*ytehsBo@Q6mq^6bI$ypBU%n_us2 zzS!w+c`yDB`yamj2=My#Kj018j7v+0<|9i+0WxU?O#Od*-G#!XBE$3zxssf8-WJ5A zIj6$zR%^hvS6W79Vg~DTro`9G65o(Timdd_Om?e{d@Z78W$~^yGxdRFkzsv`!gL=c zBX%e`+U{hNbq@u&()|b;WUOsnrnGp6X7;(2s_VH8!rZSG4VbPZd0033`?r-k&pL?&i9QuCtOlOHKQTVfGovqESt={HVZ?o?- z-F^<=y+4Lu`u~D&caL}9?Le*}4OvL_!IVYzE+{m%R;9R7u@L5)x5vBt3h41z6A1)N zpqu^-?Z+l?6Z^xsp;dQIzrSuCotSH`@<i%=DZ}zOUYHwMf?e<0$uVBWtraF)lG`VMNUF>IiQbvS;1a zVh$t-^oc}Ibz93Qgrc^E@tMdB=vd3l09YbvI8?_eVjkh!8q+O!`|%x`&kwBbZ>DIJ zVJ6*S<7GRE*-u6%focB~R;}lWn$2Ja1D=d^qb(fewnd=>NG(tr2ljfp2y?3n6Ps=W zny@|i<$Efe>uI%yo*Ziqlkk`+Mx!s^SJX<9n3+lShKz>68ZtkhM4Bx(@}0t5PXXJi zL*{!WEdmx)E40Pa86H#(H7Mq&FZfR^9h@Kz^lzOW?4m!CJ;9uy=H)=wXgXO3 zVjG6sKReOxC`u#2yZ9u}T2)Xav%9+N&b4r1S{x+;?)3CF&uLK2Tk2R1QI^NjorE4s zc5cU;9U1@$<1rb~g!#A+Sp)>ee8Sa;Rp;kqhK!$tVc~u!rxVW%CZS>iClewQDhdVy zbWaI1-CG-kJ(P!JN>9uu*AQ`HuPp2;8xI^zRIbl~s4Euk!L!a?*y>nb>{RuW9r5?n z#tf)1&y!2?9f=`ldTI~u54FZXO!EEw^j~0)xzig=x6Iy>K8=WA-11k$3DqjdV(Wm@ zLY0gYkZ7j1f|n1(Y(g3oi6*-`WOk!Z^-fm&q}g7y*;{gG}sDq7Dc2?_aV1Hbg@zSu|#8wv4y0rypN z*-K3KvL`4y#G%K{7XEh|&lbIW=Gp_N$D)9K(yEr4iAAkk3Zr?`Z$dt*HXW!j>)N!Z z@Ty&brdz#)n}2SvEw+Ab)?2?eUF+B1AB|u0j9>iG`1Kbte%+eI>pG2Bm$EHYFaH-B zmf9MW>K0{L(-5BW%Qm68tMkx}LYLbgH)s7ZC&|AzyGQxi(Zq4AbXdO@e%5c!yw++E zp$O|_KxJ|g3&o3-9M>&2RxF5~{$ULdU`HI^X-?7v4?Wfq5$#PA9>P`0Gar{3wBMQUX_4i9UcmO4>(B`zUE^ zB<-@><0b_yykkC%B<*zecsvtqG)~~(eIC#L1GYA|p2Iixn4BATy7srOwg;dY!TtvO zk?$#j*x&-4S7kboEMP=vab~)6{X}+S;*Y5v$T}D+9&CA=T3U3?r|#N+bkv$p$PBF_ z`VNRmZPTKkd>SV@bNM6=8OEr|tgHrv*ZHRbz%`dApeAeCxv2#t>2&^6DGTO)gy0bI zZY0!I+;9M}Zm)1e@ zVRB9c?F39Ff(_E7fF3(0Cc&-1BOnoUcaLEIs0-ih9_=5kcYfYK{{G{S$MEy+;o>8S^^;mSyZ1R;O4-r+AIhA#t4jx#exW z^fo(=&-f1P!aFt)>U`#0&8S_8es^m7nFoa-A2A;WG&!d1=BokKeAx*2d4wT~H$pz*(thhrh&c1&Xw*5QGkguc z)9IPM{>U4U}XS(;W!)(sib#75>sB{8+9~8u`ET zQ245SEa3l}+x-`1{{MVy`;q_OMp=PeV{XHSR^aI1&42Zb{?pq>!@`hJu>-pYyL;cG zCW$I}iHEGLn1`gUa=Ev^vG>LVrBKJOSa6?8sdVg>75Ja>jK$Wr@{^*+&dN&1{oC?6 z{rvlp6D?mF_rDcma^r;P5aa6Tlf;+c2J`oS|9OAAwEwsIThE_A?*H2;fBw1gtg{>o zXu|-WZG8UR=_ueZWs@BYcA(C;N|Sg!Q8@RB_^}IJEz-KM8Zwzc_nY0ruKMqP zch}aS_xZCP^q+tBbcYLjP)%aA9S9n{Nd=2WNB)=wX~?3{zK)fne}4Ok`YCF_9e^Ih zS)`Jkjt-zTA&EbJt52|271yN)XLPm$fBp&H8J)rB&z(H%7B^7?m9p;qY~Boa+z4H2 zRsw?>c;@AF0g~!RbpaRfd&(19{;HbOD6l`DQ$-Uy(C^p}z3MuSi3qN=jtaoduA}Zj zHL*ge{W2;DM<#agF%=Vo@ZNn|%Q>reg@w!3Ol|fqboFkzkrmX7r{~0mKz|2Mb!z)G zGsmy(xQ6Edq@komsG44ufvQ^+pc;J7!F|GG9`e!b2f766oksZrxW6HmTcfwoRF4 zPg<|0gCM&oPcjt2&g*7V7ye+rFB4_54eA+w68&=CSuCLj>iA$BMU`V_`(8VRnX~Z-C zg0(fcfQUs(UxWT;Q7Rbz8Sw;G=_F2O4tcDVt3oVTlnkLe=cU39e1(I`NU74DqfV5? z+LJRnTZbn)_Rr4i8bT_M=!ur6C|gatTIpBm*i5`aBw2-Q>lNgo{28(jl^hI{9FMYZ3SQeHx_kTW#N#@Ici5tKL z_kS;6ZElwD|6aU!y#KqEa@X$voD5?>WCAaHvfRWx3lypZrSU?`j~%)GZ~o!!xov*P z+j!P<*Pfspcx$k_YJb+z&ew{c=L#YdGez)NFmxu*oac1-bshvviWGh225B&&iPvt% zF$UW6a(K>h%Nd@M@@2-}j5#lZ!G`FZvFfTU?yT2fRnp0niakDwiJ;K^&%U?S?`?LA z^)i5{)<9Pedt=%k(1i4z`vN7On(9`s6)uZ!6Q$4~uN|ffkqz$_!e1@KH^!31JIQu9 zH}!$pS0JIoZ+7<}OCi*flEi@ccWj?7RMD$HwseLH&Z@J-uA@I`RP)?o1tATi5#-%{vrXnB2@-#yqj;kjGlpTx{I zxLeDx*=89`PQ2y!L1WwNTZ-@17UNNZkwpX8O%pDC5>nA}WyUeN?j^DJ23I?eY&WkA z8-k~>zg5AvnLUd}8`CYUvS5KK4Vso_VdLvt(x0fnuz#0Z(COoX-u8k@NJNUN&_SN2 z!{o$cG7l%%(gG%9FKcq`Jd%srC97Sr7xgQqFG+C9%VTIaFi+A8S%5BqCQ=LNOJEMS zK`Vm4H2xL}KfR%)WS%qzH+uPLcLB;DW1;(JyhDA4#gd~s%ND!b01PP)0 z=jYE|a5Ju&Pz!2UOuBs&iAlFd^bgW)*E{otlx4mr>`dsR$&RrHIR1Aqv5hFvMMnP96JMcQso*iSHa)DXX0v!igXZOW8 zw@;->o*dP!#HfJo{bjTF_kXQD`{n<9{@1h5pPzJLwSXSD zfP^0i)lw(0O?WfCcaWpKraWy#Iam>gD78|811Ja{mvRAa)pL zKojaGGyp}3LoOz2i+;2T#XCPrGNLcvqT$@~(hOWGso21vJy~Y-F-aic$d1psI0K$$c`63{Q!&-DU}{-=(>@&S z?X_P=?Yde@)0p|>N$O`m*(-pi+>Y(N*;|buQ{7A)SKG_ZYw+@6*@e|N8LO+A$8X7_ zq|oJBPDJiW$eyi!hk7VWW#NbJkPDx7*I*SjwCjRO(u6CZ%j{0mZ+7fc)7^LmIs3*P#JV z(swLMKYfgKNV&o!_lIVmk_fbdtb_8dJ3EjidCCv166vs6dVE%^VhP|sq{0jfti3=g zBwcFW#bFm`X?a$&BdEKCdSWFs=y@u9)f3u;C2v)`qhd2AQL-dlGt_Jk+oMMNJyzd= zF;q0KpSvZmg32ajnN%n`FT9`-FsZZZShdy$0!H;V3l7R6b)=cHkd9~o6CMGrL{`Ad@<^i>mNH$t1jXW%sdEN*nzjk5!k9{qt{&_5bayS1(KY|K{`O zkMX~5rQ8AiABS+PGbEXOg{90zItaBE95&>)EXaHX5=HI62pF&9s+CW2Dfm`(u(?@h zSj#I(6IsPm9f#CIki{oc`Uz%ClChU?&yH$y6Iq}yXp;To0|vwBA}hxxx@vE8vzyzp zDkJRQ)Cl`ajIf`Lurd_G6ilq|47{w4#GO|5Fs5cjswgNuI#ncfXrss34s|kosUJd? zdxIVrn@SICOAp?u0q~)#8cD1rO@cdtN6~d9Dz!eQNslR|GJ2X|I@-Qf*N0^YAB7lg}z-BU|GyChAiOiivpHE=Q!*oKy z4+)#dYZ>;psldayo8!+Xk?LOu`&Q#hvS_Y43z!qVR%ho6H^%(Dm+*~5@N^JTIp#dk zIk8a8n><|`EBWMmn*4g_V;0h0wl1c=Ss8hWDSlOIWF9pZs#OW{0)~Wz3(gf^T6*69 z(nkNoq}Mm5|MfRFEAbyUw_iT$f45TZivH(Z!^VQgiUBIB!{_o|XMbq^OvnfYSinY9 z>LalZ{;dRQu#Tx6SY$Gzk}w<)|4iz8H6jwowtZ zx;AvRVIi-g|AsDh+yzg^I!7g@N&XkR&oo2h4L4I-nj1yJ%LeAzDzLhmSvd4p^I^>e z7jwxHo&ziZvLEvXk}4G|;Q|Hzjx#bB+eJ8Potzsj*YJr3m10d-J@NEPxkAS6=C!Hd z?Pg0^?r5GFN_Ua>Zs^GEEuZdd=>E@=yekc*0;n{Un$C;MSrv=jujIy8QvJVfscw1Y zoRJpBRk`7=_MpKp53!9Gh1tGuxPL|k?BnjRWC%w(>l22pfYpG;g8C$(!49TDDeePp zYd=WCP`8ryhIwty-i`C>r5&6dJsLN0f4%wJ)>_jRZ?mNfqjJ=E=XbS%aCi!*Zk&el zmUa-bRyH=Du_)M=(4CMtf1RHZuiM*e#S7IM7JkKKJw@BDOGNGARP4i*eOZ{J!Be4? z`Qn1@mBXww&i9ftuATEGW?i<6&$E-~tgql1;{Y~yA?4NIDWR$`mJ3KD_PY&;qH{uE zRM>gza(!EyPDz;RgvZ8Aqs~%RrJS>WE1Uk>cX7$0uwdubjA!xwHD7(_7SR5?y1Sn= z+)&*3X0Ke!EsPO_d&NOu-Cf;fS;q3CX)%&{8}3VUo|{1b)%e$SFTum+f?DslzGJH_ z8gW6T`B~5@Q@?M-g4$GyEE;n7_xZoW?)me=^`_%_clv);(edtYYpcI{=3oEA{bvuq@Am$i z^#1s@_piU1Z}|^v&(>6nzqP9O~c)|7ijH z9o}0c6f<8d4n!k$s=x~zCS3>|>`0}}8d`gqw)#HOa5yW3jzleOO1j}i~?CsYPk zxqv9045(PMo^7U?2m2;-@qj`iNF>!}#;mOY8)EvI1ZHFsI{u^Cbtf_Rp_uJ80v087 zL@QUFZxrI(d(LCygQPxNZ+^>0vJ>IFH#3#R$^9zx=QumBYa9v}g`vpY3SCp2aiDa8 z*TrsFtJZ1`w4gh1R$eQdE-B>S)j}4&m2QK+&2S(Fr^t>xc_At%Yp=jHFVJ0#U=hZl;(YIO4XHpc_~XeSZsC}KRu z6(3MK@bN4*&U3|Al8YIGeb#YcpmgmD5_lW-M-ez|JAs63vtyLaAVfyy@ zFPb!RF85r?Ol>bDQX;px5d&tPesz3kcf=XBf&VdIGK2n zdmg_#0?9@Z4LYg;8!}(1V|eGK#3|FKQG*z_dFPN0^GvaBZ;Dd|+?N}vTDJ{@YQ2;+s~Qve1D!zubhnHMKTOp* zRJ%e5#pYWzTTs0ira1{^kN6YXU31ynjlb$@U)t#ZCNB0(`F}p|zuKza|G#+D|8Jw* z8U5c;=4C>nfQUfHaj(5CFeGVf=J_{jvu)@`)oAVWU;b43=#^d<*!^GXcCGNmbmm;Z z+xz>S*OpUOqPuGeR}o*BJHRv2Gt+3hV;W98Io>dqzJ_Mj4w$#A_KVJOk?(*nWFx3+ zqB&#G8_a4NnfOK7-1XCBTWhb$81>{t+sTp~Sv5URtIytZUYZ{~s>bXI73kmL*s}9$ zr9xzX+x=Qi&Wp8*qF%p_wQ=tYE2#h#)^JV}yNXcTpItw#Vi%UKwtl{N#TItm3d>em zu3C^;cKC7*%6U#1MRmy_A0NQ;2-swjCYadYyoQmfKL#RA%uP?m)c2&x_TXl3P^y5O z`o#)!xJfbb!#sxKr0*$nlg`f+0`w9-ClNkP3OSE~PpJS6%&;|c1!G)aC-{g7&+lfO zl0`lRmORDq*E4*R8CrvgJ=+UP&Yakza^0T)6vG@tD2Fnk^C6mT!YYRvqzOWa>ANNckO z7kG*BPcjK>`K8{#^BC!x?eeNjetBlSmOrGrPENHf*ZJ}~?qg{#ZS;Rl0+&|*7wZ47 zw#)e+Hn+AO<3HR=xdZw?4&m*mSP20oaw20O$0x0*b2N$-mpS$FDki}Lwna8%<+iH% zaY}_?0ae^lq>&FVxAC8Ba+D5->=Sf*O{t6b1XUWPzO=_-sxjNEU1C(0!7LxTmZpm) zV3fLobqW16dtb?JaU%NPx#z2(aY%d$-M@9Ad(yr9q)3LiVH$=r_&p_|TGv47DaH-^ zXR4uN{Y3SlMlQ2g=aNsTdn=P$4Yk=SWD{mCA(AnZuxe2C=EISyi#H)L_1Dnh+qz94 zkStn}gL{@=a3J0>9+4Wn8SQ+WtBMsvqNNl2F{2e*r?EmSyt^jECsEm*dl9`lZ zp%i6qHrB$gOAoM2g#~E3-(`qxzRJIl2=y%$jr`fYq{U`as;0(#dJbXQBZ%t8Tt(*f z%)2b3f#M%OnUk!go^s{ZzK?UpXh4ZmnzBHW_+~UA5ghwW@PI_`0uG-b%Gl zuX{`Fh9N(9UsdL*C(ScguF(@v9piA4h6(HCm7HTGun2rU8Q|+fttYk-$_S@MGFLNP zx(pA0xw@1lje-p0-I*7s?sL$mGDlHc@liL=(sDPP@bW$7)r82YAkkY}3Kebjzc%`R z9=-o^XMjcVAFBF)zrX#M|Ls=F9nk+v$vQ2y;ms(O$p%Vl8zE>wj8%}ph)rn(v7nd+ z=6FmcHGgRduXx&kNajOAwVRF7)+3(4gc=i=71tOqhX<+Olm|550=kECBG8CLygUCo zp$a1NmpvXEeMApj^K6{F;=I0~#GET$Y<>l+^pj8H1ZX^_6Dmm9lL;4Oqz@yXLLx?V zV@v-pNcIPnYuS*VSp3+#hYpyroB0^O?XK-US3=^n3*G+a=A;YV37v2;Q-5!L{hoE9 zTNz6ix-T~0v+kM&sHay3_o4*u?^})Fw!Sma)PK4Y=8Lw2-YWI7_2T7K2EG02MFHIQ z*UwuHT24q9wjA_}5~g4Dn^DtZ(6353c=fV*;YygM0ZS*B9Cd$ls~I_$7#Z+2?3YDw{jII9 z+mUvDBrw+t8FSQ0k+U;H6wpuQkJS)mb^5*uh;o2UF|hH|ujEr;5k|W}@ygF>kUfzT z6q9-DE>?9sm-Q~Jp3#}iMu3^cv)C9{wXV0(|8$U`oAUqezk2?nU$+0gc=hU0|GSNH z2lT(9`?n@GV>X#%H?c%nFqK$`>LU~1s?P1KSO>c1RI1LR`aGrB$t?TlJO}R*s2;p6 z{IHS5p3>0}NvM6K)oAQ>uj2x4-?-!N=H3!Lw6&sZ>{ECl3z%BpQD3vj>pE<*K4?8fQz_dXRrH3oK}{@jRD3YIo;uv<8dDy)s2flI8$u=oWht))g-2h?Uyks z)ydyY(|Ob0>_rN#7_b(b1qwdvnboZZK?TsS%k~Fj=&H6`(zIgzK+05~3sQW!S@Anx zE4lwfu7%Bxxz2~D*7Vm{)g5S#0XzMAQ&E}goY7e;rgKJTSK>L?Ui}x%x{r+IHW-Ui zziPt)esJy^RVB+Wu<7FMjcb|Fr~0vnH2XRbWOCv;HQE=U?qNLzb2;UR(B1$A&o}KxC{MFTtlwb5Z@3K zZ*U6#p8I&rZ2bzwLa#uwPxVSZ%?)SgV`{8O8IG9*bqZ3boU<27kZU%BYFtDV^e~?0 ze`;X=C_*bUB^%|>h6Sl%O6x$q(T-BV&X{WpZbbfnPqvn*>Lto+{W5(E-k}z`{qt;ieHummOrUTLRTFi> z{1|wINp4CX!A>yvk!HZcF8+bq@efb<{>ySz7x(IR#ZZlnei@ePs66F7&Xgh zc&d!4udh|j6rSo2PYaA$4=C?g%`G3^Lfxt=(=)B(Nj9>b*U;@k7Ol`QzS7S*pLKi( zx?R_OAiEi0LnGtj*@@gO>|I zWczD%@fvF)U?1ng=hyC6^4Bf|oa)?thAC$HmHz%~)U`;kV_OjYxD-$`pRS9i1~O?t zQ$uYhz)ulTrQ=8`jG4*i0mY+8wmV@7X*BO7!{wN>2%EQ6H)|ZOvU!7U0t2ke=N?3RuZohsoMKISE;=g%3wQQw) zN>*(th zmn#>w3!L$~g)VWJuFPZq&#M`_;MYFw0zpQ>;+nc!q;Cup8(>PH`-S3r2N%?)JLHD&;`URtUOv zbOgI!yT5kbxaTzyzMC8C(@Fi%7QJNo(!1l~mp1%A52J8H??0Zus=WVr@fiR0R>~dV z|4yd+?9xNuDP=cEncr1R#hAoWAG@T_<1AXC`9ta_AW<^!&1Fqo5GV!)sSz5L^D-EP z@}|IdaY|iyLgILSV7zLF)k9*%y<}?7OjJZf%ceH4$*X`BO3XN?U?9!^r44ka^%V`=iO%VQ^ijS~a#WmD3_4k!&woQp*Da!G#Gz zgdJA38U1@D6wenU8@VLiWMQIm0?K}BPf5M3%^jgBcR=@mx;|#Xx-La#jq<;Z{-^Jd zZ!G@Li_L2O_pO(Y?>}y(+!glHl;OoV=gi!Nd#n&e(k#%V@?*xxaK8%=*A!UnXYpJFZ&P+Qvq`26|xlhw0y zK>Z{{y0)D|a5pl?n z`AhW=OcCGhb?t5 z*KUN;;HsVECo#Ou!{JPwT&y+F}*JxgAc;#yDR&E9iBLH%s-GW+>;9 zkAvUThbJXZmXacn_f&D=ci{AD9ED;Xg+k)6Umq*<@kU9$48&m&GrV}8}k;W`f*B*B; zTn9YUeo)Wyo5|HZ9u3*(J&D)tyO<#tm_3+95c5JFg>|5x6rOCb9~-q6Rwbpd(*Jt< z)yuWT8ij8Ld~8OW;Z={gyW!)QCL~5Lf@PUn6?{f8T(cxQtW}%E%`hn2!uy6udZ4%n z9+B2u`JNpl%rSvaSfT-U-2^$^GZvv-H7M%lM~MrNW3u()rCPE_`}_kdZO(t^M5z6E z=_BBR_y7I=Rwe(#tLKmLpKhhxmGj@vn#O<(c$z?}_&x|8hRC|fXdaW!$kWt6CZjnC zQ4cv!=JDiOXzoi+j8&dn!tKM;E_iu1{WX)twwxjLLOX?i%c20fJ@M4)?9JU6o?2Hcn4baGAv#V5Kn+W6_c z`m1+LqCe8!ZXAcqM}geiBMAxlXu}4n-$>Gg3r0e@@!gO6Z+iG32_w<>%y4>YI6Z@I zDr66}TQpD3x5O6-%7n=cz@&S|LrsDYvDk=3By2@s6CU7`mwlUwe5u6bit^g4nT?%h z$^jV;ktph>N3KY@S+f6Z$epc^(`l#y+NU-~ zDO}B!!;P=x4o0~VHgmj7(2A7TYF*SR0r zLXYTTOGUT;-~aRf!qMBk9}oAB{|Ol%_-G~w`6keDE(VlF+HagCX`(|`FbUt&aAIDC zo@0^>okVOMqu{Ck#hBIF8CHkV!@!?$NI?nwAr}*ASOT%HynJNfdXi}Bau(nVIcS!jm#v6 zlq9L3QcrBcBo5tSSTH4mkwHkM*XbPP__DmfGS?mV{8ge5lR@0(4sYN*i@1=YWJ#06 zo=37%jR|h3xNaD_fmhqfJsVljwRM`rYtGTvo_W<_*PahF4|@BFOyZF4n0Qy0@5Yzx zK)X79NSRLTy%;`(T;Omi7`PMJpr`4z@*n6q7+A~U7G zg9n*2m->0*8SE*YA@}*sh7AS=K3~-s7fO9v_W1Ymx|-#Lkm5V-G^hqPAHU zhw?K8d_06oo#5$6zr@WG`G$0g8Iz)fg+PR0Q)Dr`$Ysn}J+S*7h&S;3oJ>pOva7%I zI$JXrfre9NCNB4JmTBpF`r{8?!xGfeiEUI%Ckx44#xgpVJ3x^*eOjIBqDi{S1Wl;CLsjwlU)fKT)rsf1%p4jZjsX9k8%#c9;qBs%a5#W(iFB-B@DsaRmJzy_GgmVtxc=XwR5T%9v*Lj}4apDsxNfZdOgfV!5|IgOT zt+N03_VdlHNBi$>l)Ga8-AzBSkP$I+V*Y6VA5>v+aK;~ji&A{VxsRuFf;EnL<#>%Y znso69T6jolgFe(nhw`%cUwgKvEHWvPd0Q@&@bIj7-qEMJ#xvx?VfTve*4Ypp0a^~p zk;4oIKT0afF$vu_@U^F+D#+G=n$y%Qx9}#aNIsS2*50b*^^4wqF^a1F+*NS>D4JKq zPHYbD3l13=PoX~KN}W55ZX2>MB8)X4ix=IhMUYOsx-RDkX!J3zM zS>19LzkEq0YFYU|A%c-$u+&xX0{Q>y)ywkz@Al^NNB`ejDR)Kwn@B+v-YWuo^9|U5 zMOg#_ogYE*Jq$lIm+{v`RI-GdfopOMHepk89%X~L&S6oFkrs;4g>67eJe1mrI>{e& zfQ3R7lFZ9h_RKh!VT!hoYha1(Q0iO-mUMy|r#0K&fFo%#uBbU)0i+R^v!l!(OGoSBR{OFV)xVM?)@J z$tc}1lmnv4vIBRiUU`)&f#qtKvoW$<*U(5$EwSUuzF>^GEw1{leyGLS;2>Q>xlFvs z3c=N7&aW}+Bi}BKp~h0!DvV!+XcuF;isY*zx)P4>&xQzwNR9>W({=Vj&2>V)mi3#e znfhw#w;;m2)eWGjZcOtr) zjQbJ>3JbOd-ersvEx}x~X~IroiFLoES)vKb6^#-W>>W2rxa@S9JyuZaZqOicgWIhz zg{$l{S0XL5u^QT4v$09--z8QySF`Wss^-IAu%a%PtXnbn_l2gVrR_gF8u9D3|7^e5 zuH64@zIu%Rb1UVp*ne(L#WDnYJ1Umy?9Jd>xI>F^UxxJ?u=iWVGLu+h-7l+RH9)zl zie4%+MrS=f?AdSgcwA#Hd5o@SlHuFY>fEU;GNZQc zT_u&Sk-yEKL6Oasgo{B_ujq?WU(4yox%k(&@t8P%VJj1+&Q${2KIFRLGe8xE?al!0 zm!w%gf1(l9n4onD0xfs4z@MQ}gUQp3l-AT4OTBKtR-DcpJ*TmcWdUFF5yF|`k`!~f z!^9jwS3FU;i*>h;6;=tgBbEnQ3jC#(Kq+?>7&oeb2wluU z(DkyqHRsSRxO*)B=YgARyh5UGqiW6ONDUP!RNvzba1z&<^>VDlg>bK_xOP6*$K6~+ zFUjf3P#1u@W`5`P7GIv_)p||eV{&Pn(*lxdAc?rN&0Tt!SMOn;>)V!P&;No3Y}r7d zi{5{~c(qx+|9SEJ<>UGPHp-2j|8MXD$S`kBj`Mh`u5mRbz8=OEC7cF&Csp@YzsAA1 z^j4th*|)|tYPJaGMr9TDc^zV^Jb0OnzImB-8+Xm-qh1F-&#YNGHkTSdLVPMMMs;#| z41-Dp`-7NRJ5wF28~*Vc`p?%|uS5oi2Hw<$`@RDG*TQoS4Nsg3yJ<2mz5)A6*5qz( zM!vTO-LgHu+<^(7cge?0<u_}ug%leCtXK5m-Z-87~1Km|JViqo@IvoO(lv~vp-6Gh`O zVdyQhDT{ljR|6dZ`D;{j9BD}B=&d!}Y8Q{`8a(E5tFW~!1Skmtc-reJwMdP13B^e?Zq%6c(v@!9|$cRd>v!XLh$TW_*NF>NH4MV(xg9%CeF^fj) zAWUv{E!pzlNfdNeAfh9kD`Hh2@r`#3{9kJxe2l^w22YRmkRYZ4LKac4a<*?0B)@zfnKwcr2h~L)R1hw~jZJJ`bpZ z%_bul@-U#H!zLr_ZLHL`kdHch-|v3-?(Mse-+7bZOB(_j_y6JB-8b*wdXwPleazqg z{g?fIDgImk#fz=S_aC=XRwgAmtTf838V)Z{sJH{ue&>uu!46~zD?2u~q8hm# zcVn*RVH$?oMFV4b+g6jAR$`$0Jq<$+KXVZVT|J0*oS&J{;i;s+QvoE;&4aPd^T_1? z1CGTEe$k1#nlt(23vG~NGJGXe#DYn>ww`G$RQQa9Km&#Vj4qz7sfMQ)bw5h7fpj`3 zQnY4c(zHk*mO`=wp<^1xR7f}(r$%X*Q`hb z9pGGeolcSX3LT()`sIgoKuH|S9NL-r#S7?kLld}@@B@<(A$a_g4|QUuJq@2l;D@P9 zs4#?nASj;aB}=I6bb9bZ7Ns5f-q!ZtIz0_w+xyzv{F`p04w_WKz(0>-ic7$vI88wM zV>%)DWd-(a{;`}^?G{jEmCRzg0}{_w9e{vJUobpe?Z6w{ACRTP%1@1$J_$28abF#- zI$#s^>_E1wIvr>CE$HenPqd}5r{Swz;6k?f_M#DsK6N??87%^<;Ur|_g6VtxmtMaY z&;cV+ufN$7FSk3LI1R%C9x{L7SP|L784wNrAZwW2IH5xIbT>RFGx-$rgV=E>oITZ9 zn4hleyCX~qz~Sls@B>dU-$0Z+)#K5Qk}@9<5rWJ*04$-}%rJkX_LLmXOg1ltT@6H0 zWj1`%53J^X%S2`yP_B8MPCy5#gYJR8n5&>h&jl|**brtsg>#;U0cQGAUnHeKKP7?y zi!iB|<`^gsq`OG^%1+VtuF0hk`P2X#(4EdoMwTwctrQr3kzQ65{u@?*1)SvvcO^P2 zpr>v|g#1?Rs{_UDWkMsL<~z$=$1cQ^y~EV?m9lGLc`gzZ9y=YGa6v|NFC2xcfIHMds6jzy!w zh`YURr$c<7O8K4#bkT5ku_jD-kh@Pcj=Rv&Atk}jf+h4LE`VfzEbQ!v{Xv)L;p^{M zr=t!si{Ks@Gu|Ly$e;kpO3=CFY)MBi`buFrG+tN0L2L;DD}WflL9gKP%jq_w*|!A; z?3`>-BUvy@*UV-exs3&{Fxkn>KtH3i_as(2-%c5p`8`wrG|X7%r#YiDw9 zz94BN86JgkFX9W!Qcf^p2EGoGPbeg8LZt$5P9oftBh^}KbV9oY#9T@?U?EFpx?7V> zkO^7FhEgr;>aOIRe8b=hfA95r^=hlZ%H?%dfRGGmXe1Pgy@&)5-ZG~LO9T-xKr!S2z~&mRxpIBA-u z)_S|+{0Iio7 z6a)3j_rbU}`$NT%hFthca&np7(VJ&(c*&r677Ihv?U0p`kg8*JP7_)YRi4exdLPfu|?&!NK4UB%vrq%WUCFIel;p`d>$ z6!cvb^sfg6{To0*zl;LIG~ma0BuJn%6=WgD9Qxj-x7q3Z@-c!Bd`dB`V1IiZwl=pm z*Yy#4x@Aq+m?0UDAnJHDWD%2N8sta(5lhDD0F54sc#Vm4MJ6OkWseVg$(Z(xH5hFn zOu~fx#)UOQsiU6B!Ko%}6qxMh-bb6T)P@>u%TV8QOM%&ActF6Ei9Q!GH<>`)44^g( zvRZ43m#ccJ+ggJ{C~6-Xl^1OqSyyHTz!FKr;s4Lx_4T%G1Mzo#ih~PmO%X|hy{C1^ z;3e5AtgwgJMo{#jp{27;Ocp(olIw3j-0>L8$&h7FeK8*bLq3sr$NNM6$oOberKAj9 zr09llJN=kRRKDLR|H%6uErJ$E2jyJ%sUChwk`(^5*fEMO4_(kltXUldZXLqI>5Ak5 z-o`?Dq=)4Jmd0GO7x=Hf(>h8|p!mdV735gp=_sNnt@+2+JUWQy-r3)Qz$ewpQxz z{Ebkj%v>sF{vX`Xfb&ypB0e1K!uB^Vw%>)i@RaV~@o@>4Tjet3##TrMt?J!IEz#?e zJ&ISj8?!ZQY>;MtuMS}yOc|hu5B9Jzv9W71)(WklmwY7EMsE*}q8}}Ztg8^HK~~Oz z7N1gv%KwT)3$|z)j5#S{ed0Nn+@IUfmZACH!-_(eY7U^CYtg-a$goBiKN3TAB{s@x z6N^g9ObO`Hf4eg1X!sbiqCcwUKofan<0u916P`F*U$o#f#I}2h?^9(+F0r^m8w1s6 zvs!5e9hs~$WlK97p0k&oryG9hMqjpiEu|*kJbp}DhiR!Cx(jg=`Knom$%`_01Kmih zW#Mqu)==x_FGf#)j$_l9S-0IQi-KL}0rXB8^iF%AcPgNF?$CC+M3?*48>?0A^4}N5 z4cEb}J4{KQA^#nvJL7zM1%KtK-ZTz;r>NE2iMf~dvOJXa~U`k5h(bI02dLti4 z`4wkGxtN*sE_Hran%&_&WxyXFFL;K_M;`4K2JzV-OfK6*d=~D3s;0_~rThS{T{>N9 zWi|iuf(%zd+f;I=eY{iU1|uVvAkhtm*$?oR2nO=i;NpEkhPzb<_5lk0Y=9hF@N0=X zjy?rI(O?=f#l7g_R2IxtJGINXr1G0_SYD4!&()SvQvzu#wyS~Xliusn5mbCc$z}#qE>sLD7Lm3vWsqDZ?)%xZf91WEG((C^Hwigf1;kq zlXlQ2prp{iU_66jHh^Epvto9c+!yl?)5RR_$9H$*$-KCo!SoKUr<0pvUQ8!5nEnRi z$tU=unA}_f>VPMHrMT^gLA~X5MV8DEPX>=TTR?<$3AJk0ty&|jO@~~*9(KnP_#g<@j-QE>ed~klb33 Date: Sun, 12 Oct 2025 13:00:19 +0530 Subject: [PATCH 16/26] step to build dependency for pipelines Signed-off-by: kunal-511 --- .github/workflows/helm-kustomize-comparison.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/helm-kustomize-comparison.yml b/.github/workflows/helm-kustomize-comparison.yml index 14cd4a7d14..7e9b854d40 100644 --- a/.github/workflows/helm-kustomize-comparison.yml +++ b/.github/workflows/helm-kustomize-comparison.yml @@ -38,6 +38,10 @@ jobs: rm get_helm.sh - name: Install Kustomize run: ./tests/kustomize_install.sh + - name: Build Helm dependencies for pipelines + run: | + cd experimental/helm/charts/pipelines + helm dependency build - name: Run all scenarios comparison env: VERBOSE: "true" From 120214126501b63ffb84f65a2904767983a2ff91 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sun, 19 Oct 2025 21:15:47 +0530 Subject: [PATCH 17/26] Remove cloud specific templates Signed-off-by: kunal-511 --- .../pipelines/ci/values-aws-enhanced.yaml | 355 ------------------ .../helm/charts/pipelines/ci/values-aws.yaml | 88 ----- .../pipelines/ci/values-azure-enhanced.yaml | 334 ---------------- .../charts/pipelines/ci/values-azure.yaml | 90 ----- .../pipelines/ci/values-dev-enhanced.yaml | 5 - .../pipelines/ci/values-gcp-enhanced.yaml | 343 ----------------- .../helm/charts/pipelines/ci/values-gcp.yaml | 95 ----- .../pipelines/ci/values-production.yaml | 6 +- .../charts/pipelines/templates/_helpers.tpl | 36 +- .../config/pipeline-install-config.yaml | 24 -- .../templates/config/ui-configmap.yaml | 40 -- .../templates/database/mysql-secret.yaml | 11 - .../pipelines/templates/env/aws-patches.yaml | 60 --- .../templates/env/azure-patches.yaml | 136 ------- .../pipelines/templates/env/gcp-patches.yaml | 273 -------------- .../env/gcp-proxy-agent-configmap.yaml | 12 - .../env/gcp-proxy-agent-deployment.yaml | 32 -- .../templates/env/gcp-proxy-agent-role.yaml | 18 - .../env/gcp-proxy-agent-rolebinding.yaml | 19 - .../env/gcp-proxy-agent-serviceaccount.yaml | 12 - .../profile-controller/deployment.yaml | 2 +- .../third-party/minio-deployment.yaml | 15 +- .../templates/third-party/minio-pvc.yaml | 2 +- .../templates/third-party/minio-secret.yaml | 15 - .../templates/third-party/minio-service.yaml | 2 +- .../workflow-controller-configmap.yaml | 24 -- .../helm/charts/pipelines/values.yaml | 118 +----- 27 files changed, 11 insertions(+), 2156 deletions(-) delete mode 100644 experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-aws.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-azure.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-gcp.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/env/aws-patches.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/env/azure-patches.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-configmap.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-deployment.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-role.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-rolebinding.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-serviceaccount.yaml diff --git a/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml deleted file mode 100644 index 94389e2d05..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-aws-enhanced.yaml +++ /dev/null @@ -1,355 +0,0 @@ -# AWS-Specific Configuration - Enhanced -# This configuration provides AWS integration with S3 and RDS - -global: - namespace: kubeflow - imageRegistry: ghcr.io/kubeflow - imageTag: "2.14.3" - imagePullPolicy: IfNotPresent - -# Installation mode -installMode: - type: generic - multiUser: - enabled: false - -crds: - install: false - -cacheDeployer: - enabled: true - serviceAccount: - create: false - -# Database Configuration -mysql: - enabled: false - -postgresql: - enabled: false - -externalDatabase: - type: mysql - host: "YOUR_RDS_ENDPOINT" - port: 3306 - database: mlpipeline - username: "YOUR_RDS_USERNAME" - password: "YOUR_RDS_PASSWORD" - existingSecret: "" - -# Object Storage Configuration -minio: - enabled: false - -objectStore: - provider: s3 - secure: true - bucketName: "YOUR_S3_BUCKET_NAME" - s3: - region: "YOUR_AWS_REGION" - endpoint: "s3.amazonaws.com" - accessKey: "" - secretKey: "" - -# Database names -database: - pipelinedb: mlpipeline - metadb: metadb - cachedb: cachedb - -# Pipeline Configuration -pipeline: - cache: - image: "ghcr.io/containerd/busybox" - nodeRestrictions: false - maximumStaleness: "" - defaultStaleness: "" - -# Argo Workflows Configuration -argo: - enabled: false - -# Built-in third-party components -thirdParty: - argo: - enabled: true - mysql: - enabled: true - minio: - enabled: true - portName: http - -env: - platform: aws - aws: - region: "YOUR_AWS_REGION" - rds: - enabled: true - endpoint: "YOUR_RDS_ENDPOINT" - s3: - enabled: false - bucket: "YOUR_S3_BUCKET_NAME" - region: "YOUR_AWS_REGION" - -# AWS API Server configuration -apiServer: - env: - # Object store configuration for S3 - objectStoreSecure: "true" - objectStoreBucketName: "YOUR_S3_BUCKET_NAME" - objectStoreHost: "s3.amazonaws.com" - objectStoreRegion: "YOUR_AWS_REGION" - objectStorePort: "" - - # Service account annotations for IAM roles - serviceAccount: - annotations: - eks.amazonaws.com/role-arn: "arn:aws:iam::ACCOUNT_ID:role/KubeflowPipelinesRole" - - resources: - requests: - cpu: 250m - memory: 500Mi - additionalEnv: - - name: OBJECTSTORECONFIG_SECURE - value: "true" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - configMapName: pipeline-install-config - - name: OBJECTSTORECONFIG_HOST - key: minioServiceHost - configMapName: pipeline-install-config - - name: OBJECTSTORECONFIG_REGION - key: minioServiceRegion - configMapName: pipeline-install-config - - name: OBJECTSTORECONFIG_PORT - value: "" - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - configMapName: pipeline-install-config - - name: POD_NAMESPACE - - name: DBCONFIG_USER - key: username - secretName: mysql-secret - - name: DBCONFIG_PASSWORD - key: password - secretName: mysql-secret - - name: DBCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_HOST - key: dbHost - configMapName: pipeline-install-config - - name: DBCONFIG_PORT - key: dbPort - configMapName: pipeline-install-config - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - configMapName: pipeline-install-config - - name: DB_DRIVER_NAME - key: dbType - secretName: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - secretName: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - secretName: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - configMapName: pipeline-install-config - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - secretName: mlpipeline-minio-artifact - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - -# AWS UI configuration -ui: - env: - # MinIO service configuration for S3 - minioServiceHost: "s3.amazonaws.com" - minioServiceRegion: "YOUR_AWS_REGION" - - resources: - requests: - cpu: 10m - memory: 70Mi - additionalEnv: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" - - -# Enhanced resources for AWS deployment -persistenceAgent: - additionalEnv: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 120m - memory: 500Mi - -scheduledWorkflow: - resources: - additionalEnv: - - name: LOG_LEVEL - value: "info" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -metadata: - enabled: true - grpc: - resources: - envoy: - resources: - requests: - cpu: 20m - memory: 20Mi - limits: - cpu: 100m - memory: 100Mi - -cache: - server: - additionalEnv: - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 120m - memory: 500Mi - limits: - cpu: 500m - memory: 1Gi - -visualization: - resources: - requests: - cpu: 30m - memory: 500Mi - -viewerCrd: - image: - pullPolicy: Always - additionalEnv: - - name: MAX_NUM_VIEWERS - value: 50 - - name: NAMESPACE - -# AWS Load Balancer Controller integration - - service: - type: LoadBalancer - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - -# Monitoring can be enabled with CloudWatch -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - cloudwatch: - enabled: true - -# Service mesh integration -istio: - enabled: false - -# Cert-manager with AWS integration -certManager: - enabled: true - issuer: "letsencrypt-prod" - issuerKind: "ClusterIssuer" - aws: - region: "YOUR_AWS_REGION" - hostedZoneID: "YOUR_ROUTE53_ZONE_ID" - -profileController: - additionalEnv: - - name: AWS_REGION - value: "YOUR_AWS_REGION" - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/ci/values-aws.yaml b/experimental/helm/charts/pipelines/ci/values-aws.yaml deleted file mode 100644 index e2675a59ea..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-aws.yaml +++ /dev/null @@ -1,88 +0,0 @@ -# AWS-specific values for Kubeflow Pipelines -# This configuration uses AWS S3 and RDS - -# Environment configuration -env: - platform: aws - aws: - region: us-west-2 - rds: - enabled: true - s3: - enabled: true - -# Database configuration -mysql: - enabled: false - -postgresql: - enabled: false - -externalDatabase: - type: mysql - host: "YOUR_RDS_ENDPOINT" - port: 3306 - database: "mlpipeline" - username: "root" - password: "" - existingSecret: "mysql-secret" - -# Additional databases -database: - metadb: "metadb" - cachedb: "cachedb" - -# Object storage configuration -objectStore: - provider: s3 - s3: - region: "YOUR_AWS_REGION" - bucket: "YOUR_S3_BUCKET_NAME" - existingSecret: "mlpipeline-minio-artifact" - -minio: - enabled: false - -# Argo Workflows configuration -argo: - enabled: true - -# API Server AWS-specific environment variables -apiServer: - env: - publishLogs: true - pipelineLogLevel: "1" - autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - additionalEnv: - - name: AWS_REGION - value: "YOUR_AWS_REGION" - - name: OBJECTSTORECONFIG_HOST - value: "s3.amazonaws.com" - - name: OBJECTSTORECONFIG_REGION - value: "YOUR_AWS_REGION" - -# UI AWS-specific configuration -ui: - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: AWS_REGION - value: "YOUR_AWS_REGION" - -# Monitoring configuration -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -# Certificate management -certManager: - enabled: false - -# Networking -networking: - istio: - enabled: false diff --git a/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml deleted file mode 100644 index e0f83a7151..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-azure-enhanced.yaml +++ /dev/null @@ -1,334 +0,0 @@ -# Azure-Specific Configuration - Enhanced -# This configuration provides Azure integration with Blob Storage and Azure SQL - -global: - namespace: kubeflow - imageRegistry: ghcr.io/kubeflow - imageTag: "2.14.3" - imagePullPolicy: IfNotPresent - -# Installation mode -installMode: - type: generic - multiUser: - enabled: false - -# Database Configuration -mysql: - enabled: false - -crds: - install: false - -cacheDeployer: - enabled: true - serviceAccount: - create: false - -postgresql: - enabled: false - -externalDatabase: - type: mysql - host: "[SQLSERVERNAME].mysql.database.azure.com" - port: 3306 - database: mlpipeline - username: "[SQLUSER]@[SQLSERVERNAME]" - password: "[SQLPASS]" - existingSecret: "" - -# Object Storage Configuration -minio: - enabled: false - -objectStore: - provider: azure - secure: true - bucketName: "mlpipeline" - azure: - storageAccount: "[STORAGEACCOUNTNAME]" - accountName: "[STORAGEACCOUNTNAME]" - container: "mlpipeline" - accountKey: "[STORAGEACCOUNTKEY]" - -# Database names -database: - pipelinedb: mlpipeline - metadb: metadb - cachedb: cachedb - -# Pipeline Configuration -pipeline: - cache: - image: "ghcr.io/containerd/busybox" - nodeRestrictions: false - maximumStaleness: "" - defaultStaleness: "" - -# Argo Workflows Configuration -argo: - enabled: false - # Azure-specific Argo configuration - -# Environment Configuration -env: - platform: azure - azure: - resourceGroup: "yourResourceGroup" - storageAccount: "yourstorageaccount" - sql: - enabled: true - serverName: "[SQLSERVERNAME]" - database: "mlpipeline" - blob: - enabled: true - storageAccount: "yourstorageaccount" - container: "mlpipeline" - createMinioGateway: true - createMinioSecret: true - managedIdentity: - enabled: false - -# Azure API Server configuration -apiServer: - env: - # Object store configuration for Azure Blob Storage - objectStoreSecure: "true" - objectStoreBucketName: "mlpipeline" - objectStoreHost: "yourstorageaccount.blob.core.windows.net" - objectStoreRegion: "" - objectStorePort: "" - - # Service account annotations for Azure Managed Identity - serviceAccount: - annotations: - azure.workload.identity/client-id: "YOUR_MANAGED_IDENTITY_CLIENT_ID" - - resources: - requests: - cpu: 250m - memory: 500Mi - additionalEnv: - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - - name: POD_NAMESPACE - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: DBCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_HOST - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - - name: DB_DRIVER_NAME - key: dbType - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - -# Azure UI configuration -ui: - env: - # MinIO service configuration for Azure Blob Storage - minioServiceHost: "yourstorageaccount.blob.core.windows.net" - minioServiceRegion: "" - - resources: - requests: - cpu: 10m - memory: 70Mi - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" - -# Third-party components configuration -thirdParty: - minio: - enabled: true # Enable to create the mlpipeline-minio-artifact secret - mysql: - enabled: false # Azure uses external MySQL/Azure SQL - -# Enhanced resources for Azure deployment -persistenceAgent: - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -scheduledWorkflow: - resources: - additionalEnv: - - name: LOG_LEVEL - value: "info" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - -metadata: - enabled: true - grpc: - resources: - envoy: - resources: - requests: - cpu: 20m - memory: 20Mi - limits: - cpu: 100m - memory: 100Mi - -cache: - server: - resources: - requests: - cpu: 120m - memory: 500Mi - limits: - cpu: 500m - memory: 1Gi - additionalEnv: - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -visualization: - resources: - requests: - cpu: 30m - memory: 500Mi - -viewerCrd: - resources: - image: - pullPolicy: Always - additionalEnv: - - name: MAX_NUM_VIEWERS - value: "50" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -# Azure Load Balancer integration - service: - type: LoadBalancer - annotations: - service.beta.kubernetes.io/azure-load-balancer-internal: "false" - service.beta.kubernetes.io/azure-dns-label-name: "kubeflow-pipelines" - -# Monitoring with Azure Monitor -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - azureMonitor: - enabled: true - workspaceId: "YOUR_LOG_ANALYTICS_WORKSPACE_ID" - -# Service mesh integration -istio: - enabled: false - -# Cert-manager with Azure DNS -certManager: - enabled: true - issuer: "letsencrypt-prod" - issuerKind: "ClusterIssuer" - azure: - subscriptionId: "YOUR_AZURE_SUBSCRIPTION_ID" - resourceGroup: "yourResourceGroup" - hostedZoneName: "yourdomain.com" - managedIdentityClientId: "YOUR_MANAGED_IDENTITY_CLIENT_ID" - -# Azure-specific MinIO gateway for Blob Storage compatibility -minioAzureGateway: - enabled: true - image: "minio/minio:latest" - storageAccount: "yourstorageaccount" - accessKey: "YOUR_STORAGE_ACCOUNT_KEY" - secretKey: "YOUR_STORAGE_ACCOUNT_SECRET" - service: - type: ClusterIP - port: 9000 diff --git a/experimental/helm/charts/pipelines/ci/values-azure.yaml b/experimental/helm/charts/pipelines/ci/values-azure.yaml deleted file mode 100644 index a2219b6200..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-azure.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# Azure-specific values for Kubeflow Pipelines -# This configuration uses Azure Blob Storage and Azure Database for MySQL - -# Environment configuration -env: - platform: azure - azure: - resourceGroup: "YOUR_RESOURCE_GROUP" - storageAccount: "YOUR_STORAGE_ACCOUNT" - sql: - enabled: true - blob: - enabled: true - -# Database configuration - use external Azure Database for MySQL -mysql: - enabled: false - -postgresql: - enabled: false - -externalDatabase: - type: mysql - host: "[SQLSERVERNAME].mysql.database.azure.com" - port: 3306 - database: "mlpipeline" - username: "root" - password: "" - existingSecret: "mysql-secret" - -database: - metadb: "metadb" - cachedb: "cachedb" - -# Object storage configuration - use Azure Blob Storage -objectStore: - provider: azure - azure: - storageAccount: "YOUR_STORAGE_ACCOUNT" - container: "mlpipeline" - existingSecret: "mlpipeline-minio-artifact" - -minio: - enabled: false - -# Argo Workflows configuration -argo: - enabled: true - -# API Server Azure environment variables -apiServer: - env: - publishLogs: true - pipelineLogLevel: "1" - autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - additionalEnv: - - name: AZURE_STORAGE_ACCOUNT - value: "YOUR_STORAGE_ACCOUNT" - - name: OBJECTSTORECONFIG_HOST - value: "YOUR_STORAGE_ACCOUNT.blob.core.windows.net" - -# UI Azure configuration -ui: - additionalEnv: - - name: AZURE_STORAGE_ACCOUNT - value: "YOUR_STORAGE_ACCOUNT" - -# Azure AD configuration -azureAD: - enabled: true - tenantId: "YOUR_TENANT_ID" - clientId: "YOUR_CLIENT_ID" - -# Monitoring configuration -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -# Certificate management -certManager: - enabled: false - -# Networking -networking: - istio: - enabled: false diff --git a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml index 9d5679aa1b..a660b6c72f 100644 --- a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml +++ b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml @@ -147,11 +147,6 @@ application: # Environment Configuration env: platform: platform-agnostic - gcp: - proxyAgent: - enabled: true - image: - tag: "master" # Reduced resources for development apiServer: diff --git a/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml deleted file mode 100644 index 050c2f3cc2..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-gcp-enhanced.yaml +++ /dev/null @@ -1,343 +0,0 @@ -# GCP-Specific Configuration - Enhanced -# This configuration provides GCP integration with GCS and Cloud SQL - -global: - namespace: kubeflow - imageRegistry: ghcr.io/kubeflow - imageTag: "2.14.3" - imagePullPolicy: IfNotPresent - -# Installation mode -installMode: - type: generic - multiUser: - enabled: false - legacyLabels: true - -# Database Configuration - External Cloud SQL -mysql: - enabled: false - -crds: - install: false - -cacheDeployer: - enabled: true - serviceAccount: - create: false - -postgresql: - enabled: false - -externalDatabase: - type: mysql - host: "mysql" - port: 3306 - database: pipelinedb - username: "YOUR_CLOUDSQL_USERNAME" - password: "YOUR_CLOUDSQL_PASSWORD" - existingSecret: "mysql-secret" - -# Object Storage Configuration - GCS -minio: - enabled: false - -objectStore: - provider: gcs - secure: false - bucketName: "yourGcsBucketName" - gcs: - projectId: "yourGcsProjectId" - serviceAccount: "" - -# Database names -database: - pipelinedb: pipelinedb - metadb: metadb - cachedb: cachedb - -# Pipeline Configuration -pipeline: - cache: - image: "ghcr.io/containerd/busybox" - nodeRestrictions: false - maximumStaleness: "" - defaultStaleness: "" - -# Argo Workflows Configuration -argo: - enabled: false - # GCP-specific Argo configuration - -# Built-in third-party components -thirdParty: - argo: - enabled: true - mysql: - enabled: false - minio: - enabled: false - -# Application Configuration -application: - enabled: true - controller: - enabled: true - -# Environment Configuration -env: - platform: gcp - gcp: - project: "yourGcsProjectId" - cloudsql: - enabled: true - instanceName: "yourCloudSqlInstanceName" - region: "YOUR_GCP_REGION" - gcs: - enabled: true - bucket: "mlpipeline" - projectId: "yourGcsProjectId" - proxyAgent: - enabled: true - workloadIdentity: - enabled: false - -# GCP-specific API Server configuration -apiServer: - env: - # Object store configuration for GCS - objectStoreSecure: "true" - objectStoreBucketName: "yourGcsBucketName" - objectStoreHost: "storage.googleapis.com" - objectStoreRegion: "" - objectStorePort: "" - - additionalEnv: - - name: HAS_DEFAULT_BUCKET - value: "true" - - name: BUCKET_NAME - key: bucketName - - name: PROJECT_ID - key: gcsProjectId - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: DBCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_HOST - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - - name: DB_DRIVER_NAME - key: dbType - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - # Service account annotations for Workload Identity - serviceAccount: - annotations: - iam.gke.io/gcp-service-account: "kubeflow-pipelines@yourGcsProjectId.iam.gserviceaccount.com" - - resources: - requests: - cpu: 250m - memory: 500Mi - -# GCP-specific UI configuration -ui: - env: - # MinIO service configuration for GCS - minioServiceHost: "storage.googleapis.com" - minioServiceRegion: "" - - resources: - requests: - cpu: 10m - memory: 70Mi - additionalEnv: - - name: DISABLE_GKE_METADATA - value: "false" - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - -# Enhanced resources for GCP deployment -persistenceAgent: - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -scheduledWorkflow: - resources: - additionalEnv: - - name: LOG_LEVEL - value: "info" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -metadata: - enabled: true - grpc: - resources: - envoy: - resources: - requests: - cpu: 20m - memory: 20Mi - limits: - cpu: 100m - memory: 100Mi - -cache: - server: - resources: - requests: - cpu: 120m - memory: 500Mi - limits: - cpu: 500m - memory: 1Gi - additionalEnv: - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: NAMESPACE_TO_WATCH - key: null - - -visualization: - resources: - requests: - cpu: 30m - memory: 500Mi - -viewerCrd: - additionalEnv: - - name: MAX_NUM_VIEWERS - value: "50" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: - pullPolicy: Always - -# Cloud SQL Proxy sidecar configuration -cloudsqlProxy: - enabled: true - image: "gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.0.0" - instanceConnectionName: "yourGcsProjectId:YOUR_GCP_REGION:yourCloudSqlInstanceName" - serviceAccount: - annotations: - iam.gke.io/gcp-service-account: "cloudsql-proxy@yourGcsProjectId.iam.gserviceaccount.com" - -# GCP Load Balancer integration - service: - type: LoadBalancer - annotations: - cloud.google.com/load-balancer-type: "External" - networking.gke.io/load-balancer-type: "External" - -# Monitoring with Google Cloud Monitoring -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - stackdriver: - enabled: true - projectId: "yourGcsProjectId" - -# Service mesh integration with Istio on GKE -istio: - enabled: true - gateway: "kubeflow-gateway" - gke: - enabled: true - -# Cert-manager with Google Cloud DNS -certManager: - enabled: true - issuer: "letsencrypt-prod" - issuerKind: "ClusterIssuer" - gcp: - projectId: "yourGcsProjectId" - serviceAccountKey: "" # Use Workload Identity instead diff --git a/experimental/helm/charts/pipelines/ci/values-gcp.yaml b/experimental/helm/charts/pipelines/ci/values-gcp.yaml deleted file mode 100644 index 3e5c4bd165..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-gcp.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# GCP-specific values for Kubeflow Pipelines -# This configuration uses Google Cloud Storage and Cloud SQL - -# Environment configuration -env: - platform: gcp - gcp: - project: "yourGcsProjectId" - cloudsql: - enabled: true - instanceName: "yourCloudSqlInstanceName" - gcs: - enabled: true - -# Database configuration - use external Cloud SQL -mysql: - enabled: false - -postgresql: - enabled: false - -externalDatabase: - type: mysql - host: "127.0.0.1" - port: 3306 - database: "pipelinedb" - username: "root" - password: "" - existingSecret: "mysql-secret" - -# Additional databases -database: - metadb: "metadb" - cachedb: "cachedb" - -# Object storage configuration - use GCS -objectStore: - provider: gcs - gcs: - bucket: "yourGcsBucketName" - existingSecret: "mlpipeline-minio-artifact" - -minio: - enabled: false - -# Argo Workflows configuration -argo: - enabled: true - -# API Server GCP-specific environment variables -apiServer: - env: - publishLogs: true - pipelineLogLevel: "1" - autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - additionalEnv: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/etc/gcp-service-account/application_default_credentials.json" - - name: OBJECTSTORECONFIG_HOST - value: "storage.googleapis.com" - -# UI GCP-specific configuration -ui: - additionalEnv: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/etc/gcp-service-account/application_default_credentials.json" - -# Cloud SQL Proxy sidecar configuration -cloudsqlProxy: - enabled: true - image: "gcr.io/cloudsql-docker/gce-proxy:1.33.2" - instanceConnectionName: "yourGcsProjectId:us-central1:yourCloudSqlInstanceName" - -# Workload Identity configuration -workloadIdentity: - enabled: true - serviceAccount: "kf-user@yourGcsProjectId.iam.gserviceaccount.com" - -# Monitoring configuration -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -# Certificate management -certManager: - enabled: false - -# Networking -networking: - istio: - enabled: false diff --git a/experimental/helm/charts/pipelines/ci/values-production.yaml b/experimental/helm/charts/pipelines/ci/values-production.yaml index 222695667a..d497b7452b 100644 --- a/experimental/helm/charts/pipelines/ci/values-production.yaml +++ b/experimental/helm/charts/pipelines/ci/values-production.yaml @@ -65,9 +65,9 @@ externalDatabase: # Use external object storage objectStore: - provider: s3 - s3: - region: "us-west-2" + provider: custom + custom: + endpoint: "s3.example.com" bucket: "production-kfp-artifacts" existingSecret: "s3-credentials" diff --git a/experimental/helm/charts/pipelines/templates/_helpers.tpl b/experimental/helm/charts/pipelines/templates/_helpers.tpl index acbec806e2..beb01b2158 100644 --- a/experimental/helm/charts/pipelines/templates/_helpers.tpl +++ b/experimental/helm/charts/pipelines/templates/_helpers.tpl @@ -558,12 +558,6 @@ Object storage configuration helpers {{- else -}} {{- .Values.objectStore.minio.endpoint }} {{- end -}} -{{- else if eq .Values.objectStore.provider "s3" -}} -s3.amazonaws.com -{{- else if eq .Values.objectStore.provider "gcs" -}} -storage.googleapis.com -{{- else if eq .Values.objectStore.provider "azure" -}} -{{ .Values.objectStore.azure.storageAccount }}.blob.core.windows.net {{- else -}} {{ .Values.objectStore.custom.endpoint }} {{- end -}} @@ -572,12 +566,6 @@ storage.googleapis.com {{- define "kubeflow-pipelines.objectStore.bucket" -}} {{- if eq .Values.objectStore.provider "minio" -}} {{- .Values.objectStore.minio.bucket | default "mlpipeline" }} -{{- else if eq .Values.objectStore.provider "s3" -}} -{{- .Values.objectStore.bucketName | default .Values.objectStore.s3.bucket }} -{{- else if eq .Values.objectStore.provider "gcs" -}} -{{- .Values.objectStore.bucketName | default .Values.objectStore.gcs.bucket | default "mlpipeline" }} -{{- else if eq .Values.objectStore.provider "azure" -}} -{{- .Values.objectStore.azure.container }} {{- else -}} {{- .Values.objectStore.custom.bucket }} {{- end -}} @@ -586,12 +574,6 @@ storage.googleapis.com {{- define "kubeflow-pipelines.objectStore.secure" -}} {{- if eq .Values.objectStore.provider "minio" -}} {{- .Values.objectStore.minio.secure | default "false" }} -{{- else if eq .Values.objectStore.provider "s3" -}} -"true" -{{- else if eq .Values.objectStore.provider "gcs" -}} -"true" -{{- else if eq .Values.objectStore.provider "azure" -}} -"true" {{- else -}} {{- .Values.objectStore.custom.secure | default "true" }} {{- end -}} @@ -604,26 +586,12 @@ storage.googleapis.com {{- else -}} mlpipeline-minio-artifact {{- end -}} -{{- else if eq .Values.objectStore.provider "s3" -}} -{{- if .Values.objectStore.s3.existingSecret -}} -{{- .Values.objectStore.s3.existingSecret }} {{- else -}} -mlpipeline-minio-artifact -{{- end -}} -{{- else if eq .Values.objectStore.provider "gcs" -}} -{{- if .Values.objectStore.gcs.existingSecret -}} -{{- .Values.objectStore.gcs.existingSecret }} +{{- if .Values.objectStore.custom.existingSecret -}} +{{- .Values.objectStore.custom.existingSecret }} {{- else -}} mlpipeline-minio-artifact {{- end -}} -{{- else if eq .Values.objectStore.provider "azure" -}} -{{- if .Values.objectStore.azure.existingSecret -}} -{{- .Values.objectStore.azure.existingSecret }} -{{- else -}} -mlpipeline-minio-artifact -{{- end -}} -{{- else -}} -mlpipeline-minio-artifact {{- end -}} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml b/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml index d7eba19165..ee57f67548 100644 --- a/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml +++ b/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml @@ -25,11 +25,7 @@ data: postgresPort: {{ include "kubeflow-pipelines.database.port" . | quote }} {{- else }} dbType: {{ .Values.externalDatabase.type | default "mysql" | quote }} -{{- if or (eq .Values.env.platform "azure") (eq .Values.env.platform "aws") }} - mysqlHost: "mysql" -{{- else }} mysqlHost: {{ $dbHost | quote }} -{{- end }} mysqlPort: {{ include "kubeflow-pipelines.database.port" . | quote }} {{- end }} mlmdDb: {{ .Values.database.metadb }} @@ -51,23 +47,3 @@ data: {{- if not .Values.thirdParty.postgresql.enabled }} LOG_LEVEL: {{ .Values.global.logLevel | quote }} {{- end }} -{{- if eq .Values.objectStore.provider "s3" }} - # AWS S3 configuration - minioServiceHost: {{ .Values.objectStore.s3.endpoint | default "s3.amazonaws.com" | quote }} -{{- if .Values.objectStore.s3.region }} - minioServiceRegion: {{ .Values.objectStore.s3.region | quote }} -{{- else if and .Values.env .Values.env.aws .Values.env.aws.region }} - minioServiceRegion: {{ .Values.env.aws.region | quote }} -{{- end }} -{{- end }} -{{- if eq .Values.objectStore.provider "gcs" }} - # GCP GCS configuration -{{- if .Values.objectStore.gcs.projectId }} - gcsProjectId: {{ .Values.objectStore.gcs.projectId | quote }} -{{- else if and .Values.env .Values.env.gcp .Values.env.gcp.project }} - gcsProjectId: {{ .Values.env.gcp.project | quote }} -{{- end }} -{{- if and .Values.env .Values.env.gcp .Values.env.gcp.cloudsql .Values.env.gcp.cloudsql.enabled }} - gcsCloudSqlInstanceName: {{ .Values.env.gcp.cloudsql.instanceName | quote }} -{{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml b/experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml index 4acd6987d8..bccd111691 100644 --- a/experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml +++ b/experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml @@ -8,49 +8,9 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} data: viewer-pod-template.json: |- -{{- if eq .Values.objectStore.provider "s3" }} - { - "spec": { - "containers": [ - { - "env": [ - { - "name": "AWS_ACCESS_KEY_ID", - "valueFrom": { - "secretKeyRef": { - "name": "mlpipeline-minio-artifact", - "key": "accesskey" - } - } - }, - { - "name": "AWS_SECRET_ACCESS_KEY", - "valueFrom": { - "secretKeyRef": { - "name": "mlpipeline-minio-artifact", - "key": "secretkey" - } - } - }, - { - "name": "AWS_REGION", - "valueFrom": { - "configMapKeyRef": { - "name": "pipeline-install-config", - "key": "minioServiceRegion" - } - } - } - ] - } - ] - } - } -{{- else }} { "spec": { "serviceAccountName": "{{ if and .Values.installMode.multiUser.enabled (eq .Values.installMode.type "multi-user") }}default-editor{{ else }}kubeflow-pipelines-viewer{{ end }}" } } {{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml b/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml index d6f3a54d24..f80922d5a8 100644 --- a/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml +++ b/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml @@ -4,20 +4,9 @@ kind: Secret metadata: name: mysql-secret namespace: {{ include "kubeflow-pipelines.namespace" . }} -{{- if not (or (eq .Values.env.platform "gcp") ) }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -{{- end }} -{{- if ne .Values.env.platform "platform-agnostic" }} -type: Opaque -{{- end }} -{{- if or (eq .Values.env.platform "aws") (eq .Values.env.platform "azure") }} -data: - username: {{ .Values.externalDatabase.username | b64enc }} - password: {{ .Values.externalDatabase.password | b64enc }} -{{- else }} stringData: username: {{ .Values.externalDatabase.username | quote }} password: {{ .Values.externalDatabase.password | quote }} {{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/env/aws-patches.yaml b/experimental/helm/charts/pipelines/templates/env/aws-patches.yaml deleted file mode 100644 index 38964200da..0000000000 --- a/experimental/helm/charts/pipelines/templates/env/aws-patches.yaml +++ /dev/null @@ -1,60 +0,0 @@ -{{- if eq .Values.env.platform "aws" }} -{{- if .Values.env.aws.s3.enabled }} ---- -# AWS S3 Configuration Patch for API Server -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-aws-config - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: aws-config - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -data: - # AWS S3 specific configuration - OBJECTSTORECONFIG_SECURE: "true" - OBJECTSTORECONFIG_HOST: "s3.amazonaws.com" - OBJECTSTORECONFIG_REGION: {{ .Values.env.aws.s3.region | quote }} - OBJECTSTORECONFIG_BUCKETNAME: {{ .Values.env.aws.s3.bucket | quote }} - OBJECTSTORECONFIG_PORT: "" -{{- end }} - -{{- if .Values.env.aws.iam.enabled }} ---- -# AWS IAM Role Annotation for API Server Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline - component: api-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - eks.amazonaws.com/role-arn: {{ .Values.env.aws.iam.apiServerRoleArn | quote }} -{{- end }} - -{{- if and .Values.env.aws.rds.enabled .Values.env.aws.rds.createSecret }} ---- -# AWS RDS Configuration Secret -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-aws-rds-secret - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: aws-rds - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -type: Opaque -data: - host: {{ .Values.env.aws.rds.endpoint | b64enc }} - port: {{ "3306" | b64enc }} - database: {{ .Values.database.pipelinedb | b64enc }} -{{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml b/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml deleted file mode 100644 index 4459dd91eb..0000000000 --- a/experimental/helm/charts/pipelines/templates/env/azure-patches.yaml +++ /dev/null @@ -1,136 +0,0 @@ -{{- if eq .Values.env.platform "azure" }} -{{- if and .Values.env.azure.blob.enabled (not .Values.env.azure.blob.createMinioGateway) }} ---- -# Azure Blob Storage Configuration Patch for API Server -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-azure-config - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: azure-config - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -data: - # Azure Blob Storage specific configuration - OBJECTSTORECONFIG_SECURE: "true" - OBJECTSTORECONFIG_HOST: "{{ .Values.env.azure.blob.storageAccount }}.blob.core.windows.net" - OBJECTSTORECONFIG_REGION: "" - OBJECTSTORECONFIG_BUCKETNAME: {{ .Values.env.azure.blob.container | quote }} - OBJECTSTORECONFIG_PORT: "" - AZURE_STORAGE_ACCOUNT: {{ .Values.env.azure.blob.storageAccount | quote }} -{{- end }} - -{{- if .Values.env.azure.managedIdentity.enabled }} ---- -# Azure Managed Identity Annotation for API Server Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline - component: api-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - azure.workload.identity/client-id: {{ .Values.env.azure.managedIdentity.clientId | quote }} -{{- end }} - - - - - -{{- if and .Values.env.azure.blob.enabled .Values.env.azure.blob.createMinioGateway }} ---- -# Azure MinIO Gateway for Blob Storage Compatibility -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: minio - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -spec: - selector: - matchLabels: - app: minio - application-crd-id: kubeflow-pipelines - strategy: - type: Recreate - template: - metadata: - labels: - app: minio - application-crd-id: kubeflow-pipelines - spec: - containers: - - name: minio - image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance - args: - - gateway - - azure - env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: mlpipeline-minio-artifact - key: accesskey - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: mlpipeline-minio-artifact - key: secretkey - ports: - - containerPort: 9000 - resources: - requests: - cpu: 20m - memory: 25Mi - ---- -# Azure MinIO Gateway Service -apiVersion: v1 -kind: Service -metadata: - name: minio-service - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -spec: - ports: - - port: 9000 - targetPort: 9000 - protocol: TCP - selector: - app: minio - application-crd-id: kubeflow-pipelines -{{- end }} - -{{- if and .Values.env.azure.blob.enabled .Values.env.azure.blob.createMinioGateway (not .Values.thirdParty.minio.enabled) }} ---- -# MinIO Artifact Secret -apiVersion: v1 -kind: Secret -metadata: - name: mlpipeline-minio-artifact - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -{{- if ne .Values.env.platform "platform-agnostic" }} -type: Opaque -{{- end }} -stringData: - accesskey: {{ .Values.objectStore.azure.accessKey | default "minio" | quote }} - secretkey: {{ .Values.objectStore.azure.secretKey | default "minio123" | quote }} -{{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml deleted file mode 100644 index f35780d4f7..0000000000 --- a/experimental/helm/charts/pipelines/templates/env/gcp-patches.yaml +++ /dev/null @@ -1,273 +0,0 @@ -{{- if eq .Values.env.platform "gcp" }} - -{{- if .Values.env.gcp.workloadIdentity.enabled }} ---- -# GCP Workload Identity Annotation for API Server Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: ml-pipeline - component: api-server - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - iam.gke.io/gcp-service-account: {{ .Values.env.gcp.workloadIdentity.serviceAccount | quote }} -{{- end }} - -{{- if .Values.env.gcp.cloudsql.enabled }} ---- -# GCP Cloud SQL Proxy ServiceAccount -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kubeflow-pipelines-cloudsql-proxy - namespace: {{ include "kubeflow-pipelines.namespace" . }} - {{- if .Values.installMode.legacyLabels }} - labels: - application-crd-id: kubeflow-pipelines - {{- end }} - {{- if .Values.env.gcp.workloadIdentity.enabled }} - annotations: - iam.gke.io/gcp-service-account: cloudsql-proxy@{{ .Values.env.gcp.project }}.iam.gserviceaccount.com - {{- end }} - ---- -# GCP Cloud SQL Proxy Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cloudsqlproxy - namespace: {{ include "kubeflow-pipelines.namespace" . }} - {{- if .Values.installMode.legacyLabels }} - labels: - app: cloudsqlproxy - application-crd-id: kubeflow-pipelines - {{- end }} -spec: - selector: - matchLabels: - app: cloudsqlproxy - {{- if .Values.installMode.legacyLabels }} - application-crd-id: kubeflow-pipelines - {{- end }} - replicas: 1 - template: - metadata: - labels: - app: cloudsqlproxy - {{- if .Values.installMode.legacyLabels }} - application-crd-id: kubeflow-pipelines - {{- end }} - spec: - serviceAccountName: kubeflow-pipelines-cloudsql-proxy - containers: - - image: gcr.io/cloudsql-docker/gce-proxy:1.25.0 - name: cloudsqlproxy - env: - - name: GCP_CLOUDSQL_INSTANCE_NAME - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: gcsCloudSqlInstanceName - command: ["/cloud_sql_proxy", - "-dir=/cloudsql", - "-instances=$(GCP_CLOUDSQL_INSTANCE_NAME)=tcp:0.0.0.0:3306", - "-use_http_health_check", - "term_timeout=10s"] - lifecycle: - preStop: - exec: - command: ['sleep', '10'] - livenessProbe: - httpGet: - path: /liveness - port: 8090 - initialDelaySeconds: 0 - periodSeconds: 60 - timeoutSeconds: 30 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /readiness - port: 8090 - initialDelaySeconds: 0 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 1 - startupProbe: - httpGet: - path: /startup - port: 8090 - periodSeconds: 1 - timeoutSeconds: 5 - failureThreshold: 20 - ports: - - name: mysql - containerPort: 3306 - volumeMounts: - - mountPath: /cloudsql - name: cloudsql - volumes: - - name: cloudsql - emptyDir: - ---- -# MySQL Service for Cloud SQL Proxy -apiVersion: v1 -kind: Service -metadata: - name: mysql - namespace: {{ include "kubeflow-pipelines.namespace" . }} - {{- if .Values.installMode.legacyLabels }} - labels: - application-crd-id: kubeflow-pipelines - {{- end }} -spec: - ports: - - port: 3306 - selector: - app: cloudsqlproxy - {{- if .Values.installMode.legacyLabels }} - application-crd-id: kubeflow-pipelines - {{- end }} - ---- -# MySQL Secret -apiVersion: v1 -kind: Secret -metadata: - name: mysql-secret - namespace: {{ include "kubeflow-pipelines.namespace" . }} - {{- if .Values.installMode.legacyLabels }} - labels: - application-crd-id: kubeflow-pipelines - {{- end }} -stringData: - password: "" - username: root -{{- end }} - -{{- if .Values.env.gcp.gcs.enabled }} ---- -# MinIO GCS Gateway ServiceAccount -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kubeflow-pipelines-minio-gcs-gateway - namespace: {{ include "kubeflow-pipelines.namespace" . }} - {{- if .Values.installMode.legacyLabels }} - labels: - application-crd-id: kubeflow-pipelines - {{- end }} - {{- if .Values.env.gcp.workloadIdentity.enabled }} - annotations: - iam.gke.io/gcp-service-account: {{ .Values.env.gcp.workloadIdentity.serviceAccount | quote }} - {{- end }} - ---- -# MinIO GCS Gateway Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - namespace: {{ include "kubeflow-pipelines.namespace" . }} - {{- if .Values.installMode.legacyLabels }} - labels: - app: minio - application-crd-id: kubeflow-pipelines - {{- end }} -spec: - selector: - matchLabels: - app: minio - {{- if .Values.installMode.legacyLabels }} - application-crd-id: kubeflow-pipelines - {{- end }} - strategy: - type: Recreate - template: - metadata: - labels: - app: minio - {{- if .Values.installMode.legacyLabels }} - application-crd-id: kubeflow-pipelines - {{- end }} - spec: - serviceAccountName: kubeflow-pipelines-minio-gcs-gateway - containers: - - name: minio - image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance - args: - - gateway - - gcs - - $(GCP_PROJECT_ID) - env: - - name: GCP_PROJECT_ID - valueFrom: - configMapKeyRef: - name: pipeline-install-config - key: gcsProjectId - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: mlpipeline-minio-artifact - key: accesskey - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: mlpipeline-minio-artifact - key: secretkey - resources: - requests: - cpu: 20m - memory: 25Mi - ports: - - containerPort: 9000 - ---- -# MinIO GCS Gateway Service -apiVersion: v1 -kind: Service -metadata: - name: minio-service - namespace: {{ include "kubeflow-pipelines.namespace" . }} - {{- if .Values.installMode.legacyLabels }} - labels: - application-crd-id: kubeflow-pipelines - {{- end }} -spec: - ports: - - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - {{- if .Values.installMode.legacyLabels }} - application-crd-id: kubeflow-pipelines - {{- end }} -{{- end }} - -{{- if and .Values.env.gcp.gcs.enabled (not .Values.thirdParty.minio.enabled) }} ---- -# MinIO Artifact Secret -apiVersion: v1 -kind: Secret -metadata: - name: mlpipeline-minio-artifact - namespace: {{ include "kubeflow-pipelines.namespace" . }} - {{- if .Values.installMode.legacyLabels }} - labels: - application-crd-id: kubeflow-pipelines - {{- end }} -{{- if ne .Values.env.platform "platform-agnostic" }} -type: Opaque -{{- end }} -data: - accesskey: {{ "minio" | b64enc | quote }} - secretkey: {{ "minio123" | b64enc | quote }} -{{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-configmap.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-configmap.yaml deleted file mode 100644 index 30d9322fb9..0000000000 --- a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-configmap.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.env.gcp.proxyAgent.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: inverse-proxy-config - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-deployment.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-deployment.yaml deleted file mode 100644 index b985b88467..0000000000 --- a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-deployment.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.env.gcp.proxyAgent.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: proxy-agent - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: proxy-agent -spec: - selector: - matchLabels: - app: proxy-agent - application-crd-id: kubeflow-pipelines - template: - metadata: - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 8 }} - app: proxy-agent - spec: - hostNetwork: true - serviceAccountName: proxy-agent-runner - containers: - - name: proxy-agent - image: {{ .Values.env.gcp.proxyAgent.image.repository }}:{{ .Values.env.gcp.proxyAgent.image.tag }} - imagePullPolicy: {{ .Values.env.gcp.proxyAgent.image.pullPolicy | default .Values.global.imagePullPolicy }} - {{- if .Values.env.gcp.proxyAgent.resources }} - resources: - {{- toYaml .Values.env.gcp.proxyAgent.resources | nindent 10 }} - {{- end }} -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-role.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-role.yaml deleted file mode 100644 index 1bcd20256e..0000000000 --- a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-role.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if .Values.env.gcp.proxyAgent.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: proxy-agent-runner - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: proxy-agent-runner -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-rolebinding.yaml deleted file mode 100644 index fbbf802dc3..0000000000 --- a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-rolebinding.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if .Values.env.gcp.proxyAgent.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: proxy-agent-runner - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: proxy-agent-runner -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: proxy-agent-runner -subjects: -- kind: ServiceAccount - name: proxy-agent-runner - namespace: {{ include "kubeflow-pipelines.namespace" . }} -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-serviceaccount.yaml deleted file mode 100644 index 0874c2e2cb..0000000000 --- a/experimental/helm/charts/pipelines/templates/env/gcp-proxy-agent-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.env.gcp.proxyAgent.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: proxy-agent-runner - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml index a3b321844f..d089d91983 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml @@ -42,7 +42,7 @@ spec: {{- if .value }} value: {{ .value | quote }} {{- else if and .key (ne .key "null") }} - {{- if or (eq .name "MINIO_ACCESS_KEY") (eq .name "MINIO_SECRET_KEY") (eq .name "AWS_ACCESS_KEY_ID") (eq .name "AWS_SECRET_ACCESS_KEY") }} + {{- if or (eq .name "MINIO_ACCESS_KEY") (eq .name "MINIO_SECRET_KEY") }} valueFrom: secretKeyRef: name: mlpipeline-minio-artifact diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml index e588219bf6..6ef8a70e72 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.thirdParty.minio.enabled (ne .Values.env.platform "azure") }} +{{- if .Values.thirdParty.minio.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -19,24 +19,17 @@ spec: {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: minio spec: - {{- if ne .Values.env.platform "azure" }} securityContext: fsGroup: 1000 fsGroupChangePolicy: "OnRootMismatch" seccompProfile: type: RuntimeDefault - {{- end }} containers: - name: minio image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance args: - {{- if eq .Values.env.platform "azure" }} - - gateway - - azure - {{- else }} - server - /data - {{- end }} env: - name: MINIO_ACCESS_KEY valueFrom: @@ -53,12 +46,7 @@ spec: resources: requests: cpu: 20m - {{- if eq .Values.env.platform "azure" }} - memory: 25Mi - {{- else }} memory: 100Mi - {{- end }} - {{- if ne .Values.env.platform "azure" }} securityContext: runAsUser: 1000 runAsGroup: 0 @@ -77,5 +65,4 @@ spec: - name: data persistentVolumeClaim: claimName: minio-pvc - {{- end }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml index a5b184e27c..6fc93106ac 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.thirdParty.minio.enabled (ne .Values.env.platform "azure") (not .Values.env.gcp.gcs.enabled) }} +{{- if .Values.thirdParty.minio.enabled }} apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml index eac07cfff8..7ddaf139d5 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml @@ -4,24 +4,9 @@ kind: Secret metadata: name: mlpipeline-minio-artifact namespace: {{ include "kubeflow-pipelines.namespace" . }} -{{- if not (or (eq .Values.env.platform "gcp") ) }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -{{- end }} -{{- if ne .Values.env.platform "platform-agnostic" }} -type: Opaque -{{- end }} -{{- if eq .Values.env.platform "aws" }} -data: - accesskey: {{ .Values.objectStore.s3.accessKey | default "YOUR_AWS_ACCESS_ID" | b64enc }} - secretkey: {{ .Values.objectStore.s3.secretKey | default "YOUR_AWS_SECRET_KEY" | b64enc }} -{{- else if eq .Values.env.platform "azure" }} -data: - accesskey: {{ .Values.objectStore.azure.accountName | default "YOUR_AZURE_STORAGE_ACCOUNT" | b64enc }} - secretkey: {{ .Values.objectStore.azure.accountKey | default "YOUR_AZURE_STORAGE_KEY" | b64enc }} -{{- else }} stringData: accesskey: "minio" secretkey: "minio123" {{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml index 8079d96767..dad3b4f2a3 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.thirdParty.minio.enabled (ne .Values.env.platform "azure") }} +{{- if .Values.thirdParty.minio.enabled }} apiVersion: v1 kind: Service metadata: diff --git a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml index 3cadb158f1..1a0d5f5d7a 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml @@ -1,29 +1,6 @@ {{- if .Values.thirdParty.argo.enabled }} apiVersion: v1 data: -{{- if eq .Values.env.platform "aws" }} - config: | - { - artifactRepository: - { - s3: { - bucket: {{ include "kubeflow-pipelines.objectStore.bucket" . }}, - keyPrefix: artifacts, - endpoint: s3.amazonaws.com, - insecure: true, - accessKeySecret: { - name: mlpipeline-minio-artifact, - key: accesskey - }, - secretKeySecret: { - name: mlpipeline-minio-artifact, - key: secretkey - } - }, - archiveLogs: true - } - } -{{- else }} artifactRepository: | archiveLogs: true s3: @@ -51,7 +28,6 @@ data: key: secretkey executor: | imagePullPolicy: IfNotPresent -{{- end }} kind: ConfigMap metadata: labels: diff --git a/experimental/helm/charts/pipelines/values.yaml b/experimental/helm/charts/pipelines/values.yaml index 28923ec7f2..28a4afbd04 100644 --- a/experimental/helm/charts/pipelines/values.yaml +++ b/experimental/helm/charts/pipelines/values.yaml @@ -685,7 +685,7 @@ database: # Object Storage Configuration objectStore: - # Provider: minio, s3, gcs, azure, custom + # Provider: minio, custom provider: minio minio: @@ -695,20 +695,6 @@ objectStore: endpoint: "" existingSecret: "" - s3: - region: "us-west-2" - bucket: "" - existingSecret: "" - - gcs: - bucket: "" - existingSecret: "" - - azure: - storageAccount: "" - container: "" - existingSecret: "" - custom: endpoint: "" bucket: "" @@ -751,108 +737,8 @@ pipeline: # Environment-specific Configuration env: - # -- Platform type: platform-agnostic, aws, gcp, azure + # -- Platform type: platform-agnostic platform: platform-agnostic - - # AWS-specific configuration - aws: - # -- AWS region - region: us-west-2 - # RDS configuration - rds: - # -- Enable RDS integration - enabled: false - # -- RDS endpoint - endpoint: "" - # S3 configuration - s3: - # -- Enable S3 integration - enabled: false - # -- S3 bucket name - bucket: "" - # -- S3 region - region: us-west-2 - # IAM configuration - iam: - # -- Enable IAM roles for service accounts (IRSA) - enabled: false - # -- IAM role ARN for API server - apiServerRoleArn: "" - - # GCP-specific configuration - gcp: - # -- GCP project ID - project: "" - # -- GCP region - region: us-central1 - # Cloud SQL configuration - cloudsql: - # -- Enable Cloud SQL integration - enabled: false - # -- Cloud SQL instance name - instanceName: "" - # -- Cloud SQL region - region: us-central1 - # GCS configuration - gcs: - # -- Enable GCS integration - enabled: false - # -- GCS bucket name - bucket: "" - # -- GCS project ID - projectId: "" - # Workload Identity configuration - workloadIdentity: - # -- Enable Workload Identity - enabled: false - # -- GCP service account email - serviceAccount: "" - # Inverse Proxy Agent configuration - proxyAgent: - # -- Enable inverse proxy agent for GCP - enabled: false - - image: - # -- Proxy agent image repository - repository: ghcr.io/kubeflow/kfp-inverse-proxy-agent - # -- Proxy agent image tag - tag: "2.14.3" - # -- Proxy agent image pull policy - pullPolicy: IfNotPresent - - # -- Proxy agent resource requests and limits - resources: {} - - # Azure-specific configuration - azure: - # -- Azure subscription ID - subscriptionId: "" - # -- Azure resource group - resourceGroup: "" - # -- Azure region - region: eastus - # Azure SQL configuration - sql: - # -- Enable Azure SQL integration - enabled: false - # -- Azure SQL server name - serverName: "" - # -- Azure SQL database name - database: "" - # Azure Blob Storage configuration - blob: - # -- Enable Azure Blob Storage integration - enabled: false - # -- Storage account name - storageAccount: "" - # -- Container name - container: "" - # Managed Identity configuration - managedIdentity: - # -- Enable Azure Managed Identity - enabled: false - # -- Managed Identity client ID - clientId: "" # RBAC Configuration rbac: From 47f333517fd3c56e4dd3f7ed8674c69cdf71b989 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sun, 19 Oct 2025 21:18:30 +0530 Subject: [PATCH 18/26] remove cloud speific templates from comparision Signed-off-by: kunal-511 --- tests/helm_kustomize_compare.py | 2 +- tests/helm_kustomize_compare.sh | 9 --------- tests/helm_kustomize_compare_all.sh | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/tests/helm_kustomize_compare.py b/tests/helm_kustomize_compare.py index e57b3b77ed..cf5b323425 100755 --- a/tests/helm_kustomize_compare.py +++ b/tests/helm_kustomize_compare.py @@ -276,7 +276,7 @@ def get_expected_helm_extras(component: str, scenario: str) -> set: # These get extra resources from the subchart dependency argo_subchart_scenarios = [ - 'dev', 'aws', 'gcp', 'azure', + 'dev', 'platform-agnostic-multi-user-emissary', 'platform-agnostic-multi-user', 'platform-agnostic-multi-user-legacy', diff --git a/tests/helm_kustomize_compare.sh b/tests/helm_kustomize_compare.sh index 32a53b08b5..02d96f330d 100755 --- a/tests/helm_kustomize_compare.sh +++ b/tests/helm_kustomize_compare.sh @@ -164,9 +164,6 @@ case "$COMPONENT" in declare -A KUSTOMIZE_PATHS=( ["platform-agnostic"]="$MANIFESTS_DIR/env/platform-agnostic" - ["aws"]="$MANIFESTS_DIR/env/aws" - ["gcp"]="$MANIFESTS_DIR/env/gcp" - ["azure"]="$MANIFESTS_DIR/env/azure" ["multi-user"]="$MANIFESTS_DIR/base/installs/multi-user" ["generic"]="$MANIFESTS_DIR/base/installs/generic" ["dev"]="$MANIFESTS_DIR/env/dev" @@ -181,9 +178,6 @@ case "$COMPONENT" in declare -A HELM_VALUES=( ["platform-agnostic"]="$CHART_DIR/ci/values-platform-agnostic-enhanced.yaml" - ["aws"]="$CHART_DIR/ci/values-aws-enhanced.yaml" - ["gcp"]="$CHART_DIR/ci/values-gcp-enhanced.yaml" - ["azure"]="$CHART_DIR/ci/values-azure-enhanced.yaml" ["multi-user"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" ["generic"]="$CHART_DIR/ci/values-generic.yaml" ["dev"]="$CHART_DIR/ci/values-dev-enhanced.yaml" @@ -198,9 +192,6 @@ case "$COMPONENT" in declare -A NAMESPACES=( ["platform-agnostic"]="kubeflow" - ["aws"]="kubeflow" - ["gcp"]="kubeflow" - ["azure"]="kubeflow" ["multi-user"]="kubeflow" ["generic"]="kubeflow" ["dev"]="kubeflow" diff --git a/tests/helm_kustomize_compare_all.sh b/tests/helm_kustomize_compare_all.sh index 4e9837e530..de7d5d388d 100755 --- a/tests/helm_kustomize_compare_all.sh +++ b/tests/helm_kustomize_compare_all.sh @@ -13,7 +13,7 @@ declare -A COMPONENT_SCENARIOS=( ["model-registry"]="base overlay-postgres overlay-db controller-manager controller-rbac controller-default controller-prometheus controller-network-policy ui-base ui-standalone ui-integrated ui-istio istio csi" ["kserve-models-web-app"]="base kubeflow" ["notebook-controller"]="base kubeflow standalone" - ["pipelines"]="platform-agnostic aws gcp azure multi-user generic dev plain plain-multi-user platform-agnostic-emissary platform-agnostic-multi-user platform-agnostic-multi-user-emissary platform-agnostic-multi-user-legacy platform-agnostic-postgresql" + ["pipelines"]="platform-agnostic multi-user generic dev plain plain-multi-user platform-agnostic-emissary platform-agnostic-multi-user platform-agnostic-multi-user-emissary platform-agnostic-multi-user-legacy platform-agnostic-postgresql" ) test_component() { From 8474f0d550475c1b4d7e593afea457b3b7257964 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Mon, 1 Dec 2025 23:19:52 +0530 Subject: [PATCH 19/26] Update helm charts according to v2.15.0 and support only 2 flavors Signed-off-by: kunal-511 --- experimental/helm/charts/pipelines/Chart.yaml | 10 +- .../pipelines/ci/values-dev-enhanced.yaml | 388 ------ .../helm/charts/pipelines/ci/values-dev.yaml | 190 --- .../pipelines/ci/values-external-db.yaml | 59 - .../charts/pipelines/ci/values-generic.yaml | 326 ----- .../charts/pipelines/ci/values-kubeflow.yaml | 133 -- .../ci/values-multi-user-enhanced.yaml | 402 ------ .../pipelines/ci/values-multi-user.yaml | 408 ------ .../ci/values-platform-agnostic-enhanced.yaml | 288 ----- ...platform-agnostic-multi-user-enhanced.yaml | 385 ------ ...atform-agnostic-multi-user-k8s-native.yaml | 124 ++ ...s-platform-agnostic-multi-user-legacy.yaml | 292 ----- .../values-platform-agnostic-multi-user.yaml | 95 ++ .../pipelines/ci/values-postgresql.yaml | 235 ---- .../pipelines/ci/values-production.yaml | 104 -- .../pipelines/ci/values-standalone.yaml | 226 ---- .../helm/charts/pipelines/files/sync.py | 526 ++++---- .../charts/pipelines/templates/_helpers.tpl | 193 +-- .../templates/api-server/deployment.yaml | 25 +- .../templates/api-server/service.yaml | 6 + .../templates/application/application.yaml | 52 - .../application/controller-deployment.yaml | 39 - .../application/controller-role.yaml | 27 - .../application/controller-rolebinding.yaml | 18 - .../application/controller-service.yaml | 19 - .../templates/cache-deployer/deployment.yaml | 34 - .../templates/cache-deployer/role.yaml | 20 - .../templates/cache-deployer/rolebinding.yaml | 19 - .../cache-deployer/serviceaccount.yaml | 14 - .../templates/cache/cert-issuer.yaml | 12 + .../templates/cache/certificate.yaml | 21 + .../pipelines/templates/cache/deployment.yaml | 10 +- .../templates/cache/mutating-webhook.yaml | 30 + .../config/pipeline-api-server-config.yaml | 4 +- .../config/pipeline-install-config.yaml | 15 +- .../templates/config/ui-configmap.yaml | 2 +- .../templates/crds/application-crd.yaml | 330 ----- .../templates/crds/pipelines-crd.yaml | 9 +- .../templates/crds/pipelineversions-crd.yaml | 41 +- .../templates/database/mysql-secret.yaml | 2 +- .../templates/istio/authorization-config.yaml | 47 +- .../templates/istio/virtual-service.yaml | 2 +- .../templates/metadata-writer/deployment.yaml | 12 +- .../templates/metadata/envoy-configmap.yaml | 83 ++ .../templates/metadata/envoy-deployment.yaml | 18 +- .../templates/metadata/grpc-deployment.yaml | 54 +- .../metadata/mysql-db-configmap.yaml | 14 - .../metadata/mysql-db-deployment.yaml | 62 - .../templates/metadata/mysql-db-pvc.yaml | 16 - .../templates/metadata/mysql-db-secret.yaml | 14 - .../templates/metadata/mysql-db-service.yaml | 22 - .../metadata/postgres-db-configmap.yaml | 13 - .../metadata/postgres-db-deployment.yaml | 48 - .../templates/metadata/postgres-db-pvc.yaml | 16 - .../metadata/postgres-db-secret.yaml | 14 - .../metadata/postgres-db-service.yaml | 20 - .../persistence-agent/deployment.yaml | 3 +- .../profile-controller/configmap.yaml | 2 +- .../decorator-controller.yaml | 2 +- .../profile-controller/deployment.yaml | 60 +- .../profile-controller/env-configmap.yaml | 14 + .../templates/profile-controller/service.yaml | 4 +- .../rbac/argo-workflow-controller-rbac.yaml | 2 +- .../templates/rbac/cluster-role-bindings.yaml | 19 +- .../templates/rbac/cluster-roles.yaml | 73 +- .../rbac/view-edit-cluster-roles.yaml | 6 +- .../scheduled-workflow/deployment.yaml | 11 +- .../templates/third-party/argo-role.yaml | 4 +- .../third-party/minio-deployment.yaml | 68 - .../third-party/mysql-deployment.yaml | 51 +- .../templates/third-party/mysql-pvc.yaml | 3 +- .../templates/third-party/mysql-service.yaml | 7 +- .../third-party/mysql-serviceaccount.yaml | 1 + .../third-party/postgresql-deployment.yaml | 54 - .../templates/third-party/postgresql-pvc.yaml | 16 - .../postgresql-secret-extended.yaml | 12 - .../third-party/postgresql-secret.yaml | 12 - .../third-party/postgresql-service.yaml | 18 - .../postgresql-serviceaccount.yaml | 9 - .../seaweedfs/authorization-policy.yaml | 28 + .../third-party/seaweedfs/deployment.yaml | 99 ++ .../seaweedfs/destination-rule.yaml | 15 + .../{ => seaweedfs}/minio-service.yaml | 13 +- .../third-party/seaweedfs/networkpolicy.yaml | 36 + .../{minio-pvc.yaml => seaweedfs/pvc.yaml} | 7 +- .../secret.yaml} | 9 +- .../seaweedfs/service-account.yaml} | 6 +- .../third-party/seaweedfs/service.yaml | 43 + .../workflow-controller-configmap.yaml | 3 +- .../workflow-controller-deployment.yaml | 4 +- .../pipelines/templates/ui/deployment.yaml | 3 +- .../templates/viewer-crd/deployment.yaml | 13 +- .../templates/visualization/deployment.yaml | 8 +- .../templates/webhook/certificate.yaml | 20 + .../pipelines/templates/webhook/issuer.yaml | 10 + .../webhook/mutating-webhook-config.yaml | 34 - .../templates/webhook/mutating-webhook.yaml | 32 + .../webhook/validating-webhook-config.yaml | 34 - .../templates/webhook/validating-webhook.yaml | 32 + .../webhook/webhook-server-tls-secret.yaml | 18 - .../helm/charts/pipelines/values.yaml | 1095 +++++++---------- tests/helm_kustomize_compare.sh | 37 +- tests/helm_kustomize_compare_all.sh | 2 +- 103 files changed, 1657 insertions(+), 5983 deletions(-) delete mode 100644 experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-dev.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-external-db.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-generic.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-kubeflow.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-multi-user.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml create mode 100644 experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-k8s-native.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-legacy.yaml create mode 100644 experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-postgresql.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-production.yaml delete mode 100644 experimental/helm/charts/pipelines/ci/values-standalone.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/application/application.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/application/controller-deployment.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/application/controller-role.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/application/controller-rolebinding.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/application/controller-service.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/cache-deployer/deployment.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/cache-deployer/role.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml create mode 100644 experimental/helm/charts/pipelines/templates/cache/cert-issuer.yaml create mode 100644 experimental/helm/charts/pipelines/templates/cache/certificate.yaml create mode 100644 experimental/helm/charts/pipelines/templates/cache/mutating-webhook.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/crds/application-crd.yaml create mode 100644 experimental/helm/charts/pipelines/templates/metadata/envoy-configmap.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-configmap.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-deployment.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-pvc.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-secret.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/mysql-db-service.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-configmap.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-deployment.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-pvc.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-secret.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/metadata/postgres-db-service.yaml create mode 100644 experimental/helm/charts/pipelines/templates/profile-controller/env-configmap.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-deployment.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-pvc.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-service.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/seaweedfs/authorization-policy.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/seaweedfs/deployment.yaml create mode 100644 experimental/helm/charts/pipelines/templates/third-party/seaweedfs/destination-rule.yaml rename experimental/helm/charts/pipelines/templates/third-party/{ => seaweedfs}/minio-service.yaml (52%) create mode 100644 experimental/helm/charts/pipelines/templates/third-party/seaweedfs/networkpolicy.yaml rename experimental/helm/charts/pipelines/templates/third-party/{minio-pvc.yaml => seaweedfs/pvc.yaml} (66%) rename experimental/helm/charts/pipelines/templates/third-party/{minio-secret.yaml => seaweedfs/secret.yaml} (54%) rename experimental/helm/charts/pipelines/templates/{application/controller-serviceaccount.yaml => third-party/seaweedfs/service-account.yaml} (55%) create mode 100644 experimental/helm/charts/pipelines/templates/third-party/seaweedfs/service.yaml create mode 100644 experimental/helm/charts/pipelines/templates/webhook/certificate.yaml create mode 100644 experimental/helm/charts/pipelines/templates/webhook/issuer.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/webhook/mutating-webhook-config.yaml create mode 100644 experimental/helm/charts/pipelines/templates/webhook/mutating-webhook.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/webhook/validating-webhook-config.yaml create mode 100644 experimental/helm/charts/pipelines/templates/webhook/validating-webhook.yaml delete mode 100644 experimental/helm/charts/pipelines/templates/webhook/webhook-server-tls-secret.yaml diff --git a/experimental/helm/charts/pipelines/Chart.yaml b/experimental/helm/charts/pipelines/Chart.yaml index 6c9c82a338..0ac0e3f0ed 100644 --- a/experimental/helm/charts/pipelines/Chart.yaml +++ b/experimental/helm/charts/pipelines/Chart.yaml @@ -18,20 +18,12 @@ sources: dependencies: - name: argo-workflows version: "~0.40.0" - condition: argo.enabled + condition: thirdParty.argo.enabled repository: "https://argoproj.github.io/argo-helm" - - name: minio - version: "~5.0.0" - condition: minio.enabled - repository: "https://charts.min.io/" - name: mysql version: "~14.0.0" condition: mysql.enabled repository: "https://charts.bitnami.com/bitnami" - - name: postgresql - version: "~16.0.0" - condition: postgresql.enabled - repository: "https://charts.bitnami.com/bitnami" annotations: category: Machine Learning licenses: Apache-2.0 diff --git a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml deleted file mode 100644 index a660b6c72f..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-dev-enhanced.yaml +++ /dev/null @@ -1,388 +0,0 @@ -# Development Environment Configuration - Enhanced -# This configuration is optimized for local development and testing - -global: - namespace: kubeflow - imageRegistry: ghcr.io/kubeflow - imageTag: "master" - version: "2.14.3" - imagePullPolicy: IfNotPresent - -# Installation mode -installMode: - type: generic - multiUser: - enabled: false - -crds: - install: false - application: true - -# Cache deployer for generic mode -cacheDeployer: - enabled: true - serviceAccount: - create: false - replicas: 1 - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 100m - memory: 256Mi -# Database Configuration -mysql: - enabled: false - auth: - rootPassword: "dev123" - username: "dev" - password: "dev123" - database: "mlpipeline" - primary: - persistence: - enabled: false - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - -postgresql: - enabled: false - -# Third-party components configuration -thirdParty: - # Use built-in MySQL instead of external subchart - mysql: - enabled: true - - # Use built-in MinIO instead of external subchart - minio: - enabled: true - portName: http - - # Use built-in Argo instead of external subchart - argo: - enabled: true - -# Object Storage Configuration - Lightweight MinIO -minio: - enabled: false - auth: - rootUser: "dev" - rootPassword: "dev123" - defaultBuckets: "mlpipeline" - persistence: - enabled: false - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 250m - memory: 256Mi - -objectStore: - provider: minio - secure: false - bucketName: mlpipeline - -# Database names -database: - pipelinedb: mlpipeline - metadb: metadb - cachedb: cachedb - -# Pipeline Configuration - Development settings -pipeline: - cache: - image: "ghcr.io/containerd/busybox" - nodeRestrictions: false - maximumStaleness: "" - defaultStaleness: "" - -# Argo Workflows Configuration - Minimal for dev -argo: - enabled: true - crds: - install: true - keep: true - createAggregateRoles: false - controller: - replicas: 0 - serviceAccount: - create: false - rbac: - create: false - configMap: - create: false - clusterWorkflowTemplates: - enabled: false - server: - enabled: false - replicas: 0 - serviceAccount: - create: false - rbac: - create: false - clusterWorkflowTemplates: - enabled: false - executor: - enabled: false - workflow: - serviceAccount: - create: false - rbac: - create: false - -# Application Configuration -application: - enabled: true - controller: - enabled: true - -# Environment Configuration -env: - platform: platform-agnostic - -# Reduced resources for development -apiServer: - replicas: 1 - resources: - requests: - cpu: 250m - memory: 500Mi - env: - logLevel: "2" - additionalEnv: - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:master" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:master" - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: DBCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_HOST - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - - name: DB_DRIVER_NAME - key: dbType - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - -persistenceAgent: - replicas: 1 - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -scheduledWorkflow: - replicas: 1 - image: - repository: kfp-scheduled-workflow-controller - tag: "master" - additionalEnv: - - name: LOG_LEVEL - value: "info" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -ui: - replicas: 1 - resources: - requests: - cpu: 10m - memory: 70Mi - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - key: accesskey - - name: MINIO_SECRET_KEY - key: secretkey - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" - -metadata: - enabled: true - grpc: - replicas: 1 - resources: - envoy: - replicas: 1 - resources: - requests: - cpu: 10m - memory: 16Mi - limits: - cpu: 50m - memory: 64Mi - -cache: - server: - replicas: 1 - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 512Mi - additionalEnv: - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -visualization: - replicas: 1 - resources: - requests: - cpu: 30m - memory: 500Mi - -viewerCrd: - replicas: 1 - image: - pullPolicy: Always - env: - maxNumViewers: "10" - additionalEnv: - - name: MAX_NUM_VIEWERS - value: "50" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -dev: - # Enable debug mode - debug: true - - # Faster probe intervals for development - probes: - readiness: - initialDelaySeconds: 1 - periodSeconds: 2 - liveness: - initialDelaySeconds: 5 - periodSeconds: 5 - - # Development logging - logging: - level: debug - format: text - - # Local port forwarding helpers - portForward: - ui: 8080 - api: 8888 - minio: 9000 - -# Monitoring disabled for development -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -# Service mesh disabled for development -istio: - enabled: false - -# Cert-manager disabled for development -certManager: - enabled: false - -# Development-specific service types -services: - ui: - type: NodePort - nodePort: 30080 - apiServer: - type: NodePort - nodePort: 30888 - -# Node selectors for development -nodeSelector: {} - -# Tolerations for development -tolerations: [] - -# Affinity for development -affinity: {} diff --git a/experimental/helm/charts/pipelines/ci/values-dev.yaml b/experimental/helm/charts/pipelines/ci/values-dev.yaml deleted file mode 100644 index 8f7689c312..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-dev.yaml +++ /dev/null @@ -1,190 +0,0 @@ -# Development environment values for Kubeflow Pipelines -# This configuration is optimized for local development and testing - -# Environment configuration -env: - platform: dev - -# Reduced resource requirements for development -apiServer: - replicas: 1 - resources: - requests: - cpu: 100m - memory: 200Mi - limits: - cpu: 500m - memory: 500Mi - -persistenceAgent: - replicas: 1 - resources: - requests: - cpu: 50m - memory: 100Mi - limits: - cpu: 200m - memory: 200Mi - -scheduledWorkflow: - replicas: 1 - resources: - requests: - cpu: 50m - memory: 50Mi - limits: - cpu: 200m - memory: 100Mi - -ui: - replicas: 1 - resources: - requests: - cpu: 10m - memory: 50Mi - limits: - cpu: 100m - memory: 100Mi - -viewerCrd: - replicas: 1 - resources: - requests: - cpu: 50m - memory: 50Mi - limits: - cpu: 200m - memory: 100Mi - -visualization: - replicas: 1 - resources: - requests: - cpu: 25m - memory: 100Mi - limits: - cpu: 200m - memory: 200Mi - -cache: - server: - replicas: 1 - resources: - requests: - cpu: 50m - memory: 50Mi - limits: - cpu: 200m - memory: 100Mi - -cacheDeployer: - replicas: 1 - resources: - requests: - cpu: 50m - memory: 50Mi - limits: - cpu: 200m - memory: 100Mi - -metadata: - grpc: - replicas: 1 - resources: - requests: - cpu: 50m - memory: 100Mi - limits: - cpu: 200m - memory: 200Mi - - envoy: - resources: - requests: - cpu: 10m - memory: 20Mi - limits: - cpu: 100m - memory: 50Mi - -# Database configuration - lightweight MySQL -mysql: - enabled: true - auth: - rootPassword: "dev123" - database: "mlpipeline" - username: "root" - primary: - persistence: - enabled: true - size: 5Gi - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - -postgresql: - enabled: false - -# Object storage configuration - lightweight MinIO -minio: - enabled: true - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: - enabled: true - size: 5Gi - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi - -# Argo Workflows with reduced resources -argo: - enabled: true - controller: - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi - -# Development-friendly settings -pipeline: - cache: - # Use faster cache settings for development - maximumStaleness: "P1D" - defaultStaleness: "P1H" - -# Disable production features -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -certManager: - enabled: false - -networking: - istio: - enabled: false - -# Enable debug logging -global: - logLevel: "debug" - -# Disable security restrictions for easier development -security: - podSecurityStandards: - enabled: false diff --git a/experimental/helm/charts/pipelines/ci/values-external-db.yaml b/experimental/helm/charts/pipelines/ci/values-external-db.yaml deleted file mode 100644 index 025310d008..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-external-db.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# External database values for kubeflow-pipelines. -# This configuration uses external database instead of embedded MySQL/PostgreSQL - -# Disable embedded databases -mysql: - enabled: false - -postgresql: - enabled: false - -# Configure external database -externalDatabase: - type: mysql - host: "external-db-host.example.com" - port: 3306 - database: "mlpipeline" - username: "mlpipeline" - existingSecret: "external-db-secret" - -# Additional databases for metadata and cache -database: - metadb: "metadb" - cachedb: "cachedb" - connectionMaxLifetime: "300s" - -# Use MinIO for object storage (can be changed to external) -objectStore: - provider: minio - -minio: - enabled: true - auth: - rootUser: "minio" - rootPassword: "minio123" - persistence: - enabled: true - size: 50Gi - -# Standard configuration for other components -apiServer: - replicas: 1 - resources: - requests: - cpu: 250m - memory: 500Mi - -persistenceAgent: - replicas: 1 - resources: - requests: - cpu: 120m - memory: 500Mi - -# Disable monitoring for simplicity -monitoring: - prometheus: - enabled: false - grafana: - enabled: false diff --git a/experimental/helm/charts/pipelines/ci/values-generic.yaml b/experimental/helm/charts/pipelines/ci/values-generic.yaml deleted file mode 100644 index 07b3c6b0ad..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-generic.yaml +++ /dev/null @@ -1,326 +0,0 @@ -# Generic Configuration - -global: - namespace: kubeflow - imageRegistry: ghcr.io/kubeflow - imageTag: "2.14.3" - imagePullPolicy: IfNotPresent - -# Installation mode - generic single-user -installMode: - type: generic - multiUser: - enabled: false - includeApplicationCrdId: false - -# CRDs configuration -crds: - install: false - application: false - -# Cache deployer for generic mode -cacheDeployer: - enabled: true - serviceAccount: - create: false - -# Environment configuration -env: - platform: platform-agnostic - -# Database configuration - use built-in MySQL -mysql: - enabled: false - -postgresql: - enabled: false - -externalDatabase: - type: mysql - host: mysql - port: 3306 - database: mlpipeline - username: root - password: "" - -# Database names -database: - pipelinedb: mlpipeline - metadb: metadb - cachedb: cachedb - connectionMaxLifetime: "120s" - -# Object storage configuration - use built-in MinIO -objectStore: - provider: minio - minio: - enabled: true - bucket: mlpipeline - secure: false - -minio: - enabled: false - -# Argo Workflows configuration - use built-in -argo: - enabled: false - -# Built-in third-party components -thirdParty: - mysql: - enabled: false - minio: - enabled: false - portName: http - argo: - enabled: false - -# Pipeline Configuration -pipeline: - app: - name: pipeline - version: "2.14.3" - defaultPipelineRoot: "" - autoUpdatePipelineDefaultVersion: true - cronScheduleTimezone: "UTC" - cache: - image: "ghcr.io/containerd/busybox" - nodeRestrictions: false - maximumStaleness: "" - defaultStaleness: "" - -# API Server configuration -apiServer: - enabled: true - replicas: 1 - resources: - requests: - cpu: 250m - memory: 500Mi - env: - publishLogs: true - pipelineLogLevel: "1" - autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - additionalEnv: - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: DBCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_HOST - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - - name: DB_DRIVER_NAME - key: dbType - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - -# Persistence Agent configuration -persistenceAgent: - enabled: true - replicas: 1 - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -# Scheduled Workflow configuration -scheduledWorkflow: - enabled: true - replicas: 1 - additionalEnv: - - name: LOG_LEVEL - value: "info" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - -# UI configuration -ui: - enabled: true - replicas: 1 - resources: - requests: - cpu: 10m - memory: 70Mi - env: - allowCustomVisualizations: true - argoArchiveLogs: true - disableGkeMetadata: true - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" - - -# Viewer CRD Controller configuration -viewerCrd: - enabled: true - replicas: 1 - image: - pullPolicy: Always - env: - maxNumViewers: "50" - additionalEnv: - - name: MAX_NUM_VIEWERS - value: "50" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -# Visualization Server configuration -visualization: - enabled: true - replicas: 1 - resources: - requests: - cpu: 30m - memory: 500Mi - -# Cache System Configuration -cache: - enabled: true - server: - replicas: 1 - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: NAMESPACE_TO_WATCH - key: null - -# ML Metadata Configuration -metadata: - enabled: false - grpc: - replicas: 1 - envoy: - replicas: 1 - resources: - requests: - cpu: 20m - memory: 20Mi - limits: - cpu: 100m - memory: 100Mi - -# Metadata Writer Configuration -metadataWriter: - enabled: true - replicas: 1 - -# Disable cloud-specific features -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -certManager: - enabled: false - -networking: - istio: - enabled: false - -# Disable webhooks for generic scenario -webhooks: - enabled: false - -# Profile controller disabled for single-user -profileController: - enabled: false - -# Application CRD disabled -application: - enabled: false diff --git a/experimental/helm/charts/pipelines/ci/values-kubeflow.yaml b/experimental/helm/charts/pipelines/ci/values-kubeflow.yaml deleted file mode 100644 index 004d9e00c7..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-kubeflow.yaml +++ /dev/null @@ -1,133 +0,0 @@ -# Kubeflow integration values for kubeflow-pipelines. -# This configuration is optimized for integration with full Kubeflow deployment - - -global: - namespace: kubeflow - logLevel: "info" - -# Installation mode -installMode: - type: multi-user - multiUser: - enabled: true - profileController: - enabled: true - -# Kubeflow-specific RBAC -rbac: - create: true - multiUser: - enabled: true - clusterRoles: - - name: "kubeflow-pipelines-edit" - create: true - - name: "kubeflow-pipelines-view" - create: true - -# Service accounts for Kubeflow integration -serviceAccounts: - pipelineRunner: - create: true - name: "pipeline-runner" - viewer: - create: true - name: "viewer" - containerBuilder: - create: true - name: "container-builder" - metadataWriter: - create: true - name: "metadata-writer" - -# API Server with Kubeflow integration -apiServer: - replicas: 1 - additionalEnv: - - name: MULTI_USER - value: "true" - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - value: "" - -# UI with Kubeflow integration -ui: - replicas: 1 - additionalEnv: - - name: MULTI_USER - value: "true" - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - value: "" - -# Database configuration -mysql: - enabled: true - auth: - rootPassword: "" - database: "mlpipeline" - username: "root" - primary: - persistence: - enabled: true - size: 20Gi - -# Object storage -objectStore: - provider: minio - -minio: - enabled: true - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: - enabled: true - size: 20Gi - -# Istio integration (required for Kubeflow) -networking: - istio: - enabled: true - gateway: "kubeflow-gateway" - host: "*" - virtualService: - enabled: true - prefix: "/pipeline" - -# Authorization policies for Kubeflow -authorizationPolicies: - enabled: true - policies: - - name: "ml-pipeline" - action: "ALLOW" - rules: - - from: - - source: - principals: ["cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account"] - - to: - - operation: - methods: ["GET", "POST"] - -# Webhooks for multi-user -webhook: - enabled: true - mutating: - enabled: true - multiUser: true - validating: - enabled: true - multiUser: true - -# Security configuration -security: - podSecurityStandards: - enabled: true - enforce: "restricted" - networkPolicies: - enabled: true - rbacEnforcement: - enabled: true diff --git a/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml deleted file mode 100644 index 0573cbd73e..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-multi-user-enhanced.yaml +++ /dev/null @@ -1,402 +0,0 @@ -# Multi-User Configuration - Enhanced -# This configuration provides multi-user deployment with enhanced RBAC and isolation - -global: - namespace: kubeflow - imageRegistry: ghcr.io/kubeflow - imageTag: "2.14.3" - imagePullPolicy: Always - -# Installation mode - Multi-user enabled -installMode: - type: multi-user - multiUser: - enabled: true - profileController: - enabled: true - # Legacy mode disabled - excludes application-crd-id label for regular multi-user resources - legacyLabels: false - -# Database Configuration - Enhanced for multi-user -mysql: - enabled: false - auth: - rootPassword: "root123" - username: "mlpipeline" - password: "mlpipeline123" - database: "mlpipeline" - primary: - persistence: - enabled: true - size: 50Gi # Larger storage for multi-user - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 1 - memory: 2Gi - -postgresql: - enabled: false - -# Object Storage Configuration - Enhanced for multi-user -minio: - enabled: false - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: - enabled: true - size: 100Gi # Larger storage for multi-user - resources: - requests: - cpu: 250m - memory: 256Mi - limits: - cpu: 500m - memory: 1Gi - -objectStore: - provider: minio - secure: false - bucketName: mlpipeline - -# Database names -database: - pipelinedb: mlpipeline - metadb: metadb - cachedb: cachedb - -# Pipeline Configuration -pipeline: - cache: - image: "ghcr.io/containerd/busybox" - nodeRestrictions: false - maximumStaleness: "" - defaultStaleness: "" - -# Argo Workflows Configuration - Multi-user -argo: - enabled: false - # Multi-user uses built-in minimal Argo instead of subchart - -# Third Party Components -thirdParty: - argo: - enabled: false - metacontroller: - enabled: false - -# Environment Configuration -env: - platform: platform-agnostic - -# Enhanced RBAC Configuration -rbac: - create: true - multiUser: - enabled: true - # Additional RBAC rules for multi-user scenarios - -# Multi-user API Server configuration -apiServer: - env: - # Multi-user specific environment variables - multiUser: "true" - kubeflowUserId: "user@example.com" - - resources: - requests: - cpu: 250m - memory: 500Mi - additionalEnv: - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - value: "" - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - configMapName: pipeline-install-config - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - configMapName: pipeline-install-config - - name: DBCONFIG_USER - key: username - secretName: mysql-secret - - name: DBCONFIG_PASSWORD - key: password - secretName: mysql-secret - - name: DBCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_HOST - key: dbHost - configMapName: pipeline-install-config - - name: DBCONFIG_PORT - key: dbPort - configMapName: pipeline-install-config - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - configMapName: pipeline-install-config - - name: DB_DRIVER_NAME - key: dbType - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - secretName: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - secretName: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - configMapName: pipeline-install-config - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - secretName: mlpipeline-minio-artifact - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - secretName: mlpipeline-minio-artifact - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - -# Multi-user UI configuration -ui: - env: - # Multi-user UI configuration - multiUser: "true" - - resources: - requests: - cpu: 10m - memory: 70Mi - - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: DEPLOYMENT - value: "KUBEFLOW" - - name: ARTIFACTS_SERVICE_PROXY_NAME - value: "ml-pipeline-ui-artifact" - - name: ARTIFACTS_SERVICE_PROXY_PORT - value: "80" - - name: ARTIFACTS_SERVICE_PROXY_ENABLED - value: "true" - - name: ENABLE_AUTHZ - value: "true" - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - key: null - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - key: accesskey - - name: MINIO_SECRET_KEY - key: secretkey - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" - - -# Enhanced resources for multi-user deployment -persistenceAgent: - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: NAMESPACE - value: "" - -scheduledWorkflow: - additionalEnv: - - name: NAMESPACE - value: '' - - name: LOG_LEVEL - value: "info" - valueFrom: null - -metadata: - grpc: - resources: - requests: - cpu: 250m - memory: 500Mi - limits: - cpu: 1 - memory: 2Gi - envoy: - resources: - requests: - cpu: 100m - memory: 100Mi - limits: - cpu: 200m - memory: 200Mi - -cache: - # Cache deployer disabled in multi-user mode - enabled: true - server: - resources: - requests: - cpu: 250m - memory: 1Gi - limits: - cpu: 1 - memory: 2Gi - additionalEnv: - - name: NAMESPACE_TO_WATCH - key: null - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - -cacheDeployer: - # Cache deployer is included in base multi-user install - enabled: true - -visualization: - resources: - requests: - cpu: 30m - memory: 500Mi - -viewerCrd: - env: - maxNumViewers: "" - additionalEnv: - - name: NAMESPACE - value: "" - - name: MAX_NUM_VIEWERS - value: 50 - -# Profile Controller Configuration (for multi-user) -profileController: - enabled: true - image: - repository: kfp-profile-controller - tag: "2.14.3" - resources: - requests: - cpu: 100m - memory: 200Mi - limits: - cpu: 500m - memory: 1Gi - - additionalEnv: - - name: KFP_VERSION - key: appVersion - - name: KFP_DEFAULT_PIPELINE_ROOT - key: defaultPipelineRoot - - name: MINIO_ACCESS_KEY - key: accesskey - - name: MINIO_SECRET_KEY - key: secretkey - -# Namespace Isolation Configuration -namespaceIsolation: - enabled: true - # Additional network policies and RBAC for isolation - -# Security Configuration - Enhanced for multi-user -security: - podSecurityStandards: - enabled: true - level: "restricted" - networkPolicies: - enabled: true - -# Monitoring - Enhanced for multi-user -monitoring: - prometheus: - enabled: true - multiUser: true - grafana: - enabled: true - multiUser: true - -# Service mesh - Recommended for multi-user -networking: - istio: - enabled: true - gateway: kubeflow-gateway - virtualService: - enabled: true - -# Cert-manager for TLS -certManager: - enabled: true - issuer: "kubeflow-self-signing-issuer" - issuerKind: "ClusterIssuer" - -# Additional multi-user specific configurations -multiUserConfig: - # User namespace template - userNamespaceTemplate: "kubeflow-user-{userid}" - - # Default resource quotas per user namespace - resourceQuotas: - requests.cpu: "2" - requests.memory: "4Gi" - limits.cpu: "4" - limits.memory: "8Gi" - persistentvolumeclaims: "10" - - # Network policies for user isolation - networkPolicies: - enabled: true - allowCrossNamespace: false - - # RBAC templates for user permissions - rbacTemplates: - enabled: true diff --git a/experimental/helm/charts/pipelines/ci/values-multi-user.yaml b/experimental/helm/charts/pipelines/ci/values-multi-user.yaml deleted file mode 100644 index c65769c696..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-multi-user.yaml +++ /dev/null @@ -1,408 +0,0 @@ -# Multi-user values for Kubeflow Pipelines -# This configuration enables multi-user mode with proper isolation - -# Installation mode configuration -installMode: - type: multi-user - multiUser: - enabled: true - profileController: - enabled: true - # Legacy mode - includes application-crd-id label on all resources - legacyLabels: true - -# Environment configuration -env: - platform: platform-agnostic - -# Multi-user specific components -profileController: - enabled: true - name: profiles-controller - - image: - repository: kfp-profile-controller - tag: "2.14.3" - - replicas: 1 - - # Service Account - serviceAccount: - create: true - name: "profiles-controller-service-account" - annotations: {} - additionalEnv: - - name: KFP_VERSION - key: appVersion - - name: KFP_DEFAULT_PIPELINE_ROOT - key: defaultPipelineRoot - - name: MINIO_ACCESS_KEY - key: accesskey - - name: MINIO_SECRET_KEY - key: secretkey - - # Resources - resources: - requests: - cpu: 100m - memory: 200Mi - limits: {} - - # Security context - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - - ALL - -# Enhanced API Server configuration for multi-user -apiServer: - env: - publishLogs: true - pipelineLogLevel: "1" - autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - # Multi-user specific environment variables - multiUser: true - - # Additional environment variables for multi-user mode - additionalEnv: - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - value: "" - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - configMapName: pipeline-install-config - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - configMapName: pipeline-install-config - - name: DBCONFIG_USER - key: username - secretName: mysql-secret - - name: DBCONFIG_PASSWORD - key: password - secretName: mysql-secret - - name: DBCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_HOST - key: dbHost - configMapName: pipeline-install-config - - name: DBCONFIG_PORT - key: dbPort - configMapName: pipeline-install-config - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - configMapName: pipeline-install-config - - name: DB_DRIVER_NAME - key: dbType - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - secretName: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - secretName: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - configMapName: pipeline-install-config - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - secretName: mlpipeline-minio-artifact - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - secretName: mlpipeline-minio-artifact - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - - -# Enhanced UI configuration for multi-user -ui: - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: DEPLOYMENT - value: "KUBEFLOW" - - name: ARTIFACTS_SERVICE_PROXY_NAME - value: "ml-pipeline-ui-artifact" - - name: ARTIFACTS_SERVICE_PROXY_PORT - value: "80" - - name: ARTIFACTS_SERVICE_PROXY_ENABLED - value: "true" - - name: ENABLE_AUTHZ - value: "true" - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - key: null - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - key: accesskey - - name: MINIO_SECRET_KEY - key: secretkey - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" - -persistenceAgent: - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: NAMESPACE - value: "" - -scheduledWorkflow: - additionalEnv: - - name: NAMESPACE - value: '' - - name: LOG_LEVEL - value: "info" - valueFrom: null - -# Enhanced RBAC for multi-user -rbac: - create: true - # Multi-user specific RBAC - multiUser: - enabled: true - # Cluster roles for multi-user - clusterRoles: - - name: "kubeflow-pipelines-edit" - create: true - - name: "kubeflow-pipelines-view" - create: true - -# Service accounts for multi-user -serviceAccounts: - pipelineRunner: - create: true - name: "pipeline-runner" - - viewer: - create: true - name: "viewer" - - containerBuilder: - create: true - name: "container-builder" - - metadataWriter: - create: true - name: "metadata-writer" - - # Multi-user specific service accounts - profilesController: - create: true - name: "profiles-controller-service-account" - -# Third party services configuration -thirdParty: - # Argo Workflows configuration - argo: - enabled: false - - # Database configuration - mysql: - enabled: true - auth: - rootPassword: "" - database: "mlpipeline" - username: "root" - primary: - persistence: - enabled: true - size: 20Gi - - # Object storage configuration - minio: - enabled: true - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: - enabled: true - size: 20Gi - - # Metacontroller for profile controller - metacontroller: - enabled: true - -# Metadata service configuration -metadata: - enabled: true - grpc: - image: - repository: gcr.io/tfx-oss-public/ml_metadata_store_server - tag: "1.14.0" - pullPolicy: IfNotPresent - resources: - -postgresql: - enabled: false - -cache: - # Cache deployer disabled in multi-user mode - enabled: true - server: - resources: - requests: - cpu: 250m - memory: 1Gi - limits: - cpu: 1 - memory: 2Gi - additionalEnv: - - name: NAMESPACE_TO_WATCH - key: null - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - -viewerCrd: - env: - maxNumViewers: "" - additionalEnv: - - name: NAMESPACE - value: "" - - name: MAX_NUM_VIEWERS - value: 50 - - -# Istio configuration for multi-user -networking: - istio: - enabled: true - gateway: "kubeflow-gateway" - host: "*" - - # Virtual Service configuration - virtualService: - enabled: true - prefix: "/pipeline" - -# Authorization policies for multi-user -authorizationPolicies: - enabled: true - policies: - - name: "ml-pipeline" - action: "ALLOW" - rules: - - from: - - source: - principals: ["cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account"] - - to: - - operation: - methods: ["GET", "POST"] - - - name: "ml-pipeline-ui" - action: "ALLOW" - rules: - - from: - - source: - principals: ["cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account"] - - to: - - operation: - methods: ["GET", "POST"] - -# Webhook configuration for multi-user -webhook: - enabled: true - - # Mutating webhook - mutating: - enabled: true - name: "cache-webhook-kubeflow" - # Multi-user webhook configuration - multiUser: true - - # Validating webhook - validating: - enabled: true - name: "cache-webhook-kubeflow" - # Multi-user webhook configuration - multiUser: true - -# Security configuration for multi-user -security: - # Pod Security Standards - podSecurityStandards: - enabled: true - enforce: "restricted" - - # Security Context Profile - seccompProfile: - type: "RuntimeDefault" - - # Multi-user security policies - networkPolicies: - enabled: true - - # RBAC enforcement - rbacEnforcement: - enabled: true - -# Monitoring configuration -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -# Certificate management -certManager: - enabled: false diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml deleted file mode 100644 index ec5a6a0554..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-enhanced.yaml +++ /dev/null @@ -1,288 +0,0 @@ -# Platform-Agnostic Configuration - Enhanced -# This configuration provides a complete platform-agnostic deployment -# using MinIO for object storage and MySQL for database - -global: - namespace: kubeflow - imageRegistry: ghcr.io/kubeflow - imageTag: "2.14.3" - imagePullPolicy: IfNotPresent - -# Installation mode -installMode: - type: generic - multiUser: - enabled: false - -# Database Configuration - Using built-in MySQL -mysql: - enabled: false - auth: - rootPassword: "root123" - username: "mlpipeline" - password: "mlpipeline123" - database: "mlpipeline" - primary: - persistence: - enabled: true - size: 20Gi - -postgresql: - enabled: false - -externalDatabase: - type: mysql - host: "" - port: 3306 - database: mlpipeline - username: root - password: "" - -# Object Storage Configuration - Using built-in MinIO -minio: - enabled: false - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: - enabled: true - size: 20Gi - -objectStore: - provider: minio - secure: false - bucketName: mlpipeline - -# Database names -database: - pipelinedb: mlpipeline - metadb: metadb - cachedb: cachedb - -# Pipeline Configuration -pipeline: - # Platform-agnostic specific: Empty defaultPipelineRoot - defaultPipelineRoot: "" - cache: - image: "ghcr.io/containerd/busybox" - nodeRestrictions: false - maximumStaleness: "" - defaultStaleness: "" - -# Argo Workflows Configuration - use built-in -argo: - enabled: false - # Basic Argo configuration for platform-agnostic deployment - -# Built-in third-party components -thirdParty: - mysql: - enabled: true - minio: - enabled: true - portName: http - argo: - enabled: true - -# Environment Configuration -env: - platform: platform-agnostic - -# Resource Configuration -apiServer: - resources: - requests: - cpu: 250m - memory: 500Mi - additionalEnv: - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - - name: POD_NAMESPACE - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: DBCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_HOST - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - - name: DB_DRIVER_NAME - key: dbType - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - -persistenceAgent: - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -scheduledWorkflow: - resources: - additionalEnv: - - name: LOG_LEVEL - value: "info" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -ui: - resources: - requests: - cpu: 10m - memory: 70Mi - - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" -# Disable CRDs for platform-agnostic scenario -crds: - install: false - -# Cache deployer for generic mode -cacheDeployer: - enabled: true - serviceAccount: - create: false -# Disable webhooks for platform-agnostic scenario -webhooks: - enabled: false - -metadata: - enabled: true - grpc: - resources: - envoy: - resources: - requests: - cpu: 20m - memory: 20Mi - limits: - cpu: 100m - memory: 100Mi - -cache: - server: - resources: - requests: - cpu: 120m - memory: 500Mi - limits: - cpu: 250m - memory: 1Gi - additionalEnv: - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: NAMESPACE_TO_WATCH - - -visualization: - resources: - requests: - cpu: 30m - memory: 500Mi - -viewerCrd: - image: - pullPolicy: Always - resources: - additionalEnv: - - name: MAX_NUM_VIEWERS - value: 50 - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -# Monitoring disabled for platform-agnostic -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -# Service mesh disabled for platform-agnostic -istio: - enabled: false - -# Cert-manager disabled for platform-agnostic -certManager: - enabled: false diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml deleted file mode 100644 index b8201e5f64..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-enhanced.yaml +++ /dev/null @@ -1,385 +0,0 @@ -# Platform-Agnostic Multi-User Configuration - Enhanced -# This configuration provides platform-agnostic multi-user deployment - -global: - namespace: kubeflow - imageRegistry: ghcr.io/kubeflow - imageTag: "2.14.3" - imagePullPolicy: Always - -# Installation mode - Multi-user enabled with legacy labels -installMode: - type: multi-user - multiUser: - enabled: true - profileController: - enabled: true - # Legacy mode enabled - includes application-crd-id label - legacyLabels: true - -# Database Configuration - Enhanced for multi-user -mysql: - enabled: false - auth: - rootPassword: "root123" - username: "mlpipeline" - password: "mlpipeline123" - database: "mlpipeline" - primary: - persistence: - enabled: true - size: 50Gi - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 1 - memory: 2Gi - -postgresql: - enabled: false - -# Object Storage Configuration - Enhanced for multi-user -minio: - enabled: false - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: - enabled: true - size: 100Gi - resources: - requests: - cpu: 250m - memory: 256Mi - limits: - cpu: 500m - memory: 1Gi - -objectStore: - provider: minio - secure: false - bucketName: mlpipeline - -# Database names -database: - pipelinedb: mlpipeline - metadb: metadb - cachedb: cachedb - -# Pipeline Configuration -pipeline: - cache: - image: "ghcr.io/containerd/busybox" - nodeRestrictions: false - maximumStaleness: "" - defaultStaleness: "" - -# Argo Workflows Configuration - Enable for CRDs only -argo: - enabled: true - crds: - install: true - keep: true - # Disable aggregate roles - createAggregateRoles: false - controller: - # Set replicas to 0 to prevent deployment - replicas: 0 - # Disable service account - serviceAccount: - create: false - # Disable controller RBAC - rbac: - create: false - # Disable controller ConfigMap - configMap: - create: false - # Disable cluster workflow templates - clusterWorkflowTemplates: - enabled: false - server: - # Disable server completely - enabled: false - # Set replicas to 0 to prevent deployment - replicas: 0 - # Disable service account - serviceAccount: - create: false - # Disable server RBAC - rbac: - create: false - # Disable cluster workflow templates for server - clusterWorkflowTemplates: - enabled: false - executor: - enabled: false - workflow: - serviceAccount: - create: false - rbac: - create: false - -# Built-in third-party components -thirdParty: - argo: - enabled: true - mysql: - enabled: true - minio: - enabled: true - portName: http - metacontroller: - enabled: true - -# Application Configuration -application: - enabled: false - controller: - enabled: false - -# Metadata (MLMD) Configuration -metadata: - enabled: true - grpc: - resources: - -# Environment Configuration -env: - platform: platform-agnostic - -# Enhanced RBAC Configuration -rbac: - create: true - multiUser: - enabled: true - -# Multi-user API Server configuration -apiServer: - env: - publishLogs: true - pipelineLogLevel: "1" - autoUpdatePipelineDefaultVersion: true - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - multiUser: true - - additionalEnv: - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - value: "" - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - configMapName: pipeline-install-config - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - configMapName: pipeline-install-config - - name: DBCONFIG_USER - key: username - secretName: mysql-secret - - name: DBCONFIG_PASSWORD - key: password - secretName: mysql-secret - - name: DBCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_HOST - key: dbHost - configMapName: pipeline-install-config - - name: DBCONFIG_PORT - key: dbPort - configMapName: pipeline-install-config - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - configMapName: pipeline-install-config - - name: DB_DRIVER_NAME - key: dbType - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - secretName: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - secretName: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - configMapName: pipeline-install-config - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - secretName: mlpipeline-minio-artifact - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - secretName: mlpipeline-minio-artifact - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - -# Enhanced UI configuration -ui: - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: DEPLOYMENT - value: "KUBEFLOW" - - name: ARTIFACTS_SERVICE_PROXY_NAME - value: "ml-pipeline-ui-artifact" - - name: ARTIFACTS_SERVICE_PROXY_PORT - value: "80" - - name: ARTIFACTS_SERVICE_PROXY_ENABLED - value: "true" - - name: ENABLE_AUTHZ - value: "true" - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - key: null - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - key: accesskey - - name: MINIO_SECRET_KEY - key: secretkey - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" - -persistenceAgent: - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: NAMESPACE - value: "" - -scheduledWorkflow: - additionalEnv: - - name: NAMESPACE - value: '' - - name: LOG_LEVEL - value: "info" - valueFrom: null - -cache: - server: - additionalEnv: - - name: NAMESPACE_TO_WATCH - key: null - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - -viewerCrd: - env: - maxNumViewers: "" - additionalEnv: - - name: NAMESPACE - value: "" - - name: MAX_NUM_VIEWERS - value: 50 - -# Profile Controller Configuration -profileController: - enabled: true - replicas: 1 - - image: - repository: public.ecr.aws/docker/library/python - tag: "3.12" - pullPolicy: IfNotPresent - - additionalEnv: - - name: KFP_VERSION - key: appVersion - - name: KFP_DEFAULT_PIPELINE_ROOT - key: defaultPipelineRoot - - name: MINIO_ACCESS_KEY - key: accesskey - - name: MINIO_SECRET_KEY - key: secretkey - - resources: - requests: - cpu: 100m - memory: 200Mi - limits: - cpu: 500m - memory: 1Gi - - service: - type: ClusterIP - port: 80 - - disableIstioSidecar: "false" - - - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - -# Istio configuration for multi-user -istio: - enabled: true - -# Networking configuration -networking: - istio: - enabled: true - gateway: "kubeflow-gateway" - host: "*" - diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-k8s-native.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-k8s-native.yaml new file mode 100644 index 0000000000..13d20607f0 --- /dev/null +++ b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-k8s-native.yaml @@ -0,0 +1,124 @@ +# Values for platform-agnostic multi-user k8s-native mode +# Pipeline definitions stored as Kubernetes Resources + +global: + namespace: kubeflow + imageTag: 2.15.0 + security: + allowInsecureImages: true + +installMode: + type: multi-user + pipelineDefinition: + storage: k8s-native + +# API Server configuration for k8s-native mode +apiServer: + # Use local image for k8s-native mode + image: + repository: domain.local/apiserver + tag: local + # K8s-native specific configuration + k8sNative: + enabled: true + command: + - "/bin/apiserver" + args: + - "--config=/config" + - "--sampleconfig=/config/sample_config.json" + - "-logtostderr=true" + - "--webhookTLSCertPath=/etc/webhook/certs/tls.crt" + - "--webhookTLSKeyPath=/etc/webhook/certs/tls.key" + - "--pipelinesStoreKubernetes=true" + webhookPort: 8443 + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: kfp-api-webhook-cert + +# CRDs +crds: + install: true + webhook: true + +# Enable cert-manager integration +certManager: + enabled: true + +# Enable metadata +metadata: + enabled: true + +viewerCrd: + additionalEnv: + - name: NAMESPACE + value: "" + - name: MAX_NUM_VIEWERS + value: "50" + +scheduledWorkflow: + additionalEnv: + - name: NAMESPACE + value: "" + - name: LOG_LEVEL + value: "info" + +persistenceAgent: + additionalEnv: + - name: NAMESPACE + value: "" + + + +# Third-party components +thirdParty: + argo: + enabled: true + mysql: + enabled: true + metacontroller: + enabled: true + +# MySQL configuration (disabled - using external MySQL like Kustomize) +mysql: + enabled: false + +# External MySQL configuration +externalDatabase: + enabled: true + host: "mysql" + port: 3306 + database: "mlpipeline" + username: "root" + password: "" + +# SeaweedFS configuration +seaweedfs: + enabled: true + bucket: "mlpipeline" + s3EndpointUrl: "http://seaweedfs.kubeflow:8333" + +# Profile controller +profileController: + enabled: true + disableIstioSidecar: "false" + artifactsProxyEnabled: "false" + artifactRetentionDays: "-1" + +# Networking +networking: + istio: + enabled: true + gateway: "kubeflow-gateway" + virtualService: + enabled: true + +# Webhooks enabled for k8s-native mode +webhooks: + enabled: true + failurePolicy: "Fail" + diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-legacy.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-legacy.yaml deleted file mode 100644 index 861f61a635..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-legacy.yaml +++ /dev/null @@ -1,292 +0,0 @@ -# Platform-Agnostic Multi-User Legacy Configuration - -global: - namespace: kubeflow - imageRegistry: ghcr.io/kubeflow - imageTag: "2.14.3" - imagePullPolicy: Always - -# Installation mode - Multi-user enabled with legacy labels -installMode: - type: multi-user - multiUser: - enabled: true - profileController: - enabled: true - # Legacy mode enabled - includes application-crd-id label - legacyLabels: true - -# Database Configuration - Use built-in MySQL -mysql: - enabled: false - -postgresql: - enabled: false - -# Object Storage Configuration - Use built-in MinIO -minio: - enabled: false - -objectStore: - provider: minio - secure: false - bucketName: mlpipeline - -# Database names -database: - pipelinedb: mlpipeline - metadb: metadb - cachedb: cachedb - -# Pipeline Configuration -pipeline: - cache: - image: "ghcr.io/containerd/busybox" - nodeRestrictions: false - maximumStaleness: "" - defaultStaleness: "" - -cache: - server: - additionalEnv: - - name: NAMESPACE_TO_WATCH - key: null - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - -# Argo Workflows Configuration - Enable for CRDs and subchart -argo: - enabled: true - crds: - install: true - keep: true - -# Third Party Components - Enable built-in MySQL and MinIO -thirdParty: - argo: - enabled: true - mysql: - enabled: true - secretName: "mysql-secret" - minio: - enabled: true - postgresql: - enabled: false - # Enable separate MySQL database for metadata - mysqlMetadata: - enabled: true - image: - repository: mysql - tag: "8.0.3" - database: "metadb" - username: "root" - password: "test" - persistence: - enabled: true - size: 10Gi - metacontroller: - enabled: true - -# Environment Configuration -env: - platform: platform-agnostic - -# Metadata Configuration -metadata: - enabled: true - grpc: - image: - repository: gcr.io/tfx-oss-public/ml_metadata_store_server - tag: "1.14.0" - -# Profile Controller Configuration -profileController: - enabled: true - name: ml-pipeline-profile-controller - additionalEnv: - - name: KFP_VERSION - key: appVersion - - name: KFP_DEFAULT_PIPELINE_ROOT - key: defaultPipelineRoot - - name: MINIO_ACCESS_KEY - key: accesskey - - name: MINIO_SECRET_KEY - key: secretkey - - image: - repository: kfp-profile-controller - tag: "2.14.3" - - replicas: 1 - - serviceAccount: - create: true - name: "ml-pipeline-profile-controller" - annotations: {} - - resources: - requests: - cpu: 100m - memory: 200Mi - limits: {} - -# API Server configuration -apiServer: - env: - publishLogs: true - pipelineLogLevel: "1" - # Multi-user specific headers - kubeflowUserIdHeader: "kubeflow-userid" - kubeflowUserIdPrefix: "" - additionalEnv: - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - value: "" - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - - name: DBCONFIG_USER - key: username - secretName: mysql-secret - - name: DBCONFIG_PASSWORD - key: password - secretName: mysql-secret - - name: DBCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_HOST - key: dbHost - configMapName: pipeline-install-config - - name: DBCONFIG_PORT - valueFrom: - key: dbPort - configMapName: pipeline-install-config - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - - name: DB_DRIVER_NAME - key: dbType - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - configMapName: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - configMapName: pipeline-install-config - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - secretName: mlpipeline-minio-artifact - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - secretName: mlpipeline-minio-artifact - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - -persistenceAgent: - additionalEnv: - - name: NAMESPACE - value: "" - -scheduledWorkflow: - additionalEnv: - - name: NAMESPACE - value: "" - - name: LOG_LEVEL - value: "info" - -ui: - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: DEPLOYMENT - value: "KUBEFLOW" - - name: ARTIFACTS_SERVICE_PROXY_NAME - value: "ml-pipeline-ui-artifact" - - name: ARTIFACTS_SERVICE_PROXY_PORT - value: "80" - - name: ARTIFACTS_SERVICE_PROXY_ENABLED - value: "true" - - name: ENABLE_AUTHZ - value: "true" - - name: KUBEFLOW_USERID_HEADER - value: "kubeflow-userid" - - name: KUBEFLOW_USERID_PREFIX - value: "" - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - key: accesskey - secretName: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - key: secretkey - secretName: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" - -viewerCrd: - additionalEnv: - - name: NAMESPACE - value: "" - - name: MAX_NUM_VIEWERS - value: "50" - -# Networking Configuration -networking: - istio: - enabled: true - gateway: "kubeflow-gateway" - host: "*" - virtualService: - enabled: true - prefix: "/pipeline" - diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user.yaml new file mode 100644 index 0000000000..cebb3f754c --- /dev/null +++ b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user.yaml @@ -0,0 +1,95 @@ +# Values for platform-agnostic multi-user mode +# Pipeline definitions stored in the database + +global: + namespace: kubeflow + imageTag: 2.15.0 + security: + allowInsecureImages: true + +installMode: + type: multi-user + pipelineDefinition: + storage: database + +# CRDs +crds: + install: true + application: false + webhook: false + +# Enable cert-manager integration +certManager: + enabled: true + +# Enable metadata +metadata: + enabled: true + +viewerCrd: + additionalEnv: + - name: NAMESPACE + value: "" + - name: MAX_NUM_VIEWERS + value: "50" + +scheduledWorkflow: + additionalEnv: + - name: NAMESPACE + value: "" + - name: LOG_LEVEL + value: "info" + +persistenceAgent: + additionalEnv: + - name: NAMESPACE + value: "" + +# Third-party components +thirdParty: + argo: + enabled: true + mysql: + enabled: true + metacontroller: + enabled: true + + +# MySQL configuration (disabled - using external MySQL like Kustomize) +mysql: + enabled: false + +# External MySQL configuration +externalDatabase: + enabled: true + host: "mysql" + port: 3306 + database: "mlpipeline" + username: "root" + password: "" + +# SeaweedFS configuration +seaweedfs: + enabled: true + bucket: "mlpipeline" + s3EndpointUrl: "http://seaweedfs.kubeflow:8333" + +# Profile controller +profileController: + enabled: true + disableIstioSidecar: "false" + artifactsProxyEnabled: "false" + artifactRetentionDays: "-1" + +# Networking +networking: + istio: + enabled: true + gateway: "kubeflow-gateway" + virtualService: + enabled: true + +# Webhooks disabled for database mode +webhooks: + enabled: false + diff --git a/experimental/helm/charts/pipelines/ci/values-postgresql.yaml b/experimental/helm/charts/pipelines/ci/values-postgresql.yaml deleted file mode 100644 index 6f1bab3ef5..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-postgresql.yaml +++ /dev/null @@ -1,235 +0,0 @@ -# PostgreSQL values for kubeflow-pipelines. -# This configuration uses PostgreSQL instead of MySQL - -# CRDs are not included in Kustomize platform-agnostic-postgresql -crds: - install: false - -# Cache deployer configuration -cacheDeployer: - enabled: true - serviceAccount: - create: false - -# Third-party services configuration -thirdParty: - argo: - enabled: true - mysql: - enabled: false - minio: - enabled: true - portName: http - metacontroller: - enabled: false - # Use built-in PostgreSQL deployment (not subchart) - postgresql: - enabled: true - image: - repository: postgres - tag: "14.7-alpine3.17" - database: "postgres" - username: "user" - password: "password" - secretName: "postgres-secret" - secretKey: "root_password" - persistence: - size: "20Gi" - -# Disable MySQL subchart -mysql: - enabled: false - -# Disable postgresql subchart -postgresql: - enabled: false - -# External database configuration pointing to PostgreSQL -externalDatabase: - type: postgresql - host: "" - port: 5432 - database: "mlpipeline" - username: "postgres" -pipeline: - app: - name: "pipeline" - version: "2.0.0" - -# Additional databases for PostgreSQL -database: - metadb: "metadb" - cachedb: "cachedb" - connectionMaxLifetime: "120s" - -# Standard MinIO configuration -objectStore: - provider: minio - -# Disable MinIO subchart -minio: - enabled: false - -# Metadata service configuration -metadata: - enabled: true - grpc: - image: - repository: gcr.io/tfx-oss-public/ml_metadata_store_server - tag: "1.14.0" - pullPolicy: IfNotPresent - resources: - -# Standard component configuration -apiServer: - replicas: 1 - resources: - requests: - cpu: 250m - memory: 500Mi - additionalEnv: - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - configMapName: pipeline-install-config - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - - name: DB_DRIVER_NAME - key: dbType - - name: DBCONFIG_POSTGRESQLCONFIG_USER - key: username - secretName: postgres-secret-extended - secretKey: username - - name: DBCONFIG_POSTGRESQLCONFIG_PASSWORD - key: password - secretName: postgres-secret-extended - secretKey: password - - name: DBCONFIG_POSTGRESQLCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_POSTGRESQLCONFIG_HOST - key: postgresHost - - name: DBCONFIG_POSTGRESQLCONFIG_PORT - key: postgresPort - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - -persistenceAgent: - replicas: 1 - resources: - requests: - cpu: 120m - memory: 500Mi - additionalEnv: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - -scheduledWorkflow: - replicas: 1 - resources: - additionalEnv: - - name: LOG_LEVEL - value: "info" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -cache: - server: - additionalEnv: - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "pgx" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: postgresHost - - name: DBCONFIG_PORT - key: postgresPort - - name: DBCONFIG_USER - secretName: postgres-secret-extended - secretKey: username - - name: DBCONFIG_PASSWORD - secretName: postgres-secret-extended - secretKey: password - - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -viewerCrd: - additionalEnv: - - name: MAX_NUM_VIEWERS - value: "50" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - -ui: - replicas: 1 - resources: - requests: - cpu: 10m - memory: 70Mi - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" - - - -# Disable monitoring for simplicity -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -# Standard security -security: - podSecurityStandards: - enabled: true - enforce: "restricted" \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/ci/values-production.yaml b/experimental/helm/charts/pipelines/ci/values-production.yaml deleted file mode 100644 index d497b7452b..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-production.yaml +++ /dev/null @@ -1,104 +0,0 @@ -# Production values for kubeflow-pipelines. -# This configuration is optimized for production deployments - -# Global configuration -global: - namespace: kubeflow - logLevel: "info" - -# Production resource settings -apiServer: - replicas: 3 - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 2000m - memory: 2Gi - -persistenceAgent: - replicas: 2 - resources: - requests: - cpu: 250m - memory: 500Mi - limits: - cpu: 1000m - memory: 1Gi - -scheduledWorkflow: - replicas: 2 - resources: - requests: - cpu: 250m - memory: 256Mi - limits: - cpu: 1000m - memory: 512Mi - -ui: - replicas: 2 - resources: - requests: - cpu: 100m - memory: 200Mi - limits: - cpu: 500m - memory: 500Mi - -# High availability database -mysql: - enabled: false - -postgresql: - enabled: false - -# Use external database in production -externalDatabase: - type: mysql - host: "production-db-host" - port: 3306 - database: "mlpipeline" - username: "mlpipeline" - existingSecret: "mysql-secret" - -# Use external object storage -objectStore: - provider: custom - custom: - endpoint: "s3.example.com" - bucket: "production-kfp-artifacts" - existingSecret: "s3-credentials" - -minio: - enabled: false - -# Enable monitoring -monitoring: - prometheus: - enabled: true - serviceMonitor: - enabled: true - grafana: - enabled: true - -# Enable security features -security: - podSecurityStandards: - enabled: true - enforce: "restricted" - -# Enable Istio for production -networking: - istio: - enabled: true - gateway: "kubeflow-gateway" - -# Enable webhooks -webhook: - enabled: true - mutating: - enabled: true - validating: - enabled: true diff --git a/experimental/helm/charts/pipelines/ci/values-standalone.yaml b/experimental/helm/charts/pipelines/ci/values-standalone.yaml deleted file mode 100644 index 16ea37983c..0000000000 --- a/experimental/helm/charts/pipelines/ci/values-standalone.yaml +++ /dev/null @@ -1,226 +0,0 @@ -# Platform-agnostic values for Kubeflow Pipelines -# This configuration uses local storage (MinIO) and MySQL - -# Installation mode -installMode: - type: generic - multiUser: - enabled: false - -# CRDs configuration -crds: - install: false - application: false - -# Cache deployer for generic mode -cacheDeployer: - enabled: true - serviceAccount: - create: false - -# Environment configuration -env: - platform: platform-agnostic - -# Database configuration - use built-in MySQL -mysql: - enabled: false - auth: - rootPassword: "" - database: "mlpipeline" - username: "root" - primary: - persistence: - enabled: true - size: 20Gi - -postgresql: - enabled: false - -externalDatabase: - type: mysql - -# Object storage configuration - use built-in MinIO -objectStore: - provider: minio - -minio: - enabled: false - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: - enabled: true - size: 20Gi - -# Argo Workflows configuration - use built-in -argo: - enabled: false - -# Built-in third-party components -thirdParty: - mysql: - enabled: true - minio: - enabled: true - portName: http - argo: - enabled: true - -# Workflow controller configuration -workflowController: - priorityClass: - create: false - -# Enable metadata server for standalone deployment -metadata: - enabled: true - grpc: - resources: - -# Disable cloud-specific features -monitoring: - prometheus: - enabled: false - grafana: - enabled: false - -certManager: - enabled: false - -networking: - istio: - enabled: false - -# Disable webhooks for platform-agnostic scenario -webhooks: - enabled: false - -apiServer: - additionalEnv: - - name: PUBLISH_LOGS - value: "true" - - name: LOG_LEVEL - value: "info" - - name: PIPELINE_LOG_LEVEL - value: "1" - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - key: autoUpdatePipelineDefaultVersion - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - key: bucketName - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: DBCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_HOST - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_CONMAXLIFETIME - key: ConMaxLifeTime - - name: DB_DRIVER_NAME - key: dbType - - name: DBCONFIG_MYSQLCONFIG_USER - key: username - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - key: password - - name: DBCONFIG_MYSQLCONFIG_DBNAME - key: pipelineDb - - name: DBCONFIG_MYSQLCONFIG_HOST - key: mysqlHost - - name: DBCONFIG_MYSQLCONFIG_PORT - key: mysqlPort - - name: OBJECTSTORECONFIG_ACCESSKEY - key: accesskey - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - key: secretkey - - name: V2_DRIVER_IMAGE - value: "ghcr.io/kubeflow/kfp-driver:2.14.3" - - name: V2_LAUNCHER_IMAGE - value: "ghcr.io/kubeflow/kfp-launcher:2.14.3" -cache: - server: - additionalEnv: - - name: DEFAULT_CACHE_STALENESS - key: DEFAULT_CACHE_STALENESS - - name: MAXIMUM_CACHE_STALENESS - key: MAXIMUM_CACHE_STALENESS - - name: CACHE_IMAGE - key: cacheImage - - name: CACHE_NODE_RESTRICTIONS - key: cacheNodeRestrictions - - name: DBCONFIG_DRIVER - value: "mysql" - - name: DBCONFIG_DB_NAME - key: cacheDb - - name: DBCONFIG_HOST_NAME - key: dbHost - - name: DBCONFIG_PORT - key: dbPort - - name: DBCONFIG_USER - key: username - - name: DBCONFIG_PASSWORD - key: password - - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace -scheduledWorkflow: - additionalEnv: - - name: LOG_LEVEL - value: "info" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -persistenceAgent: - additionalEnv: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -viewerCrd: - additionalEnv: - - name: MAX_NUM_VIEWERS - value: "50" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -ui: - additionalEnv: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: "/etc/config/viewer-pod-template.json" - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - - name: FRONTEND_SERVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ARGO_ARCHIVE_LOGS - value: "true" - - name: DISABLE_GKE_METADATA - value: "true" \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/files/sync.py b/experimental/helm/charts/pipelines/files/sync.py index 3d39e3187f..6ea4dc1c35 100644 --- a/experimental/helm/charts/pipelines/files/sync.py +++ b/experimental/helm/charts/pipelines/files/sync.py @@ -17,6 +17,18 @@ import os import base64 +# From awscli installed in alpine/k8s image +import botocore.session + +S3_BUCKET_NAME = 'mlpipeline' + +session = botocore.session.get_session() +# To interact with seaweedfs user management. Region does not matter. +iam = session.create_client('iam', region_name='foobar') +# S3 client for lifecycle policy management +s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", "http://seaweedfs.kubeflow:8333") +s3 = session.create_client('s3', region_name='foobar', endpoint_url=s3_endpoint_url) + def main(): settings = get_settings_from_env() @@ -25,9 +37,11 @@ def main(): def get_settings_from_env(controller_port=None, - visualization_server_image=None, frontend_image=None, - visualization_server_tag=None, frontend_tag=None, disable_istio_sidecar=None, - minio_access_key=None, minio_secret_key=None, kfp_default_pipeline_root=None): + frontend_image=None, + frontend_tag=None, + disable_istio_sidecar=None, + artifacts_proxy_enabled=None, + artifact_retention_days=None): """ Returns a dict of settings from environment variables relevant to the controller @@ -36,35 +50,30 @@ def get_settings_from_env(controller_port=None, Settings are pulled from the all-caps version of the setting name. The following defaults are used if those environment variables are not set to enable backwards compatibility with previous versions of this script: - visualization_server_image: ghcr.io/kubeflow/kfp-visualization-server - visualization_server_tag: value of KFP_VERSION environment variable frontend_image: ghcr.io/kubeflow/kfp-frontend frontend_tag: value of KFP_VERSION environment variable disable_istio_sidecar: Required (no default) - minio_access_key: Required (no default) - minio_secret_key: Required (no default) """ settings = dict() settings["controller_port"] = \ controller_port or \ os.environ.get("CONTROLLER_PORT", "8080") - settings["visualization_server_image"] = \ - visualization_server_image or \ - os.environ.get("VISUALIZATION_SERVER_IMAGE", "ghcr.io/kubeflow/kfp-visualization-server") - settings["frontend_image"] = \ frontend_image or \ os.environ.get("FRONTEND_IMAGE", "ghcr.io/kubeflow/kfp-frontend") + settings["artifacts_proxy_enabled"] = \ + artifacts_proxy_enabled or \ + os.environ.get("ARTIFACTS_PROXY_ENABLED", "false") + + settings["artifact_retention_days"] = \ + artifact_retention_days or \ + os.environ.get("ARTIFACT_RETENTION_DAYS", -1) + # Look for specific tags for each image first, falling back to # previously used KFP_VERSION environment variable for backwards # compatibility - settings["visualization_server_tag"] = \ - visualization_server_tag or \ - os.environ.get("VISUALIZATION_SERVER_TAG") or \ - os.environ["KFP_VERSION"] - settings["frontend_tag"] = \ frontend_tag or \ os.environ.get("FRONTEND_TAG") or \ @@ -74,306 +83,315 @@ def get_settings_from_env(controller_port=None, disable_istio_sidecar if disable_istio_sidecar is not None \ else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" - settings["minio_access_key"] = \ - minio_access_key or \ - base64.b64encode(bytes(os.environ.get("MINIO_ACCESS_KEY"), 'utf-8')).decode('utf-8') - - settings["minio_secret_key"] = \ - minio_secret_key or \ - base64.b64encode(bytes(os.environ.get("MINIO_SECRET_KEY"), 'utf-8')).decode('utf-8') - - # KFP_DEFAULT_PIPELINE_ROOT is optional - settings["kfp_default_pipeline_root"] = \ - kfp_default_pipeline_root or \ - os.environ.get("KFP_DEFAULT_PIPELINE_ROOT") - return settings -def server_factory(visualization_server_image, - visualization_server_tag, frontend_image, frontend_tag, - disable_istio_sidecar, minio_access_key, - minio_secret_key, kfp_default_pipeline_root=None, - url="", controller_port=8080): +def server_factory(frontend_image, + frontend_tag, + disable_istio_sidecar, + artifacts_proxy_enabled, + artifact_retention_days, + url="", + controller_port=8080): """ Returns an HTTPServer populated with Handler with customized settings """ class Controller(BaseHTTPRequestHandler): + def upsert_lifecycle_policy(self, bucket_name, artifact_retention_days): + """Configures or deletes the lifecycle policy based on the artifact_retention_days string.""" + try: + retention_days = int(artifact_retention_days) + except ValueError: + print(f"ERROR: ARTIFACT_RETENTION_DAYS value '{artifact_retention_days}' is not a valid integer. Aborting policy update.") + return + + # To disable lifecycle policy we need to delete it + if retention_days <= 0: + print(f"ARTIFACT_RETENTION_DAYS is non-positive ({retention_days} days). Attempting to delete lifecycle policy.") + try: + response = s3.get_bucket_lifecycle_configuration(Bucket=bucket_name) + # Check if there are any enabled rules + has_enabled_rules = any(rule.get('Status') == 'Enabled' for rule in response.get('Rules', [])) + + if has_enabled_rules: + s3.delete_bucket_lifecycle(Bucket=bucket_name) + print("Successfully deleted lifecycle policy.") + else: + print("No enabled lifecycle rules found to delete.") + except Exception: + print(f"Warning: No lifecycle policy exists") + return + + # Create/update lifecycle policy + life_cycle_policy = { + "Rules": [ + { + "Status": "Enabled", + "Filter": {"Prefix": "private-artifacts"}, + "Expiration": {"Days": retention_days}, + "ID": "private-artifacts", + }, + ] + } + print('upsert_lifecycle_policy:', life_cycle_policy) + + try: + api_response = s3.put_bucket_lifecycle_configuration( + Bucket=bucket_name, + LifecycleConfiguration = life_cycle_policy + ) + print('Lifecycle policy configured successfully:', api_response) + except Exception as exception: + if hasattr(exception, 'response') and 'Error' in exception.response: + print(f"ERROR: Failed to configure lifecycle policy: {exception.response['Error']['Code']} - {exception}") + else: + print(f"ERROR: Failed to configure lifecycle policy: {exception}") + + def sync(self, parent, attachments): # parent is a namespace namespace = parent.get("metadata", {}).get("name") + pipeline_enabled = parent.get("metadata", {}).get( "labels", {}).get("pipelines.kubeflow.org/enabled") if pipeline_enabled != "true": return {"status": {}, "attachments": []} - desired_configmap_count = 1 - desired_resources = [] - if kfp_default_pipeline_root: - desired_configmap_count = 2 - desired_resources += [{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "kfp-launcher", - "namespace": namespace, - }, - "data": { - "defaultPipelineRoot": kfp_default_pipeline_root, - }, - }] - - # Compute status based on observed state. desired_status = { "kubeflow-pipelines-ready": len(attachments["Secret.v1"]) == 1 and - len(attachments["ConfigMap.v1"]) == desired_configmap_count and - len(attachments["Deployment.apps/v1"]) == 2 and - len(attachments["Service.v1"]) == 2 and - len(attachments["DestinationRule.networking.istio.io/v1alpha3"]) == 1 and - len(attachments["AuthorizationPolicy.security.istio.io/v1beta1"]) == 1 and + len(attachments["ConfigMap.v1"]) == 3 and + len(attachments["Deployment.apps/v1"]) == (1 if artifacts_proxy_enabled.lower() == "true" else 0) and + len(attachments["Service.v1"]) == (1 if artifacts_proxy_enabled.lower() == "true" else 0) and "True" or "False" } # Generate the desired attachment object(s). - desired_resources += [ + desired_resources = [ { "apiVersion": "v1", "kind": "ConfigMap", "metadata": { - "name": "metadata-grpc-configmap", + "name": "kfp-launcher", "namespace": namespace, }, "data": { - "METADATA_GRPC_SERVICE_HOST": - "metadata-grpc-service.kubeflow", - "METADATA_GRPC_SERVICE_PORT": "8080", + "defaultPipelineRoot": f"minio://{S3_BUCKET_NAME}/private-artifacts/{namespace}/v2/artifacts", }, }, - # Visualization server related manifests below { - "apiVersion": "apps/v1", - "kind": "Deployment", + "apiVersion": "v1", + "kind": "ConfigMap", "metadata": { - "labels": { - "app": "ml-pipeline-visualizationserver" - }, - "name": "ml-pipeline-visualizationserver", + "name": "metadata-grpc-configmap", "namespace": namespace, }, - "spec": { - "selector": { - "matchLabels": { - "app": "ml-pipeline-visualizationserver" - }, - }, - "template": { - "metadata": { - "labels": { - "app": "ml-pipeline-visualizationserver" - }, - "annotations": disable_istio_sidecar and { - "sidecar.istio.io/inject": "false" - } or {}, - }, - "spec": { - "containers": [{ - "image": f"{visualization_server_image}:{visualization_server_tag}", - "imagePullPolicy": - "IfNotPresent", - "name": - "ml-pipeline-visualizationserver", - "ports": [{ - "containerPort": 8888 - }], - "resources": { - "requests": { - "cpu": "50m", - "memory": "200Mi" - }, - "limits": { - "cpu": "500m", - "memory": "1Gi" - }, - } - }], - "serviceAccountName": - "default-editor", - }, - }, + "data": { + "METADATA_GRPC_SERVICE_HOST": + "metadata-grpc-service.kubeflow", + "METADATA_GRPC_SERVICE_PORT": "8080", }, }, { - "apiVersion": "networking.istio.io/v1alpha3", - "kind": "DestinationRule", + "apiVersion": "v1", + "kind": "ConfigMap", "metadata": { - "name": "ml-pipeline-visualizationserver", + "name": "artifact-repositories", "namespace": namespace, - }, - "spec": { - "host": "ml-pipeline-visualizationserver", - "trafficPolicy": { - "tls": { - "mode": "ISTIO_MUTUAL" - } + "annotations": { + "workflows.argoproj.io/default-artifact-repository": "default-namespaced" } - } - }, - { - "apiVersion": "security.istio.io/v1beta1", - "kind": "AuthorizationPolicy", - "metadata": { - "name": "ml-pipeline-visualizationserver", - "namespace": namespace, }, - "spec": { - "selector": { - "matchLabels": { - "app": "ml-pipeline-visualizationserver" - } - }, - "rules": [{ - "from": [{ - "source": { - "principals": ["cluster.local/ns/kubeflow/sa/ml-pipeline"] + "data": { + "default-namespaced": json.dumps({ + "archiveLogs": True, + "s3": { + "endpoint": "minio-service.kubeflow:9000", + "bucket": S3_BUCKET_NAME, + "keyFormat": f"private-artifacts/{namespace}/{{{{workflow.name}}}}/{{{{workflow.creationTimestamp.Y}}}}/{{{{workflow.creationTimestamp.m}}}}/{{{{workflow.creationTimestamp.d}}}}/{{{{pod.name}}}}", + "insecure": True, + "accessKeySecret": { + "name": "mlpipeline-minio-artifact", + "key": "accesskey", + }, + "secretKeySecret": { + "name": "mlpipeline-minio-artifact", + "key": "secretkey", } - }] - }] + } + }) } }, - { - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "name": "ml-pipeline-visualizationserver", - "namespace": namespace, - }, - "spec": { - "ports": [{ - "name": "http", - "port": 8888, - "protocol": "TCP", - "targetPort": 8888, - }], - "selector": { - "app": "ml-pipeline-visualizationserver", - }, - }, - }, - # Artifact fetcher related resources below. - { - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "labels": { - "app": "ml-pipeline-ui-artifact" - }, - "name": "ml-pipeline-ui-artifact", - "namespace": namespace, - }, - "spec": { - "selector": { - "matchLabels": { + ] + + # Add artifact fetcher related resources if enabled + if artifacts_proxy_enabled.lower() == "true": + desired_resources.extend([ + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "labels": { "app": "ml-pipeline-ui-artifact" - } + }, + "name": "ml-pipeline-ui-artifact", + "namespace": namespace, }, - "template": { - "metadata": { - "labels": { + "spec": { + "selector": { + "matchLabels": { "app": "ml-pipeline-ui-artifact" - }, - "annotations": disable_istio_sidecar and { - "sidecar.istio.io/inject": "false" - } or {}, + } }, - "spec": { - "containers": [{ - "name": - "ml-pipeline-ui-artifact", - "image": f"{frontend_image}:{frontend_tag}", - "imagePullPolicy": - "IfNotPresent", - "ports": [{ - "containerPort": 3000 - }], - "env": [ - { - "name": "MINIO_ACCESS_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "accesskey", - "name": "mlpipeline-minio-artifact" + "template": { + "metadata": { + "labels": { + "app": "ml-pipeline-ui-artifact" + }, + "annotations": disable_istio_sidecar and { + "sidecar.istio.io/inject": "false" + } or {}, + }, + "spec": { + "containers": [{ + "name": + "ml-pipeline-ui-artifact", + "image": f"{frontend_image}:{frontend_tag}", + "imagePullPolicy": + "IfNotPresent", + "ports": [{ + "containerPort": 3000 + }], + "env": [ + { + "name": "MINIO_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "accesskey", + "name": "mlpipeline-minio-artifact" + } } - } - }, - { - "name": "MINIO_SECRET_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "secretkey", - "name": "mlpipeline-minio-artifact" + }, + { + "name": "MINIO_SECRET_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "secretkey", + "name": "mlpipeline-minio-artifact" + } } + }, + { + "name": "ML_PIPELINE_SERVICE_HOST", + "value": "ml-pipeline.kubeflow.svc.cluster.local" + }, + { + "name": "ML_PIPELINE_SERVICE_PORT", + "value": "8888" + }, + { + "name": "FRONTEND_SERVER_NAMESPACE", + "value": namespace, } + ], + "resources": { + "requests": { + "cpu": "10m", + "memory": "70Mi" + }, + "limits": { + "cpu": "100m", + "memory": "500Mi" + }, } - ], - "resources": { - "requests": { - "cpu": "10m", - "memory": "70Mi" - }, - "limits": { - "cpu": "100m", - "memory": "500Mi" - }, - } - }], - "serviceAccountName": - "default-editor" + }], + "serviceAccountName": + "default-editor" + } } } - } - }, - { - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "name": "ml-pipeline-ui-artifact", - "namespace": namespace, - "labels": { - "app": "ml-pipeline-ui-artifact" - } }, - "spec": { - "ports": [{ - "name": - "http", # name is required to let istio understand request protocol - "port": 80, - "protocol": "TCP", - "targetPort": 3000 - }], - "selector": { - "app": "ml-pipeline-ui-artifact" + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "ml-pipeline-ui-artifact", + "namespace": namespace, + "labels": { + "app": "ml-pipeline-ui-artifact" + } + }, + "spec": { + "ports": [{ + "name": + "http", # name is required to let istio understand request protocol + "port": 80, + "protocol": "TCP", + "targetPort": 3000 + }], + "selector": { + "app": "ml-pipeline-ui-artifact" + } } - } - }, - ] + }, + ]) + print('Received request:\n', json.dumps(parent, sort_keys=True)) print('Desired resources except secrets:\n', json.dumps(desired_resources, sort_keys=True)) + # Moved after the print argument because this is sensitive data. - desired_resources.append({ - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": "mlpipeline-minio-artifact", - "namespace": namespace, - }, - "data": { - "accesskey": minio_access_key, - "secretkey": minio_secret_key, - }, - }) + + # Check if secret is already there when the controller made the request. If yes, then + # use it. Else create a new credentials on seaweedfs for the namespace. + if s3_secret := attachments["Secret.v1"].get(f"{namespace}/mlpipeline-minio-artifact"): + desired_resources.append(s3_secret) + print('Using existing secret') + else: + print('Creating new access key.') + s3_access_key = iam.create_access_key(UserName=namespace) + # Use the AWS IAM API of seaweedfs to manage access policies to bucket. + # This policy ensures that a user can only access artifacts from his own profile. + iam.put_user_policy( + UserName=namespace, + PolicyName=f"KubeflowProject{namespace}", + PolicyDocument=json.dumps( + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "s3:Put*", + "s3:Get*", + "s3:List*" + ], + "Resource": [ + f"arn:aws:s3:::{S3_BUCKET_NAME}/artifacts/*", + f"arn:aws:s3:::{S3_BUCKET_NAME}/private-artifacts/{namespace}/*", + f"arn:aws:s3:::{S3_BUCKET_NAME}/private/{namespace}/*", + f"arn:aws:s3:::{S3_BUCKET_NAME}/shared/*", + ] + }] + }) + ) + + self.upsert_lifecycle_policy(S3_BUCKET_NAME, artifact_retention_days) + + desired_resources.insert( + 0, + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "mlpipeline-minio-artifact", + "namespace": namespace, + }, + "data": { + "accesskey": base64.b64encode(s3_access_key["AccessKey"]["AccessKeyId"].encode('utf-8')).decode("utf-8"), + "secretkey": base64.b64encode(s3_access_key["AccessKey"]["SecretAccessKey"].encode('utf-8')).decode("utf-8"), + }, + }) return {"status": desired_status, "attachments": desired_resources} diff --git a/experimental/helm/charts/pipelines/templates/_helpers.tpl b/experimental/helm/charts/pipelines/templates/_helpers.tpl index beb01b2158..95becc66e1 100644 --- a/experimental/helm/charts/pipelines/templates/_helpers.tpl +++ b/experimental/helm/charts/pipelines/templates/_helpers.tpl @@ -36,23 +36,19 @@ Common labels {{- define "kubeflow-pipelines.labels" -}} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if or (and (ne .Values.installMode.type "multi-user") (ne .Values.installMode.includeApplicationCrdId false)) (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines -{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} {{- end }} {{/* -Authorization Policy labels - includes application-crd-id for legacy multi-user mode +Authorization Policy labels - includes application-crd-id for multi-user mode */}} {{- define "kubeflow-pipelines.authorizationPolicyLabels" -}} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if or (and (ne .Values.installMode.type "multi-user") (ne .Values.installMode.includeApplicationCrdId false)) (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines -{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} @@ -73,24 +69,7 @@ Cache server labels app: cache-server app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if or (and (ne .Values.installMode.type "multi-user") (ne .Values.installMode.includeApplicationCrdId false)) (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines -{{- end }} -{{- with .Values.commonLabels }} -{{ toYaml . }} -{{- end }} -{{- end }} - -{{/* -Cache deployer labels -*/}} -{{- define "kubeflow-pipelines.cacheDeployerLabels" -}} -app: cache-deployer -app.kubernetes.io/component: ml-pipeline -app.kubernetes.io/name: kubeflow-pipelines -{{- if or (and (ne .Values.installMode.type "multi-user") (ne .Values.installMode.includeApplicationCrdId false)) (.Values.installMode.legacyLabels) }} -application-crd-id: kubeflow-pipelines -{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} @@ -103,9 +82,7 @@ ML Pipeline specific labels - matching original manifests app: ml-pipeline app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if or (and (ne .Values.installMode.type "multi-user") (ne .Values.installMode.includeApplicationCrdId false)) (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines -{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} @@ -118,9 +95,7 @@ ML Pipeline UI labels app: ml-pipeline-ui app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if or (and (ne .Values.installMode.type "multi-user") (ne .Values.installMode.includeApplicationCrdId false)) (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines -{{- end }} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} @@ -131,54 +106,30 @@ ML Pipeline selector labels */}} {{- define "kubeflow-pipelines.mlPipelineSelectorLabels" -}} app: ml-pipeline -{{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- else }} -{{- if ne .Values.installMode.includeApplicationCrdId false }} application-crd-id: kubeflow-pipelines {{- end }} -{{- end }} -{{- end }} {{/* Cache server selector labels */}} {{- define "kubeflow-pipelines.cacheSelectorLabels" -}} app: cache-server -{{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- else }} -{{- if ne .Values.installMode.includeApplicationCrdId false }} application-crd-id: kubeflow-pipelines {{- end }} -{{- end }} -{{- end }} {{/* UI selector labels */}} {{- define "kubeflow-pipelines.uiSelectorLabels" -}} app: ml-pipeline-ui -{{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- else }} -{{- if ne .Values.installMode.includeApplicationCrdId false }} application-crd-id: kubeflow-pipelines {{- end }} -{{- end }} -{{- end }} {{/* Viewer CRD labels @@ -208,20 +159,11 @@ application-crd-id: kubeflow-pipelines Viewer CRD selector labels */}} {{- define "kubeflow-pipelines.viewerCrdSelectorLabels" -}} -{{- if eq .Values.installMode.type "multi-user" }} app: ml-pipeline-viewer-crd app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- else }} -app: ml-pipeline-viewer-crd -{{- if ne .Values.installMode.includeApplicationCrdId false }} application-crd-id: kubeflow-pipelines {{- end }} -{{- end }} -{{- end }} {{/* Visualization server labels @@ -230,9 +172,7 @@ Visualization server labels app: ml-pipeline-visualizationserver app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if or (and (ne .Values.installMode.type "multi-user") (ne .Values.installMode.includeApplicationCrdId false)) (.Values.installMode.legacyLabels) }} application-crd-id: kubeflow-pipelines -{{- end}} {{- with .Values.commonLabels }} {{ toYaml . }} {{- end }} @@ -242,20 +182,11 @@ application-crd-id: kubeflow-pipelines Visualization server selector labels */}} {{- define "kubeflow-pipelines.visualizationSelectorLabels" -}} -{{- if eq .Values.installMode.type "multi-user" }} app: ml-pipeline-visualizationserver app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- else }} -app: ml-pipeline-visualizationserver -{{- if ne .Values.installMode.includeApplicationCrdId false }} application-crd-id: kubeflow-pipelines {{- end }} -{{- end }} -{{- end }} {{/* MySQL selector labels @@ -265,13 +196,6 @@ app: mysql application-crd-id: kubeflow-pipelines {{- end }} -{{/* -MinIO selector labels -*/}} -{{- define "kubeflow-pipelines.minioSelectorLabels" -}} -app: minio -application-crd-id: kubeflow-pipelines -{{- end }} {{/* Metadata GRPC selector labels @@ -289,77 +213,35 @@ component: metadata-envoy application-crd-id: kubeflow-pipelines {{- end }} -{{/* -Cache deployer selector labels -*/}} -{{- define "kubeflow-pipelines.cacheDeployerSelectorLabels" -}} -app: cache-deployer -{{- if eq .Values.installMode.type "multi-user" }} -app.kubernetes.io/component: ml-pipeline -app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- else }} -{{- if ne .Values.installMode.includeApplicationCrdId false }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- end }} -{{- end }} - {{/* Metadata writer selector labels */}} {{- define "kubeflow-pipelines.metadataWriterSelectorLabels" -}} app: metadata-writer -{{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- else }} -{{- if ne .Values.installMode.includeApplicationCrdId false }} application-crd-id: kubeflow-pipelines {{- end }} -{{- end }} -{{- end }} {{/* Persistence agent selector labels */}} {{- define "kubeflow-pipelines.persistenceAgentSelectorLabels" -}} app: ml-pipeline-persistenceagent -{{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- else }} -{{- if ne .Values.installMode.includeApplicationCrdId false }} application-crd-id: kubeflow-pipelines {{- end }} -{{- end }} -{{- end }} {{/* Scheduled workflow selector labels */}} {{- define "kubeflow-pipelines.scheduledWorkflowSelectorLabels" -}} app: ml-pipeline-scheduledworkflow -{{- if eq .Values.installMode.type "multi-user" }} app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} -application-crd-id: kubeflow-pipelines -{{- end }} -{{- else }} -{{- if ne .Values.installMode.includeApplicationCrdId false }} application-crd-id: kubeflow-pipelines {{- end }} -{{- end }} -{{- end }} {{/* Workflow controller selector labels @@ -426,17 +308,6 @@ Create the name of the service account to use for Cache Server {{- end }} {{- end }} -{{/* -Create the name of the service account to use for Cache Deployer -*/}} -{{- define "kubeflow-pipelines.cacheDeployer.serviceAccountName" -}} -{{- if .Values.cacheDeployer.serviceAccount.create }} -{{- default "kubeflow-pipelines-cache-deployer-sa" .Values.cacheDeployer.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.cacheDeployer.serviceAccount.name }} -{{- end }} -{{- end }} - {{/* Create the name of the service account to use for Metadata */}} @@ -482,33 +353,27 @@ Create the name of the service account to use for Metadata Writer {{- end }} {{/* -Database configuration helpers +Database configuration helpers (MySQL only) */}} {{- define "kubeflow-pipelines.database.host" -}} {{- if .Values.mysql.enabled -}} mysql -{{- else if .Values.thirdParty.postgresql.enabled -}} -postgres {{- else -}} -{{- .Values.externalDatabase.host | default "mysql" }} +{{- .Values.externalDatabase.host | default "mysql.kubeflow.svc.cluster.local" }} {{- end -}} {{- end }} {{- define "kubeflow-pipelines.database.port" -}} {{- if .Values.mysql.enabled -}} 3306 -{{- else if .Values.postgresql.enabled -}} -5432 {{- else -}} -{{- .Values.externalDatabase.port }} +{{- .Values.externalDatabase.port | default 3306 }} {{- end -}} {{- end }} {{- define "kubeflow-pipelines.database.name" -}} {{- if .Values.mysql.enabled -}} {{- .Values.mysql.auth.database | default "mlpipeline" }} -{{- else if .Values.postgresql.enabled -}} -{{- .Values.postgresql.auth.database | default "mlpipeline" }} {{- else -}} {{- .Values.externalDatabase.database }} {{- end -}} @@ -516,9 +381,7 @@ postgres {{- define "kubeflow-pipelines.database.username" -}} {{- if .Values.mysql.enabled -}} -{{- .Values.mysql.auth.username | default "mlpipeline" }} -{{- else if .Values.postgresql.enabled -}} -{{- .Values.postgresql.auth.username | default "mlpipeline" }} +{{- .Values.mysql.auth.username | default "root" }} {{- else -}} {{- .Values.externalDatabase.username }} {{- end -}} @@ -527,10 +390,6 @@ postgres {{- define "kubeflow-pipelines.database.secretName" -}} {{- if .Values.externalDatabase.existingSecret -}} {{- .Values.externalDatabase.existingSecret }} -{{- else if .Values.mysql.enabled -}} -mysql-secret -{{- else if .Values.postgresql.enabled -}} -postgresql-secret {{- else -}} mysql-secret {{- end -}} @@ -541,58 +400,20 @@ mysql-secret {{- .Values.externalDatabase.existingSecretPasswordKey | default "password" }} {{- else if .Values.mysql.enabled -}} mysql-password -{{- else if .Values.postgresql.enabled -}} -password {{- else -}} password {{- end -}} {{- end }} {{/* -Object storage configuration helpers +Object storage configuration helpers (SeaweedFS only) */}} -{{- define "kubeflow-pipelines.objectStore.endpoint" -}} -{{- if eq .Values.objectStore.provider "minio" -}} -{{- if .Values.minio.enabled -}} -{{- include "minio.fullname" .Subcharts.minio }}:9000 -{{- else -}} -{{- .Values.objectStore.minio.endpoint }} -{{- end -}} -{{- else -}} -{{ .Values.objectStore.custom.endpoint }} -{{- end -}} -{{- end }} - {{- define "kubeflow-pipelines.objectStore.bucket" -}} -{{- if eq .Values.objectStore.provider "minio" -}} -{{- .Values.objectStore.minio.bucket | default "mlpipeline" }} -{{- else -}} -{{- .Values.objectStore.custom.bucket }} -{{- end -}} -{{- end }} - -{{- define "kubeflow-pipelines.objectStore.secure" -}} -{{- if eq .Values.objectStore.provider "minio" -}} -{{- .Values.objectStore.minio.secure | default "false" }} -{{- else -}} -{{- .Values.objectStore.custom.secure | default "true" }} -{{- end -}} +{{- .Values.seaweedfs.bucket | default "mlpipeline" }} {{- end }} {{- define "kubeflow-pipelines.objectStore.secretName" -}} -{{- if eq .Values.objectStore.provider "minio" -}} -{{- if .Values.objectStore.minio.existingSecret -}} -{{- .Values.objectStore.minio.existingSecret }} -{{- else -}} -mlpipeline-minio-artifact -{{- end -}} -{{- else -}} -{{- if .Values.objectStore.custom.existingSecret -}} -{{- .Values.objectStore.custom.existingSecret }} -{{- else -}} mlpipeline-minio-artifact -{{- end -}} -{{- end -}} {{- end }} {{/* diff --git a/experimental/helm/charts/pipelines/templates/api-server/deployment.yaml b/experimental/helm/charts/pipelines/templates/api-server/deployment.yaml index 7cfcae452d..ffd1058bd1 100644 --- a/experimental/helm/charts/pipelines/templates/api-server/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/api-server/deployment.yaml @@ -20,9 +20,23 @@ spec: serviceAccountName: ml-pipeline containers: - name: ml-pipeline-api-server + {{- if and .Values.apiServer.k8sNative .Values.apiServer.k8sNative.enabled .Values.apiServer.image.repository .Values.apiServer.image.tag }} + image: "{{ .Values.apiServer.image.repository }}:{{ .Values.apiServer.image.tag }}" + {{- else }} image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.apiServer.image.repository "tag" .Values.apiServer.image.tag "context" .) }} + {{- end }} imagePullPolicy: IfNotPresent + {{- if and .Values.apiServer.k8sNative .Values.apiServer.k8sNative.enabled }} + command: + {{- toYaml .Values.apiServer.k8sNative.command | nindent 10 }} + args: + {{- toYaml .Values.apiServer.k8sNative.args | nindent 10 }} + {{- end }} ports: + {{- if and .Values.apiServer.k8sNative .Values.apiServer.k8sNative.enabled .Values.apiServer.k8sNative.webhookPort }} + - name: webhook + containerPort: {{ .Values.apiServer.k8sNative.webhookPort }} + {{- end }} - name: http containerPort: 8888 - name: grpc @@ -111,7 +125,14 @@ spec: failureThreshold: {{ .Values.apiServer.probes.startup.failureThreshold }} periodSeconds: {{ .Values.apiServer.probes.startup.periodSeconds }} timeoutSeconds: {{ .Values.apiServer.probes.startup.timeoutSeconds }} + {{- if and .Values.apiServer.k8sNative .Values.apiServer.k8sNative.enabled .Values.apiServer.k8sNative.volumeMounts }} + volumeMounts: + {{- toYaml .Values.apiServer.k8sNative.volumeMounts | nindent 10 }} + {{- end }} + {{- if and .Values.apiServer.k8sNative .Values.apiServer.k8sNative.enabled .Values.apiServer.k8sNative.volumes }} + volumes: + {{- toYaml .Values.apiServer.k8sNative.volumes | nindent 8 }} + {{- end }} securityContext: - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} + {{- toYaml .Values.podSecurityContext | nindent 8 }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/api-server/service.yaml b/experimental/helm/charts/pipelines/templates/api-server/service.yaml index a7dd68959e..f0e4157c3d 100644 --- a/experimental/helm/charts/pipelines/templates/api-server/service.yaml +++ b/experimental/helm/charts/pipelines/templates/api-server/service.yaml @@ -20,6 +20,12 @@ spec: port: {{ .Values.apiServer.service.grpcPort }} protocol: TCP targetPort: {{ .Values.apiServer.service.grpcPort }} + {{- if and .Values.apiServer.k8sNative .Values.apiServer.k8sNative.enabled .Values.apiServer.k8sNative.webhookPort }} + - name: webhook + port: {{ .Values.apiServer.k8sNative.webhookPort }} + protocol: TCP + targetPort: {{ .Values.apiServer.k8sNative.webhookPort }} + {{- end }} selector: {{- include "kubeflow-pipelines.mlPipelineSelectorLabels" . | nindent 4 }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/application/application.yaml b/experimental/helm/charts/pipelines/templates/application/application.yaml deleted file mode 100644 index 7df00136cb..0000000000 --- a/experimental/helm/charts/pipelines/templates/application/application.yaml +++ /dev/null @@ -1,52 +0,0 @@ -{{- if .Values.application.enabled }} -apiVersion: app.k8s.io/v1beta1 -kind: Application -metadata: - name: pipeline - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - kubernetes-engine.cloud.google.com/icon: >- - data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADMAAAAyCAYAAADx/eOPAAALuUlEQVRogd2afWxd9XnHP99bK4pS3yhiGUIRiiJUVVVqbq8ppdR20ibqpuIMtDRkUYERp29J57gMZVuxsrZiK7oZXVv5re1AiOuoG+N1DMkuytprsGPEVMouxqQZJVHEWIdQlGVxZlmZdb/747zcc869jpMO+seOdPz7nd/L83tev89zzjX8Bq795Rq9o17zXp+Tey+Ijkyboela29DRWkhffyT733pH/Z3este9F2cC6N0kNjxtjD+FdRD8UF9X7u97y7UbQFPAivC0BdllS381slun3s3z3xVhhqeds90tqR/oMB7u68z19ZZra0E/l1if3WOziPx3skrDPTr+bvDxfxImEIJbgX6gGBJ7EfHJX/ySReDHwO9KYAenyWCMFKw21GSeslwa2Z17+TcuzPBRr7B8m6Df5oOJqdPAR/u6cm/2lmv3At+IT3GiXZqbcaxSLsfRoTsvn7XL2jE87ZXGnwf+VGiDY86ETM1wU1+XjvSW/RlgTJADQ2QaCZKWcX1/aDIcjE8i3SdzZLjn0lm8pJXD02417BM+gLmq2Rqjr/d16Vu95dp6wc8Ra5O8NrPIcoZCvIR1H+KZkd2qLcfnRYUZOuorJO+3uQt0RerolGYZR7r5+C9ZATwPviGyQprd6Liszy3bnwVKwGMjPbnFyxJmeNpX2T4gaR/QmmSpyYZTho/2depMb9k/kNh3KawuJ1bWauHzUcyXRpZAv5Zmg7aHBLcmNN9ECAFeAO3s69KZ3nLtDuF9dnBs0IT9JO24rbPb0JfP2syCZpFfE5q1mRWcvlgMNcwMTRq9z/+OWXdx4AGjvX1deqC37DbwPwOrMgsufol5mWMWs1ivEbjTrOCtLNNb+udygqsNbUBtopR/NkuuwTJ6Hxsw67KSuvH5MPDA/nJttfGTdUFCMUlp/ALwOtIs9muBxpnFnBzuSQf21oP/BbXclVvumWuTaDN8WNBm2GizJkxPM0CDMA2WGZ72bbb/Njue2TRj9Il/PcG87SeBz4ZTNaSTsmctHcO8SqDp14d7dCFLZ2v/3OpQ023Ah4n65kohvETUCdcsfmuilD+bpNdgGZvOODuHqYGIVGCec9g7+7o031v2jaBTiD0ysxbHRnZrPktzyz1zK7f0z10nh5pWwLRhvZro1KqznVJhNB8UyDeSsU4zAOiIXV1OuEqQ2AR79nflXgcY6dGLwIvR8q39cy1b+uc2Emo6dI824BpMSxz8iVhy4m/2WiYHdV5UmOHp2mpwm52ESCdwRn+9v0tPAWzpn9sAFAQbMdc60PaHsFZEWd9uxk4z8G3seykECfObTEd2KmuZG4CWyLXkYLMwtiYt+hMsTUdAEZQzjs9apv66SHJRk73ZjBQ+iRu29s+1VEr5OImmXs4MHUahVoLWgK23wbv6OrU4OulcuHYehWsVHhpXwpE2FNRayTszX2cwDpQEzTB+QvrJHCXUaigk+c++aXZiE98YmUVgV19X7u3ypH/fgfUA5h2usY2jNjmWoGVn50nvC9T2NviA5OPBGPW91OlG+0Xa1WJhhqadk3WjpKCilQIQFP19XZocnfIHgIeFWyNh6goXyX6gdNWfU8aJ5tNjEheAHZVS/ruGj0s8k6VPhh6ms6kwgoLl1aGuCEuSpwXfHZ2qrTJ+HHkNCpOjmbdFcEcGUIhUSj/H65rPO6j+766U8i/QXV0z8cqJc4btwF8AtWgtMb1wj+j41Df/s1EYQwdEDiqM3hDes9quGY3IKoYOvCrU7HlCoZtEWapPkzEpsU8uq8b36a6uBqaBv5l45URLpZT/pmGH8LnkvlAdAOt1oeXqRsuYTjlEMJiXvWN/Z+5szfqioKcOKo7qr/nAEesKiOyv2A/q88rOx8+8bPhK5dUTAA8jbUT6MuKnbKteNVHKP23xCeD1LC0F2TWOmzoAKEiWxmC+sr8rN1OerF2HGaqXFcZhDWaYj11S4ZxcXxVqyKqPZOeNTwM7Jkr5BeDPQJ8NFQaoC/gZ26rXT5TyxxAfRx6P94d0gU0pYYama+tsbwix/AHM4fKUrwAeB68kRJ5AZsWWieGTjLipsVCgrKCwKHF7pZQ/RXf104j76i4ZMmquxkzRXb2zUsqfxdxsfCiA70hRjZbpCDHmJcRdeZPDHkVck0Ul5PeHZ81DgHxKtglXaHCxVN9fr5TyR9hW3QA8Amqp5526SyKtBEbZVv1eZeZkbqKU7xfsFJwPqRW29s+11oUxnUhnkHf2dWoB+R5Jv5dNaGHh1wog8d/ZAI+0GgVpFPTp4AfJT2Hup7u6EvMk0tpkboutEz0HMPzHyD+mu3pFpZR/Aug0Pgm0RLkvFzLWYfjDvs7cqfKUt2LuXTLhue5mdWhVDJdEzxDDcRKawceN9lRePVkDfgBcR/LKVqNpz/s08DO6q4VKKT8j8zHgJ1HyzA1P11YZjfV1arw85auBR4RalDB5lEjDKi0CgPPphKZ0QiNRwUQeg88B2ydKreew9yH1NCxe/r4GaZpt1Vsrh/JnDDcBLwPkbLVgf6s86RXYj4KvtJKJM8KsGLkSlsmUL6mSg1RJY1xD7KmU8sfprnYgBqJsGVsiEfupsca7FfMo26p/OfHKiVqllB8HyPV16VxfV66G/G1QBwY5xvCgTT7X3/MTaBbFVr0fJvqw2ASZ+yul/FN0V68CHsesiDl3UopM3CwhDZDD/Dnwj3S/sjoYAMqTtc1YX02jVqYOiuuqsAKIkqZCfFIz/IrfFY8gDrKt2gI8irSuwQezyTeNaOl+6qYb+fpYGKEXJE9GSTObK5ItrheaLHE5/XRKcHul+kYN8x2kzWlLNNuVtUqibzKW5CBjxUoszO7NWrS1E/xWvMeJjck2WQHEKJeMD+qH4gWCSvg00m3AVxv5TMRKsp9Cs0Q/Ka/1BOZQNBSXMz2b9Q5oO9JCKgkqg2aKofl8uvTPeE1w3t5KKf8y26pFxINhLRa5R9JV6huT/aZuFu7Ds+A9jBdj+VIvZz2b9BL2Xi5yJQEgUFqinI9SZBDx358o5Q/HiRGtquOEmxJu6DcbC/afQWxnvHg+Odrwm2bP5txh5OEYjOM3vaiu8qqHJw1mPmK/Xs7HJf0LRncDMF5cAL6NWUxDrX/duwbczljxjSzvTX+gtXU3MBlrRCltrsxBTgorACKrRGf5bczOiVLrhUL74B2F9oHVjBd/iLwTWEhr+CIWaLYumDjIWLHha+aSwvRs1iJmJ9Kb9ZJERTS3ACsMC8i1ZNwgXZDYWTmU/1WhfeAW8Cjo+UL7wDrGik8jfid0kYz/Z2ODepv+GPIY+FAznpcUJhAo9w5mh81CFtEsWieCTzwXkogmfKBSyh8ttA98EDPqoPouYqYLxYEPMVY8itmEeTM+KEaqZhVAkiPPIL6QDPhLFiYQSC9J7M3mGlF/24zWSvwIM1xoH2gF/sFiTcSPxQakqUJxsIPx4jGCr0AzCUYTbROJ7DPAdsbSAX9ZwgDs3qTDiMGUOxF/1DgfekLVsPf0sw8DPARsDNwy8iYBXov4p0L7wC2MF99CfBJ4rqmbJbO/qYE+x1jx5HK8Xtp/aFgHDM/FX+RM9FFjHjjj4NV3HvlPsP4g+SqQgm6zCuvJQnHgi4wVz2JuAj8RnLGEVaCf8Y8cuRQ2L0mYEBB2Gb8ZHKD4NQBx+0Qpf7LQPrAVVGqiiWTpCcEn4QcLxcF7C7+aXMDahT1YX5IS5DHE/ZfC4yULEwr0DtIOWwuuvwZ8rVLKP1soDqzHPGJoyRao9b4SXiQQ30A8eO1/PJ8D7gK+BtQSJcQM8AXGlg747LUkmi91lad8J3CuZ5OeBii0D64ET2FdH1N0omWJvgLPkvwM8LmZf7lrnm3VO4CHsM4DH2P8I8vGSfK67P9q8v9wWPAcQLH4PbBHbK6Pq+3M9+Ml+6FL2dyC+WmhOLiWseKPMDeDd12uIPBrWCZ5Xds++AHsAwGlBKnoB5747c2J+aSJEuvRL8CDv/2Zz+cqh/LL/gPD//vrfwFjcI5oX6jDBwAAAABJRU5ErkJggg== - marketplace.cloud.google.com/deploy-info: '{"partner_id": "google-cloud-ai-platform", "product_id": "kubeflow-pipelines", "partner_name": "Google Cloud AI Platform"}' -spec: - addOwnerRef: true - selector: - matchLabels: - application-crd-id: kubeflow-pipelines - descriptor: - version: {{ .Values.global.version | default .Values.global.imageTag | quote }} - type: Kubeflow Pipelines - description: |- - Reusable end-to-end ML workflow - maintainers: - - name: Google Cloud AI Platform - url: https://cloud.google.com/ai-platform/ - - name: Kubeflow Pipelines - url: https://github.com/kubeflow/pipelines - links: - - description: 'Kubeflow Pipelines Documentation' - url: https://www.kubeflow.org/docs/pipelines/ - notes: |- - Please go to [Hosted Kubeflow Pipelines Console](https://console.cloud.google.com/ai-platform/pipelines/clusters). - info: - - name: Console - value: 'https://console.cloud.google.com/ai-platform/pipelines/clusters' - componentKinds: - - group: v1 - kind: ServiceAccount - - group: rbac.authorization.k8s.io/v1 - kind: Role - - group: rbac.authorization.k8s.io/v1 - kind: RoleBinding - - group: v1 - kind: Service - - group: v1 - kind: ConfigMap - - group: v1 - kind: Secret - - group: apps/v1 - kind: Deployment -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/application/controller-deployment.yaml b/experimental/helm/charts/pipelines/templates/application/controller-deployment.yaml deleted file mode 100644 index 315dec428b..0000000000 --- a/experimental/helm/charts/pipelines/templates/application/controller-deployment.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- if .Values.application.controller.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - control-plane: controller-manager - controller-tools.k8s.io: "1.0" -spec: - selector: - matchLabels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - application-crd-id: kubeflow-pipelines - template: - metadata: - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 8 }} - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - spec: - serviceAccountName: application - containers: - - name: manager - image: {{ .Values.application.controller.image.repository }}:{{ .Values.application.controller.image.tag }} - imagePullPolicy: {{ .Values.application.controller.image.pullPolicy | default .Values.global.imagePullPolicy }} - command: - - /kube-app-manager - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - {{- toYaml .Values.application.controller.resources | nindent 10 }} -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/application/controller-role.yaml b/experimental/helm/charts/pipelines/templates/application/controller-role.yaml deleted file mode 100644 index 538c8e107e..0000000000 --- a/experimental/helm/charts/pipelines/templates/application/controller-role.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{- if .Values.application.controller.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: application-manager-role - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -rules: -- apiGroups: - - '*' - resources: - - '*' - verbs: - - get - - list - - update - - patch - - watch -- apiGroups: - - app.k8s.io - resources: - - '*' - verbs: - - '*' -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/application/controller-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/application/controller-rolebinding.yaml deleted file mode 100644 index 886726f6f7..0000000000 --- a/experimental/helm/charts/pipelines/templates/application/controller-rolebinding.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if .Values.application.controller.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: application-manager-rolebinding - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: application-manager-role -subjects: -- kind: ServiceAccount - name: application - namespace: {{ include "kubeflow-pipelines.namespace" . }} -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/application/controller-service.yaml b/experimental/helm/charts/pipelines/templates/application/controller-service.yaml deleted file mode 100644 index f96d6ac642..0000000000 --- a/experimental/helm/charts/pipelines/templates/application/controller-service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if .Values.application.controller.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: controller-manager-service - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - control-plane: controller-manager - controller-tools.k8s.io: "1.0" -spec: - selector: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - application-crd-id: kubeflow-pipelines - ports: - - port: 443 -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/cache-deployer/deployment.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/deployment.yaml deleted file mode 100644 index 4a99d1e690..0000000000 --- a/experimental/helm/charts/pipelines/templates/cache-deployer/deployment.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.cacheDeployer.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cache-deployer-deployment - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: cache-deployer -spec: - replicas: {{ .Values.cacheDeployer.replicas }} - selector: - matchLabels: - {{- include "kubeflow-pipelines.cacheDeployerSelectorLabels" . | nindent 6 }} - strategy: - type: Recreate - template: - metadata: - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 8 }} - app: cache-deployer - spec: - serviceAccountName: kubeflow-pipelines-cache-deployer-sa - containers: - - name: main - image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.cacheDeployer.image.repository "tag" .Values.cacheDeployer.image.tag "context" .) }} - imagePullPolicy: Always - env: - - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace - restartPolicy: Always -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/cache-deployer/role.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/role.yaml deleted file mode 100644 index 56bcc795b9..0000000000 --- a/experimental/helm/charts/pipelines/templates/cache-deployer/role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.cacheDeployer.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - {{- include "kubeflow-pipelines.cacheDeployerLabels" . | nindent 4 }} - name: kubeflow-pipelines-cache-deployer-role - namespace: {{ include "kubeflow-pipelines.namespace" . }} -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - get - - patch - - list -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml deleted file mode 100644 index 64d88a09c8..0000000000 --- a/experimental/helm/charts/pipelines/templates/cache-deployer/rolebinding.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if .Values.cacheDeployer.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - {{- include "kubeflow-pipelines.cacheDeployerLabels" . | nindent 4 }} - name: kubeflow-pipelines-cache-deployer-rolebinding - namespace: {{ include "kubeflow-pipelines.namespace" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kubeflow-pipelines-cache-deployer-role -subjects: -- kind: ServiceAccount - name: {{ include "kubeflow-pipelines.cacheDeployer.serviceAccountName" . }} - {{- if .Values.installMode.multiUser.enabled }} - namespace: {{ include "kubeflow-pipelines.namespace" . }} - {{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml deleted file mode 100644 index c030af479f..0000000000 --- a/experimental/helm/charts/pipelines/templates/cache-deployer/serviceaccount.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if and .Values.cacheDeployer.enabled .Values.cacheDeployer.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "kubeflow-pipelines.cacheDeployer.serviceAccountName" . }} - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - {{- with .Values.cacheDeployer.serviceAccount.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/cache/cert-issuer.yaml b/experimental/helm/charts/pipelines/templates/cache/cert-issuer.yaml new file mode 100644 index 0000000000..ab4e873b59 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/cache/cert-issuer.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.cache.enabled .Values.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: kfp-cache-selfsigned-issuer + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + app: cache-server-cert-manager +spec: + selfSigned: {} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/cache/certificate.yaml b/experimental/helm/charts/pipelines/templates/cache/certificate.yaml new file mode 100644 index 0000000000..2631d71ef9 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/cache/certificate.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.cache.enabled .Values.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kfp-cache-cert + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + app: cache-server-cert-manager +spec: + commonName: kfp-cache-cert + isCA: true + dnsNames: + - cache-server + - cache-server.{{ include "kubeflow-pipelines.namespace" . }} + - cache-server.{{ include "kubeflow-pipelines.namespace" . }}.svc + issuerRef: + kind: Issuer + name: kfp-cache-selfsigned-issuer + secretName: webhook-server-tls +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/cache/deployment.yaml b/experimental/helm/charts/pipelines/templates/cache/deployment.yaml index d8d1d0332d..299e900a2b 100644 --- a/experimental/helm/charts/pipelines/templates/cache/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/cache/deployment.yaml @@ -33,6 +33,8 @@ spec: - "--db_password=$(DBCONFIG_PASSWORD)" - "--namespace_to_watch=$(NAMESPACE_TO_WATCH)" - "--listen_port=$(WEBHOOK_PORT)" + - "--tls_cert_filename=tls.crt" + - "--tls_key_filename=tls.key" env: {{- range .Values.cache.server.additionalEnv }} - name: {{ .name }} @@ -83,13 +85,7 @@ spec: - name: webhook-api containerPort: 8443 securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsGroup: 0 - runAsNonRoot: true - runAsUser: 1000 + {{- toYaml .Values.cache.server.securityContext | nindent 10 }} volumeMounts: - name: webhook-tls-certs mountPath: /etc/webhook/certs diff --git a/experimental/helm/charts/pipelines/templates/cache/mutating-webhook.yaml b/experimental/helm/charts/pipelines/templates/cache/mutating-webhook.yaml new file mode 100644 index 0000000000..a61a299763 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/cache/mutating-webhook.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.cache.enabled .Values.certManager.mutating.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ .Values.certManager.mutating.name }} + labels: + app: cache-server-cert-manager + annotations: + cert-manager.io/inject-ca-from: {{ include "kubeflow-pipelines.namespace" . }}/kfp-cache-cert +webhooks: + - name: cache-server.{{ include "kubeflow-pipelines.namespace" . }}.svc + clientConfig: + service: + name: cache-server + namespace: {{ include "kubeflow-pipelines.namespace" . }} + path: "/mutate" + failurePolicy: Ignore + rules: + - operations: [ "CREATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + sideEffects: None + timeoutSeconds: 5 + objectSelector: + matchLabels: + pipelines.kubeflow.org/cache_enabled: "true" + admissionReviewVersions: ["v1beta1"] +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/config/pipeline-api-server-config.yaml b/experimental/helm/charts/pipelines/templates/config/pipeline-api-server-config.yaml index eabf0841e0..d6281e6dec 100644 --- a/experimental/helm/charts/pipelines/templates/config/pipeline-api-server-config.yaml +++ b/experimental/helm/charts/pipelines/templates/config/pipeline-api-server-config.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.apiServer.enabled .Values.installMode.multiUser.enabled }} +{{- if and .Values.apiServer.enabled (eq .Values.installMode.type "multi-user") }} apiVersion: v1 kind: ConfigMap metadata: @@ -7,7 +7,7 @@ metadata: labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} data: - {{- if and .Values.installMode.multiUser.enabled (eq .Values.installMode.type "multi-user") }} + {{- if eq .Values.installMode.type "multi-user" }} MULTIUSER: "true" DEFAULTPIPELINERUNNERSERVICEACCOUNT: "default-editor" {{- else }} diff --git a/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml b/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml index ee57f67548..56ce960236 100644 --- a/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml +++ b/experimental/helm/charts/pipelines/templates/config/pipeline-install-config.yaml @@ -15,19 +15,13 @@ data: namespace: `kubectl rollout restart deployment -n `. appName: {{ .Values.pipeline.app.name }} appVersion: {{ .Values.pipeline.app.version }} - # Database configuration + # Database configuration (MySQL only) {{- $dbHost := include "kubeflow-pipelines.database.host" . }} dbHost: {{ $dbHost | quote }} dbPort: {{ include "kubeflow-pipelines.database.port" . | quote }} -{{- if or (eq .Values.externalDatabase.type "postgresql") (eq .Values.externalDatabase.type "postgres") }} - dbType: "postgres" - postgresHost: {{ $dbHost | quote }} - postgresPort: {{ include "kubeflow-pipelines.database.port" . | quote }} -{{- else }} - dbType: {{ .Values.externalDatabase.type | default "mysql" | quote }} + dbType: "mysql" mysqlHost: {{ $dbHost | quote }} mysqlPort: {{ include "kubeflow-pipelines.database.port" . | quote }} -{{- end }} mlmdDb: {{ .Values.database.metadb }} cacheDb: {{ .Values.database.cachedb }} pipelineDb: {{ include "kubeflow-pipelines.database.name" . }} @@ -44,6 +38,7 @@ data: DEFAULT_CACHE_STALENESS: {{ .Values.pipeline.cache.defaultStaleness | quote }} # Connection configuration ConMaxLifeTime: {{ .Values.database.connectionMaxLifetime | quote }} -{{- if not .Values.thirdParty.postgresql.enabled }} LOG_LEVEL: {{ .Values.global.logLevel | quote }} -{{- end }} + # Profile controller configuration + ARTIFACTS_PROXY_ENABLED: {{ .Values.profileController.artifactsProxyEnabled | quote }} + ARTIFACT_RETENTION_DAYS: {{ .Values.profileController.artifactRetentionDays | quote }} diff --git a/experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml b/experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml index bccd111691..0adb9cf82a 100644 --- a/experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml +++ b/experimental/helm/charts/pipelines/templates/config/ui-configmap.yaml @@ -10,7 +10,7 @@ data: viewer-pod-template.json: |- { "spec": { - "serviceAccountName": "{{ if and .Values.installMode.multiUser.enabled (eq .Values.installMode.type "multi-user") }}default-editor{{ else }}kubeflow-pipelines-viewer{{ end }}" + "serviceAccountName": "{{ if eq .Values.installMode.type "multi-user" }}default-editor{{ else }}kubeflow-pipelines-viewer{{ end }}" } } {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/crds/application-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/application-crd.yaml deleted file mode 100644 index 70146a9e0b..0000000000 --- a/experimental/helm/charts/pipelines/templates/crds/application-crd.yaml +++ /dev/null @@ -1,330 +0,0 @@ -{{- if and .Values.crds.install .Values.crds.application }} -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: applications.app.k8s.io - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - controller-tools.k8s.io: "1.0" - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/application/pull/2 - controller-gen.kubebuilder.io/version: v0.4.0 -spec: - group: app.k8s.io - names: - categories: - - all - kind: Application - listKind: ApplicationList - plural: applications - shortNames: - - app - singular: application - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: The type of the application - jsonPath: .spec.descriptor.type - name: Type - type: string - - description: The creation date - jsonPath: .spec.descriptor.version - name: Version - type: string - - description: The application object owns the matched resources - jsonPath: .spec.addOwnerRef - name: Owner - type: boolean - - description: Numbers of components ready - jsonPath: .status.componentsReady - name: Ready - type: string - - description: The creation date - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: Application is the Schema for the applications API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object.' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents.' - type: string - metadata: - type: object - spec: - description: ApplicationSpec defines the specification for an Application. - properties: - addOwnerRef: - description: AddOwnerRef objects - flag to indicate if we need to add OwnerRefs to matching objects - type: boolean - assemblyPhase: - description: 'AssemblyPhase represents the current phase of the application''s assembly such as "Pending", "Succeeded", or "Failed".' - type: string - componentKinds: - description: ComponentGroupKinds is a list of Kinds for Application's components - items: - description: GroupKind specifies a Group and a Kind, but does not force a version. - properties: - group: - type: string - kind: - type: string - required: - - group - - kind - type: object - type: array - descriptor: - description: Descriptor regroups information and metadata about an application. - properties: - description: - description: Description is a brief string description of the Application. - type: string - icons: - description: Icons is an optional list of icons for an application. - items: - description: ImageSpec contains information about an image used as an icon. - properties: - size: - description: (optional) The size of the image in pixels (e.g., 25x25). - type: string - src: - description: The source for image represented as either an absolute URL to the image or a Data URL containing the image. - type: string - type: - description: (optional) The mine type of the image (e.g., "image/png"). - type: string - required: - - src - type: object - type: array - keywords: - description: Keywords is an optional list of key words associated with the application - items: - type: string - type: array - links: - description: Links are a list of descriptive URLs intended to be used to surface additional documentation, dashboards, etc. - items: - description: Link contains information about an URL to surface documentation, dashboards, etc. - properties: - description: - description: Description is human readable content explaining the purpose of the link. - type: string - url: - description: Url is typically an HTTP or HTTPS URL. - type: string - type: object - type: array - maintainers: - description: Maintainers is an optional list of maintainers of the application. - items: - description: ContactData contains information about an individual or organization. - properties: - email: - description: Email is the email address of the contact person/organization. - type: string - name: - description: Name is the descriptive name of the contact person/organization - type: string - url: - description: Url could typically be a website address of the contact person/organization. - type: string - type: object - type: array - notes: - description: Notes contain a human readable snippets intended as a quick start for the users of the Application. - type: string - owners: - description: Owners is an optional list of the owners of the installed application. - items: - description: ContactData contains information about an individual or organization. - properties: - email: - description: Email is the email address of the contact person/organization. - type: string - name: - description: Name is the descriptive name of the contact person/organization - type: string - url: - description: Url could typically be a website address of the contact person/organization. - type: string - type: object - type: array - type: - description: Type is the type of the application (e.g. WordPress, MySQL, Cassandra). - type: string - version: - description: Version is an optional version indicator for the Application. - type: string - type: object - info: - description: Info contains human readable key-value pairs for the Application. - items: - description: InfoItem is a human readable key-value pair containing important information about how to access the Application. - properties: - name: - description: Name is a human readable title for this piece of information. - type: string - type: - description: Type of the value for this InfoItem. - type: string - value: - description: Value is human readable content. - type: string - valueFrom: - description: ValueFrom defines a reference to derive the value from another source. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent.' - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - ingressRef: - description: Selects an Ingress. - properties: - host: - description: The host to select. - type: string - name: - description: 'Name of the referent.' - type: string - path: - description: The path to select. - type: string - required: - - name - type: object - secretKeyRef: - description: Selects a key of a secret. - properties: - key: - description: The key of the secret to select from. - type: string - name: - description: 'Name of the referent.' - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - serviceRef: - description: Selects a Service. - properties: - name: - description: 'Name of the referent.' - type: string - path: - description: The optional path to select. - type: string - port: - description: The port to select. - format: int32 - type: integer - required: - - name - type: object - type: - description: Type of the value for this InfoItem. - type: string - type: object - required: - - name - type: object - type: array - selector: - description: Selector is a label query over kinds that created by the application. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. - type: string - values: - description: values is an array of string values. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. - type: object - type: object - type: object - status: - description: ApplicationStatus defines controller's the observed state of Application - properties: - componentsReady: - description: ComponentsReady is a human readable message indicating details about the status of the application. - type: string - conditions: - description: Conditions represents the latest state of the object - items: - description: ApplicationCondition contains details for the current condition of this application. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status to another. - format: date-time - type: string - lastUpdateTime: - description: Last time the condition was probed - format: date-time - type: string - message: - description: A human readable message indicating details about the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of application condition. - type: string - required: - - status - - type - type: object - type: array - observedGeneration: - description: ObservedGeneration is the most recent generation observed. It corresponds to the Object's generation, which is updated on mutation by the API Server. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml index 14ed1d5183..a708cc8442 100644 --- a/experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml +++ b/experimental/helm/charts/pipelines/templates/crds/pipelines-crd.yaml @@ -1,14 +1,11 @@ -{{- if and .Values.crds.install .Values.webhooks.enabled }} +{{- if and .Values.crds.install (eq .Values.installMode.pipelineDefinition.storage "k8s-native") }} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.4 - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} name: pipelines.pipelines.kubeflow.org - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} spec: group: pipelines.kubeflow.org names: @@ -45,10 +42,10 @@ spec: properties: description: type: string + displayName: + type: string type: object type: object served: true storage: true - subresources: - status: {} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml b/experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml index 467539c576..6047631510 100644 --- a/experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml +++ b/experimental/helm/charts/pipelines/templates/crds/pipelineversions-crd.yaml @@ -1,14 +1,11 @@ -{{- if and .Values.crds.install .Values.webhooks.enabled }} +{{- if and .Values.crds.install (eq .Values.installMode.pipelineDefinition.storage "k8s-native") }} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.4 - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} name: pipelineversions.pipelines.kubeflow.org - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} spec: group: pipelines.kubeflow.org names: @@ -47,16 +44,50 @@ spec: type: string description: type: string + displayName: + type: string pipelineName: type: string pipelineSpec: x-kubernetes-preserve-unknown-fields: true + pipelineSpecURI: + type: string + platformSpec: + x-kubernetes-preserve-unknown-fields: true required: - - pipelineName - pipelineSpec type: object status: description: PipelineVersionStatus defines the observed state of PipelineVersion. + properties: + conditions: + items: + description: |- + SimplifiedCondition is a metav1.Condition without lastTransitionTime since the database model doesn't have such + a concept and it allows a default status in the CRD without a controller setting it. + properties: + message: + maxLength: 32768 + type: string + reason: + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array type: object type: object served: true diff --git a/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml b/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml index f80922d5a8..6124d3dac1 100644 --- a/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml +++ b/experimental/helm/charts/pipelines/templates/database/mysql-secret.yaml @@ -1,4 +1,4 @@ -{{- if and (not .Values.externalDatabase.existingSecret) (not .Values.mysql.enabled) (not .Values.postgresql.enabled) (not .Values.thirdParty.postgresql.enabled) }} +{{- if and (not .Values.externalDatabase.existingSecret) (not .Values.mysql.enabled) }} apiVersion: v1 kind: Secret metadata: diff --git a/experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml b/experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml index bfda14cc50..2d78f92f44 100644 --- a/experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml +++ b/experimental/helm/charts/pipelines/templates/istio/authorization-config.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.networking.istio.enabled .Values.installMode.multiUser.enabled }} +{{- if and .Values.networking.istio.enabled (eq .Values.installMode.type "multi-user") }} apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: @@ -92,33 +92,6 @@ spec: rules: - {} ---- -{{- if .Values.thirdParty.minio.enabled }} -apiVersion: security.istio.io/v1beta1 -kind: AuthorizationPolicy -metadata: - name: minio-service - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.authorizationPolicyLabels" . | nindent 4 }} -spec: - action: ALLOW - selector: - matchLabels: - app: minio - rules: - - from: - - source: - principals: - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline - - from: - - source: - principals: - - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-ui - # Allow traffic from User Pipeline Pods, which don't have a sidecar. - - {} -{{- end }} - --- {{- if .Values.thirdParty.mysql.enabled }} apiVersion: security.istio.io/v1beta1 @@ -211,24 +184,6 @@ spec: tls: mode: ISTIO_MUTUAL ---- -{{- if .Values.thirdParty.minio.enabled }} -apiVersion: "networking.istio.io/v1alpha3" -kind: DestinationRule -metadata: - name: ml-pipeline-minio - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines -spec: - host: minio-service.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local - trafficPolicy: - tls: - mode: ISTIO_MUTUAL -{{- end }} - --- {{- if .Values.thirdParty.mysql.enabled }} apiVersion: "networking.istio.io/v1alpha3" diff --git a/experimental/helm/charts/pipelines/templates/istio/virtual-service.yaml b/experimental/helm/charts/pipelines/templates/istio/virtual-service.yaml index 7c3af25ab6..59abb63dee 100644 --- a/experimental/helm/charts/pipelines/templates/istio/virtual-service.yaml +++ b/experimental/helm/charts/pipelines/templates/istio/virtual-service.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.networking.istio.enabled .Values.installMode.multiUser.enabled }} +{{- if and .Values.networking.istio.enabled (eq .Values.installMode.type "multi-user") }} apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: diff --git a/experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml index afd37f5b47..f868fcec04 100644 --- a/experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/metadata-writer/deployment.yaml @@ -9,9 +9,7 @@ metadata: app: metadata-writer app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} application-crd-id: kubeflow-pipelines -{{- end }} {{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: metadata-writer @@ -28,9 +26,7 @@ spec: app: metadata-writer app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} application-crd-id: kubeflow-pipelines -{{- end }} {{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: metadata-writer @@ -53,11 +49,5 @@ spec: fieldPath: metadata.namespace {{- end }} securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsGroup: 0 - runAsNonRoot: true - runAsUser: 1000 + {{- toYaml .Values.metadataWriter.securityContext | nindent 10 }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/metadata/envoy-configmap.yaml b/experimental/helm/charts/pipelines/templates/metadata/envoy-configmap.yaml new file mode 100644 index 0000000000..2e4a75274e --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/metadata/envoy-configmap.yaml @@ -0,0 +1,83 @@ +{{- if .Values.metadata.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: metadata-envoy-configmap + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +data: + envoy-config.yaml: |- + admin: + access_log: + name: admin_access + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /tmp/admin_access.log + address: + socket_address: { address: 0.0.0.0, port_value: 9901 } + + static_resources: + listeners: + - name: listener_0 + address: + socket_address: { address: 0.0.0.0, port_value: 9090 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: auto + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: [ "*" ] + routes: + - match: { prefix: "/" } + route: + cluster: metadata-cluster + max_stream_duration: + grpc_timeout_header_max: '0s' + typed_per_filter_config: + envoy.filter.http.cors: + "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.CorsPolicy + allow_origin_string_match: + - safe_regex: + regex: ".*" + allow_methods: GET, PUT, DELETE, POST, OPTIONS + allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout + max_age: "1728000" + expose_headers: custom-header-1,grpc-status,grpc-message + http_filters: + - name: envoy.filters.http.grpc_web + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb + - name: envoy.filters.http.cors + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: metadata-cluster + connect_timeout: 30.0s + type: logical_dns + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: { } + lb_policy: round_robin + load_assignment: + cluster_name: metadata-grpc + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: metadata-grpc-service + port_value: 8080 +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/metadata/envoy-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/envoy-deployment.yaml index 34837c39cc..84a25c0e6f 100644 --- a/experimental/helm/charts/pipelines/templates/metadata/envoy-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/metadata/envoy-deployment.yaml @@ -23,19 +23,19 @@ spec: containers: - name: container image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.metadata.envoy.image.repository "tag" .Values.metadata.envoy.image.tag "context" .) }} + args: ["/etc/envoy/envoy-config.yaml"] ports: - name: md-envoy containerPort: 9090 - name: envoy-admin containerPort: 9901 securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsGroup: 0 - runAsNonRoot: true - runAsUser: 1000 - seccompProfile: - type: RuntimeDefault + {{- toYaml .Values.metadata.envoy.securityContext | nindent 10 }} + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + volumes: + - name: envoy-config + configMap: + name: metadata-envoy-configmap {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml index 0dc53d8b82..ce2c36cb77 100644 --- a/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/metadata/grpc-deployment.yaml @@ -26,52 +26,7 @@ spec: - name: container image: {{ .Values.metadata.grpc.image.repository }}:{{ .Values.metadata.grpc.image.tag | default "1.14.0" }} securityContext: - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - - ALL - {{- if .Values.thirdParty.postgresql.enabled }} - env: [] - envFrom: - - configMapRef: - name: metadata-postgres-db-parameters - - secretRef: - name: metadata-postgres-db-secrets - - configMapRef: - name: metadata-grpc-configmap - command: ["/bin/metadata_store_server"] - args: - - --grpc_port=$(METADATA_GRPC_SERVICE_PORT) - - --metadata_source_config_type=postgresql - - --postgres_config_host=metadata-postgres-db - - --postgres_config_port=$(POSTGRES_PORT) - - --postgres_config_dbname=$(POSTGRES_DBNAME) - - --postgres_config_user=$(POSTGRES_USER) - - --postgres_config_password=$(POSTGRES_PASSWORD) - - --enable_database_upgrade=true - {{- else if .Values.thirdParty.mysqlMetadata.enabled }} - env: [] - envFrom: - - configMapRef: - name: metadata-db-parameters - - secretRef: - name: metadata-db-secrets - - configMapRef: - name: metadata-grpc-configmap - command: ["/bin/metadata_store_server"] - args: - - --grpc_port=$(METADATA_GRPC_SERVICE_PORT) - - --mysql_config_host=metadata-db - - --mysql_config_database=$(MYSQL_DATABASE) - - --mysql_config_port=$(MYSQL_PORT) - - --mysql_config_user=$(MYSQL_USER_NAME) - - --mysql_config_password=$(MYSQL_ROOT_PASSWORD) - {{- else }} + {{- toYaml .Values.metadata.grpc.securityContext | nindent 10 }} env: - name: DBCONFIG_USER valueFrom: @@ -108,7 +63,6 @@ spec: - --mysql_config_password=$(DBCONFIG_PASSWORD) - --enable_database_upgrade=true - --grpc_channel_arguments=grpc.max_metadata_size=16384 - {{- end }} ports: - name: grpc-api containerPort: 8080 @@ -124,12 +78,6 @@ spec: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 - {{- if and (not .Values.thirdParty.postgresql.enabled) (not .Values.thirdParty.mysqlMetadata.enabled) }} - {{- with .Values.metadata.grpc.resources }} - resources: - {{- toYaml . | nindent 10 }} - {{- end }} - {{- end }} serviceAccountName: metadata-grpc-server {{- with .Values.metadata.grpc.nodeSelector }} nodeSelector: diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-configmap.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-configmap.yaml deleted file mode 100644 index 6c63ca68f9..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: metadata-db-parameters - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -data: - MYSQL_DATABASE: {{ .Values.thirdParty.mysqlMetadata.database | default "metadb" }} - MYSQL_PORT: "3306" - MYSQL_ALLOW_EMPTY_PASSWORD: "true" -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-deployment.yaml deleted file mode 100644 index f59a53b045..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-deployment.yaml +++ /dev/null @@ -1,62 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metadata-db - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: db -spec: - selector: - matchLabels: - {{- if .Values.installMode.legacyLabels }} - application-crd-id: kubeflow-pipelines - {{- end }} - component: db - replicas: 1 - strategy: - type: Recreate - template: - metadata: - name: db - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 8 }} - component: db - sidecar.istio.io/inject: "false" - {{- if .Values.installMode.legacyLabels }} - application-crd-id: kubeflow-pipelines - {{- end }} - spec: - containers: - - name: db-container - image: {{ .Values.thirdParty.mysqlMetadata.image.repository }}:{{ .Values.thirdParty.mysqlMetadata.image.tag }} - args: - - --datadir - - /var/lib/mysql/datadir - envFrom: - - configMapRef: - name: metadata-db-parameters - - secretRef: - name: metadata-db-secrets - ports: - - name: dbapi - containerPort: 3306 - readinessProbe: - exec: - command: - - "/bin/bash" - - "-c" - - "mysql -D $$MYSQL_DATABASE -p$$MYSQL_ROOT_PASSWORD -e 'SELECT 1'" - initialDelaySeconds: 5 - periodSeconds: 2 - timeoutSeconds: 1 - volumeMounts: - - name: metadata-mysql - mountPath: /var/lib/mysql - volumes: - - name: metadata-mysql - persistentVolumeClaim: - claimName: metadata-mysql -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-pvc.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-pvc.yaml deleted file mode 100644 index a47563586f..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-pvc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: metadata-mysql - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.thirdParty.mysqlMetadata.persistence.size | default "10Gi" }} -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-secret.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-secret.yaml deleted file mode 100644 index 3049442d08..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-secret.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} -apiVersion: v1 -kind: Secret -metadata: - name: metadata-db-secrets - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -data: - MYSQL_USER_NAME: {{ .Values.thirdParty.mysqlMetadata.username | default "root" | b64enc }} - MYSQL_ROOT_PASSWORD: {{ .Values.thirdParty.mysqlMetadata.password | default "test" | b64enc }} -type: Opaque -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-service.yaml b/experimental/helm/charts/pipelines/templates/metadata/mysql-db-service.yaml deleted file mode 100644 index 3c48fe84fb..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/mysql-db-service.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.mysqlMetadata.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: metadata-db - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: db -spec: - type: ClusterIP - ports: - - port: 3306 - protocol: TCP - name: dbapi - selector: - {{- if .Values.installMode.legacyLabels }} - application-crd-id: kubeflow-pipelines - {{- end }} - component: db -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-configmap.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-configmap.yaml deleted file mode 100644 index 4f11813607..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-configmap.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: metadata-postgres-db-parameters - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -data: - POSTGRES_PORT: "5432" - POSTGRES_DBNAME: "mlmdpostgres" -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-deployment.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-deployment.yaml deleted file mode 100644 index 92b6a390f4..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-deployment.yaml +++ /dev/null @@ -1,48 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metadata-postgres-db - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: db -spec: - selector: - matchLabels: - application-crd-id: kubeflow-pipelines - component: db - replicas: 1 - strategy: - type: Recreate - template: - metadata: - name: db - labels: - component: db - sidecar.istio.io/inject: "false" - application-crd-id: kubeflow-pipelines - spec: - containers: - - name: db-container - image: {{ .Values.thirdParty.postgresql.image.repository }}:{{ .Values.thirdParty.postgresql.image.tag }} - env: - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - envFrom: - - configMapRef: - name: metadata-postgres-db-parameters - - secretRef: - name: metadata-postgres-db-secrets - ports: - - name: postgres - containerPort: 5432 - volumeMounts: - - name: metadata-postgres - mountPath: /var/lib/postgresql/data - volumes: - - name: metadata-postgres - persistentVolumeClaim: - claimName: metadata-postgres -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-pvc.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-pvc.yaml deleted file mode 100644 index 5ce8e2fe0c..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-pvc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: metadata-postgres - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-secret.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-secret.yaml deleted file mode 100644 index 968634b935..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-secret.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} -apiVersion: v1 -kind: Secret -metadata: - name: metadata-postgres-db-secrets - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -data: - POSTGRES_USER: {{ "root" | b64enc }} - POSTGRES_PASSWORD: {{ .Values.thirdParty.postgresql.password | default "password" | b64enc }} -type: Opaque -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-service.yaml b/experimental/helm/charts/pipelines/templates/metadata/postgres-db-service.yaml deleted file mode 100644 index dec27721e6..0000000000 --- a/experimental/helm/charts/pipelines/templates/metadata/postgres-db-service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if and .Values.metadata.enabled .Values.thirdParty.postgresql.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: metadata-postgres-db - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: db -spec: - type: ClusterIP - ports: - - port: 5432 - protocol: TCP - name: postgres - selector: - component: db - application-crd-id: kubeflow-pipelines -{{- end }} - diff --git a/experimental/helm/charts/pipelines/templates/persistence-agent/deployment.yaml b/experimental/helm/charts/pipelines/templates/persistence-agent/deployment.yaml index 6fa53a4b1e..8b39838054 100644 --- a/experimental/helm/charts/pipelines/templates/persistence-agent/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/persistence-agent/deployment.yaml @@ -58,8 +58,7 @@ spec: - mountPath: /var/run/secrets/kubeflow/tokens name: persistenceagent-sa-token securityContext: - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} + {{- toYaml .Values.podSecurityContext | nindent 8 }} volumes: - name: persistenceagent-sa-token projected: diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml index bbb89dfcb0..1731513b84 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.installMode.multiUser.enabled .Values.installMode.multiUser.profileController.enabled }} +{{- if and (eq .Values.installMode.type "multi-user") .Values.profileController.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml index 883e6863bc..3f24f245d9 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/decorator-controller.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.installMode.multiUser.enabled .Values.installMode.multiUser.profileController.enabled }} +{{- if and (eq .Values.installMode.type "multi-user") .Values.profileController.enabled }} apiVersion: metacontroller.k8s.io/v1alpha1 kind: DecoratorController metadata: diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml index d089d91983..8b53ed826b 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/deployment.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.installMode.multiUser.enabled .Values.installMode.multiUser.profileController.enabled }} +{{- if .Values.profileController.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -16,9 +16,7 @@ spec: app: kubeflow-pipelines-profile-controller app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} application-crd-id: kubeflow-pipelines -{{- end }} template: metadata: labels: @@ -29,33 +27,61 @@ spec: {{- include "kubeflow-pipelines.annotations" . | nindent 8 }} spec: {{- include "kubeflow-pipelines.imagePullSecrets" . | nindent 6 }} + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: profile-controller - image: public.ecr.aws/docker/library/python:3.12 + securityContext: + {{- toYaml .Values.containerSecurityContext | nindent 10 }} + image: {{ .Values.profileController.image.repository }}:{{ .Values.profileController.image.tag }} command: ["python", "/hooks/sync.py"] envFrom: - configMapRef: name: kubeflow-pipelines-profile-controller-env env: - {{- range .Values.profileController.additionalEnv }} - - name: {{ .name }} - {{- if .value }} - value: {{ .value | quote }} - {{- else if and .key (ne .key "null") }} - {{- if or (eq .name "MINIO_ACCESS_KEY") (eq .name "MINIO_SECRET_KEY") }} + - name: KFP_VERSION + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: appVersion + - name: AWS_ENDPOINT_URL + value: {{ .Values.seaweedfs.awsEndpointUrl }} + - name: S3_ENDPOINT_URL + value: {{ .Values.seaweedfs.s3EndpointUrl }} + - name: AWS_REGION + value: {{ .Values.seaweedfs.awsRegion }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: + key: secretkey name: mlpipeline-minio-artifact - key: {{ .key }} - {{- else }} + - name: KFP_DEFAULT_PIPELINE_ROOT valueFrom: configMapKeyRef: + key: defaultPipelineRoot name: pipeline-install-config - key: {{ .key }} - {{- if eq .name "KFP_DEFAULT_PIPELINE_ROOT" }} optional: true - {{- end }} - {{- end }} + - name: ARTIFACTS_PROXY_ENABLED + valueFrom: + configMapKeyRef: + key: ARTIFACTS_PROXY_ENABLED + name: pipeline-install-config + optional: true + - name: ARTIFACT_RETENTION_DAYS + valueFrom: + configMapKeyRef: + key: ARTIFACT_RETENTION_DAYS + name: pipeline-install-config + {{- range .Values.profileController.additionalEnv }} + - name: {{ .name }} + {{- if .value }} + value: {{ .value | quote }} {{- else if .valueFrom }} valueFrom: {{- toYaml .valueFrom | nindent 12 }} @@ -68,8 +94,6 @@ spec: mountPath: /hooks ports: - containerPort: 8080 - securityContext: - {{- toYaml .Values.profileController.securityContext | nindent 10 }} volumes: - name: hooks configMap: diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/env-configmap.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/env-configmap.yaml new file mode 100644 index 0000000000..81fbef92e5 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/profile-controller/env-configmap.yaml @@ -0,0 +1,14 @@ +{{- if and (eq .Values.installMode.type "multi-user") .Values.profileController.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubeflow-pipelines-profile-controller-env + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: kubeflow-pipelines-profile-controller + application-crd-id: kubeflow-pipelines +data: + DISABLE_ISTIO_SIDECAR: {{ .Values.profileController.disableIstioSidecar | quote }} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/profile-controller/service.yaml b/experimental/helm/charts/pipelines/templates/profile-controller/service.yaml index 56c79723fc..18c8e93e70 100644 --- a/experimental/helm/charts/pipelines/templates/profile-controller/service.yaml +++ b/experimental/helm/charts/pipelines/templates/profile-controller/service.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.installMode.multiUser.enabled .Values.installMode.multiUser.profileController.enabled }} +{{- if and (eq .Values.installMode.type "multi-user") .Values.profileController.enabled }} apiVersion: v1 kind: Service metadata: @@ -19,7 +19,5 @@ spec: app: kubeflow-pipelines-profile-controller app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} application-crd-id: kubeflow-pipelines {{- end }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/rbac/argo-workflow-controller-rbac.yaml b/experimental/helm/charts/pipelines/templates/rbac/argo-workflow-controller-rbac.yaml index 7d02f84856..94fbf12e16 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/argo-workflow-controller-rbac.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/argo-workflow-controller-rbac.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.thirdParty.argo.enabled .Values.installMode.multiUser.enabled }} +{{- if and .Values.thirdParty.argo.enabled (eq .Values.installMode.type "multi-user") }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml b/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml index 43cdd75680..f78edf4c2b 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml @@ -1,4 +1,4 @@ -{{- if .Values.installMode.multiUser.enabled }} +{{- if eq .Values.installMode.type "multi-user" }} # API Server ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -102,23 +102,6 @@ subjects: name: {{ include "kubeflow-pipelines.cache.serviceAccountName" . }} namespace: {{ include "kubeflow-pipelines.namespace" . }} ---- -# Cache Deployer ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubeflow-pipelines-cache-deployer-clusterrolebinding - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kubeflow-pipelines-cache-deployer-clusterrole -subjects: -- kind: ServiceAccount - name: {{ include "kubeflow-pipelines.cacheDeployer.serviceAccountName" . }} - namespace: {{ include "kubeflow-pipelines.namespace" . }} - --- # Metadata Writer ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml b/experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml index 0037f1b2c4..ebb7453940 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/cluster-roles.yaml @@ -1,4 +1,4 @@ -{{- if .Values.installMode.multiUser.enabled }} +{{- if eq .Values.installMode.type "multi-user" }} # API Server ClusterRole apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -67,6 +67,35 @@ rules: - tokenreviews verbs: - create +{{- if eq .Values.installMode.pipelineDefinition.storage "k8s-native" }} +- apiGroups: + - pipelines.kubeflow.org + resources: + - pipelines + - pipelines/finalizers + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - pipelines.kubeflow.org + resources: + - pipelineversions + - pipelineversions/status + - pipelineversions/finalizers + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +{{- end }} --- # UI ClusterRole @@ -288,47 +317,6 @@ rules: - update - patch ---- -# Cache Deployer ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: kubeflow-pipelines-cache-deployer-clusterrole - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app.kubernetes.io/component: ml-pipeline - app.kubernetes.io/name: kubeflow-pipelines - app: kubeflow-pipelines-cache-deployer-clusterrole -rules: -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - - certificatesigningrequests/approval - verbs: - - create - - delete - - get - - update -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - verbs: - - create - - delete - - get - - list - - patch -- apiGroups: - - certificates.k8s.io - resources: - - signers - resourceNames: - - kubernetes.io/* - verbs: - - approve - --- # Metadata Writer ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -396,6 +384,7 @@ rules: - "" resources: - configmaps + - namespaces verbs: - get - watch diff --git a/experimental/helm/charts/pipelines/templates/rbac/view-edit-cluster-roles.yaml b/experimental/helm/charts/pipelines/templates/rbac/view-edit-cluster-roles.yaml index 7d7b182ae8..1a14ed72fb 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/view-edit-cluster-roles.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/view-edit-cluster-roles.yaml @@ -1,4 +1,4 @@ -{{- if .Values.installMode.multiUser.enabled }} +{{- if eq .Values.installMode.type "multi-user" }} # NOTE: IMPORTANT # We need to separate out actual rules from aggregation rules due to # https://github.com/kubernetes/kubernetes/issues/65171 @@ -54,7 +54,7 @@ rules: - pipelines.kubeflow.org resources: - pipelines - - pipelines/versions + - pipelineversions verbs: - create - delete @@ -125,7 +125,7 @@ rules: - pipelines.kubeflow.org resources: - pipelines - - pipelines/versions + - pipelineversions - experiments - jobs verbs: diff --git a/experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml b/experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml index 9cca3cd1c9..05aae79308 100644 --- a/experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/scheduled-workflow/deployment.yaml @@ -9,9 +9,7 @@ metadata: app: ml-pipeline-scheduledworkflow app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} application-crd-id: kubeflow-pipelines -{{- end }} {{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 4 }} app: ml-pipeline-scheduledworkflow @@ -29,9 +27,7 @@ spec: app: ml-pipeline-scheduledworkflow app.kubernetes.io/component: ml-pipeline app.kubernetes.io/name: kubeflow-pipelines -{{- if .Values.installMode.legacyLabels }} application-crd-id: kubeflow-pipelines -{{- end }} {{- else }} {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: ml-pipeline-scheduledworkflow @@ -59,18 +55,13 @@ spec: configMapKeyRef: key: cronScheduleTimezone name: pipeline-install-config - {{- if and .Values.scheduledWorkflow.resources (or .Values.scheduledWorkflow.resources.requests .Values.scheduledWorkflow.resources.limits) }} - resources: - {{- toYaml .Values.scheduledWorkflow.resources | nindent 10 }} - {{- end }} securityContext: {{- toYaml .Values.scheduledWorkflow.securityContext | nindent 10 }} volumeMounts: - mountPath: /var/run/secrets/kubeflow/tokens name: scheduledworkflow-sa-token securityContext: - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} + {{- toYaml .Values.podSecurityContext | nindent 8 }} volumes: - name: scheduledworkflow-sa-token projected: diff --git a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml index f42901bbc0..abed726c10 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/argo-role.yaml @@ -15,7 +15,7 @@ rules: - create - get - update -{{- if not .Values.installMode.multiUser.enabled }} +{{- if ne .Values.installMode.type "multi-user" }} - apiGroups: - "" resources: @@ -94,7 +94,7 @@ rules: - secrets verbs: - get -{{- if not .Values.installMode.multiUser.enabled }} +{{- if ne .Values.installMode.type "multi-user" }} - apiGroups: - argoproj.io resources: diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml deleted file mode 100644 index 6ef8a70e72..0000000000 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-deployment.yaml +++ /dev/null @@ -1,68 +0,0 @@ -{{- if .Values.thirdParty.minio.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - app: minio -spec: - selector: - matchLabels: - {{- include "kubeflow-pipelines.minioSelectorLabels" . | nindent 6 }} - strategy: - type: Recreate - template: - metadata: - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 8 }} - app: minio - spec: - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: "OnRootMismatch" - seccompProfile: - type: RuntimeDefault - containers: - - name: minio - image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance - args: - - server - - /data - env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: mlpipeline-minio-artifact - key: accesskey - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: mlpipeline-minio-artifact - key: secretkey - ports: - - containerPort: 9000 - resources: - requests: - cpu: 20m - memory: 100Mi - securityContext: - runAsUser: 1000 - runAsGroup: 0 - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - volumeMounts: - - mountPath: /data - name: data - subPath: minio - volumes: - - name: data - persistentVolumeClaim: - claimName: minio-pvc -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml index 7a74b494a0..01159e6c50 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/mysql-deployment.yaml @@ -10,27 +10,41 @@ metadata: spec: selector: matchLabels: - {{- include "kubeflow-pipelines.mysqlSelectorLabels" . | nindent 6 }} + app: mysql + application-crd-id: kubeflow-pipelines strategy: type: Recreate template: metadata: labels: - {{- include "kubeflow-pipelines.labels" . | nindent 8 }} app: mysql + application-crd-id: kubeflow-pipelines spec: - securityContext: - fsGroup: 999 - fsGroupChangePolicy: OnRootMismatch - seccompProfile: - type: RuntimeDefault serviceAccountName: mysql containers: - - name: mysql - image: mysql:8.4 - args: + - args: + # https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_ignore-db-dir + # Ext4, Btrfs etc. volumes root directories have a lost+found directory that should not be treated as a database. + # ignore-db-dir option has been deprecated in mysql v5.7.16. + # + # If upgrading MySQL to v8.0 fails, try removing /var/lib/mysql/lost+found folder in + # mysql-pv-claim (mysql-persistent-storage): + # + # kubectl exec -it -n kubeflow -- bash + # rm -rf /var/lib/mysql/lost+found + # + # More details on upgrading MySQL can be found here: + # https://dev.mysql.com/doc/refman/8.0/en/upgrade-prerequisites.html + # https://dev.mysql.com/doc/refman/8.0/en/upgrade-docker-mysql.html - --datadir - /var/lib/mysql + # MLMD workloads (metadata-grpc-deployment and metadata-writer) depend on mysql_native_password authentication plugin. + # mysql_native_password plugin implements native authentication; that is, authentication based on the password + # hashing method in use from before the introduction of pluggable authentication in MySQL 8.0. + # + # The mysql_native_password authentication plugin is deprecated as of MySQL 8.0.34, disabled by default + # in MySQL 8.4, and removed as of MySQL 9.0.0: + # https://dev.mysql.com/doc/refman/8.4/en/native-pluggable-authentication.html - --authentication-policy=mysql_native_password - --mysql-native-password=ON # Disable binlog as the logs grow fast and eat up all disk spaces eventually. And KFP doesn't currently utilize binlog. @@ -39,26 +53,23 @@ spec: env: - name: MYSQL_ALLOW_EMPTY_PASSWORD value: "true" + image: {{ .Values.thirdParty.mysql.image }}:{{ .Values.thirdParty.mysql.imageTag }} + name: mysql ports: - containerPort: 3306 name: mysql securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 999 - runAsGroup: 999 - capabilities: - drop: - - ALL + {{- toYaml .Values.thirdParty.mysql.securityContext | nindent 10 }} volumeMounts: - mountPath: /var/lib/mysql name: mysql-persistent-storage resources: - requests: - cpu: 100m - memory: 800Mi + {{- toYaml .Values.thirdParty.mysql.resources | nindent 10 }} + securityContext: + {{- toYaml .Values.thirdParty.mysql.podSecurityContext | nindent 8 }} volumes: - name: mysql-persistent-storage persistentVolumeClaim: claimName: mysql-pv-claim {{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/mysql-pvc.yaml b/experimental/helm/charts/pipelines/templates/third-party/mysql-pvc.yaml index 828edf1e1f..6018db152a 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/mysql-pvc.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/mysql-pvc.yaml @@ -11,5 +11,6 @@ spec: - ReadWriteOnce resources: requests: - storage: 20Gi + storage: {{ .Values.thirdParty.mysql.persistence.size }} {{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/mysql-service.yaml b/experimental/helm/charts/pipelines/templates/third-party/mysql-service.yaml index 8652dc97c0..1f90606232 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/mysql-service.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/mysql-service.yaml @@ -8,9 +8,12 @@ metadata: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} spec: ports: - - port: 3306 + - # We cannot have name: mysql here, because some requests through istio fail with it. + port: 3306 protocol: TCP targetPort: 3306 selector: - {{- include "kubeflow-pipelines.mysqlSelectorLabels" . | nindent 4 }} + app: mysql + application-crd-id: kubeflow-pipelines {{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/mysql-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/third-party/mysql-serviceaccount.yaml index d618ce709e..83b0404120 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/mysql-serviceaccount.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/mysql-serviceaccount.yaml @@ -7,3 +7,4 @@ metadata: labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} {{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-deployment.yaml deleted file mode 100644 index fb3dd4d40b..0000000000 --- a/experimental/helm/charts/pipelines/templates/third-party/postgresql-deployment.yaml +++ /dev/null @@ -1,54 +0,0 @@ -{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - app: postgres - application-crd-id: kubeflow-pipelines -spec: - selector: - matchLabels: - app: postgres - application-crd-id: kubeflow-pipelines - strategy: - type: Recreate - template: - metadata: - labels: - app: postgres - application-crd-id: kubeflow-pipelines - spec: - serviceAccountName: postgresql - containers: - - image: {{ .Values.thirdParty.postgresql.image.repository }}:{{ .Values.thirdParty.postgresql.image.tag }} - name: postgres - env: - - name: POSTGRES_DB - value: {{ .Values.thirdParty.postgresql.database | default "postgres" }} - - name: POSTGRES_USER - value: {{ .Values.thirdParty.postgresql.username | default "user" }} - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.thirdParty.postgresql.secretName | default "postgres-secret" }} - key: {{ .Values.thirdParty.postgresql.secretKey | default "root_password" }} - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - ports: - - containerPort: 5432 - name: postgres - readinessProbe: - exec: - command: ["psql", "-U", "{{ .Values.thirdParty.postgresql.username | default "user" }}", "-d", "{{ .Values.thirdParty.postgresql.database | default "postgres" }}", "-c", "SELECT 1"] - initialDelaySeconds: 15 - timeoutSeconds: 2 - volumeMounts: - - name: postgres-stateful-data - mountPath: /var/lib/postgresql/data - volumes: - - name: postgres-stateful-data - persistentVolumeClaim: - claimName: postgres-pvc -{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-pvc.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-pvc.yaml deleted file mode 100644 index 857ea60a1f..0000000000 --- a/experimental/helm/charts/pipelines/templates/third-party/postgresql-pvc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: postgres-pvc - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - app: postgres - application-crd-id: kubeflow-pipelines -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.thirdParty.postgresql.persistence.size | default "20Gi" }} -{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml deleted file mode 100644 index bd7ef9a03a..0000000000 --- a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret-extended.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} -apiVersion: v1 -kind: Secret -metadata: - name: postgres-secret-extended - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -stringData: - username: {{ .Values.thirdParty.postgresql.username | default "user" }} - password: {{ .Values.thirdParty.postgresql.password | default "password" }} -{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml deleted file mode 100644 index 54757e95ef..0000000000 --- a/experimental/helm/charts/pipelines/templates/third-party/postgresql-secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} -apiVersion: v1 -kind: Secret -metadata: - name: postgres-secret - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} -type: Opaque -data: - root_password: {{ .Values.thirdParty.postgresql.password | default "password" }} -{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-service.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-service.yaml deleted file mode 100644 index 7d3de5660c..0000000000 --- a/experimental/helm/charts/pipelines/templates/third-party/postgresql-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} -apiVersion: v1 -kind: Service -metadata: - name: postgres-service - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - app: postgres - application-crd-id: kubeflow-pipelines -spec: - ports: - - protocol: TCP - port: 5432 - targetPort: 5432 - selector: - app: postgres - application-crd-id: kubeflow-pipelines -{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml deleted file mode 100644 index bf00d18d5a..0000000000 --- a/experimental/helm/charts/pipelines/templates/third-party/postgresql-serviceaccount.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if and .Values.thirdParty.postgresql.enabled (not .Values.postgresql.enabled) }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: postgresql - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - application-crd-id: kubeflow-pipelines -{{- end }} \ No newline at end of file diff --git a/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/authorization-policy.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/authorization-policy.yaml new file mode 100644 index 0000000000..14ecfecf39 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/authorization-policy.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.seaweedfs.enabled .Values.networking.istio.enabled (eq .Values.installMode.type "multi-user") }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: seaweedfs-service + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.authorizationPolicyLabels" . | nindent 4 }} +spec: + action: ALLOW + selector: + matchLabels: + app: seaweedfs + rules: + - from: + - source: + principals: + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline + - from: + - source: + principals: + - cluster.local/ns/{{ include "kubeflow-pipelines.namespace" . }}/sa/ml-pipeline-ui + # Allow traffic to s3 endpoint from User Pipeline Pods, which don't have a sidecar. + # Also needed for traffic from seaweedfs init pod. Seaweedfs gives the client an ip to connect to. This can not be + # handled well by istio (AuthPolicy). Instead, access to the sensitive ports will be limited by the NetworkPolicy. + - {} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/deployment.yaml new file mode 100644 index 0000000000..df4e61b202 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/deployment.yaml @@ -0,0 +1,99 @@ +{{- if .Values.seaweedfs.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: seaweedfs + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: seaweedfs +spec: + selector: + matchLabels: + app: seaweedfs + application-crd-id: kubeflow-pipelines + strategy: + type: Recreate + # Single container setup not scalable + replicas: 1 + template: + metadata: + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 8 }} + app: seaweedfs + application-crd-id: kubeflow-pipelines + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: seaweedfs + envFrom: + - secretRef: + name: mlpipeline-minio-artifact + image: "chrislusf/seaweedfs:{{ .Values.seaweedfs.imageTag | default "4.00" }}" + args: + - "server" + - "-dir=/data" + - "-s3" + - "-iam" + - "-filer" + - "-master.volumePreallocate=false" + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -ec + - | + # wait until seaweedfs master is ready + for i in $(seq 1 120); do + if wget -q --spider http://127.0.0.1:8333/status; then + break + fi + sleep 2 + done + # create bucket if not exists (ignore error if exists) + echo "s3.bucket.create --name mlpipeline" | /usr/bin/weed shell || true + # configure admin user using keys from secret + echo "s3.configure -user kubeflow-admin -access_key $accesskey -secret_key $secretkey -actions Admin -apply" | /usr/bin/weed shell + ports: + - containerPort: 8333 + - containerPort: 8111 + - containerPort: 9333 + - containerPort: 19333 + - containerPort: 8888 + readinessProbe: + httpGet: + path: /status + port: 8333 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 100 + timeoutSeconds: 10 + securityContext: # Using restricted profile + allowPrivilegeEscalation: false + privileged: false + runAsNonRoot: true + # image defaults to root user + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /data + name: data + resources: + requests: + cpu: 32m + memory: 128Mi + volumes: + - name: data + persistentVolumeClaim: + claimName: seaweedfs-pvc + serviceAccountName: seaweedfs +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/destination-rule.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/destination-rule.yaml new file mode 100644 index 0000000000..aab413ca00 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/destination-rule.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.seaweedfs.enabled .Values.networking.istio.enabled }} +apiVersion: "networking.istio.io/v1alpha3" +kind: DestinationRule +metadata: + name: ml-pipeline-seaweedfs + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + host: seaweedfs.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/minio-service.yaml similarity index 52% rename from experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml rename to experimental/helm/charts/pipelines/templates/third-party/seaweedfs/minio-service.yaml index dad3b4f2a3..caab285420 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-service.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/minio-service.yaml @@ -1,4 +1,5 @@ -{{- if .Values.thirdParty.minio.enabled }} +{{- if .Values.seaweedfs.enabled }} +# Backward compatibility service for Minio API apiVersion: v1 kind: Service metadata: @@ -6,12 +7,16 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} + app: seaweedfs + component: minio-compatibility spec: ports: - - name: {{ .Values.thirdParty.minio.portName | default "http" }} + - name: http port: 9000 protocol: TCP - targetPort: 9000 + targetPort: 8333 selector: - {{- include "kubeflow-pipelines.minioSelectorLabels" . | nindent 4 }} + app: seaweedfs + application-crd-id: kubeflow-pipelines {{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/networkpolicy.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/networkpolicy.yaml new file mode 100644 index 0000000000..323cbd03f1 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/networkpolicy.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.seaweedfs.enabled .Values.networking.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: seaweedfs + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + ingress: + - from: + - namespaceSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - kubeflow-profile + ports: + - port: 8333 + - from: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - istio-system + podSelector: + matchExpressions: + - key: app + operator: In + values: + - seaweedfs + policyTypes: + - Ingress +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/pvc.yaml similarity index 66% rename from experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml rename to experimental/helm/charts/pipelines/templates/third-party/seaweedfs/pvc.yaml index 6fc93106ac..574499e382 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-pvc.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/pvc.yaml @@ -1,8 +1,8 @@ -{{- if .Values.thirdParty.minio.enabled }} +{{- if .Values.seaweedfs.enabled }} apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: minio-pvc + name: seaweedfs-pvc namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} @@ -11,5 +11,6 @@ spec: - ReadWriteOnce resources: requests: - storage: 20Gi + storage: {{ dig "persistence" "size" "20Gi" .Values.seaweedfs }} {{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/secret.yaml similarity index 54% rename from experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml rename to experimental/helm/charts/pipelines/templates/third-party/seaweedfs/secret.yaml index 7ddaf139d5..ab13f3af3f 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/minio-secret.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/secret.yaml @@ -1,12 +1,13 @@ -{{- if .Values.thirdParty.minio.enabled }} -apiVersion: v1 +{{- if .Values.seaweedfs.enabled }} kind: Secret +apiVersion: v1 metadata: name: mlpipeline-minio-artifact namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} stringData: - accesskey: "minio" - secretkey: "minio123" + accesskey: {{ dig "credentials" "accesskey" "minio" .Values.seaweedfs }} + secretkey: {{ dig "credentials" "secretkey" "minio123" .Values.seaweedfs }} {{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/application/controller-serviceaccount.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/service-account.yaml similarity index 55% rename from experimental/helm/charts/pipelines/templates/application/controller-serviceaccount.yaml rename to experimental/helm/charts/pipelines/templates/third-party/seaweedfs/service-account.yaml index b586cc37a8..5e1620de1b 100644 --- a/experimental/helm/charts/pipelines/templates/application/controller-serviceaccount.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/service-account.yaml @@ -1,12 +1,10 @@ -{{- if .Values.application.controller.enabled }} +{{- if .Values.seaweedfs.enabled }} apiVersion: v1 kind: ServiceAccount metadata: - name: application + name: seaweedfs namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/service.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/service.yaml new file mode 100644 index 0000000000..660ec0268b --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/service.yaml @@ -0,0 +1,43 @@ +{{- if .Values.seaweedfs.enabled }} +# Separate service for new ports of seaweedfs. If we add them to the existing minio-service there will be a problem +# with the mlpipeline api server because it relies on MINIO_SERVICE_SERVICE_PORT pointing to the S3 port. +# But with multiple ports on a service that is not really reliable. So we use the existing minio-service for +# backwards-compatibility, but everything new, seaweedfs related is here. +apiVersion: v1 +kind: Service +metadata: + name: seaweedfs + namespace: {{ include "kubeflow-pipelines.namespace" . }} + labels: + {{- include "kubeflow-pipelines.labels" . | nindent 4 }} +spec: + ports: + - name: http-iam + port: 8111 + protocol: TCP + targetPort: 8111 + - name: http-s3 + port: 8333 + protocol: TCP + targetPort: 8333 + - name: http-master + port: 9333 + protocol: TCP + targetPort: 9333 + - name: grpc-master + port: 19333 + protocol: TCP + targetPort: 19333 + - name: grpc-filer + port: 18888 + protocol: TCP + targetPort: 18888 + - name: http-filer + port: 8888 + protocol: TCP + targetPort: 8888 + selector: + app: seaweedfs + application-crd-id: kubeflow-pipelines +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml index 1a0d5f5d7a..b40d4de938 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-configmap.yaml @@ -17,8 +17,7 @@ data: # The following format looks like: # artifacts/my-workflow-abc123/2018/08/23/my-workflow-abc123-1234567890 # Adding date into the path greatly reduces the chance of {{`{{pod.name}}`}} collision. - keyFormat: "artifacts/{{`{{workflow.name}}`}}/{{`{{workflow.creationTimestamp.Y}}`}}/{{`{{workflow.creationTimestamp.m}}`}}/{{`{{workflow.creationTimestamp.d}}`}}/{{`{{pod.name}}`}}" - # insecure will disable TLS. Primarily used for minio installs not configured with TLS + keyFormat: "private-artifacts/{{`{{workflow.namespace}}`}}/{{`{{workflow.name}}`}}/{{`{{workflow.creationTimestamp.Y}}`}}/{{`{{workflow.creationTimestamp.m}}`}}/{{`{{workflow.creationTimestamp.d}}`}}/{{`{{pod.name}}`}}" insecure: true accessKeySecret: name: mlpipeline-minio-artifact diff --git a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml index 84adb61b74..6c7a7f5d23 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/workflow-controller-deployment.yaml @@ -21,7 +21,7 @@ spec: - --configmap - workflow-controller-configmap - --executor-image - - quay.io/argoproj/argoexec:v3.6.7 + - quay.io/argoproj/argoexec:v3.7.3 {{- if ne .Values.installMode.type "multi-user" }} - --namespaced {{- end }} @@ -33,7 +33,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.name - image: quay.io/argoproj/workflow-controller:v3.6.7 + image: quay.io/argoproj/workflow-controller:v3.7.3 livenessProbe: failureThreshold: 3 httpGet: diff --git a/experimental/helm/charts/pipelines/templates/ui/deployment.yaml b/experimental/helm/charts/pipelines/templates/ui/deployment.yaml index 8a89408e3e..67957762f2 100644 --- a/experimental/helm/charts/pipelines/templates/ui/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/ui/deployment.yaml @@ -85,6 +85,5 @@ spec: securityContext: {{- toYaml .Values.ui.securityContext | nindent 10 }} securityContext: - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} + {{- toYaml .Values.podSecurityContext | nindent 8 }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/viewer-crd/deployment.yaml b/experimental/helm/charts/pipelines/templates/viewer-crd/deployment.yaml index 8338bc0bd7..c3e4acc2c6 100644 --- a/experimental/helm/charts/pipelines/templates/viewer-crd/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/viewer-crd/deployment.yaml @@ -6,6 +6,9 @@ metadata: namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.viewerCrdLabels" . | nindent 4 }} +{{- if eq .Values.installMode.type "multi-user" }} + application-crd-id: kubeflow-pipelines +{{- end }} spec: selector: matchLabels: @@ -16,6 +19,9 @@ spec: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" labels: {{- include "kubeflow-pipelines.viewerCrdLabels" . | nindent 8 }} +{{- if eq .Values.installMode.type "multi-user" }} + application-crd-id: kubeflow-pipelines +{{- end }} spec: serviceAccountName: ml-pipeline-viewer-crd-service-account containers: @@ -38,13 +44,8 @@ spec: value: "" {{- end }} {{- end }} - {{- if and .Values.viewerCrd.resources (or .Values.viewerCrd.resources.requests .Values.viewerCrd.resources.limits) }} - resources: - {{- toYaml .Values.viewerCrd.resources | nindent 10 }} - {{- end }} securityContext: {{- toYaml .Values.viewerCrd.securityContext | nindent 10 }} securityContext: - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} + {{- toYaml .Values.podSecurityContext | nindent 8 }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/visualization/deployment.yaml b/experimental/helm/charts/pipelines/templates/visualization/deployment.yaml index f50653a85f..e452d95bd2 100644 --- a/experimental/helm/charts/pipelines/templates/visualization/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/visualization/deployment.yaml @@ -18,10 +18,13 @@ spec: {{- include "kubeflow-pipelines.visualizationLabels" . | nindent 8 }} spec: serviceAccountName: ml-pipeline-visualizationserver + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: ml-pipeline-visualizationserver image: {{ include "kubeflow-pipelines.image" (dict "repository" .Values.visualization.image.repository "tag" .Values.visualization.image.tag "context" .) }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.global.imagePullPolicy }} ports: - name: http containerPort: 8888 @@ -43,7 +46,4 @@ spec: {{- toYaml .Values.visualization.resources | nindent 10 }} securityContext: {{- toYaml .Values.visualization.securityContext | nindent 10 }} - securityContext: - seccompProfile: - type: {{ .Values.security.seccompProfile.type }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/webhook/certificate.yaml b/experimental/helm/charts/pipelines/templates/webhook/certificate.yaml new file mode 100644 index 0000000000..644bb82625 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/webhook/certificate.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.certManager.enabled (eq .Values.installMode.pipelineDefinition.storage "k8s-native") }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kfp-api-webhook-cert + namespace: {{ include "kubeflow-pipelines.namespace" . }} +spec: + commonName: kfp-api-webhook-cert + isCA: true + dnsNames: + - ml-pipeline + - ml-pipeline.{{ include "kubeflow-pipelines.namespace" . }} + - ml-pipeline.{{ include "kubeflow-pipelines.namespace" . }}.svc + - ml-pipeline.{{ include "kubeflow-pipelines.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: kfp-api-webhook-selfsigned-issuer + secretName: kfp-api-webhook-cert +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/webhook/issuer.yaml b/experimental/helm/charts/pipelines/templates/webhook/issuer.yaml new file mode 100644 index 0000000000..60465cddd8 --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/webhook/issuer.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.certManager.enabled (eq .Values.installMode.pipelineDefinition.storage "k8s-native") }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: kfp-api-webhook-selfsigned-issuer + namespace: {{ include "kubeflow-pipelines.namespace" . }} +spec: + selfSigned: {} +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/webhook/mutating-webhook-config.yaml b/experimental/helm/charts/pipelines/templates/webhook/mutating-webhook-config.yaml deleted file mode 100644 index bd2285d3a0..0000000000 --- a/experimental/helm/charts/pipelines/templates/webhook/mutating-webhook-config.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.webhooks.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-pipelineversions-mutating - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: webhook - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -webhooks: -- name: pipelineversions.pipelines.kubeflow.org - rules: - - operations: - - CREATE - - UPDATE - apiGroups: - - pipelines.kubeflow.org - apiVersions: - - v2beta1 - resources: - - pipelineversions - scope: Namespaced - admissionReviewVersions: - - v1 - sideEffects: None - failurePolicy: {{ .Values.webhooks.failurePolicy }} - clientConfig: - service: - name: {{ include "kubeflow-pipelines.fullname" . }}-api-server - namespace: {{ include "kubeflow-pipelines.namespace" . }} - path: /webhooks/mutate-pipelineversion - port: 8443 -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/webhook/mutating-webhook.yaml b/experimental/helm/charts/pipelines/templates/webhook/mutating-webhook.yaml new file mode 100644 index 0000000000..e5e63b6f9c --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/webhook/mutating-webhook.yaml @@ -0,0 +1,32 @@ +{{- if and (eq .Values.installMode.type "multi-user") (eq .Values.installMode.pipelineDefinition.storage "k8s-native") }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: pipelineversions.pipelines.kubeflow.org + annotations: + cert-manager.io/inject-ca-from: {{ include "kubeflow-pipelines.namespace" . }}/kfp-api-webhook-cert +webhooks: + - name: pipelineversions.pipelines.kubeflow.org + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - pipelines.kubeflow.org + apiVersions: + - v2beta1 + resources: + - pipelineversions + scope: Namespaced + admissionReviewVersions: + - v1 + sideEffects: None + failurePolicy: {{ .Values.webhooks.failurePolicy | default "Fail" }} + clientConfig: + service: + name: ml-pipeline + namespace: {{ include "kubeflow-pipelines.namespace" . }} + path: /webhooks/mutate-pipelineversion + port: 8443 +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/webhook/validating-webhook-config.yaml b/experimental/helm/charts/pipelines/templates/webhook/validating-webhook-config.yaml deleted file mode 100644 index 58ea502ced..0000000000 --- a/experimental/helm/charts/pipelines/templates/webhook/validating-webhook-config.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.webhooks.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-pipelineversions-validating - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: webhook - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} -webhooks: -- name: pipelineversions.pipelines.kubeflow.org - rules: - - operations: - - CREATE - - UPDATE - apiGroups: - - pipelines.kubeflow.org - apiVersions: - - v2beta1 - resources: - - pipelineversions - scope: Namespaced - admissionReviewVersions: - - v1 - sideEffects: None - failurePolicy: {{ .Values.webhooks.failurePolicy }} - clientConfig: - service: - name: {{ include "kubeflow-pipelines.fullname" . }}-api-server - namespace: {{ include "kubeflow-pipelines.namespace" . }} - path: /webhooks/validate-pipelineversion - port: 8443 -{{- end }} diff --git a/experimental/helm/charts/pipelines/templates/webhook/validating-webhook.yaml b/experimental/helm/charts/pipelines/templates/webhook/validating-webhook.yaml new file mode 100644 index 0000000000..8f2387267a --- /dev/null +++ b/experimental/helm/charts/pipelines/templates/webhook/validating-webhook.yaml @@ -0,0 +1,32 @@ +{{- if and (eq .Values.installMode.type "multi-user") (eq .Values.installMode.pipelineDefinition.storage "k8s-native") }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: pipelineversions.pipelines.kubeflow.org + annotations: + cert-manager.io/inject-ca-from: {{ include "kubeflow-pipelines.namespace" . }}/kfp-api-webhook-cert +webhooks: + - name: pipelineversions.pipelines.kubeflow.org + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - pipelines.kubeflow.org + apiVersions: + - v2beta1 + resources: + - pipelineversions + scope: Namespaced + admissionReviewVersions: + - v1 + sideEffects: None + failurePolicy: {{ .Values.webhooks.failurePolicy | default "Fail" }} + clientConfig: + service: + name: ml-pipeline + namespace: {{ include "kubeflow-pipelines.namespace" . }} + path: /webhooks/validate-pipelineversion + port: 8443 +{{- end }} + diff --git a/experimental/helm/charts/pipelines/templates/webhook/webhook-server-tls-secret.yaml b/experimental/helm/charts/pipelines/templates/webhook/webhook-server-tls-secret.yaml deleted file mode 100644 index e15584f081..0000000000 --- a/experimental/helm/charts/pipelines/templates/webhook/webhook-server-tls-secret.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if false }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "kubeflow-pipelines.fullname" . }}-webhook-server-tls - namespace: {{ include "kubeflow-pipelines.namespace" . }} - labels: - {{- include "kubeflow-pipelines.labels" . | nindent 4 }} - component: webhook - annotations: - {{- include "kubeflow-pipelines.annotations" . | nindent 4 }} - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-5" -type: kubernetes.io/tls -data: - tls.crt: {{ .Values.webhooks.tls.cert | b64enc }} - tls.key: {{ .Values.webhooks.tls.key | b64enc }} -{{- end }} diff --git a/experimental/helm/charts/pipelines/values.yaml b/experimental/helm/charts/pipelines/values.yaml index 28a4afbd04..da4f7e6bf1 100644 --- a/experimental/helm/charts/pipelines/values.yaml +++ b/experimental/helm/charts/pipelines/values.yaml @@ -1,17 +1,7 @@ -# Default values for kubeflow-pipelines. - -# -- String to partially override release name. -nameOverride: "" - -# -- String to fully override release name. -fullnameOverride: "" - -# -- Common labels to add to the resources. -commonLabels: {} - -# -- Common annotations to add to the resources. -commonAnnotations: {} - +# Default values for kubeflow-pipelines +# This configuration supports 2 flavors: +# 1. platform-agnostic-multi-user (pipelineDefinition.storage: database) +# 2. platform-agnostic-multi-user-k8s-native (pipelineDefinition.storage: k8s-native) # Global settings global: # -- Namespace to install Kubeflow Pipelines @@ -19,109 +9,71 @@ global: # -- Image registry for Kubeflow Pipelines images imageRegistry: ghcr.io/kubeflow # -- Global image tag for all Kubeflow Pipelines components - imageTag: 2.14.3 - # -- Application version - version: "" + imageTag: 2.15.0 # -- Global image pull policy imagePullPolicy: IfNotPresent - # -- Global image pull secrets - imagePullSecrets: [] - # -- Global annotations to add to the resources - annotations: {} - # -- Global logging level + # -- Global log level logLevel: "info" +podSecurityContext: + seccompProfile: + type: RuntimeDefault + +containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + # Installation mode configuration installMode: - # -- Installation type (generic or multi-user) - type: generic - - # Multi-user specific configuration - multiUser: - # -- Enable multi-user mode - enabled: false - profileController: - # -- Enable profile controller for multi-user mode - enabled: true + # -- Installation type: multi-user (only mode supported) + type: multi-user + + # Pipeline definition storage mode + pipelineDefinition: + storage: database # Custom Resource Definitions crds: - # -- Install CRDs + # -- Install Pipeline CRDs install: true - # -- Install Application CRD - application: false + # -- Install Webhook CRDs (required for k8s-native mode) + webhook: false # API Server configuration apiServer: - # -- Enable the Kubeflow Pipelines API Server enabled: true - # -- Number of API Server replicas replicas: 1 image: - # -- API Server image repository repository: kfp-api-server - # -- API Server image tag (overrides global.imageTag if set) - tag: "" - # -- API Server image pull policy (overrides global.imagePullPolicy if set) - pullPolicy: "" - - # -- API Server resource requests and limits + tag: "" # Uses global.imageTag if not set + resources: requests: cpu: 250m memory: 500Mi - # Service Account - serviceAccount: - # -- Create service account for API Server - create: true - # -- Service account name (uses default if not set) - name: "ml-pipeline" - # -- Service account annotations - annotations: {} - - # Service configuration - service: - # -- Service type - type: ClusterIP - # -- HTTP service port - port: 8888 - # -- gRPC service port - grpcPort: 8887 - - # Environment variables - env: - # -- Whether to publish component logs to object store - publishLogs: true - # -- Driver/launcher log level during pipeline execution - pipelineLogLevel: "1" - # -- Auto update pipeline default version - autoUpdatePipelineDefaultVersion: true - # -- V2 driver image - v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.14.3" - # -- V2 launcher image - v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.14.3" - - # Additional environment variables - additionalEnv: [] - # Probes configuration probes: - readiness: + liveness: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 - liveness: + readiness: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 startup: + initialDelaySeconds: 3 failureThreshold: 12 periodSeconds: 5 timeoutSeconds: 2 - # Security context securityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -129,78 +81,124 @@ apiServer: runAsGroup: 0 capabilities: drop: - - ALL - + - ALL + + serviceAccount: + create: true + name: "ml-pipeline" + + service: + type: ClusterIP + port: 8888 + grpcPort: 8887 + + env: + # -- V2 driver image + v2DriverImage: "ghcr.io/kubeflow/kfp-driver:2.15.0" + # -- V2 launcher image + v2LauncherImage: "ghcr.io/kubeflow/kfp-launcher:2.15.0" + + additionalEnv: + - name: KUBEFLOW_USERID_HEADER + value: kubeflow-userid + - name: KUBEFLOW_USERID_PREFIX + value: "" + - name: PUBLISH_LOGS + value: true + - name: LOG_LEVEL + value: "info" + - name: PIPELINE_LOG_LEVEL + value: "1" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: false + - name: OBJECTSTORECONFIG_BUCKETNAME + key: bucketName + - name: OBJECTSTORECONFIG_HOST + value: "minio-service.kubeflow" + - name: OBJECTSTORECONFIG_PORT + value: "9000" + - name: DBCONFIG_USER + key: username + - name: DBCONFIG_PASSWORD + key: password + - name: DBCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_HOST + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_CONMAXLIFETIME + key: ConMaxLifeTime + - name: DB_DRIVER_NAME + key: dbType + - name: DBCONFIG_MYSQLCONFIG_USER + key: username + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + key: password + - name: DBCONFIG_MYSQLCONFIG_DBNAME + key: pipelineDb + - name: DBCONFIG_MYSQLCONFIG_HOST + key: mysqlHost + - name: DBCONFIG_MYSQLCONFIG_PORT + key: mysqlPort + - name: OBJECTSTORECONFIG_ACCESSKEY + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + key: secretkey + - name: V2_DRIVER_IMAGE + value: ghcr.io/kubeflow/kfp-driver:2.15.0 + - name: V2_LAUNCHER_IMAGE + value: ghcr.io/kubeflow/kfp-launcher:2.15.0 + - name: COMPILED_PIPELINE_SPEC_PATCH + value: "{}" +# Persistence Agent configuration persistenceAgent: - # -- Enable Persistence Agent enabled: true - # -- Number of Persistence Agent replicas replicas: 1 image: - # -- Persistence Agent image repository repository: kfp-persistence-agent - # -- Persistence Agent image tag (overrides global.imageTag if set) tag: "" - # -- Persistence Agent image pull policy (overrides global.imagePullPolicy if set) - pullPolicy: "" - - # -- Persistence Agent resource requests and limits + resources: requests: cpu: 120m memory: 500Mi - limits: {} - # Service Account + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + serviceAccount: - # -- Create service account for Persistence Agent create: true - # -- Service account name (uses default if not set) name: "ml-pipeline-persistenceagent" - # -- Service account annotations - annotations: {} - # Environment variables env: - # -- TTL seconds after workflow finish ttlSecondsAfterWorkflowFinish: "86400" - # -- Number of workers numWorkers: "2" - # Additional environment variables additionalEnv: [] - - # Security context - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - - ALL +# Scheduled Workflow Controller configuration scheduledWorkflow: enabled: true - name: ml-pipeline-scheduledworkflow + replicas: 1 image: repository: kfp-scheduled-workflow-controller - tag: "2.14.3" - - replicas: 1 - - # Service Account - serviceAccount: - create: true - name: "ml-pipeline-scheduledworkflow" - annotations: {} - - # Resources - resources: {} + tag: "" - # Security context securityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -208,69 +206,48 @@ scheduledWorkflow: runAsGroup: 0 capabilities: drop: - - ALL + - ALL + + serviceAccount: + create: true + name: "ml-pipeline-scheduledworkflow" + + resources: + requests: + cpu: 120m + memory: 100Mi +# UI configuration ui: - # -- Enable UI enabled: true - # -- Number of UI replicas replicas: 1 image: - # -- UI image repository repository: kfp-frontend - # -- UI image tag (overrides global.imageTag if set) tag: "" - # -- UI image pull policy (overrides global.imagePullPolicy if set) - pullPolicy: "" - - # -- UI resource requests and limits + resources: requests: cpu: 10m memory: 70Mi - limits: {} - - # Service Account - serviceAccount: - # -- Create service account for UI - create: true - # -- Service account name (uses default if not set) - name: "ml-pipeline-ui" - # -- Service account annotations - annotations: {} - - # Service configuration - service: - # -- Service type - type: ClusterIP - # -- Service port - port: 80 - - # Environment variables - env: - # -- Allow custom visualizations - allowCustomVisualizations: true - # -- Archive Argo logs - argoArchiveLogs: true - # -- Disable GKE metadata - disableGkeMetadata: true - - # Additional environment variables - additionalEnv: [] # Probes configuration probes: - readiness: + liveness: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 - liveness: + readiness: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 - # Security context + # Environment variables + env: + allowCustomVisualizations: "true" + argoArchiveLogs: "true" + disableGkeMetadata: "true" + securityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -278,104 +255,118 @@ ui: runAsGroup: 0 capabilities: drop: - - ALL - + - ALL + + serviceAccount: + create: true + name: "ml-pipeline-ui" + + service: + type: ClusterIP + port: 80 + + additionalEnv: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: "/etc/config/viewer-pod-template.json" + - name: DEPLOYMENT + value: "KUBEFLOW" + - name: ARTIFACTS_SERVICE_PROXY_NAME + value: "ml-pipeline-ui-artifact" + - name: ARTIFACTS_SERVICE_PROXY_PORT + value: "80" + - name: ARTIFACTS_SERVICE_PROXY_ENABLED + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: ARTIFACTS_PROXY_ENABLED + optional: true + - name: ENABLE_AUTHZ + value: "true" + - name: KUBEFLOW_USERID_HEADER + value: "kubeflow-userid" + - name: KUBEFLOW_USERID_PREFIX + value: "" + - name: MINIO_NAMESPACE + value: "" + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: DISABLE_GKE_METADATA + value: "true" + +# Viewer CRD Controller configuration viewerCrd: - # -- Enable Viewer CRD Controller enabled: true - # -- Number of Viewer CRD Controller replicas replicas: 1 image: - # -- Viewer CRD Controller image repository repository: kfp-viewer-crd-controller - # -- Viewer CRD Controller image tag (overrides global.imageTag if set) tag: "" - # -- Viewer CRD Controller image pull policy (overrides global.imagePullPolicy if set) pullPolicy: "Always" + + resources: + requests: + cpu: 100m + memory: 100Mi - # -- Viewer CRD Controller resource requests and limits - resources: {} + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL - # Service Account serviceAccount: - # -- Create service account for Viewer CRD Controller create: true - # -- Service account name (uses default if not set) name: "ml-pipeline-viewer-crd-service-account" - # -- Service account annotations - annotations: {} - # Environment variables env: - # -- Maximum number of viewers maxNumViewers: "50" - # Additional environment variables additionalEnv: [] - - # Security context - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - - ALL +# Visualization Server configuration visualization: - # -- Enable Visualization Server enabled: true - # -- Number of Visualization Server replicas replicas: 1 image: - # -- Visualization Server image repository repository: kfp-visualization-server - # -- Visualization Server image tag (overrides global.imageTag if set) tag: "" - # -- Visualization Server image pull policy (overrides global.imagePullPolicy if set) - pullPolicy: "" - - # -- Visualization Server resource requests and limits + resources: requests: cpu: 30m memory: 500Mi - limits: {} - - # Service Account - serviceAccount: - # -- Create service account for Visualization Server - create: true - # -- Service account name (uses default if not set) - name: "ml-pipeline-visualizationserver" - # -- Service account annotations - annotations: {} - - # Service configuration - service: - # -- Service type - type: ClusterIP - # -- Service port - port: 8888 - - # Additional environment variables - additionalEnv: [] # Probes configuration probes: - readiness: + liveness: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 - liveness: + readiness: initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 - # Security context securityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -383,52 +374,38 @@ visualization: runAsGroup: 0 capabilities: drop: - - ALL + - ALL + + serviceAccount: + create: true + name: "ml-pipeline-visualizationserver" + + service: + type: ClusterIP + port: 8888 + + additionalEnv: [] -# Cache System Configuration +# Cache System Configuration (with cert-manager integration) cache: - # -- Enable Cache System enabled: true - # Service Account serviceAccount: - # -- Create service account for Cache Server create: true - # -- Service account name (uses default if not set) name: "kubeflow-pipelines-cache" - # -- Service account annotations - annotations: {} server: - # -- Number of Cache Server replicas replicas: 1 image: - # -- Cache Server image repository repository: kfp-cache-server - # -- Cache Server image tag (overrides global.imageTag if set) tag: "" - # -- Cache Server image pull policy (overrides global.imagePullPolicy if set) - pullPolicy: "" - # -- Cache Server resource requests and limits resources: requests: cpu: 120m memory: 500Mi - limits: {} - # Service configuration - service: - # -- Service type - type: ClusterIP - # -- Service port - port: 443 - - # Additional environment variables - additionalEnv: [] - - # Security context securityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -436,554 +413,332 @@ cache: runAsGroup: 0 capabilities: drop: - - ALL - -cacheDeployer: - # -- Enable Cache Deployer - enabled: true - # -- Number of Cache Deployer replicas - replicas: 1 - - image: - # -- Cache Deployer image repository - repository: kfp-cache-deployer - # -- Cache Deployer image tag (overrides global.imageTag if set) - tag: "" - # -- Cache Deployer image pull policy (overrides global.imagePullPolicy if set) - pullPolicy: "" - - # -- Cache Deployer resource requests and limits - resources: - requests: - cpu: 120m - memory: 100Mi - limits: {} - - # Service Account - serviceAccount: - # -- Create service account for Cache Deployer - create: true - # -- Service account name (uses default if not set) - name: "kubeflow-pipelines-cache-deployer-sa" - # -- Service account annotations - annotations: {} - - # Additional environment variables - additionalEnv: [] - - # Security context - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - ALL + + service: + type: ClusterIP + port: 443 + + additionalEnv: + - name: NAMESPACE_TO_WATCH + value: "" + - name: DEFAULT_CACHE_STALENESS + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: mysql + - name: DBCONFIG_DB_NAME + key: cacheDb + - name: DBCONFIG_HOST_NAME + key: dbHost + - name: DBCONFIG_PORT + key: dbPort + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + name: mysql-secret + key: username + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secret + key: password # ML Metadata Configuration metadata: - # -- Enable ML Metadata - enabled: false + enabled: true grpc: - # -- Number of metadata gRPC server replicas replicas: 1 image: - # -- Metadata gRPC server image repository (external image) repository: gcr.io/tfx-oss-public/ml_metadata_store_server - # -- Metadata gRPC server image tag tag: "1.14.0" - # -- Metadata gRPC server image pull policy pullPolicy: "IfNotPresent" - # -- Metadata gRPC server resource requests and limits resources: requests: cpu: 100m memory: 300Mi - limits: {} - # Service Account + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 0 + capabilities: + drop: + - ALL + serviceAccount: - # -- Create service account for Metadata gRPC server create: true - # -- Service account name (uses default if not set) name: "metadata-grpc-server" - # -- Service account annotations - annotations: {} - # Service configuration service: - # -- Service type type: ClusterIP - # -- Service port port: 8080 - # Probes configuration - probes: - readiness: - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - liveness: - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - - # Additional environment variables additionalEnv: [] - - # Security context - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault envoy: - # -- Number of metadata Envoy replicas replicas: 1 image: - # -- Metadata Envoy image repository repository: kfp-metadata-envoy - # -- Metadata Envoy image tag (overrides global.imageTag if set) tag: "" - # -- Metadata Envoy image pull policy (overrides global.imagePullPolicy if set) - pullPolicy: "" - # -- Metadata Envoy resource requests and limits resources: requests: cpu: 20m memory: 20Mi - limits: {} - # Service configuration - service: - # -- Service type - type: ClusterIP - # -- Service port - port: 9090 - - # Additional environment variables - additionalEnv: [] - - # Security context securityContext: allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 capabilities: drop: - - ALL + - ALL + runAsGroup: 0 + runAsNonRoot: true + runAsUser: 1000 seccompProfile: type: RuntimeDefault + + service: + type: ClusterIP + port: 9090 + + additionalEnv: [] -# Third-party Dependencies Configuration -argo: - enabled: false - # Argo Workflows subchart values will be merged here +# Metadata Writer Configuration +metadataWriter: + enabled: true + replicas: 1 + + image: + repository: kfp-metadata-writer + tag: "" + + resources: + requests: + cpu: 100m + memory: 200Mi + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 0 + runAsNonRoot: true + runAsUser: 1000 + + serviceAccount: + create: true + name: "kubeflow-pipelines-metadata-writer" + + additionalEnv: [] -# Built-in third-party components +# Third-party Dependencies thirdParty: - # Use built-in minimal Argo instead of external subchart + # Argo Workflows argo: enabled: true - - # Use built-in minimal MySQL instead of external subchart + + # MySQL database mysql: - enabled: false - - # Use built-in minimal MinIO instead of external subchart - minio: - enabled: false - - # Use built-in minimal PostgreSQL instead of external subchart - postgresql: - enabled: false - image: - repository: postgres - tag: "13.2" - password: "password" - - # Use built-in MySQL for metadata database (separate from main MySQL) - mysqlMetadata: - enabled: false - image: - repository: mysql - tag: "8.0.3" - database: "metadb" - username: "root" - password: "test" + enabled: true + image: "mysql" + imageTag: "8.4" + resources: + requests: + cpu: 100m + memory: 800Mi + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + capabilities: + drop: + - ALL + podSecurityContext: + fsGroup: 999 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault persistence: - enabled: true - size: 10Gi - - # Metacontroller configuration + size: 20Gi + + # Metacontroller (required for multi-user mode) metacontroller: - enabled: false + enabled: true -# Database Configuration +# MySQL Configuration mysql: - enabled: false - global: - security: - allowInsecureImages: true + enabled: true auth: + # -- MySQL root password (leave empty to auto-generate) rootPassword: "" + # -- MySQL database name database: "mlpipeline" + # -- MySQL username username: "root" primary: persistence: enabled: true size: 20Gi -postgresql: - enabled: false - global: - security: - allowInsecureImages: true - auth: - postgresPassword: "" - database: "mlpipeline" - username: "postgres" - primary: - persistence: - enabled: true - size: 20Gi - -# External database configuration (when not using subcharts) +# SeaweedFS Configuration +seaweedfs: + enabled: true + # SeaweedFS image tag + imageTag: "4.00" + # Credentials for S3 access + credentials: + accesskey: "minio" + secretkey: "minio123" + # Storage configuration + persistence: + enabled: true + size: 20Gi + # -- SeaweedFS bucket name + bucket: "mlpipeline" + # -- S3 endpoint URL for profile controller + s3EndpointUrl: "http://seaweedfs.kubeflow:8333" + # -- AWS endpoint URL for profile controller + awsEndpointUrl: "http://seaweedfs.kubeflow:8111" + # -- AWS region + awsRegion: "us-east-1" + +# External MySQL database configuration (when using external database) externalDatabase: - # Database type: mysql or postgresql - type: mysql + enabled: false host: "" port: 3306 database: "mlpipeline" username: "root" password: "" - # Existing secret containing database credentials + # -- Existing secret containing database credentials existingSecret: "" existingSecretPasswordKey: "password" -# Database connection configuration +# Database configuration database: - # Connection max lifetime (important for external databases) + # -- Connection max lifetime (important for external databases) connectionMaxLifetime: "120s" - - # Additional databases + # -- Metadata database name metadb: "metadb" + # -- Cache database name cachedb: "cachedb" -# Object Storage Configuration -objectStore: - # Provider: minio, custom - provider: minio - - minio: - enabled: true - bucket: "mlpipeline" - secure: false - endpoint: "" - existingSecret: "" - - custom: - endpoint: "" - bucket: "" - secure: true - existingSecret: "" - -# MinIO subchart configuration -minio: - enabled: false - auth: - rootUser: "minio" - rootPassword: "minio123" - defaultBuckets: "mlpipeline" - persistence: - enabled: true - size: 20Gi - -# Pipeline Configuration +# Pipeline configuration pipeline: - # Application configuration app: name: "pipeline" - version: "2.14.3" - - # Pipeline root configuration + version: "2.15.0" + # -- Default pipeline root defaultPipelineRoot: "" - - # Auto update pipeline default version + # -- Auto-update pipeline default version autoUpdatePipelineDefaultVersion: true - - # Cron schedule timezone + # -- Cron schedule timezone cronScheduleTimezone: "UTC" - - # Cache configuration cache: + # -- Cache image image: "ghcr.io/containerd/busybox" + # -- Cache node restrictions nodeRestrictions: false + # -- Maximum cache staleness maximumStaleness: "" + # -- Default cache staleness defaultStaleness: "" -# Environment-specific Configuration -env: - # -- Platform type: platform-agnostic - platform: platform-agnostic - -# RBAC Configuration -rbac: - create: true - -# Service Account Configuration -serviceAccounts: - # Additional service accounts - pipelineRunner: - create: true - name: "pipeline-runner" - - viewer: - create: true - name: "viewer" - - containerBuilder: - create: true - name: "container-builder" - - metadataWriter: - create: true - name: "metadata-writer" - -# Networking Configuration -networking: - istio: - enabled: false - gateway: "kubeflow-gateway" - host: "*" - - # Virtual Service configuration - virtualService: - enabled: false - prefix: "/pipeline" - -# Security Configuration -security: - # Pod Security Standards - podSecurityStandards: - enabled: true - enforce: "restricted" - - # Security Context Profile - seccompProfile: - type: "RuntimeDefault" - -# Monitoring Configuration -monitoring: - prometheus: - enabled: false - serviceMonitor: - enabled: false - - grafana: - enabled: false - -# Webhook Configuration -webhooks: - # -- Enable admission webhooks - enabled: false - # -- Webhook failure policy - failurePolicy: "Fail" - - # TLS configuration for webhooks - tls: - # -- Use existing secret for TLS certificates - existingSecret: "" - # -- TLS certificate (base64 encoded) - cert: "" - # -- TLS private key (base64 encoded) - key: "" - -# Metadata Writer Configuration -metadataWriter: - # -- Enable Metadata Writer +# Profile Controller Configuration (Multi-User) +profileController: enabled: true - # -- Number of Metadata Writer replicas replicas: 1 image: - # -- Metadata Writer image repository - repository: kfp-metadata-writer - # -- Metadata Writer image tag (overrides global.imageTag if set) - tag: "" - # -- Metadata Writer image pull policy (overrides global.imagePullPolicy if set) - pullPolicy: "" + repository: docker.io/alpine/k8s + tag: "1.32.3" + pullPolicy: IfNotPresent - # -- Metadata Writer resource requests and limits - resources: - requests: - cpu: 100m - memory: 200Mi - limits: - cpu: 500m - memory: 1Gi + service: + type: ClusterIP + port: 80 - # Service Account - serviceAccount: - # -- Create service account for Metadata Writer - create: true - # -- Service account name (uses default if not set) - name: "kubeflow-pipelines-metadata-writer" - # -- Service account annotations - annotations: {} + # -- Disable Istio sidecar injection + disableIstioSidecar: "false" - # Additional environment variables - additionalEnv: [] + # -- Enable artifacts proxy in user namespaces + artifactsProxyEnabled: "false" - # Security context - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - - ALL + # -- Artifact retention days (-1 to disable) + artifactRetentionDays: "-1" + + # -- Default pipeline root for KFP v2 + defaultPipelineRoot: "" + + additionalEnv: [] -# Additional Service Accounts Configuration +# Additional Service Accounts additionalServiceAccounts: - # Container Builder Service Account containerBuilder: - # -- Create container builder service account create: true - # -- Container builder service account name name: "kubeflow-pipelines-container-builder" - # -- Container builder service account annotations - annotations: {} - # Pipeline Runner Service Account pipelineRunner: - # -- Create pipeline runner service account create: true - # -- Pipeline runner service account name name: "pipeline-runner" - # -- Pipeline runner service account annotations - annotations: {} - # Viewer Service Account viewer: - # -- Create viewer service account create: true - # -- Viewer service account name name: "kubeflow-pipelines-viewer" - # -- Viewer service account annotations - annotations: {} -# Profile Controller Configuration (Multi-User) -profileController: - # -- Enable Profile Controller (only for multi-user mode) - enabled: false - # -- Number of Profile Controller replicas - replicas: 1 - - image: - # -- Profile Controller image repository - repository: public.ecr.aws/docker/library/python - # -- Profile Controller image tag - tag: "3.12" - # -- Profile Controller image pull policy - pullPolicy: IfNotPresent - - # -- Profile Controller resource requests and limits - resources: - requests: - cpu: 100m - memory: 200Mi - limits: - cpu: 500m - memory: 1Gi - - # Service configuration - service: - # -- Service type - type: ClusterIP - # -- Service port - port: 80 - - # -- Disable Istio sidecar injection - disableIstioSidecar: "false" - - # Additional environment variables - additionalEnv: [] - - # Security context - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 0 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault +# RBAC Configuration +rbac: + create: true -# Application Configuration -application: - # -- Enable Application resource (for Google Cloud Console integration) - enabled: false +# Networking Configuration +networking: + istio: + enabled: true + gateway: "kubeflow-gateway" + host: "*" - # Application controller configuration - controller: - # -- Enable Application controller deployment (controller-manager) - enabled: false - - image: - # -- Application controller image repository - repository: gcr.io/ml-pipeline/application-crd-controller - # -- Application controller image tag - tag: "20231101" - # -- Application controller image pull policy - pullPolicy: IfNotPresent - - # -- Application controller resource requests and limits - resources: - requests: - cpu: 100m - memory: 20Mi - limits: - cpu: 100m - memory: 30Mi + virtualService: + enabled: true + prefix: "/pipeline" + + networkPolicy: + enabled: true # Certificate Management certManager: # -- Enable cert-manager integration - enabled: false + enabled: true - # Mutating webhook mutating: enabled: true name: "cache-webhook-kubeflow" - # Validating webhook validating: enabled: true name: "cache-webhook-kubeflow" +# Webhook Configuration (for k8s-native mode) +webhooks: + # -- Enable admission webhooks + enabled: false + # -- Webhook failure policy + failurePolicy: "Fail" -# Node selector, tolerations, and affinity -nodeSelector: {} -tolerations: [] -affinity: {} - -# Pod annotations +# Pod annotations (applied to all pods) podAnnotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" diff --git a/tests/helm_kustomize_compare.sh b/tests/helm_kustomize_compare.sh index 02d96f330d..d75509bdc1 100755 --- a/tests/helm_kustomize_compare.sh +++ b/tests/helm_kustomize_compare.sh @@ -163,45 +163,18 @@ case "$COMPONENT" in MANIFESTS_DIR="$ROOT_DIR/applications/pipeline/upstream" declare -A KUSTOMIZE_PATHS=( - ["platform-agnostic"]="$MANIFESTS_DIR/env/platform-agnostic" - ["multi-user"]="$MANIFESTS_DIR/base/installs/multi-user" - ["generic"]="$MANIFESTS_DIR/base/installs/generic" - ["dev"]="$MANIFESTS_DIR/env/dev" - ["plain"]="$MANIFESTS_DIR/env/plain" - ["plain-multi-user"]="$MANIFESTS_DIR/env/plain-multi-user" - ["platform-agnostic-emissary"]="$MANIFESTS_DIR/env/platform-agnostic-emissary" - ["platform-agnostic-multi-user"]="$MANIFESTS_DIR/env/platform-agnostic-multi-user" - ["platform-agnostic-multi-user-emissary"]="$MANIFESTS_DIR/env/platform-agnostic-multi-user-emissary" - ["platform-agnostic-multi-user-legacy"]="$MANIFESTS_DIR/env/platform-agnostic-multi-user-legacy" - ["platform-agnostic-postgresql"]="$MANIFESTS_DIR/env/platform-agnostic-postgresql" + ["platform-agnostic-multi-user"]="$MANIFESTS_DIR/env/cert-manager/platform-agnostic-multi-user" + ["platform-agnostic-multi-user-k8s-native"]="$MANIFESTS_DIR/env/cert-manager/platform-agnostic-multi-user-k8s-native" ) declare -A HELM_VALUES=( - ["platform-agnostic"]="$CHART_DIR/ci/values-platform-agnostic-enhanced.yaml" - ["multi-user"]="$CHART_DIR/ci/values-multi-user-enhanced.yaml" - ["generic"]="$CHART_DIR/ci/values-generic.yaml" - ["dev"]="$CHART_DIR/ci/values-dev-enhanced.yaml" - ["plain"]="$CHART_DIR/ci/values-standalone.yaml" - ["plain-multi-user"]="$CHART_DIR/ci/values-multi-user.yaml" - ["platform-agnostic-emissary"]="$CHART_DIR/ci/values-platform-agnostic-enhanced.yaml" - ["platform-agnostic-multi-user"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" - ["platform-agnostic-multi-user-emissary"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-enhanced.yaml" - ["platform-agnostic-multi-user-legacy"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-legacy.yaml" - ["platform-agnostic-postgresql"]="$CHART_DIR/ci/values-postgresql.yaml" + ["platform-agnostic-multi-user"]="$CHART_DIR/ci/values-platform-agnostic-multi-user.yaml" + ["platform-agnostic-multi-user-k8s-native"]="$CHART_DIR/ci/values-platform-agnostic-multi-user-k8s-native.yaml" ) declare -A NAMESPACES=( - ["platform-agnostic"]="kubeflow" - ["multi-user"]="kubeflow" - ["generic"]="kubeflow" - ["dev"]="kubeflow" - ["plain"]="kubeflow" - ["plain-multi-user"]="kubeflow" - ["platform-agnostic-emissary"]="kubeflow" ["platform-agnostic-multi-user"]="kubeflow" - ["platform-agnostic-multi-user-emissary"]="kubeflow" - ["platform-agnostic-multi-user-legacy"]="kubeflow" - ["platform-agnostic-postgresql"]="kubeflow" + ["platform-agnostic-multi-user-k8s-native"]="kubeflow" ) ;; diff --git a/tests/helm_kustomize_compare_all.sh b/tests/helm_kustomize_compare_all.sh index de7d5d388d..4f0f0fd59a 100755 --- a/tests/helm_kustomize_compare_all.sh +++ b/tests/helm_kustomize_compare_all.sh @@ -13,7 +13,7 @@ declare -A COMPONENT_SCENARIOS=( ["model-registry"]="base overlay-postgres overlay-db controller-manager controller-rbac controller-default controller-prometheus controller-network-policy ui-base ui-standalone ui-integrated ui-istio istio csi" ["kserve-models-web-app"]="base kubeflow" ["notebook-controller"]="base kubeflow standalone" - ["pipelines"]="platform-agnostic multi-user generic dev plain plain-multi-user platform-agnostic-emissary platform-agnostic-multi-user platform-agnostic-multi-user-emissary platform-agnostic-multi-user-legacy platform-agnostic-postgresql" + ["pipelines"]="platform-agnostic-multi-user platform-agnostic-multi-user-k8s-native" ) test_component() { From 51a960b8925df860a906faf81d54aca975d5ec63 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Mon, 1 Dec 2025 23:29:07 +0530 Subject: [PATCH 20/26] update pod security Signed-off-by: kunal-511 --- .../pipelines/templates/third-party/seaweedfs/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/deployment.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/deployment.yaml index df4e61b202..592cffda6c 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/deployment.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/deployment.yaml @@ -26,6 +26,8 @@ spec: securityContext: seccompProfile: type: RuntimeDefault + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" containers: - name: seaweedfs envFrom: From 821913199552fe751168ce7914314224f2a58bdc Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sat, 6 Dec 2025 00:49:07 +0530 Subject: [PATCH 21/26] Update katib helm charts with v0.19.0 Signed-off-by: kunal-511 --- experimental/helm/charts/katib/Chart.yaml | 4 +- .../charts/katib/ci/values-cert-manager.yaml | 33 ++++----- .../charts/katib/ci/values-external-db.yaml | 67 ++++++++++++++++++- .../helm/charts/katib/ci/values-kubeflow.yaml | 35 +++++----- .../katib/ci/values-leader-election.yaml | 32 ++++----- .../charts/katib/ci/values-openshift.yaml | 32 ++++----- .../helm/charts/katib/ci/values-postgres.yaml | 67 ++++++++++++++++++- .../charts/katib/ci/values-production.yaml | 2 +- .../charts/katib/ci/values-standalone.yaml | 2 +- .../katib/templates/controller/rbac.yaml | 10 ++- experimental/helm/charts/katib/values.yaml | 47 ++++++------- experimental/helm/charts/pipelines/Chart.yaml | 4 +- ...atform-agnostic-multi-user-k8s-native.yaml | 1 + .../values-platform-agnostic-multi-user.yaml | 1 + tests/helm_kustomize_compare.py | 21 +++--- 15 files changed, 248 insertions(+), 110 deletions(-) diff --git a/experimental/helm/charts/katib/Chart.yaml b/experimental/helm/charts/katib/Chart.yaml index 3646b219cf..c58101c11c 100644 --- a/experimental/helm/charts/katib/Chart.yaml +++ b/experimental/helm/charts/katib/Chart.yaml @@ -4,9 +4,9 @@ name: katib description: A Helm chart for Katib - AutoML on Kubernetes -version: 0.16.0 +version: 0.19.0 -appVersion: 0.16.0 +appVersion: 0.19.0 keywords: - automl diff --git a/experimental/helm/charts/katib/ci/values-cert-manager.yaml b/experimental/helm/charts/katib/ci/values-cert-manager.yaml index 34d7754920..a2e1728195 100644 --- a/experimental/helm/charts/katib/ci/values-cert-manager.yaml +++ b/experimental/helm/charts/katib/ci/values-cert-manager.yaml @@ -3,7 +3,7 @@ global: namespace: kubeflow - imageTag: v0.18.0 + imageTag: v0.19.0 # Enable kustomize compatibility mode kustomizeMode: @@ -77,6 +77,7 @@ config: controller: webhookPort: 8443 trialResources: + - TrainJob.v1alpha1.trainer.kubeflow.org - Job.v1.batch - TFJob.v1.kubeflow.org - PyTorchJob.v1.kubeflow.org @@ -85,40 +86,40 @@ config: runtime: metricsCollectors: - kind: StdOut - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: File - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: TensorFlowEvent - image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.19.0 resources: limits: memory: 1Gi suggestions: - algorithmName: random - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: tpe - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: grid - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: hyperband - image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.19.0 - algorithmName: bayesianoptimization - image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.19.0 - algorithmName: cmaes - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: sobol - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: multivariate-tpe - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: enas - image: ghcr.io/kubeflow/katib/suggestion-enas:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-enas:v0.19.0 resources: limits: memory: 400Mi - algorithmName: darts - image: ghcr.io/kubeflow/katib/suggestion-darts:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-darts:v0.19.0 - algorithmName: pbt - image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.19.0 persistentVolumeClaimSpec: accessModes: - ReadWriteMany @@ -127,7 +128,7 @@ config: storage: 5Gi earlyStoppings: - algorithmName: medianstop - image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.18.0 + image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.19.0 # RBAC rbac: diff --git a/experimental/helm/charts/katib/ci/values-external-db.yaml b/experimental/helm/charts/katib/ci/values-external-db.yaml index b3542cfae9..477a153c00 100644 --- a/experimental/helm/charts/katib/ci/values-external-db.yaml +++ b/experimental/helm/charts/katib/ci/values-external-db.yaml @@ -3,7 +3,7 @@ global: namespace: kubeflow - imageTag: v0.18.0 + imageTag: v0.19.0 # Enable kustomize compatibility mode kustomizeMode: @@ -57,4 +57,67 @@ namespaceCreate: rbac: create: true kubeflow: - enabled: false \ No newline at end of file + enabled: false + +# Katib configuration +config: + katibConfig: | + apiVersion: config.kubeflow.org/v1beta1 + kind: KatibConfig + init: + certGenerator: + enable: true + controller: + webhookPort: 8443 + trialResources: + - Job.v1.batch + - TFJob.v1.kubeflow.org + - PyTorchJob.v1.kubeflow.org + - MPIJob.v1.kubeflow.org + - XGBoostJob.v1.kubeflow.org + runtime: + metricsCollectors: + - kind: StdOut + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 + - kind: File + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 + - kind: TensorFlowEvent + image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.19.0 + resources: + limits: + memory: 1Gi + suggestions: + - algorithmName: random + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 + - algorithmName: tpe + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 + - algorithmName: grid + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 + - algorithmName: hyperband + image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.19.0 + - algorithmName: bayesianoptimization + image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.19.0 + - algorithmName: cmaes + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 + - algorithmName: sobol + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 + - algorithmName: multivariate-tpe + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 + - algorithmName: enas + image: ghcr.io/kubeflow/katib/suggestion-enas:v0.19.0 + resources: + limits: + memory: 400Mi + - algorithmName: darts + image: ghcr.io/kubeflow/katib/suggestion-darts:v0.19.0 + - algorithmName: pbt + image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.19.0 + persistentVolumeClaimSpec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 5Gi + earlyStoppings: + - algorithmName: medianstop + image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.19.0 \ No newline at end of file diff --git a/experimental/helm/charts/katib/ci/values-kubeflow.yaml b/experimental/helm/charts/katib/ci/values-kubeflow.yaml index c244d3f32c..69c214b07e 100644 --- a/experimental/helm/charts/katib/ci/values-kubeflow.yaml +++ b/experimental/helm/charts/katib/ci/values-kubeflow.yaml @@ -3,7 +3,7 @@ global: namespace: kubeflow - imageTag: v0.18.0 + imageTag: v0.19.0 # Enable kustomize compatibility mode kustomizeMode: @@ -111,6 +111,7 @@ config: controller: webhookPort: 8443 trialResources: + - TrainJob.v1alpha1.trainer.kubeflow.org - Job.v1.batch - TFJob.v1.kubeflow.org - PyTorchJob.v1.kubeflow.org @@ -119,40 +120,40 @@ config: runtime: metricsCollectors: - kind: StdOut - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: File - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: TensorFlowEvent - image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.18.0 - resources: + image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.19.0 + resources: limits: memory: 1Gi suggestions: - algorithmName: random - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: tpe - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: grid - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: hyperband - image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.19.0 - algorithmName: bayesianoptimization - image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.19.0 - algorithmName: cmaes - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: sobol - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: multivariate-tpe - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: enas - image: ghcr.io/kubeflow/katib/suggestion-enas:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-enas:v0.19.0 resources: limits: memory: 400Mi - algorithmName: darts - image: ghcr.io/kubeflow/katib/suggestion-darts:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-darts:v0.19.0 - algorithmName: pbt - image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.19.0 persistentVolumeClaimSpec: accessModes: - ReadWriteMany @@ -161,7 +162,7 @@ config: storage: 5Gi earlyStoppings: - algorithmName: medianstop - image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.18.0 + image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.19.0 # Security features - disabled for CI simplicity networkPolicy: diff --git a/experimental/helm/charts/katib/ci/values-leader-election.yaml b/experimental/helm/charts/katib/ci/values-leader-election.yaml index 1c813d1169..86a546f0a4 100644 --- a/experimental/helm/charts/katib/ci/values-leader-election.yaml +++ b/experimental/helm/charts/katib/ci/values-leader-election.yaml @@ -3,7 +3,7 @@ global: namespace: kubeflow - imageTag: v0.18.0 + imageTag: v0.19.0 # Enable kustomize compatibility mode kustomizeMode: @@ -110,40 +110,40 @@ config: runtime: metricsCollectors: - kind: StdOut - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: File - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: TensorFlowEvent - image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.19.0 resources: limits: memory: 1Gi suggestions: - algorithmName: random - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: tpe - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: grid - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: hyperband - image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.19.0 - algorithmName: bayesianoptimization - image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.19.0 - algorithmName: cmaes - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: sobol - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: multivariate-tpe - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: enas - image: ghcr.io/kubeflow/katib/suggestion-enas:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-enas:v0.19.0 resources: limits: memory: 400Mi - algorithmName: darts - image: ghcr.io/kubeflow/katib/suggestion-darts:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-darts:v0.19.0 - algorithmName: pbt - image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.19.0 persistentVolumeClaimSpec: accessModes: - ReadWriteMany @@ -152,4 +152,4 @@ config: storage: 5Gi earlyStoppings: - algorithmName: medianstop - image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.18.0 \ No newline at end of file + image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.19.0 \ No newline at end of file diff --git a/experimental/helm/charts/katib/ci/values-openshift.yaml b/experimental/helm/charts/katib/ci/values-openshift.yaml index 6509c4d606..56714d2658 100644 --- a/experimental/helm/charts/katib/ci/values-openshift.yaml +++ b/experimental/helm/charts/katib/ci/values-openshift.yaml @@ -3,7 +3,7 @@ global: namespace: kubeflow - imageTag: v0.18.0 + imageTag: v0.19.0 # Enable kustomize compatibility mode kustomizeMode: @@ -82,40 +82,40 @@ config: runtime: metricsCollectors: - kind: StdOut - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: File - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: TensorFlowEvent - image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.19.0 resources: limits: memory: 1Gi suggestions: - algorithmName: random - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: tpe - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: grid - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: hyperband - image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.19.0 - algorithmName: bayesianoptimization - image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.19.0 - algorithmName: cmaes - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: sobol - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: multivariate-tpe - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: enas - image: ghcr.io/kubeflow/katib/suggestion-enas:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-enas:v0.19.0 resources: limits: memory: 400Mi - algorithmName: darts - image: ghcr.io/kubeflow/katib/suggestion-darts:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-darts:v0.19.0 - algorithmName: pbt - image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.19.0 persistentVolumeClaimSpec: accessModes: - ReadWriteMany @@ -124,7 +124,7 @@ config: storage: 5Gi earlyStoppings: - algorithmName: medianstop - image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.18.0 + image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.19.0 # Namespace creation with security labels namespaceCreate: diff --git a/experimental/helm/charts/katib/ci/values-postgres.yaml b/experimental/helm/charts/katib/ci/values-postgres.yaml index 8432682521..3e1cb687a8 100644 --- a/experimental/helm/charts/katib/ci/values-postgres.yaml +++ b/experimental/helm/charts/katib/ci/values-postgres.yaml @@ -3,7 +3,7 @@ global: namespace: kubeflow - imageTag: v0.18.0 + imageTag: v0.19.0 # Enable kustomize compatibility mode kustomizeMode: @@ -95,4 +95,67 @@ autoscaling: # Namespace creation namespaceCreate: - enabled: true \ No newline at end of file + enabled: true + +# Katib configuration +config: + katibConfig: | + apiVersion: config.kubeflow.org/v1beta1 + kind: KatibConfig + init: + certGenerator: + enable: true + controller: + webhookPort: 8443 + trialResources: + - Job.v1.batch + - TFJob.v1.kubeflow.org + - PyTorchJob.v1.kubeflow.org + - MPIJob.v1.kubeflow.org + - XGBoostJob.v1.kubeflow.org + runtime: + metricsCollectors: + - kind: StdOut + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 + - kind: File + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 + - kind: TensorFlowEvent + image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.19.0 + resources: + limits: + memory: 1Gi + suggestions: + - algorithmName: random + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 + - algorithmName: tpe + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 + - algorithmName: grid + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 + - algorithmName: hyperband + image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.19.0 + - algorithmName: bayesianoptimization + image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.19.0 + - algorithmName: cmaes + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 + - algorithmName: sobol + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 + - algorithmName: multivariate-tpe + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 + - algorithmName: enas + image: ghcr.io/kubeflow/katib/suggestion-enas:v0.19.0 + resources: + limits: + memory: 400Mi + - algorithmName: darts + image: ghcr.io/kubeflow/katib/suggestion-darts:v0.19.0 + - algorithmName: pbt + image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.19.0 + persistentVolumeClaimSpec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 5Gi + earlyStoppings: + - algorithmName: medianstop + image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.19.0 \ No newline at end of file diff --git a/experimental/helm/charts/katib/ci/values-production.yaml b/experimental/helm/charts/katib/ci/values-production.yaml index 11ba9f0cb3..fdd066404a 100644 --- a/experimental/helm/charts/katib/ci/values-production.yaml +++ b/experimental/helm/charts/katib/ci/values-production.yaml @@ -3,7 +3,7 @@ global: namespace: kubeflow - imageTag: v0.18.0 + imageTag: v0.19.0 # Controller configuration - with leader election for HA controller: diff --git a/experimental/helm/charts/katib/ci/values-standalone.yaml b/experimental/helm/charts/katib/ci/values-standalone.yaml index e366d526f5..c88f96941b 100644 --- a/experimental/helm/charts/katib/ci/values-standalone.yaml +++ b/experimental/helm/charts/katib/ci/values-standalone.yaml @@ -3,7 +3,7 @@ global: namespace: kubeflow - imageTag: v0.18.0 + imageTag: v0.19.0 # Enable kustomize compatibility mode kustomizeMode: diff --git a/experimental/helm/charts/katib/templates/controller/rbac.yaml b/experimental/helm/charts/katib/templates/controller/rbac.yaml index 3fbd311473..54415caee7 100644 --- a/experimental/helm/charts/katib/templates/controller/rbac.yaml +++ b/experimental/helm/charts/katib/templates/controller/rbac.yaml @@ -92,15 +92,23 @@ rules: - "watch" - "create" - "delete" +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - "get" + - "list" + - "watch" - apiGroups: - trainer.kubeflow.org resources: - trainjobs verbs: + - "create" - "get" - "list" - "watch" - - "create" - "delete" - apiGroups: - kubeflow.org diff --git a/experimental/helm/charts/katib/values.yaml b/experimental/helm/charts/katib/values.yaml index bde3bc6ecc..6b38ec3f29 100644 --- a/experimental/helm/charts/katib/values.yaml +++ b/experimental/helm/charts/katib/values.yaml @@ -16,7 +16,7 @@ global: # -- Image registry for Katib images imageRegistry: ghcr.io/kubeflow/katib # -- Global image tag for all Katib components - imageTag: v0.18.0 + imageTag: v0.19.0 # -- Global image pull policy imagePullPolicy: IfNotPresent # -- Global image pull secrets @@ -536,16 +536,16 @@ ui: # -- Affinity rules for UI pods affinity: {} - # -- Security context for UI pods - podSecurityContext: - seccompProfile: - type: RuntimeDefault + # -- Security context for UI pods + podSecurityContext: {} # -- Security context for UI containers securityContext: runAsNonRoot: true allowPrivilegeEscalation: false runAsUser: 1000 + seccompProfile: + type: RuntimeDefault capabilities: drop: - ALL @@ -771,6 +771,7 @@ config: controller: webhookPort: 8443 trialResources: + - TrainJob.v1alpha1.trainer.kubeflow.org - Job.v1.batch - TFJob.v1.kubeflow.org - PyTorchJob.v1.kubeflow.org @@ -779,40 +780,40 @@ config: runtime: metricsCollectors: - kind: StdOut - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: File - image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0 - kind: TensorFlowEvent - image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.18.0 + image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.19.0 resources: limits: memory: 1Gi suggestions: - algorithmName: random - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: tpe - image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0 - algorithmName: grid - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: hyperband - image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.19.0 - algorithmName: bayesianoptimization - image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.19.0 - algorithmName: cmaes - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: sobol - image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0 - algorithmName: multivariate-tpe - image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0 - algorithmName: enas - image: ghcr.io/kubeflow/katib/suggestion-enas:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-enas:v0.19.0 resources: limits: memory: 400Mi - algorithmName: darts - image: ghcr.io/kubeflow/katib/suggestion-darts:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-darts:v0.19.0 - algorithmName: pbt - image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.18.0 + image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.19.0 persistentVolumeClaimSpec: accessModes: - ReadWriteMany @@ -821,24 +822,24 @@ config: storage: 5Gi earlyStoppings: - algorithmName: medianstop - image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.18.0 + image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.19.0 # Trial templates configuration trialTemplates: # Default trial template configuration defaultTrialTemplate: # -- Default trial template container image - image: "ghcr.io/kubeflow/katib/pytorch-mnist-cpu:v0.18.0" + image: "ghcr.io/kubeflow/katib/pytorch-mnist-cpu:v0.19.0" # ENAS CPU template configuration enasCPUTemplate: # -- ENAS CPU template container image - image: "ghcr.io/kubeflow/katib/enas-cnn-cifar10-cpu:v0.18.0" + image: "ghcr.io/kubeflow/katib/enas-cnn-cifar10-cpu:v0.19.0" # PyTorch Job template configuration pytorchJobTemplate: # -- PyTorch Job template container image - image: "ghcr.io/kubeflow/katib/pytorch-mnist-cpu:v0.18.0" + image: "ghcr.io/kubeflow/katib/pytorch-mnist-cpu:v0.19.0" # -- Number of worker replicas for PyTorch Job workerReplicas: 2 diff --git a/experimental/helm/charts/pipelines/Chart.yaml b/experimental/helm/charts/pipelines/Chart.yaml index 0ac0e3f0ed..60671a60b2 100644 --- a/experimental/helm/charts/pipelines/Chart.yaml +++ b/experimental/helm/charts/pipelines/Chart.yaml @@ -4,9 +4,9 @@ name: kubeflow-pipelines description: A Helm chart for Kubeflow Pipelines - ML Workflows on Kubernetes -version: 2.14.3 +version: 2.15.0 -appVersion: 2.14.3 +appVersion: 2.15.0 home: https://github.com/kubeflow/pipelines diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-k8s-native.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-k8s-native.yaml index 13d20607f0..77be03bcb8 100644 --- a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-k8s-native.yaml +++ b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user-k8s-native.yaml @@ -43,6 +43,7 @@ apiServer: # CRDs crds: install: true + # webhook should match pipelineDefinition.storage: false for database mode, true for k8s-native mode webhook: true # Enable cert-manager integration diff --git a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user.yaml b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user.yaml index cebb3f754c..1800dd9e23 100644 --- a/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user.yaml +++ b/experimental/helm/charts/pipelines/ci/values-platform-agnostic-multi-user.yaml @@ -16,6 +16,7 @@ installMode: crds: install: true application: false + # webhook should match pipelineDefinition.storage: false for database mode, true for k8s-native mode webhook: false # Enable cert-manager integration diff --git a/tests/helm_kustomize_compare.py b/tests/helm_kustomize_compare.py index cf5b323425..b4fcead60f 100755 --- a/tests/helm_kustomize_compare.py +++ b/tests/helm_kustomize_compare.py @@ -138,6 +138,8 @@ def normalize_manifest(manifest: Dict, component: str = "katib") -> Dict: # Remove leading --- if present (common in ConfigMap YAML data) if normalized_value.startswith('---'): normalized_value = normalized_value[3:].strip() + lines = normalized_value.split('\n') + normalized_value = '\n'.join(line.rstrip() for line in lines) normalized_data[key] = normalized_value else: normalized_data[key] = value @@ -361,17 +363,15 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena if only_in_kustomize: print(f"Resources only in Kustomize: {len(only_in_kustomize)}") - if verbose: - for resource in sorted(only_in_kustomize): - print(f" - {resource}") + for resource in sorted(only_in_kustomize): + print(f" - {resource}") success = False differences_found.extend(only_in_kustomize) if unexpected_helm_extras: print(f"Unexpected resources only in Helm: {len(unexpected_helm_extras)}") - if verbose: - for resource in sorted(unexpected_helm_extras): - print(f" - {resource}") + for resource in sorted(unexpected_helm_extras): + print(f" - {resource}") success = False differences_found.extend(unexpected_helm_extras) @@ -384,11 +384,10 @@ def compare_manifests(kustomize_file: str, helm_file: str, component: str, scena if differences: print(f"Differences in {key}: {len(differences)} fields") - if verbose: - print(f" Detailed differences for {key}:") - for diff in differences: - print(f" {diff}") - print() + print(f" Detailed differences for {key}:") + for diff in differences: + print(f" {diff}") + print() differences_found.append(key) success = False From 1c5dae4a8ced82a910669df29a220ad26326285a Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sat, 6 Dec 2025 00:54:38 +0530 Subject: [PATCH 22/26] Update model registry helm charts for v0.3.3 Signed-off-by: kunal-511 --- experimental/helm/charts/model-registry/ci/ci-values.yaml | 2 +- experimental/helm/charts/model-registry/ci/values-db.yaml | 2 +- .../helm/charts/model-registry/ci/values-postgres.yaml | 2 +- .../helm/charts/model-registry/ci/values-ui-integrated.yaml | 2 +- .../helm/charts/model-registry/ci/values-ui-istio.yaml | 4 ++-- .../helm/charts/model-registry/ci/values-ui-standalone.yaml | 2 +- experimental/helm/charts/model-registry/ci/values-ui.yaml | 2 +- experimental/helm/charts/model-registry/values.yaml | 4 ++-- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/experimental/helm/charts/model-registry/ci/ci-values.yaml b/experimental/helm/charts/model-registry/ci/ci-values.yaml index 352e35ff12..cedd00b594 100644 --- a/experimental/helm/charts/model-registry/ci/ci-values.yaml +++ b/experimental/helm/charts/model-registry/ci/ci-values.yaml @@ -12,7 +12,7 @@ server: dataStoreType: embedmd image: - tag: "v0.3.0" + tag: "v0.3.3" # Configure readiness probe rest: diff --git a/experimental/helm/charts/model-registry/ci/values-db.yaml b/experimental/helm/charts/model-registry/ci/values-db.yaml index d5098a5e99..96d2eb3b00 100644 --- a/experimental/helm/charts/model-registry/ci/values-db.yaml +++ b/experimental/helm/charts/model-registry/ci/values-db.yaml @@ -10,7 +10,7 @@ server: replicas: 1 dataStoreType: embedmd image: - tag: "v0.3.0" + tag: "v0.3.3" resources: limits: cpu: 200m diff --git a/experimental/helm/charts/model-registry/ci/values-postgres.yaml b/experimental/helm/charts/model-registry/ci/values-postgres.yaml index 5b61c2ce98..88e890bdd8 100644 --- a/experimental/helm/charts/model-registry/ci/values-postgres.yaml +++ b/experimental/helm/charts/model-registry/ci/values-postgres.yaml @@ -10,7 +10,7 @@ server: replicas: 1 dataStoreType: embedmd image: - tag: "v0.3.0" + tag: "v0.3.3" resources: limits: cpu: 200m diff --git a/experimental/helm/charts/model-registry/ci/values-ui-integrated.yaml b/experimental/helm/charts/model-registry/ci/values-ui-integrated.yaml index 3da56db4c9..5dcf2b22f0 100644 --- a/experimental/helm/charts/model-registry/ci/values-ui-integrated.yaml +++ b/experimental/helm/charts/model-registry/ci/values-ui-integrated.yaml @@ -5,7 +5,7 @@ ui: image: repository: ui - tag: "v0.3.0" + tag: "v0.3.3" pullPolicy: Always containerPort: 8080 diff --git a/experimental/helm/charts/model-registry/ci/values-ui-istio.yaml b/experimental/helm/charts/model-registry/ci/values-ui-istio.yaml index 7d423d94c7..125c5e2272 100644 --- a/experimental/helm/charts/model-registry/ci/values-ui-istio.yaml +++ b/experimental/helm/charts/model-registry/ci/values-ui-istio.yaml @@ -8,8 +8,8 @@ ui: replicas: 1 image: - repository: ui - tag: "v0.3.0" + repository: ui + tag: "v0.3.3" pullPolicy: Always containerPort: 8080 diff --git a/experimental/helm/charts/model-registry/ci/values-ui-standalone.yaml b/experimental/helm/charts/model-registry/ci/values-ui-standalone.yaml index 6c97c0f4ac..8e2a42e75b 100644 --- a/experimental/helm/charts/model-registry/ci/values-ui-standalone.yaml +++ b/experimental/helm/charts/model-registry/ci/values-ui-standalone.yaml @@ -9,7 +9,7 @@ ui: image: repository: ui - tag: "v0.3.0" + tag: "v0.3.3" pullPolicy: Always containerPort: 8080 diff --git a/experimental/helm/charts/model-registry/ci/values-ui.yaml b/experimental/helm/charts/model-registry/ci/values-ui.yaml index 98dac486e9..660161436f 100644 --- a/experimental/helm/charts/model-registry/ci/values-ui.yaml +++ b/experimental/helm/charts/model-registry/ci/values-ui.yaml @@ -5,7 +5,7 @@ ui: image: repository: ui - tag: "v0.3.0" + tag: "v0.3.3" pullPolicy: Always containerPort: 8080 diff --git a/experimental/helm/charts/model-registry/values.yaml b/experimental/helm/charts/model-registry/values.yaml index 3281ef6ff8..64cab7973b 100644 --- a/experimental/helm/charts/model-registry/values.yaml +++ b/experimental/helm/charts/model-registry/values.yaml @@ -42,7 +42,7 @@ server: # -- Server image repository repository: server # -- Server image tag (overrides global.imageTag if set) - tag: "v0.3.0" + tag: "v0.3.3" # -- Server image pull policy (overrides global.imagePullPolicy if set) pullPolicy: "" @@ -178,7 +178,7 @@ ui: # -- UI image repository repository: model-registry-ui # -- UI image tag - tag: "v0.3.0" + tag: "v0.3.3" # -- UI image pull policy pullPolicy: "" From 8e02defdc04760de3bb8dc43f6961d6b52b6df5e Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Tue, 6 Jan 2026 16:02:12 +0530 Subject: [PATCH 23/26] Remove configurable names Signed-off-by: kunal-511 --- .../templates/api-server/rolebinding.yaml | 2 +- .../pipelines/templates/cache/rolebinding.yaml | 2 +- .../templates/metadata-writer/rolebinding.yaml | 2 +- .../templates/persistence-agent/rolebinding.yaml | 2 +- .../templates/rbac/additional-serviceaccounts.yaml | 6 +++--- .../templates/rbac/cluster-role-bindings.yaml | 14 +++++++------- .../rbac/pipeline-runner-rolebinding.yaml | 2 +- .../templates/scheduled-workflow/rolebinding.yaml | 2 +- .../charts/pipelines/templates/ui/rolebinding.yaml | 2 +- .../templates/viewer-crd/rolebinding.yaml | 2 +- experimental/helm/charts/pipelines/values.yaml | 12 ------------ 11 files changed, 18 insertions(+), 30 deletions(-) diff --git a/experimental/helm/charts/pipelines/templates/api-server/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/api-server/rolebinding.yaml index 0c0d58531f..e78d816707 100644 --- a/experimental/helm/charts/pipelines/templates/api-server/rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/api-server/rolebinding.yaml @@ -14,6 +14,6 @@ roleRef: name: ml-pipeline subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml index 3ecbddb453..83851724e5 100644 --- a/experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/cache/rolebinding.yaml @@ -12,6 +12,6 @@ roleRef: name: kubeflow-pipelines-cache-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + name: kubeflow-pipelines-cache namespace: {{ include "kubeflow-pipelines.namespace" . }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/metadata-writer/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/metadata-writer/rolebinding.yaml index 5e3689b3b5..0c3d645c35 100644 --- a/experimental/helm/charts/pipelines/templates/metadata-writer/rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/metadata-writer/rolebinding.yaml @@ -14,6 +14,6 @@ roleRef: name: kubeflow-pipelines-metadata-writer-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} + name: kubeflow-pipelines-metadata-writer namespace: {{ include "kubeflow-pipelines.namespace" . }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/persistence-agent/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/persistence-agent/rolebinding.yaml index c884606168..cea8ab5541 100644 --- a/experimental/helm/charts/pipelines/templates/persistence-agent/rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/persistence-agent/rolebinding.yaml @@ -14,6 +14,6 @@ roleRef: name: ml-pipeline-persistenceagent-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} + name: ml-pipeline-persistenceagent namespace: {{ include "kubeflow-pipelines.namespace" . }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/rbac/additional-serviceaccounts.yaml b/experimental/helm/charts/pipelines/templates/rbac/additional-serviceaccounts.yaml index 4b348811fa..8f45434f42 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/additional-serviceaccounts.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/additional-serviceaccounts.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ .Values.additionalServiceAccounts.containerBuilder.name }} + name: kubeflow-pipelines-container-builder namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} @@ -19,7 +19,7 @@ metadata: apiVersion: v1 kind: ServiceAccount metadata: - name: {{ .Values.additionalServiceAccounts.pipelineRunner.name }} + name: pipeline-runner namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} @@ -34,7 +34,7 @@ metadata: apiVersion: v1 kind: ServiceAccount metadata: - name: {{ .Values.additionalServiceAccounts.viewer.name }} + name: kubeflow-pipelines-viewer namespace: {{ include "kubeflow-pipelines.namespace" . }} labels: {{- include "kubeflow-pipelines.labels" . | nindent 4 }} diff --git a/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml b/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml index f78edf4c2b..15382e9934 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/cluster-role-bindings.yaml @@ -12,7 +12,7 @@ roleRef: name: ml-pipeline subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.apiServer.serviceAccountName" . }} + name: ml-pipeline namespace: {{ include "kubeflow-pipelines.namespace" . }} --- @@ -30,7 +30,7 @@ roleRef: name: ml-pipeline-ui subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.ui.serviceAccountName" . }} + name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} --- @@ -47,7 +47,7 @@ roleRef: name: ml-pipeline-scheduledworkflow-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} + name: ml-pipeline-scheduledworkflow namespace: {{ include "kubeflow-pipelines.namespace" . }} --- @@ -64,7 +64,7 @@ roleRef: name: ml-pipeline-persistenceagent-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.persistenceAgent.serviceAccountName" . }} + name: ml-pipeline-persistenceagent namespace: {{ include "kubeflow-pipelines.namespace" . }} --- @@ -81,7 +81,7 @@ roleRef: name: ml-pipeline-viewer-controller-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} + name: ml-pipeline-viewer-crd-service-account namespace: {{ include "kubeflow-pipelines.namespace" . }} --- @@ -99,7 +99,7 @@ roleRef: name: kubeflow-pipelines-cache-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.cache.serviceAccountName" . }} + name: kubeflow-pipelines-cache namespace: {{ include "kubeflow-pipelines.namespace" . }} --- @@ -116,7 +116,7 @@ roleRef: name: kubeflow-pipelines-metadata-writer-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.metadataWriter.serviceAccountName" . }} + name: kubeflow-pipelines-metadata-writer namespace: {{ include "kubeflow-pipelines.namespace" . }} {{- if .Values.thirdParty.argo.enabled }} diff --git a/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-rolebinding.yaml b/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-rolebinding.yaml index 8f7c1a48b1..ade0e8030a 100644 --- a/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/rbac/pipeline-runner-rolebinding.yaml @@ -14,6 +14,6 @@ roleRef: name: pipeline-runner subjects: - kind: ServiceAccount - name: {{ .Values.additionalServiceAccounts.pipelineRunner.name }} + name: pipeline-runner namespace: {{ include "kubeflow-pipelines.namespace" . }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/scheduled-workflow/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/scheduled-workflow/rolebinding.yaml index 7a3af29d4b..9b437f0612 100644 --- a/experimental/helm/charts/pipelines/templates/scheduled-workflow/rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/scheduled-workflow/rolebinding.yaml @@ -14,6 +14,6 @@ roleRef: name: ml-pipeline-scheduledworkflow-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.scheduledWorkflow.serviceAccountName" . }} + name: ml-pipeline-scheduledworkflow namespace: {{ include "kubeflow-pipelines.namespace" . }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/ui/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/ui/rolebinding.yaml index a53196a948..a139fc2fd5 100644 --- a/experimental/helm/charts/pipelines/templates/ui/rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/ui/rolebinding.yaml @@ -12,6 +12,6 @@ roleRef: name: ml-pipeline-ui subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.ui.serviceAccountName" . }} + name: ml-pipeline-ui namespace: {{ include "kubeflow-pipelines.namespace" . }} {{- end }} diff --git a/experimental/helm/charts/pipelines/templates/viewer-crd/rolebinding.yaml b/experimental/helm/charts/pipelines/templates/viewer-crd/rolebinding.yaml index f48104909f..6c5349461e 100644 --- a/experimental/helm/charts/pipelines/templates/viewer-crd/rolebinding.yaml +++ b/experimental/helm/charts/pipelines/templates/viewer-crd/rolebinding.yaml @@ -12,6 +12,6 @@ roleRef: name: ml-pipeline-viewer-controller-role subjects: - kind: ServiceAccount - name: {{ include "kubeflow-pipelines.viewerCrd.serviceAccountName" . }} + name: ml-pipeline-viewer-crd-service-account namespace: {{ include "kubeflow-pipelines.namespace" . }} {{- end }} diff --git a/experimental/helm/charts/pipelines/values.yaml b/experimental/helm/charts/pipelines/values.yaml index da4f7e6bf1..321ba62a6f 100644 --- a/experimental/helm/charts/pipelines/values.yaml +++ b/experimental/helm/charts/pipelines/values.yaml @@ -85,7 +85,6 @@ apiServer: serviceAccount: create: true - name: "ml-pipeline" service: type: ClusterIP @@ -182,7 +181,6 @@ persistenceAgent: serviceAccount: create: true - name: "ml-pipeline-persistenceagent" env: ttlSecondsAfterWorkflowFinish: "86400" @@ -210,7 +208,6 @@ scheduledWorkflow: serviceAccount: create: true - name: "ml-pipeline-scheduledworkflow" resources: requests: @@ -259,7 +256,6 @@ ui: serviceAccount: create: true - name: "ml-pipeline-ui" service: type: ClusterIP @@ -335,7 +331,6 @@ viewerCrd: serviceAccount: create: true - name: "ml-pipeline-viewer-crd-service-account" env: maxNumViewers: "50" @@ -378,7 +373,6 @@ visualization: serviceAccount: create: true - name: "ml-pipeline-visualizationserver" service: type: ClusterIP @@ -392,7 +386,6 @@ cache: serviceAccount: create: true - name: "kubeflow-pipelines-cache" server: replicas: 1 @@ -479,7 +472,6 @@ metadata: serviceAccount: create: true - name: "metadata-grpc-server" service: type: ClusterIP @@ -541,7 +533,6 @@ metadataWriter: serviceAccount: create: true - name: "kubeflow-pipelines-metadata-writer" additionalEnv: [] @@ -691,15 +682,12 @@ profileController: additionalServiceAccounts: containerBuilder: create: true - name: "kubeflow-pipelines-container-builder" pipelineRunner: create: true - name: "pipeline-runner" viewer: create: true - name: "kubeflow-pipelines-viewer" # RBAC Configuration rbac: From 1f843bab008300c1cd09ad64acb54ae8990ec489 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Tue, 6 Jan 2026 16:38:35 +0530 Subject: [PATCH 24/26] Update katib helm charts Signed-off-by: kunal-511 --- .../templates/database/mysql-deployment.yaml | 16 ++++------------ experimental/helm/charts/katib/values.yaml | 7 +------ 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/experimental/helm/charts/katib/templates/database/mysql-deployment.yaml b/experimental/helm/charts/katib/templates/database/mysql-deployment.yaml index a482302e95..4ac0b9caa9 100644 --- a/experimental/helm/charts/katib/templates/database/mysql-deployment.yaml +++ b/experimental/helm/charts/katib/templates/database/mysql-deployment.yaml @@ -47,27 +47,19 @@ spec: command: - "/bin/bash" - "-c" - - "mysql -D ${MYSQL_DATABASE} -u root -p${MYSQL_ROOT_PASSWORD} -e 'SELECT 1'" + - "mysql -h 127.0.0.1 -D ${MYSQL_DATABASE} -u root -p${MYSQL_ROOT_PASSWORD} -e 'SELECT 1'" initialDelaySeconds: {{ .Values.database.mysql.probes.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.database.mysql.probes.readiness.periodSeconds }} failureThreshold: {{ .Values.database.mysql.probes.readiness.failureThreshold }} livenessProbe: exec: command: - - "/bin/bash" - - "-c" - - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}" + - "/bin/bash" + - "-c" + - "mysql -h 127.0.0.1 -D ${MYSQL_DATABASE} -u root -p${MYSQL_ROOT_PASSWORD} -e 'SELECT 1'" initialDelaySeconds: {{ .Values.database.mysql.probes.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.database.mysql.probes.liveness.periodSeconds }} failureThreshold: {{ .Values.database.mysql.probes.liveness.failureThreshold }} - startupProbe: - exec: - command: - - "/bin/bash" - - "-c" - - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}" - periodSeconds: {{ .Values.database.mysql.probes.startup.periodSeconds }} - failureThreshold: {{ .Values.database.mysql.probes.startup.failureThreshold }} volumeMounts: - name: katib-mysql mountPath: /var/lib/mysql diff --git a/experimental/helm/charts/katib/values.yaml b/experimental/helm/charts/katib/values.yaml index 6b38ec3f29..9e6e15c109 100644 --- a/experimental/helm/charts/katib/values.yaml +++ b/experimental/helm/charts/katib/values.yaml @@ -186,7 +186,7 @@ database: # -- MySQL image repository repository: mysql # -- MySQL image tag - tag: "8.0.29" + tag: "8.0" # -- MySQL image pull policy pullPolicy: IfNotPresent @@ -223,11 +223,6 @@ database: periodSeconds: 5 # -- Liveness probe failure threshold failureThreshold: 10 - startup: - # -- Startup probe period - periodSeconds: 15 - # -- Startup probe failure threshold - failureThreshold: 60 # Security context podSecurityContext: From 3a4a7fe734c668a3da0dc055fd75b5c6c5867c0c Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Tue, 6 Jan 2026 16:47:13 +0530 Subject: [PATCH 25/26] Update model registry helm charts Signed-off-by: kunal-511 --- experimental/helm/charts/model-registry/ci/ci-values.yaml | 2 +- experimental/helm/charts/model-registry/ci/values-db.yaml | 2 +- .../helm/charts/model-registry/ci/values-postgres.yaml | 4 ++-- .../helm/charts/model-registry/ci/values-ui-integrated.yaml | 2 +- .../helm/charts/model-registry/ci/values-ui-istio.yaml | 2 +- .../helm/charts/model-registry/ci/values-ui-standalone.yaml | 2 +- experimental/helm/charts/model-registry/ci/values-ui.yaml | 2 +- .../model-registry/templates/database/mysql/deployment.yaml | 2 +- experimental/helm/charts/model-registry/values.yaml | 6 +++--- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/experimental/helm/charts/model-registry/ci/ci-values.yaml b/experimental/helm/charts/model-registry/ci/ci-values.yaml index cedd00b594..e152abc5c5 100644 --- a/experimental/helm/charts/model-registry/ci/ci-values.yaml +++ b/experimental/helm/charts/model-registry/ci/ci-values.yaml @@ -12,7 +12,7 @@ server: dataStoreType: embedmd image: - tag: "v0.3.3" + tag: "v0.3.4" # Configure readiness probe rest: diff --git a/experimental/helm/charts/model-registry/ci/values-db.yaml b/experimental/helm/charts/model-registry/ci/values-db.yaml index 96d2eb3b00..b49f9bb55c 100644 --- a/experimental/helm/charts/model-registry/ci/values-db.yaml +++ b/experimental/helm/charts/model-registry/ci/values-db.yaml @@ -10,7 +10,7 @@ server: replicas: 1 dataStoreType: embedmd image: - tag: "v0.3.3" + tag: "v0.3.4" resources: limits: cpu: 200m diff --git a/experimental/helm/charts/model-registry/ci/values-postgres.yaml b/experimental/helm/charts/model-registry/ci/values-postgres.yaml index 88e890bdd8..951e97a289 100644 --- a/experimental/helm/charts/model-registry/ci/values-postgres.yaml +++ b/experimental/helm/charts/model-registry/ci/values-postgres.yaml @@ -10,7 +10,7 @@ server: replicas: 1 dataStoreType: embedmd image: - tag: "v0.3.3" + tag: "v0.3.4" resources: limits: cpu: 200m @@ -39,7 +39,7 @@ database: enabled: true image: repository: postgres - tag: "14.7-alpine3.17" + tag: "16-alpine" auth: database: metadb username: root diff --git a/experimental/helm/charts/model-registry/ci/values-ui-integrated.yaml b/experimental/helm/charts/model-registry/ci/values-ui-integrated.yaml index 5dcf2b22f0..c400a1c486 100644 --- a/experimental/helm/charts/model-registry/ci/values-ui-integrated.yaml +++ b/experimental/helm/charts/model-registry/ci/values-ui-integrated.yaml @@ -5,7 +5,7 @@ ui: image: repository: ui - tag: "v0.3.3" + tag: "v0.3.4" pullPolicy: Always containerPort: 8080 diff --git a/experimental/helm/charts/model-registry/ci/values-ui-istio.yaml b/experimental/helm/charts/model-registry/ci/values-ui-istio.yaml index 125c5e2272..c50e2d84ed 100644 --- a/experimental/helm/charts/model-registry/ci/values-ui-istio.yaml +++ b/experimental/helm/charts/model-registry/ci/values-ui-istio.yaml @@ -9,7 +9,7 @@ ui: image: repository: ui - tag: "v0.3.3" + tag: "v0.3.4" pullPolicy: Always containerPort: 8080 diff --git a/experimental/helm/charts/model-registry/ci/values-ui-standalone.yaml b/experimental/helm/charts/model-registry/ci/values-ui-standalone.yaml index 8e2a42e75b..c70fda14e5 100644 --- a/experimental/helm/charts/model-registry/ci/values-ui-standalone.yaml +++ b/experimental/helm/charts/model-registry/ci/values-ui-standalone.yaml @@ -9,7 +9,7 @@ ui: image: repository: ui - tag: "v0.3.3" + tag: "v0.3.4" pullPolicy: Always containerPort: 8080 diff --git a/experimental/helm/charts/model-registry/ci/values-ui.yaml b/experimental/helm/charts/model-registry/ci/values-ui.yaml index 660161436f..d6a8696958 100644 --- a/experimental/helm/charts/model-registry/ci/values-ui.yaml +++ b/experimental/helm/charts/model-registry/ci/values-ui.yaml @@ -5,7 +5,7 @@ ui: image: repository: ui - tag: "v0.3.3" + tag: "v0.3.4" pullPolicy: Always containerPort: 8080 diff --git a/experimental/helm/charts/model-registry/templates/database/mysql/deployment.yaml b/experimental/helm/charts/model-registry/templates/database/mysql/deployment.yaml index b9361449cc..572f2f86db 100644 --- a/experimental/helm/charts/model-registry/templates/database/mysql/deployment.yaml +++ b/experimental/helm/charts/model-registry/templates/database/mysql/deployment.yaml @@ -38,7 +38,7 @@ spec: command: - /bin/bash - -c - - mysql -D $$MYSQL_DATABASE -u$$MYSQL_USER_NAME -p$$MYSQL_ROOT_PASSWORD + - mysql -h 127.0.0.1 -D $$MYSQL_DATABASE -u$$MYSQL_USER_NAME -p$$MYSQL_ROOT_PASSWORD -e 'SELECT 1' initialDelaySeconds: 5 periodSeconds: 2 diff --git a/experimental/helm/charts/model-registry/values.yaml b/experimental/helm/charts/model-registry/values.yaml index 64cab7973b..3154e7591a 100644 --- a/experimental/helm/charts/model-registry/values.yaml +++ b/experimental/helm/charts/model-registry/values.yaml @@ -42,7 +42,7 @@ server: # -- Server image repository repository: server # -- Server image tag (overrides global.imageTag if set) - tag: "v0.3.3" + tag: "v0.3.4" # -- Server image pull policy (overrides global.imagePullPolicy if set) pullPolicy: "" @@ -178,7 +178,7 @@ ui: # -- UI image repository repository: model-registry-ui # -- UI image tag - tag: "v0.3.3" + tag: "v0.3.4" # -- UI image pull policy pullPolicy: "" @@ -449,7 +449,7 @@ database: enabled: false image: repository: postgres - tag: "14.7-alpine3.17" + tag: "16-alpine" auth: database: mlmdpostgres username: root From af44b95a216f6c41edec5c07708f91895c496510 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Tue, 6 Jan 2026 16:57:32 +0530 Subject: [PATCH 26/26] Update the ingress list Signed-off-by: kunal-511 --- .../templates/third-party/seaweedfs/networkpolicy.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/networkpolicy.yaml b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/networkpolicy.yaml index 323cbd03f1..b09e402e9a 100644 --- a/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/networkpolicy.yaml +++ b/experimental/helm/charts/pipelines/templates/third-party/seaweedfs/networkpolicy.yaml @@ -24,6 +24,10 @@ spec: operator: In values: - istio-system + - from: + - podSelector: {} + ports: + - port: 8333 podSelector: matchExpressions: - key: app