values.yaml

# Default values for airflow.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

airflow:
  # group of airflow user
  gid: 50000

  # Airflow home directory; Used for mount paths
  airflowHome: "/usr/local/airflow"

  # Airflow executor
  # Options: SequentialExecutor, LocalExecutor, CeleryExecutor, KubernetesExecutor
  executor: "KubernetesExecutor"

  # Airflow version
  airflowVersion: 2.0.0

  # Default airflow repository
  defaultAirflowRepository: quay.io/astronomer/ap-airflow

  # Default airflow tag to deploy
  defaultAirflowTag: 2.0.0-buster

  # Astronomer Airflow images
  images:
    airflow:
      repository: quay.io/astronomer/ap-airflow
      tag: ~
    statsd:
      repository: quay.io/astronomer/ap-statsd-exporter
      tag: 0.27.2
    redis:
      repository: quay.io/astronomer/ap-redis
      tag: 7.2.6
    pgbouncer:
      repository: quay.io/astronomer/ap-pgbouncer
      tag: 1.23.1-1
    pgbouncerExporter:
      repository: quay.io/astronomer/ap-pgbouncer-exporter
      tag: 0.18.0
    gitSync:
      repository: quay.io/astronomer/ap-git-sync
      tag: 3.6.9
  # Airflow scheduler settings
  scheduler:
    livenessProbe:
      timeoutSeconds: 30
    strategy:
      type: Recreate
    affinity:
      podAntiAffinity:
        preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                  - key: component
                    operator: In
                    values:
                      - scheduler
              topologyKey: "kubernetes.io/hostname"
    logGroomerSidecar:
      args: ["/usr/local/bin/clean-airflow-logs"]

  triggerer:
    persistence:
      enabled: false
    logGroomerSidecar:
      args: ["/usr/local/bin/clean-airflow-logs"]

  dagProcessor:
    logGroomerSidecar:
      args: ["/usr/local/bin/clean-airflow-logs"]

  # Airflow webserver settings
  webserver:
    allowPodLogReading: false
    args: ["airflow", "webserver"]
    webserverConfig: |
      import os
      from airflow import configuration as conf
      from flask_appbuilder.security.manager import AUTH_REMOTE_USER
      basedir = os.path.abspath(os.path.dirname(__file__))

      # The SQLAlchemy connection string.
      SQLALCHEMY_DATABASE_URI = conf.get('core', 'SQL_ALCHEMY_CONN')

      # Flask-WTF flag for CSRF
      CSRF_ENABLED = True

      # ----------------------------------------------------
      # AUTHENTICATION CONFIG
      # ----------------------------------------------------
      {{- if .Values.useAstroSecurityManager }}
      AUTH_TYPE = AUTH_REMOTE_USER

      from astronomer.flask_appbuilder.security import AirflowAstroSecurityManager
      SECURITY_MANAGER_CLASS = AirflowAstroSecurityManager
      {{- end }}

  # Airflow redis settings
  redis:
    securityContext:
      # Set user to redis:999
      runAsUser: 999
      fsGroup: 999

  # Airflow worker settings
  workers:
    persistence:
      # Enable persistent volumes
      enabled: false
    logGroomerSidecar:
      args: ["/usr/local/bin/clean-airflow-logs"]

  # Pgbouncer settings
  pgbouncer:
    # Pool sizes
    metadataPoolSize: 3
    resultBackendPoolSize: 2

  # Elasticsearch logging configuration
  elasticsearch:
    connection:
      scheme: http

  cleanup:
    command: ["airflow-cleanup-pods", "--namespace={{ .Release.Namespace }}"]
    args: []

  airflowLocalSettings: |
    {{- if semverCompare "<1.10.12" .Values.airflowVersion }}

    from airflow.contrib.kubernetes.pod import Pod
    from airflow.configuration import conf

    def pod_mutation_hook(pod: Pod):

        extra_labels = {
            "kubernetes_executor": "False",
            "kubernetes_pod_operator": "False"
        }

        if 'airflow-worker' in pod.labels.keys() or \
                conf.get('core', 'EXECUTOR') == "KubernetesExecutor":
            extra_labels["kubernetes_executor"] = "True"
        else:
            extra_labels["kubernetes_pod_operator"] = "True"

        pod.labels.update(extra_labels)
        pod.tolerations += {{ toJson .Values.podMutation.tolerations }}
        pod.affinity.update({{ toJson .Values.podMutation.affinity }})
    {{- else }}
    from kubernetes.client import models as k8s
    from airflow.configuration import conf

    def pod_mutation_hook(pod: k8s.V1Pod):

        extra_labels = {
            "kubernetes_executor": "False",
            "kubernetes_pod_operator": "False"
        }

        if 'airflow-worker' in pod.metadata.labels.keys() or \
                conf.get('core', 'EXECUTOR') == "KubernetesExecutor":
            extra_labels["kubernetes_executor"] = "True"
        else:
            extra_labels["kubernetes_pod_operator"] = "True"

        pod.metadata.labels.update(extra_labels)
        if pod.spec.tolerations:
            pod.spec.tolerations += {{ toJson .Values.podMutation.tolerations }}
        else:
            pod.spec.tolerations = {{ toJson .Values.podMutation.tolerations }}

        if pod.spec.affinity:
            pod.spec.affinity = pod.spec.affinity.to_dict().update({{ toJson .Values.podMutation.affinity }})
        else:
            pod.spec.affinity = {{ toJson .Values.podMutation.affinity }}

    {{- end }}
  podTemplate: |
    # Licensed to the Apache Software Foundation (ASF) under one
    # or more contributor license agreements.  See the NOTICE file
    # distributed with this work for additional information
    # regarding copyright ownership.  The ASF licenses this file
    # to you under the Apache License, Version 2.0 (the
    # "License"); you may not use this file except in compliance
    # with the License.  You may obtain a copy of the License at
    #
    #   http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing,
    # software distributed under the License is distributed on an
    # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    # KIND, either express or implied.  See the License for the
    # specific language governing permissions and limitations
    # under the License.
    ---
    {{- $nodeSelector := or .Values.nodeSelector .Values.workers.nodeSelector }}
    {{- $affinity := or .Values.affinity .Values.workers.affinity }}
    {{- $tolerations := or .Values.tolerations .Values.workers.tolerations }}
    apiVersion: v1
    kind: Pod
    metadata:
      name: astronomer-pod-template-file
      labels:
        tier: airflow
        component: worker
        release: {{ .Release.Name }}
    {{- if or (.Values.labels) (.Values.workers.labels) }}
      {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
    {{- end }}
      {{- if or .Values.airflowPodAnnotations .Values.workers.podAnnotations }}
      annotations:
        {{- if .Values.airflowPodAnnotations }}
          {{- toYaml .Values.airflowPodAnnotations | nindent 4 }}
        {{- end }}
        {{- if .Values.workers.podAnnotations }}
          {{- toYaml .Values.workers.podAnnotations | nindent 4 }}
        {{- end }}
      {{- end }}
    spec:
      {{- if or (and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled)) .Values.workers.extraInitContainers }}
      initContainers:
        {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }}
        {{- include "git_sync_container" (dict "Values" .Values "is_init" "true") | nindent 4 }}
        {{- end }}
        {{- if .Values.workers.extraInitContainers }}
        {{- toYaml .Values.workers.extraInitContainers | nindent 4 }}
        {{- end }}
      {{- end }}
      containers:
        - args: []
          command: []
          envFrom:
          {{- include "custom_airflow_environment_from" . | default "\n  []" | indent 6 }}
          env:
            - name: AIRFLOW__CORE__EXECUTOR
              value: LocalExecutor
    {{- include "standard_airflow_environment" . | indent 6}}
    {{- include "custom_airflow_environment" . | indent 6 }}
    {{- include "container_extra_envs" (list . .Values.workers.env) | indent 6 }}
          image: {{ template "pod_template_image" . }}
          imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
          name: base
          ports: []
          volumeMounts:
            - mountPath: {{ template "airflow_logs" . }}
              name: logs
            - name: config
              mountPath: {{ template "airflow_config_path" . }}
              subPath: airflow.cfg
              readOnly: true
    {{- if .Values.airflowLocalSettings }}
            - name: config
              mountPath: {{ template "airflow_local_setting_path" . }}
              subPath: airflow_local_settings.py
              readOnly: true
    {{- end }}
    {{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}
            {{- include "airflow_dags_mount" . | nindent 8 }}
    {{- end }}
    {{- if .Values.workers.extraVolumeMounts }}
    {{ toYaml .Values.workers.extraVolumeMounts | indent 8 }}
    {{- end }}
    {{- if .Values.workers.extraContainers }}
    {{- toYaml .Values.workers.extraContainers | nindent 4 }}
    {{- end }}
      hostNetwork: false
      {{- if .Values.workers.priorityClassName }}
      priorityClassName: {{ .Values.workers.priorityClassName }}
      {{- end }}
      {{- if .Values.workers.runtimeClassName }}
      runtimeClassName: {{ .Values.workers.runtimeClassName }}
      {{- end }}
      {{- if .Values.workers.hostAliases }}
      hostAliases: {{- toYaml .Values.workers.hostAliases | nindent 4 }}
      {{- end }}
      {{- if or .Values.registry.secretName .Values.registry.connection }}
      imagePullSecrets:
        - name: {{ template "registry_secret" . }}
      {{- end }}
      restartPolicy: Never
      securityContext:
        runAsUser: {{ .Values.uid }}
        fsGroup: {{ .Values.gid }}
      nodeSelector: {{ toYaml $nodeSelector | nindent 4 }}
      affinity: {{ toYaml $affinity | nindent 4 }}
      tolerations: {{ toYaml $tolerations | nindent 4 }}
      serviceAccountName: {{ include "worker.serviceAccountName" . }}
      volumes:
      {{- if .Values.dags.persistence.enabled }}
      - name: dags
        persistentVolumeClaim:
          claimName: {{ template "airflow_dags_volume_claim" . }}
      {{- else if .Values.dags.gitSync.enabled }}
      - name: dags
        emptyDir: {}
      {{- end }}
      {{- if .Values.logs.persistence.enabled }}
      - name: logs
        persistentVolumeClaim:
          claimName: {{ template "airflow_logs_volume_claim" . }}
      {{- else }}
      - emptyDir: {}
        name: logs
      {{- end }}
      {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }}
      {{- include "git_sync_ssh_key_volume" . | nindent 2 }}
      {{- end }}
      - configMap:
          name: {{ include "airflow_config" . }}
        name: config
      {{- if .Values.workers.extraVolumes }}
      {{ toYaml .Values.workers.extraVolumes | nindent 2 }}
      {{- end }}

  # Config Settings for pod_mutation_hook
  podMutation:
    # Tolerations provided here would be applied using pod_mutation_hook
    # So any pods spun up using KubernetesExecutor or KubernetesPodOperator will contain these tolerations.
    tolerations: []
    #  - key: "dynamic-pods"
    #    operator: "Equal"
    #    value: "true"
    #    effect: "NoSchedule"

    # Pods spun up would land in the node that matches the affinity
    affinity: {}
    #  nodeAffinity:
    #    requiredDuringSchedulingIgnoredDuringExecution:
    #      nodeSelectorTerms:
    #        - matchExpressions:
    #            - key: "astronomer.io/dynamic-pods"
    #              operator: In
    #              values:
    #                - "true"

  migrateDatabaseJob:
    annotations:
      "sidecar.istio.io/inject": "false"

  config:
    api:
      # if astroSecurity manager is enabled and the airflow version is greater than 2 we want this to equal astronomer.flask_appbuilder.current_user_backend
      # if astroSecurity manager is disabled and the airflow version is greater than 2 we want this to equal airflow.api.auth.backend.deny_all
      # if astroSecurity manager is disabled and the airflow version is less than 2 we want this to equal airflow.api.auth.backend.default
      auth_backend: '{{ ternary "astronomer.flask_appbuilder.current_user_backend" (ternary "airflow.api.auth.backend.deny_all" "airflow.api.auth.backend.default"  (semverCompare ">=2.0.0" .Values.airflowVersion)) (and (.Values.useAstroSecurityManager) (semverCompare ">=2.0.0" .Values.airflowVersion)) }}'
    operators:
      default_queue: celery
    celery: # compat
      default_queue: celery

      # Set Flower URL prefix to empty string
      # Remove after https://github.com/apache/airflow/issues/27402 is resolved
      flower_url_prefix: ""

    webserver:
      expose_config: True
    elasticsearch:
      write_stdout: True
      elasticsearch_write_stdout: True
      elasticsearch_json_format: True
      # "{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}" for airflow >= 2.3.0 ref https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#log-id-template
      # "{dag_id}_{task_id}_{execution_date}_{try_number}" for airflow < 2.3.0
      log_id_template: '{{ternary "{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}" "{dag_id}_{task_id}_{execution_date}_{try_number}" (semverCompare ">=2.3.0" .Values.airflowVersion)}}'
      # For Airflow <1.10.4
      elasticsearch_log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}"
    # The following kubernetes config is required to support Airflow 1.10.10
    kubernetes:
      dags_in_image: True
      worker_service_account_name: '{{ include "worker.serviceAccountName" . }}'
      image_pull_secrets: '{{ template "registry_secret" . }}'
    kubernetes_labels:
      tier: airflow
      component: worker
      release: "{{ .Release.Name }}"
    kubernetes_secrets:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: '{{ printf "%s=connection" (include "airflow_metadata_secret" .) }}'
      AIRFLOW_CONN_AIRFLOW_DB: '{{ printf "%s=connection" (include "airflow_metadata_secret" .) }}'
      AIRFLOW__CORE__FERNET_KEY: '{{ printf "%s=fernet-key" (include "fernet_key_secret" .) }}'
    # Only required when running on the platform, but works elsewhere too
    astronomer:
      jwt_signing_cert: /etc/airflow/tls/tls.crt

  # Should the Astronomer Security Manager be used for the webserver
  useAstroSecurityManager: False

  statsd:
    overrideMappings:
      # Map dot separated stats to labels
      - match: airflow.operator_successes_(.*)
        match_type: regex
        name: "airflow_operator_successes"
        labels:
          # This is here to make the Astro UI work
          # because it's expecting an operator label
          operator: "Value"

      - match: airflow.operator_failures_(.*)
        match_type: regex
        name: "airflow_operator_failures"
        labels:
          operator: "Value"

      - match: airflow.scheduler_heartbeat
        match_type: regex
        name: "airflow_scheduler_heartbeat"
        labels:
          type: counter

      - match: airflow.dag.*.*.duration
        name: "airflow_task_duration"
        labels: {}

      - match: airflow.dagrun.duration.success.*
        name: "airflow_dagrun_duration"
        labels: {}

      - match: airflow.dagrun.duration.failed.*
        name: "airflow_dagrun_failed"
        labels: {}

      - match: airflow.dagrun.schedule_delay.*
        name: "airflow_dagrun_schedule_delay"
        labels: {}

      - match: airflow.dag_processing.last_runtime.*
        name: "airflow_dag_processing_last_runtime"
        labels: {}

      - match: airflow.dag_processing.last_run.seconds_ago.*
        name: "airflow_dag_processing_last_run_seconds_ago"
        labels: {}

      - match: airflow.pool.open_slots.*
        name: "airflow_pool_open_slots"
        labels:
          pool: "$1"

      - match: airflow.pool.used_slots.*
        name: "airflow_pool_used_slots"
        labels:
          pool: "$1"

      - match: airflow.pool.starving_tasks.*
        name: "airflow_pool_starving_tasks"
        labels:
          pool: "$1"

      - match: airflow.zombies_killed
        name: "airflow_zombies_killed"
        labels: {}

      - match: airflow.executor.running_tasks
        name: "airflow_executor_running_tasks"
        labels: {}

      - match: airflow.executor.queued_tasks
        name: "airflow_executor_queued_tasks"
        labels: {}

      - match: airflow.ti_failures
        name: "airflow_ti_failures"
        labels: {}

      - match: airflow.ti_successes
        name: "airflow_ti_successes"
        labels: {}

      - match: airflow.dagbag_size
        name: "airflow_dagbag_size"
        labels: {}

      # drop any metric not matched
      - match: "."
        match_type: regex
        action: drop
        name: "dropped"

# Enable security context constraints required for OpenShift
sccEnabled: false

# Extra objects to deploy (these are passed through `tpl`)
extraObjects: []

# Enable nginx auth sidecar
authSidecar:
  enabled: false
  repository: quay.io/astronomer/ap-auth-sidecar
  tag: 1.27.1
  pullPolicy: IfNotPresent
  port: 8084
  securityContext: {}
  resources: {}

loggingSidecar:
  enabled: false
  name: sidecar-log-consumer
  customConfig: false
  indexPattern: "%Y.%m.%d"
  indexNamePrefix: vector
  image: quay.io/astronomer/ap-vector:0.40.2-1

# Ingress configuration
ingress:
  # Enable ingress resource
  enabled: false

  # Enable for cert-manager or kube-lego
  acme: false

  # Name of tls secret to use on ingress
  tlsSecretName: ~

  # Annotations always injected when configuring webserver Ingress object
  webserverAnnotations: {}

  # Annotations always injected when configuring Flower Ingress object
  flowerAnnotations: {}

  # Annotations always injected when configuring Dag Server Ingress object
  dagServerAnnotations: {}

  # Base domain for ingress vhosts
  baseDomain: ~

  # Enable platform authentication
  auth:
    enabled: true

#####################################
# Leftovers

# Pgbouncer settings
#pgbouncer:
#  networkPolicies:
#    enabled: true
#    # TODO: Originated from https://github.com/astronomer/issues/issues/2516

# This section describes the astronomer platform, and is used to share values
# from astronomer that are useful here, like networkPolicies that need to know
# what namespace astronomer is running in.
platform:
  release: ~
  namespace: ~

astronomer:
  images:
    certgenerator:
      repository: quay.io/astronomer/ap-certgenerator
      tag: 0.1.7

dagDeploy:
  enabled: false
  images:
    dagServer:
      repository: quay.io/astronomer/ap-dag-deploy
      tag: 0.5.1
      imagePullPolicy: IfNotPresent
  livenessProbe:
    initialDelaySeconds: 30
    timeoutSeconds: 30
  terminationGracePeriodSeconds: 30
  podAnnotations: {}
  extraEnv: []
  extraContainers: []
  persistence:
    enabled: true
    persistentVolumeClaimRetentionPolicy: ~
    size: 10Gi
    storageClassName: ~
    annotations: {}

  ports:
    dagServerHttp: 8000

  serviceAccount:
    annotations: {}
    name: ~
    create: true

  resources: {}
  #  limits:
  #   cpu: 100m
  #   memory: 128Mi
  #  requests:
  #   cpu: 100m
  #   memory: 128Mi

  # This setting tells kubernetes that its ok to evict
  # when it wants to scale a node down.
  safeToEvict: true

  securityContexts:
    pod: {}
    container: {}

gitSyncRelay:
  enabled: ~
  gitSyncResources: {}
  gitDaemonResources: {}
  securityContext: {}
  images:
    gitDaemon:
      repository: "quay.io/astronomer/ap-git-daemon"
      tag: "3.18.8"
    gitSync:
      repository: "quay.io/astronomer/ap-git-sync-relay"
      tag: "0.0.3-5"
  repo:
    url: ~
    branch: main
    depth: 1 # default to a shallow clone because it is faster and this isn't a dev environment
    wait: 60 # seconds between synchronizations with upstream git repo
    subPath: ~ # if your dags dir is not the repo root, specify the path relative to the repo root
    sshPrivateKeySecretName: ~ # The name of a secret that holds the private key. This secret is not maintained by this chart.
    gitSyncTimeout: 300 # How long to wait for the clone or fetch to take before aborting
    # knownHosts can be generated with: ssh-keyscan -t ed25519 github.com 2>/dev/null
    knownHosts: |
      # https://api.github.com/meta
      github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
      github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
      github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
      # https://docs.gitlab.com/ee/user/gitlab_com/#ssh-host-keys-fingerprints
      gitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9
      gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY=
      gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf
  extraContainers: []

certgenerator:
  extraAnnotations: {}
  affinity: {}