Prepare for upgrade to Cloud Composer version 2.6.5 with airflow 2.5.3 (

#3682)

* Update local Cloud Composer testing version to 2.6.5 with airflow 2.5.3
* Remove old composer requirements file

---------

Co-authored-by: V <[email protected]>

airflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.4.3-python3.8
+FROM apache/airflow:2.5.3-python3.8
 
 # install gcloud as root, then switch back to airflow
 USER root

airflow/Dockerfile.composer

@@ -15,8 +15,8 @@ RUN gcloud components install gke-gcloud-auth-plugin
 
 USER airflow
 
-COPY requirements-composer-2.6.0-airflow-2.5.3.txt /tmp/requirements-composer-2.6.0-airflow-2.5.3.txt
-RUN pip install --no-cache-dir --user -r /tmp/requirements-composer-2.6.0-airflow-2.5.3.txt
+COPY requirements-composer-2.6.5-airflow-2.5.3.txt /tmp/requirements-composer-2.6.5-airflow-2.5.3.txt
+RUN pip install --no-cache-dir --user -r /tmp/requirements-composer-2.6.5-airflow-2.5.3.txt
 
 COPY requirements.txt /tmp/requirements.txt
 RUN pip install --no-cache-dir --user -r /tmp/requirements.txt

airflow/requirements-composer-2.6.0-airflow-2.5.3.txt → airflow/requirements-composer-2.6.5-airflow-2.5.3.txt

@@ -1,9 +1,9 @@
-#composer-2.6.0-airflow-2.5.3
-absl-py==2.0.0
+#composer-2.6.5-airflow-2.5.3
+absl-py==2.1.0
 agate==1.6.3
 aiodebug==2.3.0
 aiofiles==23.2.1
-aiohttp==3.8.6
+aiohttp==3.9.3
 aiosignal==1.3.1
 alembic==1.10.2
 amqp==5.1.1
@@ -18,12 +18,12 @@ apache-airflow-providers-google==10.12.0
 apache-airflow-providers-hashicorp==3.5.0
 apache-airflow-providers-http==4.7.0
 apache-airflow-providers-imap==3.4.0
-apache-airflow-providers-mysql==5.2.0
+# apache-airflow-providers-mysql==5.2.0
 apache-airflow-providers-postgres==5.8.0
 apache-airflow-providers-sendgrid==3.3.0
 apache-airflow-providers-sqlite==3.5.0
 apache-airflow-providers-ssh==3.8.1
-apache-beam==2.51.0
+apache-beam==2.53.0
 apispec==3.3.2
 appdirs==1.4.4
 argcomplete==3.0.5
@@ -37,7 +37,7 @@ backports.zoneinfo==0.2.1
 bcrypt==4.0.1
 billiard==3.6.4.0
 blinker==1.5
-cachecontrol==0.13.1
+cachecontrol==0.14.0
 cachelib==0.9.0
 cachetools==5.3.0
 cattrs==22.2.0
@@ -59,25 +59,25 @@ connexion==2.14.2
 crcmod==1.7
 cron-descriptor==1.2.35
 croniter==1.3.8
-cryptography==41.0.5
-db-dtypes==1.1.1
+cryptography==41.0.7
+db-dtypes==1.2.0
 dbt-bigquery==1.5.4
-dbt-core==1.5.4
+dbt-core==1.5.9
 dbt-extractor==0.4.1
 decorator==5.1.1
-Deprecated==1.2.13
-diff-cover==8.0.1
+Deprecated==1.2.14
+diff-cover==8.0.3
 dill==0.3.1.1
 distlib==0.3.6
 dnspython==2.3.0
 docopt==0.6.2
 docutils==0.19
 email-validator==1.3.1
 exceptiongroup==1.1.1
-fastavro==1.9.0
+fastavro==1.9.3
 fasteners==0.19
 filelock==3.10.7
-firebase-admin==6.2.0
+firebase-admin==6.4.0
 Flask==2.2.5
 Flask-AppBuilder==4.1.4
 Flask-Babel==2.0.0
@@ -91,85 +91,86 @@ Flask-WTF==1.1.1
 flatbuffers==23.5.26
 flower==2.0.1
 frozenlist==1.3.3
-fsspec==2023.12.0
+fsspec==2024.2.0
 future==0.18.3
 gast==0.4.0
 gcloud-aio-auth==4.2.3
 gcloud-aio-bigquery==7.0.0
-gcloud-aio-storage==9.0.0
-gcsfs==2023.12.0
-google-ads==22.1.0
-google-api-core==2.14.0
-google-api-python-client==2.109.0
+gcloud-aio-storage==9.1.0
+gcsfs==2024.2.0
+google-ads==23.0.0
+google-api-core==2.17.0
+google-api-python-client==2.117.0
 google-apitools==0.5.32
 google-auth==2.23.4
-google-auth-httplib2==0.1.1
+google-auth-httplib2==0.2.0
 google-auth-oauthlib==1.0.0
 google-cloud-access-context-manager==0.1.16
-google-cloud-aiplatform==1.36.4
-google-cloud-appengine-logging==1.3.2
-google-cloud-asset==3.20.1
+google-cloud-aiplatform==1.41.0
+google-cloud-appengine-logging==1.4.1
+google-cloud-asset==3.24.1
 google-cloud-audit-log==0.2.5
-google-cloud-automl==2.11.4
-google-cloud-batch==0.17.5
-google-cloud-bigquery==3.13.0
-google-cloud-bigquery-datatransfer==3.12.1
-google-cloud-bigquery-storage==2.23.0
-google-cloud-bigtable==2.21.0
-google-cloud-build==3.21.0
-google-cloud-common==1.2.0
-google-cloud-compute==1.14.1
-google-cloud-container==2.35.0
-google-cloud-core==2.3.3
-google-cloud-datacatalog==3.16.0
-# google-cloud-datacatalog-lineage==0.3.1
+google-cloud-automl==2.13.1
+google-cloud-batch==0.17.11
+google-cloud-bigquery==3.17.2
+google-cloud-bigquery-datatransfer==3.14.1
+google-cloud-bigquery-storage==2.24.0
+google-cloud-bigtable==2.23.0
+google-cloud-build==3.23.1
+google-cloud-common==1.3.1
+google-cloud-compute==1.16.1
+google-cloud-container==2.40.0
+google-cloud-core==2.4.1
+google-cloud-datacatalog==3.18.1
+google-cloud-datacatalog-lineage==0.3.1
 # google-cloud-datacatalog-lineage-producer-client==0.1.0
-google-cloud-dataflow-client==0.8.5
-google-cloud-dataform==0.5.4
-google-cloud-dataplex==1.9.0
-google-cloud-dataproc==5.7.0
-google-cloud-dataproc-metastore==1.13.0
-google-cloud-datastore==2.18.0
-google-cloud-dlp==3.13.0
-google-cloud-documentai==2.20.2
-google-cloud-filestore==1.7.0
-google-cloud-firestore==2.13.1
-google-cloud-kms==2.19.2
-google-cloud-language==2.11.1
-google-cloud-logging==3.8.0
-google-cloud-memcache==1.7.3
-google-cloud-monitoring==2.16.0
-google-cloud-orchestration-airflow==1.9.2
-google-cloud-org-policy==1.8.3
-google-cloud-os-config==1.15.3
-google-cloud-os-login==2.11.0
-google-cloud-pubsub==2.18.4
+google-cloud-dataflow-client==0.8.8
+google-cloud-dataform==0.5.7
+google-cloud-dataplex==1.12.1
+google-cloud-dataproc==5.9.1
+google-cloud-dataproc-metastore==1.15.1
+google-cloud-datastore==2.19.0
+google-cloud-dlp==3.15.1
+google-cloud-documentai==2.23.0
+google-cloud-filestore==1.9.1
+google-cloud-firestore==2.14.0
+google-cloud-kms==2.21.1
+google-cloud-language==2.13.1
+google-cloud-logging==3.9.0
+google-cloud-memcache==1.9.1
+google-cloud-monitoring==2.19.1
+google-cloud-orchestration-airflow==1.11.1
+google-cloud-org-policy==1.10.0
+google-cloud-os-config==1.17.1
+google-cloud-os-login==2.14.1
+google-cloud-pubsub==2.19.4
 google-cloud-pubsublite==0.6.1
-google-cloud-redis==2.13.2
-google-cloud-resource-manager==1.10.4
-google-cloud-run==0.10.0
-google-cloud-secret-manager==2.16.4
-google-cloud-spanner==3.40.1
-google-cloud-speech==2.22.0
-google-cloud-storage==2.13.0
-google-cloud-storage-transfer==1.9.2
-google-cloud-tasks==2.14.2
-google-cloud-texttospeech==2.14.2
-google-cloud-translate==3.12.1
-google-cloud-videointelligence==2.11.4
-google-cloud-vision==3.4.5
-google-cloud-workflows==1.12.1
+google-cloud-redis==2.15.1
+google-cloud-resource-manager==1.12.1
+google-cloud-run==0.10.3
+google-cloud-secret-manager==2.18.1
+google-cloud-spanner==3.42.0
+google-cloud-speech==2.24.1
+google-cloud-storage==2.14.0
+google-cloud-storage-transfer==1.11.1
+google-cloud-tasks==2.16.1
+google-cloud-texttospeech==2.16.1
+google-cloud-translate==3.15.1
+google-cloud-videointelligence==2.13.1
+google-cloud-vision==3.7.0
+google-cloud-workflows==1.14.1
 google-crc32c==1.5.0
 google-pasta==0.2.0
 google-re2==1.1
-google-resumable-media==2.6.0
-googleapis-common-protos==1.61.0
+google-resumable-media==2.7.0
+googleapis-common-protos==1.62.0
 graphviz==0.20.1
 greenlet==2.0.2
-grpc-google-iam-v1==0.12.7
-grpcio==1.59.3
+grpc-google-iam-v1==0.13.0
+grpc-interceptor==0.15.4
+grpcio==1.60.1
 grpcio-gcp==0.2.2
-grpcio-status==1.59.3
+grpcio-status==1.60.1
 gunicorn==20.1.0
 h11==0.14.0
 h5py==3.10.0
@@ -179,21 +180,22 @@ httpcore==0.16.3
 httplib2==0.22.0
 httpx==0.23.3
 humanize==4.6.0
-hvac==2.0.0
+hvac==2.1.0
 idna==3.4
 importlib-metadata==4.13.0
 importlib-resources==5.12.0
 inflection==0.5.1
 iniconfig==2.0.0
 isodate==0.6.1
 itsdangerous==2.1.2
-jaraco.classes==3.3.0
+jaraco.classes==3.3.1
 jeepney==0.8.0
 Jinja2==3.1.2
 Js2Py==0.74
 json-merge-patch==0.2
+jsonpickle==3.0.2
 jsonschema==4.19.0
-jsonschema-specifications==2023.11.2
+jsonschema-specifications==2023.12.1
 keras==2.13.1
 keyring==24.3.0
 keyrings.google-artifactregistry-auth==1.1.2
@@ -206,7 +208,7 @@ libclang==16.0.6
 linkify-it-py==2.0.0
 lockfile==0.12.2
 Logbook==1.5.3
-looker-sdk==23.20.0
+looker-sdk==24.0.0
 Mako==1.2.4
 Markdown==3.4.3
 markdown-it-py==2.2.0
@@ -219,39 +221,40 @@ mashumaro==3.6
 mdit-py-plugins==0.3.5
 mdurl==0.1.2
 minimal-snowplow-tracker==0.0.2
-more-itertools==10.1.0
+more-itertools==10.2.0
 msgpack==1.0.5
 multidict==6.0.4
-mysqlclient==2.2.0
+# mysqlclient==2.2.4
 networkx==2.8.8
 numpy==1.24.2
 oauth2client==4.1.3
 oauthlib==3.2.2
 objsize==0.6.1
 opt-einsum==3.3.0
-orjson==3.9.10
+orjson==3.9.13
 overrides==6.5.0
 packaging==23.0
 pandas==1.5.3
-pandas-gbq==0.19.2
-paramiko==3.3.1
+pandas-gbq==0.21.0
+paramiko==3.4.0
 parsedatetime==2.4
 pathspec==0.9.0
 pendulum==2.1.2
 pip==20.2.4
-pipdeptree==2.13.1
+pipdeptree==2.13.2
 pkgutil-resolve-name==1.3.10
 platformdirs==3.2.0
-pluggy==1.0.0
+pluggy==1.4.0
 prison==0.2.1
 prometheus-client==0.16.0
 prompt-toolkit==3.0.38
-proto-plus==1.22.3
-protobuf==4.24.4
+proto-plus==1.23.0
+protobuf==4.25.0
 psutil==5.9.4
 psycopg2==2.9.9
 psycopg2-binary==2.9.9
-pyarrow==11.0.0
+pyarrow==14.0.2
+pyarrow-hotfix==0.6
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pycparser==2.21
@@ -262,9 +265,9 @@ pyjsparser==2.7.1
 PyJWT==2.6.0
 pymongo==4.6.1
 PyNaCl==1.5.0
-pyOpenSSL==23.3.0
+pyOpenSSL==24.0.0
 pyparsing==3.1.1
-pytest==7.4.3
+pytest==8.0.0
 python-daemon==3.0.1
 python-dateutil==2.8.2
 python-http-client==3.3.7
@@ -275,15 +278,15 @@ pytz==2023.3
 pytzdata==2020.1
 PyYAML==6.0
 redis==3.5.3
-referencing==0.31.1
-regex==2023.10.3
+referencing==0.33.0
+regex==2023.12.25
 requests==2.28.2
 requests-oauthlib==1.3.1
 requests-toolbelt==1.0.0
 rfc3339-validator==0.1.4
 rfc3986==1.5.0
 rich==13.3.3
-rpds-py==0.13.2
+rpds-py==0.17.1
 rsa==4.9
 SecretStorage==3.3.3
 sendgrid==6.11.0
@@ -293,7 +296,7 @@ shapely==2.0.2
 six==1.16.0
 sniffio==1.3.0
 SQLAlchemy==1.4.47
-sqlalchemy-bigquery==1.8.0
+sqlalchemy-bigquery==1.9.0
 SQLAlchemy-JSONField==1.0.1.post0
 sqlalchemy-spanner==1.6.2
 SQLAlchemy-Utils==0.40.0
@@ -316,7 +319,7 @@ text-unidecode==1.3
 toml==0.10.2
 tomli==2.0.1
 tornado==6.2
-tqdm==4.66.1
+tqdm==4.66.2
 typing-extensions==4.5.0
 tzlocal==5.2
 uc-micro-py==1.0.1