From d2241cfc9465448cc7f21aa385660943a0a743dc Mon Sep 17 00:00:00 2001 From: Ankit Chaurasia <8670962+sunank200@users.noreply.github.com> Date: Thu, 12 Dec 2024 23:38:57 +0545 Subject: [PATCH 1/6] Add ``airflow config lint`` cli command for linting the configuration changes from Airflow 2.x to Airflow 3.0 --- airflow/cli/cli_config.py | 29 ++ .../remote_commands/config_command.py | 346 ++++++++++++++++++ .../remote_commands/test_config_command.py | 108 ++++++ 3 files changed, 483 insertions(+) diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 503397064cbf4..5b5099f7fdbe9 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -857,6 +857,23 @@ def string_lower_type(val): ("option",), help="The option name", ) + +ARG_LINT_CONFIG_SECTION = Arg( + ("--section",), + help="The section name to lint in the airflow config.", +) +ARG_LINT_CONFIG_OPTION = Arg( + ("--option",), + help="The option name to lint in the airflow config.", +) +ARG_LINT_CONFIG_IGNORE_SECTION = Arg( + ("--ignore-section",), + help="The section name to ignore to lint in the airflow config.", +) +ARG_LINT_CONFIG_IGNORE_OPTION = Arg( + ("--ignore-option",), + help="The option name to ignore to lint in the airflow config.", +) ARG_OPTIONAL_SECTION = Arg( ("--section",), help="The section name", @@ -1733,6 +1750,18 @@ class GroupCommand(NamedTuple): ARG_VERBOSE, ), ), + ActionCommand( + name="lint", + help="lint options for the configuration changes while migrating from airflow 2.x to airflow 3.0", + func=lazy_load_command("airflow.cli.commands.remote_commands.config_command.lint_config"), + args=( + ARG_LINT_CONFIG_SECTION, + ARG_LINT_CONFIG_OPTION, + ARG_LINT_CONFIG_IGNORE_SECTION, + ARG_LINT_CONFIG_IGNORE_OPTION, + ARG_VERBOSE, + ), + ), ) JOBS_COMMANDS = ( diff --git a/airflow/cli/commands/remote_commands/config_command.py b/airflow/cli/commands/remote_commands/config_command.py index 82f1943d4cf84..8d8233f9cc176 100644 --- a/airflow/cli/commands/remote_commands/config_command.py +++ b/airflow/cli/commands/remote_commands/config_command.py @@ -64,3 +64,349 @@ def get_value(args): print(value) except AirflowConfigException: pass + + +class ConfigChange: + """Class representing the configuration changes in Airflow 3.0.""" + + def __init__( + self, section: str, option: str, suggestion: str = "", renamed_to: tuple[str, str] | None = None + ) -> None: + """ + Initialize a RemovedConfig instance. + + :param section: The section of the configuration. + :param option: The option within the section that is removed or deprecated. + :param suggestion: A suggestion for replacing or handling the removed configuration. + :param renamed_to: The new section and option if the configuration is renamed. + """ + self.section = section + self.option = option + self.suggestion = suggestion + self.renamed_to = renamed_to + + def get_message(self) -> str: + """Generate a message for this configuration change.""" + lint_message = f"Removed deprecated `{self.option}` configuration parameter from `{self.section}` section. {self.suggestion}" + + if self.renamed_to: + new_section, new_option = self.renamed_to + rename_message = f" Please use `{new_option}` from section `{new_section}` instead." + lint_message = lint_message + rename_message + return lint_message + + +CONFIGS_CHANGES = [ + ConfigChange( + section="admin", + option="hide_sensitive_variable_fields", + renamed_to=("core", "hide_sensitive_var_conn_fields"), + ), + ConfigChange( + section="admin", + option="sensitive_variable_fields", + renamed_to=("core", "sensitive_var_conn_names"), + ), + ConfigChange( + section="core", + option="check_slas", + suggestion="The SLA feature is removed in Airflow 3.0, to be replaced with Airflow Alerts in " + "Airflow 3.1", + ), + ConfigChange( + section="core", + option="strict_asset_uri_validation", + suggestion="Asset URI with a defined scheme will now always be validated strictly, " + "raising a hard error on validation failure.", + ), + ConfigChange( + section="core", + option="worker_precheck", + renamed_to=("celery", "worker_precheck"), + ), + ConfigChange( + section="core", + option="non_pooled_task_slot_count", + renamed_to=("core", "default_pool_task_slot_count"), + ), + ConfigChange( + section="core", + option="dag_concurrency", + renamed_to=("core", "max_active_tasks_per_dag"), + ), + ConfigChange( + section="core", + option="sql_alchemy_conn", + renamed_to=("database", "sql_alchemy_conn"), + ), + ConfigChange( + section="core", + option="sql_engine_encoding", + renamed_to=("database", "sql_engine_encoding"), + ), + ConfigChange( + section="core", + option="sql_engine_collation_for_ids", + renamed_to=("database", "sql_engine_collation_for_ids"), + ), + ConfigChange( + section="core", + option="sql_alchemy_pool_enabled", + renamed_to=("database", "sql_alchemy_pool_enabled"), + ), + ConfigChange( + section="core", + option="sql_alchemy_pool_size", + renamed_to=("database", "sql_alchemy_pool_size"), + ), + ConfigChange( + section="core", + option="sql_alchemy_max_overflow", + renamed_to=("database", "sql_alchemy_max_overflow"), + ), + ConfigChange( + section="core", + option="sql_alchemy_pool_recycle", + renamed_to=("database", "sql_alchemy_pool_recycle"), + ), + ConfigChange( + section="core", + option="sql_alchemy_pool_pre_ping", + renamed_to=("database", "sql_alchemy_pool_pre_ping"), + ), + ConfigChange( + section="core", + option="sql_alchemy_schema", + renamed_to=("database", "sql_alchemy_schema"), + ), + ConfigChange( + section="core", + option="sql_alchemy_connect_args", + renamed_to=("database", "sql_alchemy_connect_args"), + ), + ConfigChange( + section="core", + option="load_default_connections", + renamed_to=("database", "load_default_connections"), + ), + ConfigChange( + section="core", + option="max_db_retries", + renamed_to=("database", "max_db_retries"), + ), + ConfigChange( + section="api", + option="access_control_allow_origin", + renamed_to=("api", "access_control_allow_origins"), + ), + ConfigChange( + section="api", + option="auth_backend", + renamed_to=("api", "auth_backends"), + ), + ConfigChange( + section="logging", + option="enable_task_context_logger", + suggestion="Remove TaskContextLogger: Replaced by the Log table for better handling of task log " + "messages outside the execution context.", + ), + ConfigChange( + section="metrics", + option="metrics_use_pattern_match", + ), + ConfigChange( + section="metrics", + option="timer_unit_consistency", + suggestion="In Airflow 3.0, the ``timer_unit_consistency`` setting in the ``metrics`` section is " + "removed as it is now the default behaviour. This is done to standardize all timer and " + "timing metrics to milliseconds across all metric loggers", + ), + ConfigChange( + section="metrics", + option="statsd_allow_list", + renamed_to=("metrics", "metrics_allow_list"), + ), + ConfigChange( + section="metrics", + option="statsd_block_list", + renamed_to=("metrics", "metrics_block_list"), + ), + ConfigChange( + section="traces", + option="otel_task_log_event", + ), + ConfigChange( + section="operators", + option="allow_illegal_arguments", + ), + ConfigChange( + section="webserver", + option="allow_raw_html_descriptions", + ), + ConfigChange( + section="webserver", + option="session_lifetime_days", + suggestion="Please use ``session_lifetime_minutes``.", + ), + ConfigChange(section="webserver", option="update_fab_perms", renamed_to=("fab", "update_fab_perms")), + ConfigChange(section="webserver", option="auth_rate_limited", renamed_to=("fab", "auth_rate_limited")), + ConfigChange(section="webserver", option="auth_rate_limit", renamed_to=("fab", "auth_rate_limit")), + ConfigChange( + section="webserver", + option="session_lifetime_days", + renamed_to=("webserver", "session_lifetime_minutes"), + ), + ConfigChange( + section="webserver", + option="force_log_out_after", + renamed_to=("webserver", "session_lifetime_minutes"), + ), + ConfigChange(section="policy", option="airflow_local_settings", renamed_to=("policy", "task_policy")), + ConfigChange( + section="scheduler", + option="dependency_detector", + ), + ConfigChange( + section="scheduler", + option="processor_poll_interval", + renamed_to=("scheduler", "scheduler_idle_sleep_time"), + ), + ConfigChange( + section="scheduler", + option="deactivate_stale_dags_interval", + renamed_to=("scheduler", "parsing_cleanup_interval"), + ), + ConfigChange(section="scheduler", option="statsd_on", renamed_to=("metrics", "statsd_on")), + ConfigChange(section="scheduler", option="max_threads", renamed_to=("scheduler", "parsing_processes")), + ConfigChange(section="scheduler", option="statsd_host", renamed_to=("metrics", "statsd_host")), + ConfigChange(section="scheduler", option="statsd_port", renamed_to=("metrics", "statsd_port")), + ConfigChange(section="scheduler", option="statsd_prefix", renamed_to=("metrics", "statsd_prefix")), + ConfigChange( + section="scheduler", option="statsd_allow_list", renamed_to=("metrics", "statsd_allow_list") + ), + ConfigChange( + section="scheduler", option="stat_name_handler", renamed_to=("metrics", "stat_name_handler") + ), + ConfigChange( + section="scheduler", option="statsd_datadog_enabled", renamed_to=("metrics", "statsd_datadog_enabled") + ), + ConfigChange( + section="scheduler", option="statsd_datadog_tags", renamed_to=("metrics", "statsd_datadog_tags") + ), + ConfigChange( + section="scheduler", + option="statsd_datadog_metrics_tags", + renamed_to=("metrics", "statsd_datadog_metrics_tags"), + ), + ConfigChange( + section="scheduler", + option="statsd_custom_client_path", + renamed_to=("metrics", "statsd_custom_client_path"), + ), + ConfigChange( + section="celery", option="stalled_task_timeout", renamed_to=("scheduler", "task_queued_timeout") + ), + ConfigChange(section="celery", option="default_queue", renamed_to=("operators", "default_queue")), + ConfigChange( + section="celery", option="task_adoption_timeout", renamed_to=("scheduler", "task_queued_timeout") + ), + ConfigChange( + section="kubernetes_executor", + option="worker_pods_pending_timeout", + renamed_to=("scheduler", "task_queued_timeout"), + ), + ConfigChange( + section="kubernetes_executor", + option="worker_pods_pending_timeout_check_interval", + renamed_to=("scheduler", "task_queued_timeout_check_interval"), + ), + ConfigChange( + section="smtp", option="smtp_user", suggestion="Please use the SMTP connection (`smtp_default`)." + ), + ConfigChange( + section="smtp", option="smtp_password", suggestion="Please use the SMTP connection (`smtp_default`)." + ), +] + + +@providers_configuration_loaded +def lint_config(args) -> None: + """ + Lint the airflow.cfg file for removed, or renamed configurations. + + This function scans the Airflow configuration file for parameters that are removed or renamed in + Airflow 3.0. It provides suggestions for alternative parameters or settings where applicable. + CLI Arguments: + --section: str (optional) + The specific section of the configuration to lint. + Example: --section core + + --option: str (optional) + The specific option within a section to lint. + Example: --option check_slas + + --ignore-section: str (optional) + A section to ignore during linting. + Example: --ignore-section webserver + + --ignore-option: str (optional) + An option to ignore during linting. + Example: --ignore-option smtp_user + + --verbose: flag (optional) + Enables detailed output, including the list of ignored sections and options. + Example: --verbose + + Examples: + 1. Lint all sections and options: + airflow config lint + + 2. Lint a specific section: + airflow config lint --section core + + 3. Lint a specific section and option: + airflow config lint --section smtp --option smtp_user + + 4. Ignore a section: + airflow config lint --ignore-section webserver + + 5. Ignore an option: + airflow config lint --ignore-option smtp_user + + 6. Enable verbose output: + airflow config lint --verbose + + :param args: The CLI arguments for linting configurations. + """ + lint_issues = [] + + section_to_check_if_provided = args.section + option_to_check_if_provided = args.option + + ignore_sections = [args.ignore_section] if args.ignore_section else [] + ignore_options = [args.ignore_option] if args.ignore_option else [] + + for config in CONFIGS_CHANGES: + if section_to_check_if_provided and config.section != section_to_check_if_provided: + continue + + if option_to_check_if_provided and config.option != option_to_check_if_provided: + continue + + if config.section in ignore_sections or config.option in ignore_options: + continue + + if conf.has_option(config.section, config.option): + lint_issues.append(config.get_message()) + + if lint_issues: + print("Found issues in your airflow.cfg:") + for issue in lint_issues: + print(f" - {issue}") + if args.verbose: + print("\nDetailed Information:") + print(f"Ignored sections: {', '.join(ignore_sections)}") + print(f"Ignored options: {', '.join(ignore_options)}") + print("\nPlease update your configuration file accordingly.") + else: + print("No issues found in your airflow.cfg. It is ready for Airflow 3!") diff --git a/tests/cli/commands/remote_commands/test_config_command.py b/tests/cli/commands/remote_commands/test_config_command.py index 57354fda7694a..d8b189747a892 100644 --- a/tests/cli/commands/remote_commands/test_config_command.py +++ b/tests/cli/commands/remote_commands/test_config_command.py @@ -20,6 +20,8 @@ from io import StringIO from unittest import mock +import pytest + from airflow.cli import cli_parser from airflow.cli.commands.remote_commands import config_command @@ -232,3 +234,109 @@ def test_should_raise_exception_when_option_is_missing(self, caplog): self.parser.parse_args(["config", "get-value", "missing-section", "dags_folder"]) ) assert "section/key [missing-section/dags_folder] not found in config" in caplog.text + + +class TestConfigLint: + from airflow.cli.commands.remote_commands.config_command import CONFIGS_CHANGES + + @pytest.mark.parametrize("removed_config", CONFIGS_CHANGES) + def test_lint_detects_removed_configs(self, removed_config): + with mock.patch("airflow.configuration.conf.has_option", return_value=True): + with contextlib.redirect_stdout(StringIO()) as temp_stdout: + config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) + + output = temp_stdout.getvalue() + assert ( + f"Removed deprecated `{removed_config.option}` configuration parameter from `{removed_config.section}` section." + in output + ) + + if removed_config.suggestion: + assert removed_config.suggestion in output + + @pytest.mark.parametrize( + "section, option, suggestion", + [ + ( + "core", + "check_slas", + "The SLA feature is removed in Airflow 3.0, to be replaced with Airflow Alerts in Airflow 3.1", + ), + ( + "core", + "strict_asset_uri_validation", + "Asset URI with a defined scheme will now always be validated strictly, raising a hard error on validation failure.", + ), + ( + "logging", + "enable_task_context_logger", + "Remove TaskContextLogger: Replaced by the Log table for better handling of task log messages outside the execution context.", + ), + ], + ) + def test_lint_with_specific_removed_configs(self, section, option, suggestion): + with mock.patch("airflow.configuration.conf.has_option", return_value=True): + with contextlib.redirect_stdout(StringIO()) as temp_stdout: + config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) + + output = temp_stdout.getvalue() + assert ( + f"Removed deprecated `{option}` configuration parameter from `{section}` section." in output + ) + + if suggestion: + assert suggestion in output + + def test_lint_specific_section_option(self): + with mock.patch("airflow.configuration.conf.has_option", return_value=True): + with contextlib.redirect_stdout(StringIO()) as temp_stdout: + config_command.lint_config( + cli_parser.get_parser().parse_args( + ["config", "lint", "--section", "core", "--option", "check_slas"] + ) + ) + + output = temp_stdout.getvalue() + assert "Removed deprecated `check_slas` configuration parameter from `core` section." in output + + def test_lint_with_invalid_section_option(self): + with mock.patch("airflow.configuration.conf.has_option", return_value=False): + with contextlib.redirect_stdout(StringIO()) as temp_stdout: + config_command.lint_config( + cli_parser.get_parser().parse_args( + ["config", "lint", "--section", "invalid_section", "--option", "invalid_option"] + ) + ) + + output = temp_stdout.getvalue() + assert "No issues found in your airflow.cfg." in output + + def test_lint_detects_multiple_issues(self): + with mock.patch( + "airflow.configuration.conf.has_option", + side_effect=lambda s, o: o in ["check_slas", "strict_asset_uri_validation"], + ): + with contextlib.redirect_stdout(StringIO()) as temp_stdout: + config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) + + output = temp_stdout.getvalue() + assert "Removed deprecated `check_slas` configuration parameter from `core` section." in output + assert ( + "Removed deprecated `strict_asset_uri_validation` configuration parameter from `core` section." + in output + ) + + def test_lint_detects_renamed_configs(self): + renamed_config = config_command.CONFIGS_CHANGES[0] + with mock.patch("airflow.configuration.conf.has_option", return_value=True): + with contextlib.redirect_stdout(StringIO()) as temp_stdout: + config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) + + output = temp_stdout.getvalue() + if renamed_config.renamed_to: + new_section, new_option = renamed_config.renamed_to + assert ( + f"Removed deprecated `{renamed_config.option}` configuration parameter from `{renamed_config.section}` section." + in output + ) + assert f"Please use `{new_option}` from section `{new_section}` instead." in output From 5cc2dc5da22090a3d637615d52b4a072ed75407c Mon Sep 17 00:00:00 2001 From: Ankit Chaurasia <8670962+sunank200@users.noreply.github.com> Date: Sat, 14 Dec 2024 01:47:40 +0545 Subject: [PATCH 2/6] Fix doc-string for ConfigChange init --- airflow/cli/commands/remote_commands/config_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/cli/commands/remote_commands/config_command.py b/airflow/cli/commands/remote_commands/config_command.py index 8d8233f9cc176..e9b32b2451c60 100644 --- a/airflow/cli/commands/remote_commands/config_command.py +++ b/airflow/cli/commands/remote_commands/config_command.py @@ -73,7 +73,7 @@ def __init__( self, section: str, option: str, suggestion: str = "", renamed_to: tuple[str, str] | None = None ) -> None: """ - Initialize a RemovedConfig instance. + Initialize a ConfigChange instance. :param section: The section of the configuration. :param option: The option within the section that is removed or deprecated. From bd70bf28553031bb73b8b5cf4a27b3f93840a638 Mon Sep 17 00:00:00 2001 From: Ankit Chaurasia <8670962+sunank200@users.noreply.github.com> Date: Tue, 17 Dec 2024 02:51:07 +0545 Subject: [PATCH 3/6] Add option for multiple sections and options Add newsfragments --- airflow/cli/cli_config.py | 17 +- .../remote_commands/config_command.py | 198 +++++++++++------- newsfragments/44908.significant.rst | 15 ++ .../remote_commands/test_config_command.py | 134 +++++++++--- 4 files changed, 251 insertions(+), 113 deletions(-) create mode 100644 newsfragments/44908.significant.rst diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 5b5099f7fdbe9..c6ca88f7882b4 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -860,23 +860,28 @@ def string_lower_type(val): ARG_LINT_CONFIG_SECTION = Arg( ("--section",), - help="The section name to lint in the airflow config.", + help="The section name(s) to lint in the airflow config.", + type=string_list_type, ) ARG_LINT_CONFIG_OPTION = Arg( ("--option",), - help="The option name to lint in the airflow config.", + help="The option name(s) to lint in the airflow config.", + type=string_list_type, ) ARG_LINT_CONFIG_IGNORE_SECTION = Arg( ("--ignore-section",), - help="The section name to ignore to lint in the airflow config.", + help="The section name(s) to ignore to lint in the airflow config.", + type=string_list_type, ) ARG_LINT_CONFIG_IGNORE_OPTION = Arg( ("--ignore-option",), - help="The option name to ignore to lint in the airflow config.", + help="The option name(s) to ignore to lint in the airflow config.", + type=string_list_type, ) ARG_OPTIONAL_SECTION = Arg( ("--section",), - help="The section name", + help="The section name(s).", + type=string_list_type, ) # jobs check @@ -1752,7 +1757,7 @@ class GroupCommand(NamedTuple): ), ActionCommand( name="lint", - help="lint options for the configuration changes while migrating from airflow 2.x to airflow 3.0", + help="lint options for the configuration changes while migrating from Airflow 2.x to Airflow 3.0", func=lazy_load_command("airflow.cli.commands.remote_commands.config_command.lint_config"), args=( ARG_LINT_CONFIG_SECTION, diff --git a/airflow/cli/commands/remote_commands/config_command.py b/airflow/cli/commands/remote_commands/config_command.py index e9b32b2451c60..67e8140f8ac8b 100644 --- a/airflow/cli/commands/remote_commands/config_command.py +++ b/airflow/cli/commands/remote_commands/config_command.py @@ -18,11 +18,14 @@ from __future__ import annotations +from dataclasses import dataclass from io import StringIO +from typing import NamedTuple import pygments from pygments.lexers.configs import IniLexer +from airflow.cli.simple_table import AirflowConsole from airflow.configuration import conf from airflow.exceptions import AirflowConfigException from airflow.utils.cli import should_use_colors @@ -66,11 +69,19 @@ def get_value(args): pass +class RenamedTo(NamedTuple): + """Represents a configuration parameter that has been renamed.""" + + section: str + option: str + + +@dataclass class ConfigChange: """Class representing the configuration changes in Airflow 3.0.""" def __init__( - self, section: str, option: str, suggestion: str = "", renamed_to: tuple[str, str] | None = None + self, section: str, option: str, suggestion: str = "", renamed_to: RenamedTo | None = None ) -> None: """ Initialize a ConfigChange instance. @@ -85,33 +96,43 @@ def __init__( self.suggestion = suggestion self.renamed_to = renamed_to - def get_message(self) -> str: + @property + def message(self) -> str: """Generate a message for this configuration change.""" - lint_message = f"Removed deprecated `{self.option}` configuration parameter from `{self.section}` section. {self.suggestion}" - if self.renamed_to: - new_section, new_option = self.renamed_to - rename_message = f" Please use `{new_option}` from section `{new_section}` instead." - lint_message = lint_message + rename_message - return lint_message + if self.section != self.renamed_to.section: + return ( + f"`{self.option}` configuration parameter moved from `{self.section}` section to `" + f"{self.renamed_to.section}` section as `{self.renamed_to.option}`." + ) + else: + return ( + f"`{self.option}` configuration parameter renamed to `{self.renamed_to.option}` " + f"in the `{self.section}` section." + ) + else: + return ( + f"Removed deprecated `{self.option}` configuration parameter from `{self.section}` section. " + f"{self.suggestion}" + ) CONFIGS_CHANGES = [ ConfigChange( section="admin", option="hide_sensitive_variable_fields", - renamed_to=("core", "hide_sensitive_var_conn_fields"), + renamed_to=RenamedTo("core", "hide_sensitive_var_conn_fields"), ), ConfigChange( section="admin", option="sensitive_variable_fields", - renamed_to=("core", "sensitive_var_conn_names"), + renamed_to=RenamedTo("core", "sensitive_var_conn_names"), ), ConfigChange( section="core", option="check_slas", suggestion="The SLA feature is removed in Airflow 3.0, to be replaced with Airflow Alerts in " - "Airflow 3.1", + "future", ), ConfigChange( section="core", @@ -122,87 +143,87 @@ def get_message(self) -> str: ConfigChange( section="core", option="worker_precheck", - renamed_to=("celery", "worker_precheck"), + renamed_to=RenamedTo("celery", "worker_precheck"), ), ConfigChange( section="core", option="non_pooled_task_slot_count", - renamed_to=("core", "default_pool_task_slot_count"), + renamed_to=RenamedTo("core", "default_pool_task_slot_count"), ), ConfigChange( section="core", option="dag_concurrency", - renamed_to=("core", "max_active_tasks_per_dag"), + renamed_to=RenamedTo("core", "max_active_tasks_per_dag"), ), ConfigChange( section="core", option="sql_alchemy_conn", - renamed_to=("database", "sql_alchemy_conn"), + renamed_to=RenamedTo("database", "sql_alchemy_conn"), ), ConfigChange( section="core", option="sql_engine_encoding", - renamed_to=("database", "sql_engine_encoding"), + renamed_to=RenamedTo("database", "sql_engine_encoding"), ), ConfigChange( section="core", option="sql_engine_collation_for_ids", - renamed_to=("database", "sql_engine_collation_for_ids"), + renamed_to=RenamedTo("database", "sql_engine_collation_for_ids"), ), ConfigChange( section="core", option="sql_alchemy_pool_enabled", - renamed_to=("database", "sql_alchemy_pool_enabled"), + renamed_to=RenamedTo("database", "sql_alchemy_pool_enabled"), ), ConfigChange( section="core", option="sql_alchemy_pool_size", - renamed_to=("database", "sql_alchemy_pool_size"), + renamed_to=RenamedTo("database", "sql_alchemy_pool_size"), ), ConfigChange( section="core", option="sql_alchemy_max_overflow", - renamed_to=("database", "sql_alchemy_max_overflow"), + renamed_to=RenamedTo("database", "sql_alchemy_max_overflow"), ), ConfigChange( section="core", option="sql_alchemy_pool_recycle", - renamed_to=("database", "sql_alchemy_pool_recycle"), + renamed_to=RenamedTo("database", "sql_alchemy_pool_recycle"), ), ConfigChange( section="core", option="sql_alchemy_pool_pre_ping", - renamed_to=("database", "sql_alchemy_pool_pre_ping"), + renamed_to=RenamedTo("database", "sql_alchemy_pool_pre_ping"), ), ConfigChange( section="core", option="sql_alchemy_schema", - renamed_to=("database", "sql_alchemy_schema"), + renamed_to=RenamedTo("database", "sql_alchemy_schema"), ), ConfigChange( section="core", option="sql_alchemy_connect_args", - renamed_to=("database", "sql_alchemy_connect_args"), + renamed_to=RenamedTo("database", "sql_alchemy_connect_args"), ), ConfigChange( section="core", option="load_default_connections", - renamed_to=("database", "load_default_connections"), + renamed_to=RenamedTo("database", "load_default_connections"), ), ConfigChange( section="core", option="max_db_retries", - renamed_to=("database", "max_db_retries"), + renamed_to=RenamedTo("database", "max_db_retries"), ), ConfigChange( section="api", option="access_control_allow_origin", - renamed_to=("api", "access_control_allow_origins"), + renamed_to=RenamedTo("api", "access_control_allow_origins"), ), ConfigChange( section="api", option="auth_backend", - renamed_to=("api", "auth_backends"), + renamed_to=RenamedTo("api", "auth_backends"), ), ConfigChange( section="logging", @@ -217,19 +238,19 @@ def get_message(self) -> str: ConfigChange( section="metrics", option="timer_unit_consistency", - suggestion="In Airflow 3.0, the ``timer_unit_consistency`` setting in the ``metrics`` section is " + suggestion="In Airflow 3.0, the `timer_unit_consistency` setting in the `metrics` section is " "removed as it is now the default behaviour. This is done to standardize all timer and " "timing metrics to milliseconds across all metric loggers", ), ConfigChange( section="metrics", option="statsd_allow_list", - renamed_to=("metrics", "metrics_allow_list"), + renamed_to=RenamedTo("metrics", "metrics_allow_list"), ), ConfigChange( section="metrics", option="statsd_block_list", - renamed_to=("metrics", "metrics_block_list"), + renamed_to=RenamedTo("metrics", "metrics_block_list"), ), ConfigChange( section="traces", @@ -246,22 +267,30 @@ def get_message(self) -> str: ConfigChange( section="webserver", option="session_lifetime_days", - suggestion="Please use ``session_lifetime_minutes``.", + suggestion="Please use `session_lifetime_minutes`.", + ), + ConfigChange( + section="webserver", option="update_fab_perms", renamed_to=RenamedTo("fab", "update_fab_perms") + ), + ConfigChange( + section="webserver", option="auth_rate_limited", renamed_to=RenamedTo("fab", "auth_rate_limited") + ), + ConfigChange( + section="webserver", option="auth_rate_limit", renamed_to=RenamedTo("fab", "auth_rate_limit") ), - ConfigChange(section="webserver", option="update_fab_perms", renamed_to=("fab", "update_fab_perms")), - ConfigChange(section="webserver", option="auth_rate_limited", renamed_to=("fab", "auth_rate_limited")), - ConfigChange(section="webserver", option="auth_rate_limit", renamed_to=("fab", "auth_rate_limit")), ConfigChange( section="webserver", option="session_lifetime_days", - renamed_to=("webserver", "session_lifetime_minutes"), + renamed_to=RenamedTo("webserver", "session_lifetime_minutes"), ), ConfigChange( section="webserver", option="force_log_out_after", - renamed_to=("webserver", "session_lifetime_minutes"), + renamed_to=RenamedTo("webserver", "session_lifetime_minutes"), + ), + ConfigChange( + section="policy", option="airflow_local_settings", renamed_to=RenamedTo("policy", "task_policy") ), - ConfigChange(section="policy", option="airflow_local_settings", renamed_to=("policy", "task_policy")), ConfigChange( section="scheduler", option="dependency_detector", @@ -269,56 +298,70 @@ def get_message(self) -> str: ConfigChange( section="scheduler", option="processor_poll_interval", - renamed_to=("scheduler", "scheduler_idle_sleep_time"), + renamed_to=RenamedTo("scheduler", "scheduler_idle_sleep_time"), ), ConfigChange( section="scheduler", option="deactivate_stale_dags_interval", - renamed_to=("scheduler", "parsing_cleanup_interval"), + renamed_to=RenamedTo("scheduler", "parsing_cleanup_interval"), + ), + ConfigChange(section="scheduler", option="statsd_on", renamed_to=RenamedTo("metrics", "statsd_on")), + ConfigChange( + section="scheduler", option="max_threads", renamed_to=RenamedTo("scheduler", "parsing_processes") + ), + ConfigChange(section="scheduler", option="statsd_host", renamed_to=RenamedTo("metrics", "statsd_host")), + ConfigChange(section="scheduler", option="statsd_port", renamed_to=RenamedTo("metrics", "statsd_port")), + ConfigChange( + section="scheduler", option="statsd_prefix", renamed_to=RenamedTo("metrics", "statsd_prefix") ), - ConfigChange(section="scheduler", option="statsd_on", renamed_to=("metrics", "statsd_on")), - ConfigChange(section="scheduler", option="max_threads", renamed_to=("scheduler", "parsing_processes")), - ConfigChange(section="scheduler", option="statsd_host", renamed_to=("metrics", "statsd_host")), - ConfigChange(section="scheduler", option="statsd_port", renamed_to=("metrics", "statsd_port")), - ConfigChange(section="scheduler", option="statsd_prefix", renamed_to=("metrics", "statsd_prefix")), ConfigChange( - section="scheduler", option="statsd_allow_list", renamed_to=("metrics", "statsd_allow_list") + section="scheduler", option="statsd_allow_list", renamed_to=RenamedTo("metrics", "statsd_allow_list") ), ConfigChange( - section="scheduler", option="stat_name_handler", renamed_to=("metrics", "stat_name_handler") + section="scheduler", option="stat_name_handler", renamed_to=RenamedTo("metrics", "stat_name_handler") ), ConfigChange( - section="scheduler", option="statsd_datadog_enabled", renamed_to=("metrics", "statsd_datadog_enabled") + section="scheduler", + option="statsd_datadog_enabled", + renamed_to=RenamedTo("metrics", "statsd_datadog_enabled"), ), ConfigChange( - section="scheduler", option="statsd_datadog_tags", renamed_to=("metrics", "statsd_datadog_tags") + section="scheduler", + option="statsd_datadog_tags", + renamed_to=RenamedTo("metrics", "statsd_datadog_tags"), ), ConfigChange( section="scheduler", option="statsd_datadog_metrics_tags", - renamed_to=("metrics", "statsd_datadog_metrics_tags"), + renamed_to=RenamedTo("metrics", "statsd_datadog_metrics_tags"), ), ConfigChange( section="scheduler", option="statsd_custom_client_path", - renamed_to=("metrics", "statsd_custom_client_path"), + renamed_to=RenamedTo("metrics", "statsd_custom_client_path"), + ), + ConfigChange( + section="celery", + option="stalled_task_timeout", + renamed_to=RenamedTo("scheduler", "task_queued_timeout"), ), ConfigChange( - section="celery", option="stalled_task_timeout", renamed_to=("scheduler", "task_queued_timeout") + section="celery", option="default_queue", renamed_to=RenamedTo("operators", "default_queue") ), - ConfigChange(section="celery", option="default_queue", renamed_to=("operators", "default_queue")), ConfigChange( - section="celery", option="task_adoption_timeout", renamed_to=("scheduler", "task_queued_timeout") + section="celery", + option="task_adoption_timeout", + renamed_to=RenamedTo("scheduler", "task_queued_timeout"), ), ConfigChange( section="kubernetes_executor", option="worker_pods_pending_timeout", - renamed_to=("scheduler", "task_queued_timeout"), + renamed_to=RenamedTo("scheduler", "task_queued_timeout"), ), ConfigChange( section="kubernetes_executor", option="worker_pods_pending_timeout_check_interval", - renamed_to=("scheduler", "task_queued_timeout_check_interval"), + renamed_to=RenamedTo("scheduler", "task_queued_timeout_check_interval"), ), ConfigChange( section="smtp", option="smtp_user", suggestion="Please use the SMTP connection (`smtp_default`)." @@ -361,52 +404,53 @@ def lint_config(args) -> None: 1. Lint all sections and options: airflow config lint - 2. Lint a specific section: - airflow config lint --section core + 2. Lint a specific sections: + airflow config lint --section core,webserver - 3. Lint a specific section and option: + 3. Lint a specific sections and options: airflow config lint --section smtp --option smtp_user - 4. Ignore a section: - airflow config lint --ignore-section webserver + 4. Ignore a sections: + irflow config lint --ignore-section webserver,api - 5. Ignore an option: - airflow config lint --ignore-option smtp_user + 5. Ignore an options: + airflow config lint --ignore-option smtp_user,session_lifetime_days 6. Enable verbose output: airflow config lint --verbose :param args: The CLI arguments for linting configurations. """ + console = AirflowConsole() lint_issues = [] - section_to_check_if_provided = args.section - option_to_check_if_provided = args.option + section_to_check_if_provided = args.section or [] + option_to_check_if_provided = args.option or [] - ignore_sections = [args.ignore_section] if args.ignore_section else [] - ignore_options = [args.ignore_option] if args.ignore_option else [] + ignore_sections = args.ignore_section or [] + ignore_options = args.ignore_option or [] for config in CONFIGS_CHANGES: - if section_to_check_if_provided and config.section != section_to_check_if_provided: + if section_to_check_if_provided and config.section not in section_to_check_if_provided: continue - if option_to_check_if_provided and config.option != option_to_check_if_provided: + if option_to_check_if_provided and config.option not in option_to_check_if_provided: continue if config.section in ignore_sections or config.option in ignore_options: continue if conf.has_option(config.section, config.option): - lint_issues.append(config.get_message()) + lint_issues.append(config.message) if lint_issues: - print("Found issues in your airflow.cfg:") + console.print("[red]Found issues in your airflow.cfg:[/red]") for issue in lint_issues: - print(f" - {issue}") + console.print(f" - [yellow]{issue}[/yellow]") if args.verbose: - print("\nDetailed Information:") - print(f"Ignored sections: {', '.join(ignore_sections)}") - print(f"Ignored options: {', '.join(ignore_options)}") - print("\nPlease update your configuration file accordingly.") + console.print("\n[blue]Detailed Information:[/blue]") + console.print(f"Ignored sections: [green]{', '.join(ignore_sections)}[/green]") + console.print(f"Ignored options: [green]{', '.join(ignore_options)}[/green]") + console.print("\n[red]Please update your configuration file accordingly.[/red]") else: - print("No issues found in your airflow.cfg. It is ready for Airflow 3!") + console.print("[green]No issues found in your airflow.cfg. It is ready for Airflow 3![/green]") diff --git a/newsfragments/44908.significant.rst b/newsfragments/44908.significant.rst new file mode 100644 index 0000000000000..bcd907701aa1d --- /dev/null +++ b/newsfragments/44908.significant.rst @@ -0,0 +1,15 @@ +Airflow CLI command ``airflow config lint`` added + +The ``airflow config lint`` command has been introduced to help users migrate from Airflow 2.x to 3.0 by identifying removed or renamed configuration parameters in airflow.cfg. + +This command provides actionable feedback and highlights renamed parameters with guidance for transitioning to their new names or sections. + +* Types of change + + * [ ] DAG changes + * [ ] Config changes + * [ ] API changes + * [x] CLI changes + * [ ] Behaviour changes + * [ ] Plugin changes + * [ ] Dependency change diff --git a/tests/cli/commands/remote_commands/test_config_command.py b/tests/cli/commands/remote_commands/test_config_command.py index d8b189747a892..f35ecc6c62da7 100644 --- a/tests/cli/commands/remote_commands/test_config_command.py +++ b/tests/cli/commands/remote_commands/test_config_command.py @@ -17,6 +17,7 @@ from __future__ import annotations import contextlib +import re from io import StringIO from unittest import mock @@ -237,22 +238,18 @@ def test_should_raise_exception_when_option_is_missing(self, caplog): class TestConfigLint: - from airflow.cli.commands.remote_commands.config_command import CONFIGS_CHANGES - - @pytest.mark.parametrize("removed_config", CONFIGS_CHANGES) + @pytest.mark.parametrize("removed_config", config_command.CONFIGS_CHANGES) def test_lint_detects_removed_configs(self, removed_config): with mock.patch("airflow.configuration.conf.has_option", return_value=True): with contextlib.redirect_stdout(StringIO()) as temp_stdout: config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) output = temp_stdout.getvalue() - assert ( - f"Removed deprecated `{removed_config.option}` configuration parameter from `{removed_config.section}` section." - in output - ) - if removed_config.suggestion: - assert removed_config.suggestion in output + normalized_output = re.sub(r"\s+", " ", output.strip()) + normalized_message = re.sub(r"\s+", " ", removed_config.message.strip()) + + assert normalized_message in normalized_output @pytest.mark.parametrize( "section, option, suggestion", @@ -260,7 +257,7 @@ def test_lint_detects_removed_configs(self, removed_config): ( "core", "check_slas", - "The SLA feature is removed in Airflow 3.0, to be replaced with Airflow Alerts in Airflow 3.1", + "The SLA feature is removed in Airflow 3.0, to be replaced with Airflow Alerts in future", ), ( "core", @@ -280,12 +277,14 @@ def test_lint_with_specific_removed_configs(self, section, option, suggestion): config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) output = temp_stdout.getvalue() - assert ( - f"Removed deprecated `{option}` configuration parameter from `{section}` section." in output - ) - if suggestion: - assert suggestion in output + normalized_output = re.sub(r"\s+", " ", output.strip()) + + expected_message = f"Removed deprecated `{option}` configuration parameter from `{section}` section." + assert expected_message in normalized_output + + if suggestion: + assert suggestion in normalized_output def test_lint_specific_section_option(self): with mock.patch("airflow.configuration.conf.has_option", return_value=True): @@ -297,7 +296,13 @@ def test_lint_specific_section_option(self): ) output = temp_stdout.getvalue() - assert "Removed deprecated `check_slas` configuration parameter from `core` section." in output + + normalized_output = re.sub(r"\s+", " ", output.strip()) + + assert ( + "Removed deprecated `check_slas` configuration parameter from `core` section." + in normalized_output + ) def test_lint_with_invalid_section_option(self): with mock.patch("airflow.configuration.conf.has_option", return_value=False): @@ -309,7 +314,10 @@ def test_lint_with_invalid_section_option(self): ) output = temp_stdout.getvalue() - assert "No issues found in your airflow.cfg." in output + + normalized_output = re.sub(r"\s+", " ", output.strip()) + + assert "No issues found in your airflow.cfg." in normalized_output def test_lint_detects_multiple_issues(self): with mock.patch( @@ -320,23 +328,89 @@ def test_lint_detects_multiple_issues(self): config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) output = temp_stdout.getvalue() - assert "Removed deprecated `check_slas` configuration parameter from `core` section." in output - assert ( - "Removed deprecated `strict_asset_uri_validation` configuration parameter from `core` section." - in output + + normalized_output = re.sub(r"\s+", " ", output.strip()) + + assert ( + "Removed deprecated `check_slas` configuration parameter from `core` section." + in normalized_output + ) + assert ( + "Removed deprecated `strict_asset_uri_validation` configuration parameter from `core` section." + in normalized_output + ) + + @pytest.mark.parametrize( + "removed_configs", + [ + [ + ( + "core", + "check_slas", + "The SLA feature is removed in Airflow 3.0, to be replaced with Airflow Alerts in future", + ), + ( + "core", + "strict_asset_uri_validation", + "Asset URI with a defined scheme will now always be validated strictly, raising a hard error on validation failure.", + ), + ( + "logging", + "enable_task_context_logger", + "Remove TaskContextLogger: Replaced by the Log table for better handling of task log messages outside the execution context.", + ), + ], + [ + ("webserver", "allow_raw_html_descriptions", ""), + ("webserver", "session_lifetime_days", "Please use `session_lifetime_minutes`."), + ], + ], + ) + def test_lint_detects_multiple_removed_configs(self, removed_configs): + with mock.patch("airflow.configuration.conf.has_option", return_value=True): + with contextlib.redirect_stdout(StringIO()) as temp_stdout: + config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) + + output = temp_stdout.getvalue() + + normalized_output = re.sub(r"\s+", " ", output.strip()) + + for section, option, suggestion in removed_configs: + expected_message = ( + f"Removed deprecated `{option}` configuration parameter from `{section}` section." ) + assert expected_message in normalized_output - def test_lint_detects_renamed_configs(self): - renamed_config = config_command.CONFIGS_CHANGES[0] + if suggestion: + assert suggestion in normalized_output + + @pytest.mark.parametrize( + "renamed_configs", + [ + # Case 1: Renamed configurations within the same section + [ + ("core", "non_pooled_task_slot_count", "core", "default_pool_task_slot_count"), + ("scheduler", "processor_poll_interval", "scheduler", "scheduler_idle_sleep_time"), + ], + # Case 2: Renamed configurations across sections + [ + ("admin", "hide_sensitive_variable_fields", "core", "hide_sensitive_var_conn_fields"), + ("core", "worker_precheck", "celery", "worker_precheck"), + ], + ], + ) + def test_lint_detects_renamed_configs(self, renamed_configs): with mock.patch("airflow.configuration.conf.has_option", return_value=True): with contextlib.redirect_stdout(StringIO()) as temp_stdout: config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) output = temp_stdout.getvalue() - if renamed_config.renamed_to: - new_section, new_option = renamed_config.renamed_to - assert ( - f"Removed deprecated `{renamed_config.option}` configuration parameter from `{renamed_config.section}` section." - in output - ) - assert f"Please use `{new_option}` from section `{new_section}` instead." in output + + normalized_output = re.sub(r"\s+", " ", output.strip()) + + for old_section, old_option, new_section, new_option in renamed_configs: + if old_section == new_section: + expected_message = f"`{old_option}` configuration parameter renamed to `{new_option}` in the `{old_section}` section." + else: + expected_message = f"`{old_option}` configuration parameter moved from `{old_section}` section to `{new_section}` section as `{new_option}`." + assert expected_message in normalized_output From 96bedeb99fc9adc00be3d8592d5638c91c976f6e Mon Sep 17 00:00:00 2001 From: Ankit Chaurasia <8670962+sunank200@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:25:09 +0545 Subject: [PATCH 4/6] Refactor RenamedTo to ConfigParameter rename newfragment as feature --- airflow/cli/cli_config.py | 3 +- .../remote_commands/config_command.py | 304 ++++++++---------- newsfragments/44908.feature.rst | 1 + newsfragments/44908.significant.rst | 15 - 4 files changed, 138 insertions(+), 185 deletions(-) create mode 100644 newsfragments/44908.feature.rst delete mode 100644 newsfragments/44908.significant.rst diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index c6ca88f7882b4..66c5770db5188 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -880,8 +880,7 @@ def string_lower_type(val): ) ARG_OPTIONAL_SECTION = Arg( ("--section",), - help="The section name(s).", - type=string_list_type, + help="The section name", ) # jobs check diff --git a/airflow/cli/commands/remote_commands/config_command.py b/airflow/cli/commands/remote_commands/config_command.py index 67e8140f8ac8b..aec987b6313cc 100644 --- a/airflow/cli/commands/remote_commands/config_command.py +++ b/airflow/cli/commands/remote_commands/config_command.py @@ -69,8 +69,8 @@ def get_value(args): pass -class RenamedTo(NamedTuple): - """Represents a configuration parameter that has been renamed.""" +class ConfigParameter(NamedTuple): + """Represents a configuration parameter.""" section: str option: str @@ -78,296 +78,264 @@ class RenamedTo(NamedTuple): @dataclass class ConfigChange: - """Class representing the configuration changes in Airflow 3.0.""" - - def __init__( - self, section: str, option: str, suggestion: str = "", renamed_to: RenamedTo | None = None - ) -> None: - """ - Initialize a ConfigChange instance. - - :param section: The section of the configuration. - :param option: The option within the section that is removed or deprecated. - :param suggestion: A suggestion for replacing or handling the removed configuration. - :param renamed_to: The new section and option if the configuration is renamed. - """ - self.section = section - self.option = option - self.suggestion = suggestion - self.renamed_to = renamed_to + """ + Class representing the configuration changes in Airflow 3.0. + + :param config: The configuration parameter being changed. + :param suggestion: A suggestion for replacing or handling the removed configuration. + :param renamed_to: The new section and option if the configuration is renamed. + """ + + config: ConfigParameter + suggestion: str = "" + renamed_to: ConfigParameter | None = None @property def message(self) -> str: """Generate a message for this configuration change.""" if self.renamed_to: - if self.section != self.renamed_to.section: + if self.config.section != self.renamed_to.section: return ( - f"`{self.option}` configuration parameter moved from `{self.section}` section to `" + f"`{self.config.option}` configuration parameter moved from `{self.config.section}` section to `" f"{self.renamed_to.section}` section as `{self.renamed_to.option}`." ) - else: - return ( - f"`{self.option}` configuration parameter renamed to `{self.renamed_to.option}` " - f"in the `{self.section}` section." - ) - else: return ( - f"Removed deprecated `{self.option}` configuration parameter from `{self.section}` section. " - f"{self.suggestion}" + f"`{self.config.option}` configuration parameter renamed to `{self.renamed_to.option}` " + f"in the `{self.config.section}` section." ) + return ( + f"Removed deprecated `{self.config.option}` configuration parameter from `{self.config.section}` section. " + f"{self.suggestion}" + ) CONFIGS_CHANGES = [ ConfigChange( - section="admin", - option="hide_sensitive_variable_fields", - renamed_to=RenamedTo("core", "hide_sensitive_var_conn_fields"), + config=ConfigParameter("admin", "hide_sensitive_variable_fields"), + renamed_to=ConfigParameter("core", "hide_sensitive_var_conn_fields"), ), ConfigChange( - section="admin", - option="sensitive_variable_fields", - renamed_to=RenamedTo("core", "sensitive_var_conn_names"), + config=ConfigParameter("admin", "sensitive_variable_fields"), + renamed_to=ConfigParameter("core", "sensitive_var_conn_names"), ), ConfigChange( - section="core", - option="check_slas", + config=ConfigParameter("core", "check_slas"), suggestion="The SLA feature is removed in Airflow 3.0, to be replaced with Airflow Alerts in " "future", ), ConfigChange( - section="core", - option="strict_asset_uri_validation", + config=ConfigParameter("core", "strict_asset_uri_validation"), suggestion="Asset URI with a defined scheme will now always be validated strictly, " "raising a hard error on validation failure.", ), ConfigChange( - section="core", - option="worker_precheck", - renamed_to=RenamedTo("celery", "worker_precheck"), + config=ConfigParameter("core", "worker_precheck"), + renamed_to=ConfigParameter("celery", "worker_precheck"), ), ConfigChange( - section="core", - option="non_pooled_task_slot_count", - renamed_to=RenamedTo("core", "default_pool_task_slot_count"), + config=ConfigParameter("core", "non_pooled_task_slot_count"), + renamed_to=ConfigParameter("core", "default_pool_task_slot_count"), ), ConfigChange( - section="core", - option="dag_concurrency", - renamed_to=RenamedTo("core", "max_active_tasks_per_dag"), + config=ConfigParameter("core", "dag_concurrency"), + renamed_to=ConfigParameter("core", "max_active_tasks_per_dag"), ), ConfigChange( - section="core", - option="sql_alchemy_conn", - renamed_to=RenamedTo("database", "sql_alchemy_conn"), + config=ConfigParameter("core", "sql_alchemy_conn"), + renamed_to=ConfigParameter("database", "sql_alchemy_conn"), ), ConfigChange( - section="core", - option="sql_engine_encoding", - renamed_to=RenamedTo("database", "sql_engine_encoding"), + config=ConfigParameter("core", "sql_engine_encoding"), + renamed_to=ConfigParameter("database", "sql_engine_encoding"), ), ConfigChange( - section="core", - option="sql_engine_collation_for_ids", - renamed_to=RenamedTo("database", "sql_engine_collation_for_ids"), + config=ConfigParameter("core", "sql_engine_collation_for_ids"), + renamed_to=ConfigParameter("database", "sql_engine_collation_for_ids"), ), ConfigChange( - section="core", - option="sql_alchemy_pool_enabled", - renamed_to=RenamedTo("database", "sql_alchemy_pool_enabled"), + config=ConfigParameter("core", "sql_alchemy_pool_enabled"), + renamed_to=ConfigParameter("database", "sql_alchemy_pool_enabled"), ), ConfigChange( - section="core", - option="sql_alchemy_pool_size", - renamed_to=RenamedTo("database", "sql_alchemy_pool_size"), + config=ConfigParameter("core", "sql_alchemy_pool_size"), + renamed_to=ConfigParameter("database", "sql_alchemy_pool_size"), ), ConfigChange( - section="core", - option="sql_alchemy_max_overflow", - renamed_to=RenamedTo("database", "sql_alchemy_max_overflow"), + config=ConfigParameter("core", "sql_alchemy_max_overflow"), + renamed_to=ConfigParameter("database", "sql_alchemy_max_overflow"), ), ConfigChange( - section="core", - option="sql_alchemy_pool_recycle", - renamed_to=RenamedTo("database", "sql_alchemy_pool_recycle"), + config=ConfigParameter("core", "sql_alchemy_pool_recycle"), + renamed_to=ConfigParameter("database", "sql_alchemy_pool_recycle"), ), ConfigChange( - section="core", - option="sql_alchemy_pool_pre_ping", - renamed_to=RenamedTo("database", "sql_alchemy_pool_pre_ping"), + config=ConfigParameter("core", "sql_alchemy_pool_pre_ping"), + renamed_to=ConfigParameter("database", "sql_alchemy_pool_pre_ping"), ), ConfigChange( - section="core", - option="sql_alchemy_schema", - renamed_to=RenamedTo("database", "sql_alchemy_schema"), + config=ConfigParameter("core", "sql_alchemy_schema"), + renamed_to=ConfigParameter("database", "sql_alchemy_schema"), ), ConfigChange( - section="core", - option="sql_alchemy_connect_args", - renamed_to=RenamedTo("database", "sql_alchemy_connect_args"), + config=ConfigParameter("core", "sql_alchemy_connect_args"), + renamed_to=ConfigParameter("database", "sql_alchemy_connect_args"), ), ConfigChange( - section="core", - option="load_default_connections", - renamed_to=RenamedTo("database", "load_default_connections"), + config=ConfigParameter("core", "load_default_connections"), + renamed_to=ConfigParameter("database", "load_default_connections"), ), ConfigChange( - section="core", - option="max_db_retries", - renamed_to=RenamedTo("database", "max_db_retries"), + config=ConfigParameter("core", "max_db_retries"), + renamed_to=ConfigParameter("database", "max_db_retries"), ), ConfigChange( - section="api", - option="access_control_allow_origin", - renamed_to=RenamedTo("api", "access_control_allow_origins"), + config=ConfigParameter("api", "access_control_allow_origin"), + renamed_to=ConfigParameter("api", "access_control_allow_origins"), ), ConfigChange( - section="api", - option="auth_backend", - renamed_to=RenamedTo("api", "auth_backends"), + config=ConfigParameter("api", "auth_backend"), + renamed_to=ConfigParameter("api", "auth_backends"), ), ConfigChange( - section="logging", - option="enable_task_context_logger", + config=ConfigParameter("logging", "enable_task_context_logger"), suggestion="Remove TaskContextLogger: Replaced by the Log table for better handling of task log " "messages outside the execution context.", ), ConfigChange( - section="metrics", - option="metrics_use_pattern_match", + config=ConfigParameter("metrics", "metrics_use_pattern_match"), ), ConfigChange( - section="metrics", - option="timer_unit_consistency", + config=ConfigParameter("metrics", "timer_unit_consistency"), suggestion="In Airflow 3.0, the `timer_unit_consistency` setting in the `metrics` section is " "removed as it is now the default behaviour. This is done to standardize all timer and " "timing metrics to milliseconds across all metric loggers", ), ConfigChange( - section="metrics", - option="statsd_allow_list", - renamed_to=RenamedTo("metrics", "metrics_allow_list"), + config=ConfigParameter("metrics", "statsd_allow_list"), + renamed_to=ConfigParameter("metrics", "metrics_allow_list"), ), ConfigChange( - section="metrics", - option="statsd_block_list", - renamed_to=RenamedTo("metrics", "metrics_block_list"), + config=ConfigParameter("metrics", "statsd_block_list"), + renamed_to=ConfigParameter("metrics", "metrics_block_list"), ), ConfigChange( - section="traces", - option="otel_task_log_event", + config=ConfigParameter("traces", "otel_task_log_event"), ), ConfigChange( - section="operators", - option="allow_illegal_arguments", + config=ConfigParameter("operators", "allow_illegal_arguments"), ), ConfigChange( - section="webserver", - option="allow_raw_html_descriptions", + config=ConfigParameter("webserver", "allow_raw_html_descriptions"), ), ConfigChange( - section="webserver", - option="session_lifetime_days", + config=ConfigParameter("webserver", "session_lifetime_days"), suggestion="Please use `session_lifetime_minutes`.", ), ConfigChange( - section="webserver", option="update_fab_perms", renamed_to=RenamedTo("fab", "update_fab_perms") + config=ConfigParameter("webserver", "update_fab_perms"), + renamed_to=ConfigParameter("fab", "update_fab_perms"), + ), + ConfigChange( + config=ConfigParameter("webserver", "auth_rate_limited"), + renamed_to=ConfigParameter("fab", "auth_rate_limited"), + ), + ConfigChange( + config=ConfigParameter("webserver", option="auth_rate_limit"), + renamed_to=ConfigParameter("fab", "auth_rate_limit"), + ), + ConfigChange( + config=ConfigParameter("webserver", "session_lifetime_days"), + renamed_to=ConfigParameter("webserver", "session_lifetime_minutes"), ), ConfigChange( - section="webserver", option="auth_rate_limited", renamed_to=RenamedTo("fab", "auth_rate_limited") + config=ConfigParameter("webserver", "force_log_out_after"), + renamed_to=ConfigParameter("webserver", "session_lifetime_minutes"), ), ConfigChange( - section="webserver", option="auth_rate_limit", renamed_to=RenamedTo("fab", "auth_rate_limit") + config=ConfigParameter("policy", "airflow_local_settings"), + renamed_to=ConfigParameter("policy", "task_policy"), ), ConfigChange( - section="webserver", - option="session_lifetime_days", - renamed_to=RenamedTo("webserver", "session_lifetime_minutes"), + config=ConfigParameter("scheduler", "dependency_detector"), ), ConfigChange( - section="webserver", - option="force_log_out_after", - renamed_to=RenamedTo("webserver", "session_lifetime_minutes"), + config=ConfigParameter("scheduler", "processor_poll_interval"), + renamed_to=ConfigParameter("scheduler", "scheduler_idle_sleep_time"), ), ConfigChange( - section="policy", option="airflow_local_settings", renamed_to=RenamedTo("policy", "task_policy") + config=ConfigParameter("scheduler", "deactivate_stale_dags_interval"), + renamed_to=ConfigParameter("scheduler", "parsing_cleanup_interval"), ), ConfigChange( - section="scheduler", - option="dependency_detector", + config=ConfigParameter("scheduler", "statsd_on"), renamed_to=ConfigParameter("metrics", "statsd_on") ), ConfigChange( - section="scheduler", - option="processor_poll_interval", - renamed_to=RenamedTo("scheduler", "scheduler_idle_sleep_time"), + config=ConfigParameter("scheduler", "max_threads"), + renamed_to=ConfigParameter("scheduler", "parsing_processes"), ), ConfigChange( - section="scheduler", - option="deactivate_stale_dags_interval", - renamed_to=RenamedTo("scheduler", "parsing_cleanup_interval"), + config=ConfigParameter("scheduler", "statsd_host"), + renamed_to=ConfigParameter("metrics", "statsd_host"), ), - ConfigChange(section="scheduler", option="statsd_on", renamed_to=RenamedTo("metrics", "statsd_on")), ConfigChange( - section="scheduler", option="max_threads", renamed_to=RenamedTo("scheduler", "parsing_processes") + config=ConfigParameter("scheduler", "statsd_port"), + renamed_to=ConfigParameter("metrics", "statsd_port"), ), - ConfigChange(section="scheduler", option="statsd_host", renamed_to=RenamedTo("metrics", "statsd_host")), - ConfigChange(section="scheduler", option="statsd_port", renamed_to=RenamedTo("metrics", "statsd_port")), ConfigChange( - section="scheduler", option="statsd_prefix", renamed_to=RenamedTo("metrics", "statsd_prefix") + config=ConfigParameter("scheduler", "statsd_prefix"), + renamed_to=ConfigParameter("metrics", "statsd_prefix"), ), ConfigChange( - section="scheduler", option="statsd_allow_list", renamed_to=RenamedTo("metrics", "statsd_allow_list") + config=ConfigParameter("scheduler", "statsd_allow_list"), + renamed_to=ConfigParameter("metrics", "statsd_allow_list"), ), ConfigChange( - section="scheduler", option="stat_name_handler", renamed_to=RenamedTo("metrics", "stat_name_handler") + config=ConfigParameter("scheduler", "stat_name_handler"), + renamed_to=ConfigParameter("metrics", "stat_name_handler"), ), ConfigChange( - section="scheduler", - option="statsd_datadog_enabled", - renamed_to=RenamedTo("metrics", "statsd_datadog_enabled"), + config=ConfigParameter("scheduler", "statsd_datadog_enabled"), + renamed_to=ConfigParameter("metrics", "statsd_datadog_enabled"), ), ConfigChange( - section="scheduler", - option="statsd_datadog_tags", - renamed_to=RenamedTo("metrics", "statsd_datadog_tags"), + config=ConfigParameter("scheduler", "statsd_datadog_tags"), + renamed_to=ConfigParameter("metrics", "statsd_datadog_tags"), ), ConfigChange( - section="scheduler", - option="statsd_datadog_metrics_tags", - renamed_to=RenamedTo("metrics", "statsd_datadog_metrics_tags"), + config=ConfigParameter("scheduler", "statsd_datadog_metrics_tags"), + renamed_to=ConfigParameter("metrics", "statsd_datadog_metrics_tags"), ), ConfigChange( - section="scheduler", - option="statsd_custom_client_path", - renamed_to=RenamedTo("metrics", "statsd_custom_client_path"), + config=ConfigParameter("scheduler", "statsd_custom_client_path"), + renamed_to=ConfigParameter("metrics", "statsd_custom_client_path"), ), ConfigChange( - section="celery", - option="stalled_task_timeout", - renamed_to=RenamedTo("scheduler", "task_queued_timeout"), + config=ConfigParameter("celery", "stalled_task_timeout"), + renamed_to=ConfigParameter("scheduler", "task_queued_timeout"), ), ConfigChange( - section="celery", option="default_queue", renamed_to=RenamedTo("operators", "default_queue") + config=ConfigParameter("celery", "default_queue"), + renamed_to=ConfigParameter("operators", "default_queue"), ), ConfigChange( - section="celery", - option="task_adoption_timeout", - renamed_to=RenamedTo("scheduler", "task_queued_timeout"), + config=ConfigParameter("celery", "task_adoption_timeout"), + renamed_to=ConfigParameter("scheduler", "task_queued_timeout"), ), ConfigChange( - section="kubernetes_executor", - option="worker_pods_pending_timeout", - renamed_to=RenamedTo("scheduler", "task_queued_timeout"), + config=ConfigParameter("kubernetes_executor", "worker_pods_pending_timeout"), + renamed_to=ConfigParameter("scheduler", "task_queued_timeout"), ), ConfigChange( - section="kubernetes_executor", - option="worker_pods_pending_timeout_check_interval", - renamed_to=RenamedTo("scheduler", "task_queued_timeout_check_interval"), + config=ConfigParameter("kubernetes_executor", "worker_pods_pending_timeout_check_interval"), + renamed_to=ConfigParameter("scheduler", "task_queued_timeout_check_interval"), ), ConfigChange( - section="smtp", option="smtp_user", suggestion="Please use the SMTP connection (`smtp_default`)." + config=ConfigParameter("smtp", "smtp_user"), + suggestion="Please use the SMTP connection (`smtp_default`).", ), ConfigChange( - section="smtp", option="smtp_password", suggestion="Please use the SMTP connection (`smtp_default`)." + config=ConfigParameter("smtp", "smtp_password"), + suggestion="Please use the SMTP connection (`smtp_default`).", ), ] @@ -430,18 +398,18 @@ def lint_config(args) -> None: ignore_sections = args.ignore_section or [] ignore_options = args.ignore_option or [] - for config in CONFIGS_CHANGES: - if section_to_check_if_provided and config.section not in section_to_check_if_provided: + for configuration in CONFIGS_CHANGES: + if section_to_check_if_provided and configuration.config.section not in section_to_check_if_provided: continue - if option_to_check_if_provided and config.option not in option_to_check_if_provided: + if option_to_check_if_provided and configuration.config.option not in option_to_check_if_provided: continue - if config.section in ignore_sections or config.option in ignore_options: + if configuration.config.section in ignore_sections or configuration.config.option in ignore_options: continue - if conf.has_option(config.section, config.option): - lint_issues.append(config.message) + if conf.has_option(configuration.config.section, configuration.config.option): + lint_issues.append(configuration.message) if lint_issues: console.print("[red]Found issues in your airflow.cfg:[/red]") diff --git a/newsfragments/44908.feature.rst b/newsfragments/44908.feature.rst new file mode 100644 index 0000000000000..2a711a0149322 --- /dev/null +++ b/newsfragments/44908.feature.rst @@ -0,0 +1 @@ +The ``airflow config lint`` command has been introduced to help users migrate from Airflow 2.x to 3.0 by identifying removed or renamed configuration parameters in airflow.cfg. diff --git a/newsfragments/44908.significant.rst b/newsfragments/44908.significant.rst deleted file mode 100644 index bcd907701aa1d..0000000000000 --- a/newsfragments/44908.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -Airflow CLI command ``airflow config lint`` added - -The ``airflow config lint`` command has been introduced to help users migrate from Airflow 2.x to 3.0 by identifying removed or renamed configuration parameters in airflow.cfg. - -This command provides actionable feedback and highlights renamed parameters with guidance for transitioning to their new names or sections. - -* Types of change - - * [ ] DAG changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency change From d17c40cf7a12b964fbac79d51fd00eaa53d47a73 Mon Sep 17 00:00:00 2001 From: Ankit Chaurasia <8670962+sunank200@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:12:00 +0545 Subject: [PATCH 5/6] Add more tests with mocked env variables --- .../remote_commands/test_config_command.py | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/cli/commands/remote_commands/test_config_command.py b/tests/cli/commands/remote_commands/test_config_command.py index f35ecc6c62da7..cac338c7c8955 100644 --- a/tests/cli/commands/remote_commands/test_config_command.py +++ b/tests/cli/commands/remote_commands/test_config_command.py @@ -17,6 +17,7 @@ from __future__ import annotations import contextlib +import os import re from io import StringIO from unittest import mock @@ -25,6 +26,7 @@ from airflow.cli import cli_parser from airflow.cli.commands.remote_commands import config_command +from airflow.cli.commands.remote_commands.config_command import ConfigChange, ConfigParameter from tests_common.test_utils.config import conf_vars @@ -283,8 +285,7 @@ def test_lint_with_specific_removed_configs(self, section, option, suggestion): expected_message = f"Removed deprecated `{option}` configuration parameter from `{section}` section." assert expected_message in normalized_output - if suggestion: - assert suggestion in normalized_output + assert suggestion in normalized_output def test_lint_specific_section_option(self): with mock.patch("airflow.configuration.conf.has_option", return_value=True): @@ -414,3 +415,37 @@ def test_lint_detects_renamed_configs(self, renamed_configs): else: expected_message = f"`{old_option}` configuration parameter moved from `{old_section}` section to `{new_section}` section as `{new_option}`." assert expected_message in normalized_output + + @pytest.mark.parametrize( + "env_var, config_change, expected_message", + [ + ( + "AIRFLOW__CORE__CHECK_SLAS", + ConfigChange( + config=ConfigParameter("core", "check_slas"), + suggestion="The SLA feature is removed in Airflow 3.0, to be replaced with Airflow Alerts in future", + ), + "Removed deprecated `check_slas` configuration parameter from `core` section.", + ), + ( + "AIRFLOW__CORE__STRICT_ASSET_URI_VALIDATION", + ConfigChange( + config=ConfigParameter("core", "strict_asset_uri_validation"), + suggestion="Asset URI with a defined scheme will now always be validated strictly, raising a hard error on validation failure.", + ), + "Removed deprecated `strict_asset_uri_validation` configuration parameter from `core` section.", + ), + ], + ) + def test_lint_detects_configs_with_env_vars(self, env_var, config_change, expected_message): + with mock.patch.dict(os.environ, {env_var: "some_value"}): + with mock.patch("airflow.configuration.conf.has_option", return_value=True): + with contextlib.redirect_stdout(StringIO()) as temp_stdout: + config_command.lint_config(cli_parser.get_parser().parse_args(["config", "lint"])) + + output = temp_stdout.getvalue() + + normalized_output = re.sub(r"\s+", " ", output.strip()) + + assert expected_message in normalized_output + assert config_change.suggestion in normalized_output From 75f95caab855e3d23a06977f83d08acbb6f4937c Mon Sep 17 00:00:00 2001 From: Ankit Chaurasia <8670962+sunank200@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:20:19 +0545 Subject: [PATCH 6/6] Remove duplicate config changes --- airflow/cli/commands/remote_commands/config_command.py | 4 ---- tests/cli/commands/remote_commands/test_config_command.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/airflow/cli/commands/remote_commands/config_command.py b/airflow/cli/commands/remote_commands/config_command.py index aec987b6313cc..17e7b62321761 100644 --- a/airflow/cli/commands/remote_commands/config_command.py +++ b/airflow/cli/commands/remote_commands/config_command.py @@ -227,10 +227,6 @@ def message(self) -> str: ConfigChange( config=ConfigParameter("webserver", "allow_raw_html_descriptions"), ), - ConfigChange( - config=ConfigParameter("webserver", "session_lifetime_days"), - suggestion="Please use `session_lifetime_minutes`.", - ), ConfigChange( config=ConfigParameter("webserver", "update_fab_perms"), renamed_to=ConfigParameter("fab", "update_fab_perms"), diff --git a/tests/cli/commands/remote_commands/test_config_command.py b/tests/cli/commands/remote_commands/test_config_command.py index cac338c7c8955..f932b1851d227 100644 --- a/tests/cli/commands/remote_commands/test_config_command.py +++ b/tests/cli/commands/remote_commands/test_config_command.py @@ -361,10 +361,6 @@ def test_lint_detects_multiple_issues(self): "Remove TaskContextLogger: Replaced by the Log table for better handling of task log messages outside the execution context.", ), ], - [ - ("webserver", "allow_raw_html_descriptions", ""), - ("webserver", "session_lifetime_days", "Please use `session_lifetime_minutes`."), - ], ], ) def test_lint_detects_multiple_removed_configs(self, removed_configs):