From 18291bd1e3105046592d221426704c2418c34b7c Mon Sep 17 00:00:00 2001 From: Eric L Frederich Date: Fri, 22 Mar 2024 12:24:03 -0400 Subject: [PATCH 01/67] feat: new terraform_fmt_v2 with better Windows support --- .gitignore | 2 ++ .pre-commit-hooks.yaml | 8 +++++ hooks/__init__.py | 4 --- hooks/common.py | 61 +++++++++++++++++++++++++++++++++ hooks/terraform_docs_replace.py | 5 +++ hooks/terraform_fmt.py | 32 +++++++++++++++++ setup.py | 1 + 7 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 hooks/common.py create mode 100644 hooks/terraform_fmt.py diff --git a/.gitignore b/.gitignore index 0bbeada90..1de9c2b4a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ tests/results/* +__pycache__/ +*.py[cod] diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index cbe506dd3..f08dbb6c0 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -15,6 +15,14 @@ files: (\.tf|\.tfvars)$ exclude: \.terraform/.*$ +- id: terraform_fmt_py + name: Terraform fmt + description: Rewrites all Terraform configuration files to a canonical format. + entry: terraform_fmt + language: python + files: \.tf(vars)?$ + exclude: \.terraform/.*$ + - id: terraform_docs name: Terraform docs description: Inserts input and output documentation into README.md (using terraform-docs). diff --git a/hooks/__init__.py b/hooks/__init__.py index aeb6f9b27..e69de29bb 100644 --- a/hooks/__init__.py +++ b/hooks/__init__.py @@ -1,4 +0,0 @@ -print( - '`terraform_docs_replace` hook is DEPRECATED.' - 'For migration instructions see https://github.com/antonbabenko/pre-commit-terraform/issues/248#issuecomment-1290829226' -) diff --git a/hooks/common.py b/hooks/common.py new file mode 100644 index 000000000..28bef23ef --- /dev/null +++ b/hooks/common.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import argparse +import logging +import os +from collections.abc import Sequence + +logger = logging.getLogger(__name__) + + +def setup_logging(): + logging.basicConfig( + level={ + "error": logging.ERROR, + "warn": logging.WARNING, + "warning": logging.WARNING, + "info": logging.INFO, + "debug": logging.DEBUG, + }[os.environ.get("PRE_COMMIT_TERRAFORM_LOG_LEVEL", "warning").lower()] + ) + + +def parse_env_vars(env_var_strs: list[str]) -> dict[str, str]: + ret = {} + for env_var_str in env_var_strs: + name, val = env_var_str.split("=", 1) + if val.startswith('"') and val.endswith('"'): + val = val[1:-1] + ret[name] = val + return ret + + +def parse_cmdline( + argv: Sequence[str] | None = None, +) -> tuple[list[str], list[str], list[str], list[str], dict[str, str]]: + parser = argparse.ArgumentParser( + add_help=False, # Allow the use of `-h` for compatiblity with Bash version of the hook + ) + parser.add_argument("-a", "--args", action="append", help="Arguments") + parser.add_argument("-h", "--hook-config", action="append", help="Hook Config") + parser.add_argument("-i", "--init-args", "--tf-init-args", action="append", help="Init Args") + parser.add_argument("-e", "--envs", "--env-vars", action="append", help="Environment Variables") + parser.add_argument("FILES", nargs="*", help="Files") + + parsed_args = parser.parse_args(argv) + + args = parsed_args.args or [] + hook_config = parsed_args.hook_config or [] + files = parsed_args.FILES or [] + tf_init_args = parsed_args.init_args or [] + env_vars = parsed_args.envs or [] + + env_var_dict = parse_env_vars(env_vars) + + if hook_config: + raise NotImplementedError("TODO: implement: hook_config") + + if tf_init_args: + raise NotImplementedError("TODO: implement: tf_init_args") + + return args, hook_config, files, tf_init_args, env_var_dict diff --git a/hooks/terraform_docs_replace.py b/hooks/terraform_docs_replace.py index a9cf6c9bc..600cb08bf 100644 --- a/hooks/terraform_docs_replace.py +++ b/hooks/terraform_docs_replace.py @@ -3,6 +3,11 @@ import subprocess import sys +print( + '`terraform_docs_replace` hook is DEPRECATED.' + 'For migration instructions see https://github.com/antonbabenko/pre-commit-terraform/issues/248#issuecomment-1290829226' +) + def main(argv=None): parser = argparse.ArgumentParser( diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py new file mode 100644 index 000000000..bd07aba41 --- /dev/null +++ b/hooks/terraform_fmt.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import logging +import os +import shlex +import sys +from subprocess import PIPE, run +from typing import Sequence + +from .common import parse_cmdline, setup_logging + +logger = logging.getLogger(__name__) + + +def main(argv: Sequence[str] | None = None) -> int: + setup_logging() + logger.debug(sys.version_info) + args, hook_config, files, tf_init_args, env_vars = parse_cmdline(argv) + if os.environ.get("PRE_COMMIT_COLOR") == "never": + args.append("-no-color") + cmd = ["terraform", "fmt", *args, *files] + logger.info("calling %s", shlex.join(cmd)) + logger.debug("env_vars: %r", env_vars) + logger.debug("args: %r", args) + completed_process = run(cmd, env={**os.environ, **env_vars}, text=True, stdout=PIPE) + if completed_process.stdout: + print(completed_process.stdout) + return completed_process.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.py b/setup.py index 2d88425b9..ce1ee2623 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ entry_points={ 'console_scripts': [ 'terraform_docs_replace = hooks.terraform_docs_replace:main', + 'terraform_fmt = hooks.terraform_fmt:main', ], }, ) From 806a87a3943a29984a07b786a3ff399474c6b90f Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 26 Apr 2024 20:32:51 +0300 Subject: [PATCH 02/67] Add linters andd apply part which not requiers tests --- .gitignore | 1 + .pre-commit-config.yaml | 68 ++++++++++++++++++++++++- hooks/common.py | 89 +++++++++++++++++++++++++-------- hooks/terraform_docs_replace.py | 43 ++++++++++------ hooks/terraform_fmt.py | 32 ++++++++---- 5 files changed, 186 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index 1de9c2b4a..820940260 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ tests/results/* __pycache__/ *.py[cod] +node_modules/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6944a0adb..a44d55046 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,7 +38,7 @@ repos: # Dockerfile linter - repo: https://github.com/hadolint/hadolint - rev: v2.12.1-beta + rev: v2.13.0-beta hooks: - id: hadolint args: [ @@ -54,8 +54,72 @@ repos: # JSON5 Linter - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.1.0 + rev: v4.0.0-alpha.8 hooks: - id: prettier # https://prettier.io/docs/en/options.html#parser files: '.json5$' + + +########## +# PYTHON # +########## + +- repo: https://github.com/asottile/reorder_python_imports + rev: v3.12.0 + hooks: + - id: reorder-python-imports + +- repo: https://github.com/asottile/add-trailing-comma + rev: v3.1.0 + hooks: + - id: add-trailing-comma + +- repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v2.0.4 + hooks: + - id: autopep8 + args: + - -i + - --max-line-length=100 + +# Usage: http://pylint.pycqa.org/en/latest/user_guide/message-control.html +- repo: https://github.com/PyCQA/pylint + rev: v3.1.0 + hooks: + - id: pylint + args: + - --disable=import-error # E0401. Locally you could not have all imports. + - --disable=fixme # W0511. 'TODO' notations. + - --disable=logging-fstring-interpolation # Conflict with "use a single formatting" WPS323 + - --disable=ungrouped-imports # ignore `if TYPE_CHECKING` case. Other do reorder-python-imports + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.10.0 + hooks: + - id: mypy + args: [ + --ignore-missing-imports, + --disallow-untyped-calls, + --warn-redundant-casts, + ] + +- repo: https://github.com/pycqa/flake8.git + rev: 7.0.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-2020 + - flake8-docstrings + - flake8-pytest-style + - wemake-python-styleguide + args: + - --max-returns=2 # Default settings + - --max-arguments=4 # Default settings + # https://www.flake8rules.com/ + # https://wemake-python-stylegui.de/en/latest/pages/usage/violations/index.html + - --extend-ignore= + WPS305, + E501, + I, + # RST, diff --git a/hooks/common.py b/hooks/common.py index 28bef23ef..f981f89bd 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -1,3 +1,8 @@ +""" +Here located common functions for hooks. + +It not executed directly, but imported by other hooks. +""" from __future__ import annotations import argparse @@ -9,38 +14,80 @@ def setup_logging(): - logging.basicConfig( - level={ - "error": logging.ERROR, - "warn": logging.WARNING, - "warning": logging.WARNING, - "info": logging.INFO, - "debug": logging.DEBUG, - }[os.environ.get("PRE_COMMIT_TERRAFORM_LOG_LEVEL", "warning").lower()] - ) + """ + Set up the logging configuration based on the value of the 'PCT_LOG' environment variable. + + The 'PCT_LOG' environment variable determines the logging level to be used. + The available levels are: + - 'error': Only log error messages. + - 'warn' or 'warning': Log warning messages and above. + - 'info': Log informational messages and above. + - 'debug': Log debug messages and above. + + If the 'PCT_LOG' environment variable is not set or has an invalid value, + the default logging level is 'warning'. + + Returns: + None + """ + log_level = { + 'error': logging.ERROR, + 'warn': logging.WARNING, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG, + }[os.environ.get('PCT_LOG', 'warning').lower()] + + logging.basicConfig(level=log_level) def parse_env_vars(env_var_strs: list[str]) -> dict[str, str]: + """ + Expand environment variables definition into their values in '--args'. + + Args: + env_var_strs (list[str]): A list of environment variable strings in the format "name=value". + + Returns: + dict[str, str]: A dictionary mapping variable names to their corresponding values. + """ ret = {} for env_var_str in env_var_strs: - name, val = env_var_str.split("=", 1) - if val.startswith('"') and val.endswith('"'): - val = val[1:-1] - ret[name] = val + name, env_var_value = env_var_str.split('=', 1) + if env_var_value.startswith('"') and env_var_value.endswith('"'): + env_var_value = env_var_value[1:-1] + ret[name] = env_var_value return ret def parse_cmdline( argv: Sequence[str] | None = None, ) -> tuple[list[str], list[str], list[str], list[str], dict[str, str]]: + """ + Parse the command line arguments and return a tuple containing the parsed values. + + Args: + argv (Sequence[str] | None): The command line arguments to parse. + If None, the arguments from sys.argv will be used. + + Returns: + tuple[list[str], list[str], list[str], list[str], dict[str, str]]: + A tuple containing the parsed values: + - args (list[str]): The parsed arguments. + - hook_config (list[str]): The parsed hook configurations. + - files (list[str]): The parsed files. + - tf_init_args (list[str]): The parsed Terraform initialization arguments. + - env_var_dict (dict[str, str]): The parsed environment variables as a dictionary. + """ + parser = argparse.ArgumentParser( - add_help=False, # Allow the use of `-h` for compatiblity with Bash version of the hook + add_help=False, # Allow the use of `-h` for compatibility with the Bash version of the hook ) - parser.add_argument("-a", "--args", action="append", help="Arguments") - parser.add_argument("-h", "--hook-config", action="append", help="Hook Config") - parser.add_argument("-i", "--init-args", "--tf-init-args", action="append", help="Init Args") - parser.add_argument("-e", "--envs", "--env-vars", action="append", help="Environment Variables") - parser.add_argument("FILES", nargs="*", help="Files") + parser.add_argument('-a', '--args', action='append', help='Arguments') + parser.add_argument('-h', '--hook-config', action='append', help='Hook Config') + parser.add_argument('-i', '--init-args', '--tf-init-args', action='append', help='Init Args') + parser.add_argument('-e', '--envs', '--env-vars', action='append', help='Environment Variables') + parser.add_argument('FILES', nargs='*', help='Files') parsed_args = parser.parse_args(argv) @@ -53,9 +100,9 @@ def parse_cmdline( env_var_dict = parse_env_vars(env_vars) if hook_config: - raise NotImplementedError("TODO: implement: hook_config") + raise NotImplementedError('TODO: implement: hook_config') if tf_init_args: - raise NotImplementedError("TODO: implement: tf_init_args") + raise NotImplementedError('TODO: implement: tf_init_args') return args, hook_config, files, tf_init_args, env_var_dict diff --git a/hooks/terraform_docs_replace.py b/hooks/terraform_docs_replace.py index 600cb08bf..204034e35 100644 --- a/hooks/terraform_docs_replace.py +++ b/hooks/terraform_docs_replace.py @@ -1,3 +1,4 @@ +"""Deprecated hook to replace README.md with the output of terraform-docs.""" import argparse import os import subprocess @@ -5,15 +6,25 @@ print( '`terraform_docs_replace` hook is DEPRECATED.' - 'For migration instructions see https://github.com/antonbabenko/pre-commit-terraform/issues/248#issuecomment-1290829226' + 'For migration instructions see ' + + 'https://github.com/antonbabenko/pre-commit-terraform/issues/248#issuecomment-1290829226', ) def main(argv=None): + """ + TODO: Add docstring. + + Args: + argv (list): List of command-line arguments (default: None) + + Returns: + int: The return value indicating the success or failure of the function + """ parser = argparse.ArgumentParser( description="""Run terraform-docs on a set of files. Follows the standard convention of pulling the documentation from main.tf in order to replace the entire - README.md file each time.""" + README.md file each time.""", ) parser.add_argument( '--dest', dest='dest', default='README.md', @@ -34,25 +45,27 @@ def main(argv=None): dirs = [] for filename in args.filenames: - if (os.path.realpath(filename) not in dirs and - (filename.endswith(".tf") or filename.endswith(".tfvars"))): + if ( + os.path.realpath(filename) not in dirs and + (filename.endswith('.tf') or filename.endswith('.tfvars')) + ): dirs.append(os.path.dirname(filename)) retval = 0 - for dir in dirs: + for directory in dirs: try: - procArgs = [] - procArgs.append('terraform-docs') + proc_args = [] + proc_args.append('terraform-docs') if args.sort: - procArgs.append('--sort-by-required') - procArgs.append('md') - procArgs.append("./{dir}".format(dir=dir)) - procArgs.append('>') - procArgs.append("./{dir}/{dest}".format(dir=dir, dest=args.dest)) - subprocess.check_call(" ".join(procArgs), shell=True) - except subprocess.CalledProcessError as e: - print(e) + proc_args.append('--sort-by-required') + proc_args.append('md') + proc_args.append(f'./{directory}') + proc_args.append('>') + proc_args.append(f'./{directory}/{args.dest}') + subprocess.check_call(' '.join(proc_args), shell=True) + except subprocess.CalledProcessError as exeption: + print(exeption) retval = 1 return retval diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index bd07aba41..7a6dbe62e 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -1,32 +1,46 @@ +""" +Pre-commit hook for terraform fmt. +""" from __future__ import annotations import logging import os import shlex import sys -from subprocess import PIPE, run +from subprocess import PIPE +from subprocess import run from typing import Sequence -from .common import parse_cmdline, setup_logging +from .common import parse_cmdline +from .common import setup_logging logger = logging.getLogger(__name__) def main(argv: Sequence[str] | None = None) -> int: + setup_logging() + logger.debug(sys.version_info) + args, hook_config, files, tf_init_args, env_vars = parse_cmdline(argv) - if os.environ.get("PRE_COMMIT_COLOR") == "never": - args.append("-no-color") - cmd = ["terraform", "fmt", *args, *files] - logger.info("calling %s", shlex.join(cmd)) - logger.debug("env_vars: %r", env_vars) - logger.debug("args: %r", args) + + if os.environ.get('PRE_COMMIT_COLOR') == 'never': + args.append('-no-color') + + cmd = ['terraform', 'fmt', *args, *files] + + logger.info('calling %s', shlex.join(cmd)) + logger.debug('env_vars: %r', env_vars) + logger.debug('args: %r', args) + completed_process = run(cmd, env={**os.environ, **env_vars}, text=True, stdout=PIPE) + if completed_process.stdout: print(completed_process.stdout) + return completed_process.returncode -if __name__ == "__main__": +if __name__ == '__main__': raise SystemExit(main()) From 3f3fdd97d74634a46179f9dbe004681c9221edab Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Fri, 26 Apr 2024 20:35:11 +0300 Subject: [PATCH 03/67] Apply suggestions from code review --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a44d55046..27f92d4c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,7 +54,7 @@ repos: # JSON5 Linter - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 + rev: v3.1.0 hooks: - id: prettier # https://prettier.io/docs/en/options.html#parser From a37971b9f179ab4c539b455b075068446d8d0b59 Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Fri, 26 Apr 2024 20:35:29 +0300 Subject: [PATCH 04/67] Apply suggestions from code review --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27f92d4c5..f6894f8f9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,7 +54,7 @@ repos: # JSON5 Linter - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.1.0 + rev: v3.1.0 hooks: - id: prettier # https://prettier.io/docs/en/options.html#parser From 85269df0250ab9ce85f2fa1bd29217bb32cbdeb2 Mon Sep 17 00:00:00 2001 From: Eric L Frederich Date: Wed, 1 May 2024 09:41:22 -0400 Subject: [PATCH 05/67] fix mypy andy pylint issues --- hooks/common.py | 2 +- hooks/terraform_docs_replace.py | 2 +- hooks/terraform_fmt.py | 11 +++++++++-- setup.py | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/hooks/common.py b/hooks/common.py index f981f89bd..9ea175d02 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -def setup_logging(): +def setup_logging() -> None: """ Set up the logging configuration based on the value of the 'PCT_LOG' environment variable. diff --git a/hooks/terraform_docs_replace.py b/hooks/terraform_docs_replace.py index 204034e35..2561fdc3f 100644 --- a/hooks/terraform_docs_replace.py +++ b/hooks/terraform_docs_replace.py @@ -11,7 +11,7 @@ ) -def main(argv=None): +def main(argv=None) -> int: """ TODO: Add docstring. diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index 7a6dbe62e..44fb7c956 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -18,12 +18,16 @@ def main(argv: Sequence[str] | None = None) -> int: + """ + Main entry point for terraform_fmt_py pre-commit hook. + Parses args and calls `terraform fmt` on list of files provided by pre-commit. + """ setup_logging() logger.debug(sys.version_info) - args, hook_config, files, tf_init_args, env_vars = parse_cmdline(argv) + args, _hook_config, files, _tf_init_args, env_vars = parse_cmdline(argv) if os.environ.get('PRE_COMMIT_COLOR') == 'never': args.append('-no-color') @@ -34,7 +38,10 @@ def main(argv: Sequence[str] | None = None) -> int: logger.debug('env_vars: %r', env_vars) logger.debug('args: %r', args) - completed_process = run(cmd, env={**os.environ, **env_vars}, text=True, stdout=PIPE) + completed_process = run( + cmd, env={**os.environ, **env_vars}, + text=True, stdout=PIPE, check=False, + ) if completed_process.stdout: print(completed_process.stdout) diff --git a/setup.py b/setup.py index ce1ee2623..58535a0fb 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +# pylint: skip-file from setuptools import find_packages from setuptools import setup From 04218016965ff9d22cd3ae043cf33a6ae2a28841 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Thu, 26 Dec 2024 18:11:36 +0200 Subject: [PATCH 06/67] Add proto of per-dir execution and fix style --- .pre-commit-config.yaml | 32 +++++++++++++++---- hooks/terraform_fmt.py | 68 +++++++++++++++++++++++++++++++++++------ 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f03614c5..8f69cc1f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,10 +65,22 @@ repos: # PYTHON # ########## -- repo: https://github.com/asottile/reorder_python_imports - rev: v3.12.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.4 hooks: - - id: reorder-python-imports + - id: ruff + args: [--fix] + types_or: [python, pyi] + - id: ruff-format + types_or: [python, pyi] + args: [--config, format.quote-style = 'single', --config, line-length = 100] + +- repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort + args: [--force-single-line, --profile=black] - repo: https://github.com/asottile/add-trailing-comma rev: v3.1.0 @@ -119,7 +131,15 @@ repos: # https://www.flake8rules.com/ # https://wemake-python-stylegui.de/en/latest/pages/usage/violations/index.html - --extend-ignore= - WPS305, - E501, - I, + WPS210 + WPS300 + WPS305 + WPS323 + I + S404 + S603 + + # WPS211 + # WPS420 # RST, diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index 44fb7c956..24a8a6e2a 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -1,6 +1,5 @@ -""" -Pre-commit hook for terraform fmt. -""" +"""Pre-commit hook for terraform fmt.""" + from __future__ import annotations import logging @@ -17,12 +16,32 @@ logger = logging.getLogger(__name__) +def get_unique_dirs(files: list[str]) -> set[str]: + """ + Get unique directories from a list of files. + + Args: + files: list of file paths. + + Returns: + Set of unique directories. + """ + unique_dirs = set() + + for file_path in files: + dir_path = os.path.dirname(file_path) + unique_dirs.add(dir_path) + + return unique_dirs + + def main(argv: Sequence[str] | None = None) -> int: + # noqa: DAR101, DAR201 # TODO: Add docstrings when will end up with final implementation """ - Main entry point for terraform_fmt_py pre-commit hook. + Execute terraform_fmt_py pre-commit hook. + Parses args and calls `terraform fmt` on list of files provided by pre-commit. """ - setup_logging() logger.debug(sys.version_info) @@ -32,19 +51,50 @@ def main(argv: Sequence[str] | None = None) -> int: if os.environ.get('PRE_COMMIT_COLOR') == 'never': args.append('-no-color') - cmd = ['terraform', 'fmt', *args, *files] + # TODO: Per-dir execution + # consume modified files passed from pre-commit so that + # hook runs against only those relevant directories + unique_dirs = get_unique_dirs(files) + + final_exit_code = 0 + for dir_path in unique_dirs: + # TODO: per_dir_hook_unique_part call here + exit_code = per_dir_hook_unique_part(dir_path, args, env_vars) + + if exit_code != 0: + final_exit_code = exit_code + + return final_exit_code + + +def per_dir_hook_unique_part(dir_path: str, args: list[str], env_vars: dict[str, str]) -> int: + """ + Run the hook against a single directory. + + Args: + dir_path: The directory to run the hook against. + args: The arguments to pass to the hook + env_vars: The environment variables to pass to the hook + + Returns: + int: The exit code of the hook. + """ + cmd = ['terraform', 'fmt', *args, dir_path] logger.info('calling %s', shlex.join(cmd)) logger.debug('env_vars: %r', env_vars) logger.debug('args: %r', args) completed_process = run( - cmd, env={**os.environ, **env_vars}, - text=True, stdout=PIPE, check=False, + cmd, + env={**os.environ, **env_vars}, + text=True, + stdout=PIPE, + check=False, ) if completed_process.stdout: - print(completed_process.stdout) + sys.stdout.write(completed_process.stdout) return completed_process.returncode From 1f956dc4213d8652bba672b535aba10042981499 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Thu, 26 Dec 2024 19:20:58 +0200 Subject: [PATCH 07/67] Add first PyTests --- .pre-commit-config.yaml | 4 +++- hooks/terraform_fmt.py | 5 +++-- hooks/test_terraform_fmt.py | 39 +++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 hooks/test_terraform_fmt.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f69cc1f1..c92683ca0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -105,6 +105,7 @@ repos: - --disable=fixme # W0511. 'TODO' notations. - --disable=logging-fstring-interpolation # Conflict with "use a single formatting" WPS323 - --disable=ungrouped-imports # ignore `if TYPE_CHECKING` case. Other do reorder-python-imports + exclude: test_.+\.py$ - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.10.0 @@ -142,4 +143,5 @@ repos: # WPS211 # WPS420 - # RST, + # RST, + exclude: test_.+\.py$ diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index 24a8a6e2a..dbaa5f778 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -58,7 +58,6 @@ def main(argv: Sequence[str] | None = None) -> int: final_exit_code = 0 for dir_path in unique_dirs: - # TODO: per_dir_hook_unique_part call here exit_code = per_dir_hook_unique_part(dir_path, args, env_vars) if exit_code != 0: @@ -79,7 +78,9 @@ def per_dir_hook_unique_part(dir_path: str, args: list[str], env_vars: dict[str, Returns: int: The exit code of the hook. """ - cmd = ['terraform', 'fmt', *args, dir_path] + # Just in case is someone somehow will add something like "; rm -rf" in the args + quoted_args = [shlex.quote(arg) for arg in args] + cmd = ['terraform', 'fmt', *quoted_args, dir_path] logger.info('calling %s', shlex.join(cmd)) logger.debug('env_vars: %r', env_vars) diff --git a/hooks/test_terraform_fmt.py b/hooks/test_terraform_fmt.py new file mode 100644 index 000000000..06dbf2ae3 --- /dev/null +++ b/hooks/test_terraform_fmt.py @@ -0,0 +1,39 @@ +from os.path import join + +import pytest + +from .terraform_fmt import get_unique_dirs + + +def test_get_unique_dirs_empty(): + files = [] + result = get_unique_dirs(files) + assert result == set() + + +def test_get_unique_dirs_single_file(): + files = [join('path', 'to', 'file1.tf')] + result = get_unique_dirs(files) + assert result == {join('path', 'to')} + + +def test_get_unique_dirs_multiple_files_same_dir(): + files = [join('path', 'to', 'file1.tf'), join('path', 'to', 'file2.tf')] + result = get_unique_dirs(files) + assert result == {join('path', 'to')} + + +def test_get_unique_dirs_multiple_files_different_dirs(): + files = [join('path', 'to', 'file1.tf'), join('another', 'path', 'file2.tf')] + result = get_unique_dirs(files) + assert result == {join('path', 'to'), join('another', 'path')} + + +def test_get_unique_dirs_nested_dirs(): + files = [join('path', 'to', 'file1.tf'), join('path', 'to', 'nested', 'file2.tf')] + result = get_unique_dirs(files) + assert result == {join('path', 'to'), join('path', 'to', 'nested')} + + +if __name__ == '__main__': + pytest.main() From 59b6bacf22341c7bf8fb5fea5a084ef75a45a91b Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Thu, 26 Dec 2024 22:04:56 +0200 Subject: [PATCH 08/67] Separate per_dir_hook logic --- hooks/terraform_fmt.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index dbaa5f778..15c46df33 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -51,7 +51,20 @@ def main(argv: Sequence[str] | None = None) -> int: if os.environ.get('PRE_COMMIT_COLOR') == 'never': args.append('-no-color') - # TODO: Per-dir execution + return per_dir_hook(files, args, env_vars) + +def per_dir_hook(files: list[str], args: list[str], env_vars: dict[str, str]) -> int: + """ + Run hook boilerplate logic which is common to hooks, that run on per dir basis. + + Args: + files: The list of files to run the hook against. + args: The arguments to pass to the hook. + env_vars: The environment variables to pass to the hook. + + Returns: + The exit code of the hook execution for all directories. + """ # consume modified files passed from pre-commit so that # hook runs against only those relevant directories unique_dirs = get_unique_dirs(files) From 79ac4154bd6f17773d0672b42b6150282a4c0a74 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Thu, 26 Dec 2024 22:31:46 +0200 Subject: [PATCH 09/67] Add simple tests for per_dir_hook --- .github/CONTRIBUTING.md | 10 ++++ hooks/test_terraform_fmt.py | 103 +++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 8066f6f30..3d1fe9d50 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -18,6 +18,7 @@ Enjoy the clean, valid, and documented code! * [Prepare basic documentation](#prepare-basic-documentation) * [Add code](#add-code) * [Finish with the documentation](#finish-with-the-documentation) +* [Testing](#testing) ## Run and debug hooks locally @@ -154,3 +155,12 @@ You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/ 1. Add hook description to [Available Hooks](../README.md#available-hooks). 2. Create and populate a new hook section in [Hooks usage notes and examples](../README.md#hooks-usage-notes-and-examples). + + +## Testing + +``` +pip install pytest pytest-mock + +pytest -vv +``` diff --git a/hooks/test_terraform_fmt.py b/hooks/test_terraform_fmt.py index 06dbf2ae3..796d29ba4 100644 --- a/hooks/test_terraform_fmt.py +++ b/hooks/test_terraform_fmt.py @@ -1,8 +1,14 @@ +import os from os.path import join import pytest -from .terraform_fmt import get_unique_dirs +from hooks.terraform_fmt import get_unique_dirs +from hooks.terraform_fmt import per_dir_hook + +# +# get_unique_dirs +# def test_get_unique_dirs_empty(): @@ -35,5 +41,100 @@ def test_get_unique_dirs_nested_dirs(): assert result == {join('path', 'to'), join('path', 'to', 'nested')} +# +# per_dir_hook +# + + +# from unittest.mock import patch, call + + +@pytest.fixture +def mock_per_dir_hook_unique_part(mocker): + return mocker.patch('hooks.terraform_fmt.per_dir_hook_unique_part') + + +def test_per_dir_hook_empty_files(mock_per_dir_hook_unique_part): + files = [] + args = [] + env_vars = {} + result = per_dir_hook(files, args, env_vars) + assert result == 0 + mock_per_dir_hook_unique_part.assert_not_called() + + +def test_per_dir_hook_single_file(mock_per_dir_hook_unique_part): + files = [os.path.join('path', 'to', 'file1.tf')] + args = [] + env_vars = {} + mock_per_dir_hook_unique_part.return_value = 0 + result = per_dir_hook(files, args, env_vars) + assert result == 0 + mock_per_dir_hook_unique_part.assert_called_once_with( + os.path.join('path', 'to'), + args, + env_vars, + ) + + +def test_per_dir_hook_multiple_files_same_dir(mock_per_dir_hook_unique_part): + files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('path', 'to', 'file2.tf')] + args = [] + env_vars = {} + mock_per_dir_hook_unique_part.return_value = 0 + result = per_dir_hook(files, args, env_vars) + assert result == 0 + mock_per_dir_hook_unique_part.assert_called_once_with( + os.path.join('path', 'to'), + args, + env_vars, + ) + + +def test_per_dir_hook_multiple_files_different_dirs(mocker, mock_per_dir_hook_unique_part): + files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('another', 'path', 'file2.tf')] + args = [] + env_vars = {} + mock_per_dir_hook_unique_part.return_value = 0 + result = per_dir_hook(files, args, env_vars) + assert result == 0 + expected_calls = [ + mocker.call(os.path.join('path', 'to'), args, env_vars), + mocker.call(os.path.join('another', 'path'), args, env_vars), + ] + mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) + + +def test_per_dir_hook_nested_dirs(mocker, mock_per_dir_hook_unique_part): + files = [ + os.path.join('path', 'to', 'file1.tf'), + os.path.join('path', 'to', 'nested', 'file2.tf'), + ] + args = [] + env_vars = {} + mock_per_dir_hook_unique_part.return_value = 0 + result = per_dir_hook(files, args, env_vars) + assert result == 0 + expected_calls = [ + mocker.call(os.path.join('path', 'to'), args, env_vars), + mocker.call(os.path.join('path', 'to', 'nested'), args, env_vars), + ] + mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) + + +def test_per_dir_hook_with_errors(mocker, mock_per_dir_hook_unique_part): + files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('another', 'path', 'file2.tf')] + args = [] + env_vars = {} + mock_per_dir_hook_unique_part.side_effect = [0, 1] + result = per_dir_hook(files, args, env_vars) + assert result == 1 + expected_calls = [ + mocker.call(os.path.join('path', 'to'), args, env_vars), + mocker.call(os.path.join('another', 'path'), args, env_vars), + ] + mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) + + if __name__ == '__main__': pytest.main() From 93cddb7859796f2b4ea87dc33fb394b8e3a27b48 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Thu, 26 Dec 2024 22:42:55 +0200 Subject: [PATCH 10/67] Fix common issues --- hooks/common.py | 4 +--- hooks/terraform_fmt.py | 1 + setup.py | 11 ++--------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/hooks/common.py b/hooks/common.py index 9ea175d02..c0032ff3f 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -3,6 +3,7 @@ It not executed directly, but imported by other hooks. """ + from __future__ import annotations import argparse @@ -26,9 +27,6 @@ def setup_logging() -> None: If the 'PCT_LOG' environment variable is not set or has an invalid value, the default logging level is 'warning'. - - Returns: - None """ log_level = { 'error': logging.ERROR, diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index 15c46df33..b46b36ee3 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -53,6 +53,7 @@ def main(argv: Sequence[str] | None = None) -> int: return per_dir_hook(files, args, env_vars) + def per_dir_hook(files: list[str], args: list[str], env_vars: dict[str, str]) -> int: """ Run hook boilerplate logic which is common to hooks, that run on per dir basis. diff --git a/setup.py b/setup.py index 58535a0fb..d47288cf5 100644 --- a/setup.py +++ b/setup.py @@ -2,26 +2,19 @@ from setuptools import find_packages from setuptools import setup - setup( name='pre-commit-terraform', - description='Pre-commit hooks for terraform_docs_replace', + description='Pre-commit hooks for terraform', url='https://github.com/antonbabenko/pre-commit-terraform', version_format='{tag}+{gitsha}', - author='Contributors', - classifiers=[ 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], - packages=find_packages(exclude=('tests*', 'testing*')), install_requires=[ 'setuptools-git-version', From 0280a60480f1bb90acd48875f5bcdfeff405fd79 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 27 Dec 2024 00:01:27 +0200 Subject: [PATCH 11/67] Fix violations to have "greenfield" --- .pre-commit-config.yaml | 1 + hooks/common.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c92683ca0..7a9bcf3e9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -133,6 +133,7 @@ repos: # https://wemake-python-stylegui.de/en/latest/pages/usage/violations/index.html - --extend-ignore= WPS210 + WPS226 WPS300 WPS305 WPS323 S404 S603 + + + RST201 RST203 RST301 + WPS201 # TODO # WPS211 # WPS420 diff --git a/hooks/common.py b/hooks/common.py index 0dfcd9ade..a2e1543e1 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -37,7 +37,14 @@ def setup_logging() -> None: 'debug': logging.DEBUG, }[os.environ.get('PCT_LOG', 'warning').lower()] - logging.basicConfig(level=log_level) + log_format = '%(levelname)s:%(funcName)s:%(message)s' + if log_level == logging.DEBUG: + log_format = ( + '\n%(levelname)s:\t%(asctime)s.%(msecs)03d %(filename)s:%(lineno)s -> %(funcName)s()' + + '\n%(message)s' + ) + + logging.basicConfig(level=log_level, format=log_format, datefmt='%H:%M:%S') def parse_env_vars(env_var_strs: list[str]) -> dict[str, str]: @@ -96,6 +103,15 @@ def parse_cmdline( tf_init_args = parsed_args.tf_init_args or [] env_vars = parsed_args.env_vars or [] + logger.debug( + 'Parsed values:\nargs: %r\nhook_config: %r\nfiles: %r\ntf_init_args: %r\nenv_vars: %r', + args, + hook_config, + files, + tf_init_args, + env_vars, + ) + return args, hook_config, files, tf_init_args, env_vars @@ -118,6 +134,33 @@ def _get_unique_dirs(files: list[str]) -> set[str]: return unique_dirs +def expand_env_vars(args: list[str], env_vars: dict[str, str]) -> list[str]: + """ + Expand environment variables definition into their values in '--args'. + + Support expansion only for ${ENV_VAR} vars, not $ENV_VAR. + + Args: + args: The arguments to expand environment variables in. + env_vars: The environment variables to expand. + + Returns: + The arguments with expanded environment variables. + """ + expanded_args = [] + + for arg in args: + for env_var_name, env_var_value in env_vars.items(): + if f'${{{env_var_name}}}' in arg: + logger.info('Expanding ${%s} in "%s"', env_var_name, arg) + arg = arg.replace(f'${{{env_var_name}}}', env_var_value) + logger.debug('After ${%s} expansion: "%s"', env_var_name, arg) + + expanded_args.append(arg) + + return expanded_args + + def per_dir_hook( files: list[str], args: list[str], diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index d3853a269..5dd99f484 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -10,6 +10,7 @@ from subprocess import run from typing import Sequence +from .common import expand_env_vars from .common import parse_cmdline from .common import parse_env_vars from .common import per_dir_hook @@ -29,12 +30,14 @@ def main(argv: Sequence[str] | None = None) -> int: logger.debug(sys.version_info) # FIXME: WPS236 args, _hook_config, files, _tf_init_args, env_vars_strs = parse_cmdline(argv) # noqa: WPS236 - env_vars = parse_env_vars(env_vars_strs) + + all_env_vars = {**os.environ, **parse_env_vars(env_vars_strs)} + expanded_args = expand_env_vars(args, all_env_vars) if os.environ.get('PRE_COMMIT_COLOR') == 'never': args.append('-no-color') - return per_dir_hook(files, args, env_vars, per_dir_hook_unique_part) + return per_dir_hook(files, expanded_args, all_env_vars, per_dir_hook_unique_part) def per_dir_hook_unique_part(dir_path: str, args: list[str], env_vars: dict[str, str]) -> int: @@ -44,7 +47,8 @@ def per_dir_hook_unique_part(dir_path: str, args: list[str], env_vars: dict[str, Args: dir_path: The directory to run the hook against. args: The arguments to pass to the hook - env_vars: The custom environment variables defined by user in hook config. + env_vars: All environment variables provided to hook from system and + defined by user in hook config. Returns: int: The exit code of the hook. @@ -54,12 +58,10 @@ def per_dir_hook_unique_part(dir_path: str, args: list[str], env_vars: dict[str, cmd = ['terraform', 'fmt', *quoted_args, dir_path] logger.info('calling %s', shlex.join(cmd)) - logger.debug('env_vars: %r', env_vars) - logger.debug('args: %r', args) completed_process = run( cmd, - env={**os.environ, **env_vars}, + env=env_vars, text=True, stdout=PIPE, check=False, diff --git a/hooks/test_common.py b/hooks/test_common.py index 6cde9da24..10ca88a80 100644 --- a/hooks/test_common.py +++ b/hooks/test_common.py @@ -5,14 +5,15 @@ import pytest from hooks.common import _get_unique_dirs +from hooks.common import expand_env_vars from hooks.common import parse_cmdline from hooks.common import parse_env_vars from hooks.common import per_dir_hook -# -# get_unique_dirs -# +# ? +# ? get_unique_dirs +# ? def test_get_unique_dirs_empty(): files = [] result = _get_unique_dirs(files) @@ -43,9 +44,9 @@ def test_get_unique_dirs_nested_dirs(): assert result == {join('path', 'to'), join('path', 'to', 'nested')} -# -# per_dir_hook -# +# ? +# ? per_dir_hook +# ? @pytest.fixture def mock_per_dir_hook_unique_part(mocker): return mocker.patch('hooks.terraform_fmt.per_dir_hook_unique_part') @@ -133,9 +134,9 @@ def test_per_dir_hook_with_errors(mocker, mock_per_dir_hook_unique_part): mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) -# -# parse_env_vars -# +# ? +# ? parse_env_vars +# ? def test_parse_env_vars_empty(): env_var_strs = [] result = parse_env_vars(env_var_strs) @@ -222,13 +223,59 @@ def test_parse_cmdline_with_files(): assert env_vars_strs == [] -# def test_parse_cmdline_with_hook_config(): -# argv = ['-h', 'hook1'] -# with pytest.raises(NotImplementedError, match='TODO: implement: hook_config'): -# parse_cmdline(argv) +def test_parse_cmdline_with_hook_config(): + argv = ['-h', 'hook1', '-h', 'hook2'] + args, hook_config, files, tf_init_args, env_vars_strs = parse_cmdline(argv) + assert args == [] + assert hook_config == ['hook1', 'hook2'] + assert files == [] + assert tf_init_args == [] + assert env_vars_strs == [] + + +# ? +# ? expand_env_vars +# ? +def test_expand_env_vars_no_vars(): + args = ['arg1', 'arg2'] + env_vars = {} + result = expand_env_vars(args, env_vars) + assert result == ['arg1', 'arg2'] + + +def test_expand_env_vars_single_var(): + args = ['arg1', '${VAR1}', 'arg3'] + env_vars = {'VAR1': 'value1'} + result = expand_env_vars(args, env_vars) + assert result == ['arg1', 'value1', 'arg3'] + + +def test_expand_env_vars_multiple_vars(): + args = ['${VAR1}', 'arg2', '${VAR2}'] + env_vars = {'VAR1': 'value1', 'VAR2': 'value2'} + result = expand_env_vars(args, env_vars) + assert result == ['value1', 'arg2', 'value2'] + + +def test_expand_env_vars_no_expansion(): + args = ['arg1', 'arg2'] + env_vars = {'VAR1': 'value1'} + result = expand_env_vars(args, env_vars) + assert result == ['arg1', 'arg2'] + + +def test_expand_env_vars_partial_expansion(): + args = ['arg1', '${VAR1}', '${VAR2}'] + env_vars = {'VAR1': 'value1'} + result = expand_env_vars(args, env_vars) + assert result == ['arg1', 'value1', '${VAR2}'] + -# def test_parse_cmdline_with_tf_init_args_not_implemented(): -# argv = ['-i', 'init1'] +def test_expand_env_vars_with_special_chars(): + args = ['arg1', '${VAR_1}', 'arg3'] + env_vars = {'VAR_1': 'value1'} + result = expand_env_vars(args, env_vars) + assert result == ['arg1', 'value1', 'arg3'] if __name__ == '__main__': From c033767b1ae27d28f318ba8b1de126e6fd698280 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 27 Dec 2024 19:34:46 +0200 Subject: [PATCH 21/67] Implement "colorify" in sligtly different (better?) way --- hooks/common.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/hooks/common.py b/hooks/common.py index a2e1543e1..f9e1639ef 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -10,11 +10,58 @@ import logging import os from collections.abc import Sequence +from copy import copy from typing import Callable logger = logging.getLogger(__name__) +class ColoredFormatter(logging.Formatter): + """A logging formatter that adds color to the log messages.""" + + def __init__(self, pattern: str) -> None: + """ + Initialize the formatter with the given pattern. + + Args: + pattern (str): The log message format pattern. + """ + super().__init__(pattern) + self.disable_color = os.environ.get('PRE_COMMIT_COLOR') == 'never' + + def format(self, record: logging.LogRecord) -> str: + """ + Format the log record and add color to the levelname. + + Args: + record (logging.LogRecord): The log record to format. + + Returns: + str: The formatted log message. + """ + if self.disable_color: + return super().format(record) + + color_mapping = { + 'DEBUG': 37, # white + 'INFO': 36, # cyan + 'WARNING': 33, # yellow + 'ERROR': 31, # red + 'CRITICAL': 41, # white on red background + } + + prefix = '\033[' + suffix = '\033[0m' + + colored_record = copy(record) + levelname = colored_record.levelname + seq = color_mapping.get(levelname, 37) # default white # noqa: WPS432 + colored_levelname = f'{prefix}{seq}m{levelname}{suffix}' + colored_record.levelname = colored_levelname + + return super().format(colored_record) + + def setup_logging() -> None: """ Set up the logging configuration based on the value of the 'PCT_LOG' environment variable. @@ -44,7 +91,13 @@ def setup_logging() -> None: + '\n%(message)s' ) - logging.basicConfig(level=log_level, format=log_format, datefmt='%H:%M:%S') + formatter = ColoredFormatter(log_format) + log_handler = logging.StreamHandler() + log_handler.setFormatter(formatter) + + log = logging.getLogger() + log.setLevel(log_level) + log.addHandler(log_handler) def parse_env_vars(env_var_strs: list[str]) -> dict[str, str]: From 01a58505327302f8e8ebdf8b55432abc524b4cf7 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 27 Dec 2024 22:15:45 +0200 Subject: [PATCH 22/67] Add get_tf_binary_path --- .pre-commit-hooks.yaml | 1 + hooks/common.py | 82 +++++++++++++++++++++++++++++++++++++-- hooks/terraform_fmt.py | 14 +++++-- hooks/test_common.py | 87 +++++++++++++++++++++++++++++++++++------- 4 files changed, 163 insertions(+), 21 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 9c2836517..3244db675 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -18,6 +18,7 @@ - id: terraform_fmt_py name: Terraform fmt description: Rewrites all Terraform configuration files to a canonical format. + require_serial: true entry: terraform_fmt language: python files: \.tf(vars)?$ diff --git a/hooks/common.py b/hooks/common.py index f9e1639ef..c4199bc7d 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -9,6 +9,7 @@ import argparse import logging import os +import shutil from collections.abc import Sequence from copy import copy from typing import Callable @@ -214,16 +215,18 @@ def expand_env_vars(args: list[str], env_vars: dict[str, str]) -> list[str]: return expanded_args -def per_dir_hook( +def per_dir_hook( # noqa: WPS211 # Found too many arguments # TODO: Maybe refactor? + hook_config: list[str], files: list[str], args: list[str], env_vars: dict[str, str], - per_dir_hook_unique_part: Callable[[str, list[str], dict[str, str]], int], # noqa: WPS221 + per_dir_hook_unique_part: Callable[[str, str, list[str], dict[str, str]], int], # noqa: WPS221 ) -> int: """ Run hook boilerplate logic which is common to hooks, that run on per dir basis. Args: + hook_config: Arguments that configure hook behavior. files: The list of files to run the hook against. args: The arguments to pass to the hook. env_vars: The environment variables to pass to the hook. @@ -236,11 +239,84 @@ def per_dir_hook( # hook runs against only those relevant directories unique_dirs = _get_unique_dirs(files) + tf_path = get_tf_binary_path(hook_config) + + logger.debug( + 'Iterate per_dir_hook_unique_part with values:' + + '\ntf_path: %s\nunique_dirs: %r\nargs: %r\nenv_vars: %r', + tf_path, + unique_dirs, + args, + env_vars, + ) final_exit_code = 0 for dir_path in unique_dirs: - exit_code = per_dir_hook_unique_part(dir_path, args, env_vars) + exit_code = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) if exit_code != 0: final_exit_code = exit_code return final_exit_code + + +class BinaryNotFoundError(Exception): + """Exception raised when neither Terraform nor OpenTofu binary could be found.""" + + +def get_tf_binary_path(hook_config: list[str]) -> str: # noqa: WPS212 - Not Applicable + """ + Get Terraform/OpenTofu binary path. + + Allows user to set the path to custom Terraform or OpenTofu binary. + + Args: + hook_config (list[str]): Arguments that configure hook behavior. + + Environment Variables: + PCT_TFPATH: Path to Terraform or OpenTofu binary. + TERRAGRUNT_TFPATH: Path to Terraform or OpenTofu binary provided by Terragrunt. + + Returns: + str: The path to the Terraform or OpenTofu binary. + + Raises: + BinaryNotFoundError: If neither Terraform nor OpenTofu binary could be found. + + """ + hook_config_tf_path = None + + for config in hook_config: + if config.startswith('--tf-path='): + hook_config_tf_path = config.split('=', 1)[1].rstrip(';') + break + + # direct hook config, has the highest precedence + if hook_config_tf_path: + return hook_config_tf_path + + # environment variable + pct_tfpath = os.getenv('PCT_TFPATH') + if pct_tfpath: + return pct_tfpath + + # Maybe there is a similar setting for Terragrunt already + terragrunt_tfpath = os.getenv('TERRAGRUNT_TFPATH') + if terragrunt_tfpath: + return terragrunt_tfpath + + # check if Terraform binary is available + terraform_path = shutil.which('terraform') + if terraform_path: + return terraform_path + + # finally, check if Tofu binary is available + tofu_path = shutil.which('tofu') + if tofu_path: + return tofu_path + + # If no binary is found, raise an exception + raise BinaryNotFoundError( + 'Neither Terraform nor OpenTofu binary could be found. Please either set the "--tf-path"' + + ' hook configuration argument, or set the "PCT_TFPATH" environment variable, or set the' + + ' "TERRAGRUNT_TFPATH" environment variable, or install Terraform or OpenTofu globally.', + ) diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index 5dd99f484..8032e452e 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -29,7 +29,7 @@ def main(argv: Sequence[str] | None = None) -> int: setup_logging() logger.debug(sys.version_info) # FIXME: WPS236 - args, _hook_config, files, _tf_init_args, env_vars_strs = parse_cmdline(argv) # noqa: WPS236 + args, hook_config, files, _tf_init_args, env_vars_strs = parse_cmdline(argv) # noqa: WPS236 all_env_vars = {**os.environ, **parse_env_vars(env_vars_strs)} expanded_args = expand_env_vars(args, all_env_vars) @@ -37,14 +37,20 @@ def main(argv: Sequence[str] | None = None) -> int: if os.environ.get('PRE_COMMIT_COLOR') == 'never': args.append('-no-color') - return per_dir_hook(files, expanded_args, all_env_vars, per_dir_hook_unique_part) + return per_dir_hook(hook_config, files, expanded_args, all_env_vars, per_dir_hook_unique_part) -def per_dir_hook_unique_part(dir_path: str, args: list[str], env_vars: dict[str, str]) -> int: +def per_dir_hook_unique_part( + tf_path: str, + dir_path: str, + args: list[str], + env_vars: dict[str, str], +) -> int: """ Run the hook against a single directory. Args: + tf_path: The path to the terraform binary. dir_path: The directory to run the hook against. args: The arguments to pass to the hook env_vars: All environment variables provided to hook from system and @@ -55,7 +61,7 @@ def per_dir_hook_unique_part(dir_path: str, args: list[str], env_vars: dict[str, """ # Just in case is someone somehow will add something like "; rm -rf" in the args quoted_args = [shlex.quote(arg) for arg in args] - cmd = ['terraform', 'fmt', *quoted_args, dir_path] + cmd = [tf_path, 'fmt', *quoted_args, dir_path] logger.info('calling %s', shlex.join(cmd)) diff --git a/hooks/test_common.py b/hooks/test_common.py index 10ca88a80..afe5fb882 100644 --- a/hooks/test_common.py +++ b/hooks/test_common.py @@ -4,8 +4,10 @@ import pytest +from hooks.common import BinaryNotFoundError from hooks.common import _get_unique_dirs from hooks.common import expand_env_vars +from hooks.common import get_tf_binary_path from hooks.common import parse_cmdline from hooks.common import parse_env_vars from hooks.common import per_dir_hook @@ -53,36 +55,41 @@ def mock_per_dir_hook_unique_part(mocker): def test_per_dir_hook_empty_files(mock_per_dir_hook_unique_part): + hook_config = [] files = [] args = [] env_vars = {} - result = per_dir_hook(files, args, env_vars, mock_per_dir_hook_unique_part) + result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) assert result == 0 mock_per_dir_hook_unique_part.assert_not_called() -def test_per_dir_hook_single_file(mock_per_dir_hook_unique_part): +def test_per_dir_hook_single_file(mocker, mock_per_dir_hook_unique_part): + hook_config = [] files = [os.path.join('path', 'to', 'file1.tf')] args = [] env_vars = {} mock_per_dir_hook_unique_part.return_value = 0 - result = per_dir_hook(files, args, env_vars, mock_per_dir_hook_unique_part) + result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) assert result == 0 mock_per_dir_hook_unique_part.assert_called_once_with( + mocker.ANY, # Terraform binary path os.path.join('path', 'to'), args, env_vars, ) -def test_per_dir_hook_multiple_files_same_dir(mock_per_dir_hook_unique_part): +def test_per_dir_hook_multiple_files_same_dir(mocker, mock_per_dir_hook_unique_part): + hook_config = [] files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('path', 'to', 'file2.tf')] args = [] env_vars = {} mock_per_dir_hook_unique_part.return_value = 0 - result = per_dir_hook(files, args, env_vars, mock_per_dir_hook_unique_part) + result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) assert result == 0 mock_per_dir_hook_unique_part.assert_called_once_with( + mocker.ANY, # Terraform binary path os.path.join('path', 'to'), args, env_vars, @@ -90,20 +97,22 @@ def test_per_dir_hook_multiple_files_same_dir(mock_per_dir_hook_unique_part): def test_per_dir_hook_multiple_files_different_dirs(mocker, mock_per_dir_hook_unique_part): + hook_config = [] files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('another', 'path', 'file2.tf')] args = [] env_vars = {} mock_per_dir_hook_unique_part.return_value = 0 - result = per_dir_hook(files, args, env_vars, mock_per_dir_hook_unique_part) + result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) assert result == 0 expected_calls = [ - mocker.call(os.path.join('path', 'to'), args, env_vars), - mocker.call(os.path.join('another', 'path'), args, env_vars), + mocker.call(mocker.ANY, os.path.join('path', 'to'), args, env_vars), + mocker.call(mocker.ANY, os.path.join('another', 'path'), args, env_vars), ] mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) def test_per_dir_hook_nested_dirs(mocker, mock_per_dir_hook_unique_part): + hook_config = [] files = [ os.path.join('path', 'to', 'file1.tf'), os.path.join('path', 'to', 'nested', 'file2.tf'), @@ -111,25 +120,26 @@ def test_per_dir_hook_nested_dirs(mocker, mock_per_dir_hook_unique_part): args = [] env_vars = {} mock_per_dir_hook_unique_part.return_value = 0 - result = per_dir_hook(files, args, env_vars, mock_per_dir_hook_unique_part) + result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) assert result == 0 expected_calls = [ - mocker.call(os.path.join('path', 'to'), args, env_vars), - mocker.call(os.path.join('path', 'to', 'nested'), args, env_vars), + mocker.call(mocker.ANY, os.path.join('path', 'to'), args, env_vars), + mocker.call(mocker.ANY, os.path.join('path', 'to', 'nested'), args, env_vars), ] mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) def test_per_dir_hook_with_errors(mocker, mock_per_dir_hook_unique_part): + hook_config = [] files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('another', 'path', 'file2.tf')] args = [] env_vars = {} mock_per_dir_hook_unique_part.side_effect = [0, 1] - result = per_dir_hook(files, args, env_vars, mock_per_dir_hook_unique_part) + result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) assert result == 1 expected_calls = [ - mocker.call(os.path.join('path', 'to'), args, env_vars), - mocker.call(os.path.join('another', 'path'), args, env_vars), + mocker.call(mocker.ANY, os.path.join('path', 'to'), args, env_vars), + mocker.call(mocker.ANY, os.path.join('another', 'path'), args, env_vars), ] mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) @@ -278,5 +288,54 @@ def test_expand_env_vars_with_special_chars(): assert result == ['arg1', 'value1', 'arg3'] +# ? +# ? get_tf_binary_path +# ? +def test_get_tf_binary_path_from_hook_config(): + hook_config = ['--tf-path=/custom/path/to/terraform'] + result = get_tf_binary_path(hook_config) + assert result == '/custom/path/to/terraform' + + +def test_get_tf_binary_path_from_pct_tfpath_env_var(mocker): + hook_config = [] + mocker.patch.dict(os.environ, {'PCT_TFPATH': '/env/path/to/terraform'}) + result = get_tf_binary_path(hook_config) + assert result == '/env/path/to/terraform' + + +def test_get_tf_binary_path_from_terragrunt_tfpath_env_var(mocker): + hook_config = [] + mocker.patch.dict(os.environ, {'TERRAGRUNT_TFPATH': '/env/path/to/terragrunt'}) + result = get_tf_binary_path(hook_config) + assert result == '/env/path/to/terragrunt' + + +def test_get_tf_binary_path_from_system_path_terraform(mocker): + hook_config = [] + mocker.patch('shutil.which', return_value='/usr/local/bin/terraform') + result = get_tf_binary_path(hook_config) + assert result == '/usr/local/bin/terraform' + + +def test_get_tf_binary_path_from_system_path_tofu(mocker): + hook_config = [] + mocker.patch('shutil.which', side_effect=[None, '/usr/local/bin/tofu']) + result = get_tf_binary_path(hook_config) + assert result == '/usr/local/bin/tofu' + + +def test_get_tf_binary_path_not_found(mocker): + hook_config = [] + mocker.patch('shutil.which', return_value=None) + with pytest.raises( + BinaryNotFoundError, + match='Neither Terraform nor OpenTofu binary could be found. Please either set the "--tf-path"' + + ' hook configuration argument, or set the "PCT_TFPATH" environment variable, or set the' + + ' "TERRAGRUNT_TFPATH" environment variable, or install Terraform or OpenTofu globally.', + ): + get_tf_binary_path(hook_config) + + if __name__ == '__main__': pytest.main() From c7c8e569ffcfb6ae2be2af7d517735932bfa10d7 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 27 Dec 2024 22:25:25 +0200 Subject: [PATCH 23/67] Import all common functions at once --- .pre-commit-config.yaml | 1 - hooks/terraform_fmt.py | 22 ++++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de96d4ae8..f5a3df7e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -144,7 +144,6 @@ repos: RST201 RST203 RST301 - WPS201 # TODO # WPS211 # WPS420 diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index 8032e452e..317b2865f 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -10,11 +10,7 @@ from subprocess import run from typing import Sequence -from .common import expand_env_vars -from .common import parse_cmdline -from .common import parse_env_vars -from .common import per_dir_hook -from .common import setup_logging +from . import common logger = logging.getLogger(__name__) @@ -26,18 +22,24 @@ def main(argv: Sequence[str] | None = None) -> int: Parses args and calls `terraform fmt` on list of files provided by pre-commit. """ - setup_logging() + common.setup_logging() logger.debug(sys.version_info) # FIXME: WPS236 - args, hook_config, files, _tf_init_args, env_vars_strs = parse_cmdline(argv) # noqa: WPS236 + args, hook_config, files, _tf_init_args, env_vars_strs = common.parse_cmdline(argv) # noqa: WPS236 - all_env_vars = {**os.environ, **parse_env_vars(env_vars_strs)} - expanded_args = expand_env_vars(args, all_env_vars) + all_env_vars = {**os.environ, **common.parse_env_vars(env_vars_strs)} + expanded_args = common.expand_env_vars(args, all_env_vars) if os.environ.get('PRE_COMMIT_COLOR') == 'never': args.append('-no-color') - return per_dir_hook(hook_config, files, expanded_args, all_env_vars, per_dir_hook_unique_part) + return common.per_dir_hook( + hook_config, + files, + expanded_args, + all_env_vars, + per_dir_hook_unique_part, + ) def per_dir_hook_unique_part( From fe92d45f744f98ed5bf4deed597d4844ca2df028 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 27 Dec 2024 22:50:20 +0200 Subject: [PATCH 24/67] Update linter settings to common project limitations --- .pre-commit-config.yaml | 6 ++++-- hooks/common.py | 4 ++-- hooks/terraform_fmt.py | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5a3df7e4..0f56d5968 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -127,13 +127,14 @@ repos: - flake8-pytest-style - wemake-python-styleguide args: - - --max-returns=2 # Default settings - - --max-arguments=4 # Default settings + - --max-returns=5 # Default to 2 + - --max-arguments=5 # Default to 4 # https://www.flake8rules.com/ # https://wemake-python-stylegui.de/en/latest/pages/usage/violations/index.html - --extend-ignore= WPS210 WPS226 + WPS236 WPS300 WPS305 WPS323 RST201 RST203 RST301 + # WPS211 # WPS420 # RST, diff --git a/hooks/common.py b/hooks/common.py index c4199bc7d..d6875b490 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -215,7 +215,7 @@ def expand_env_vars(args: list[str], env_vars: dict[str, str]) -> list[str]: return expanded_args -def per_dir_hook( # noqa: WPS211 # Found too many arguments # TODO: Maybe refactor? +def per_dir_hook( hook_config: list[str], files: list[str], args: list[str], @@ -263,7 +263,7 @@ class BinaryNotFoundError(Exception): """Exception raised when neither Terraform nor OpenTofu binary could be found.""" -def get_tf_binary_path(hook_config: list[str]) -> str: # noqa: WPS212 - Not Applicable +def get_tf_binary_path(hook_config: list[str]) -> str: """ Get Terraform/OpenTofu binary path. diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index 317b2865f..d523c4507 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -24,8 +24,8 @@ def main(argv: Sequence[str] | None = None) -> int: """ common.setup_logging() logger.debug(sys.version_info) - # FIXME: WPS236 - args, hook_config, files, _tf_init_args, env_vars_strs = common.parse_cmdline(argv) # noqa: WPS236 + + args, hook_config, files, _tf_init_args, env_vars_strs = common.parse_cmdline(argv) all_env_vars = {**os.environ, **common.parse_env_vars(env_vars_strs)} expanded_args = common.expand_env_vars(args, all_env_vars) From e010c8a91fcf725e6b9f07f5c6a81b35a5217746 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 27 Dec 2024 23:18:11 +0200 Subject: [PATCH 25/67] Split logger logic to separate file --- hooks/common.py | 85 ---------------------------------------- hooks/logger.py | 89 ++++++++++++++++++++++++++++++++++++++++++ hooks/terraform_fmt.py | 3 +- 3 files changed, 91 insertions(+), 86 deletions(-) create mode 100644 hooks/logger.py diff --git a/hooks/common.py b/hooks/common.py index d6875b490..714e8a6b4 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -11,96 +11,11 @@ import os import shutil from collections.abc import Sequence -from copy import copy from typing import Callable logger = logging.getLogger(__name__) -class ColoredFormatter(logging.Formatter): - """A logging formatter that adds color to the log messages.""" - - def __init__(self, pattern: str) -> None: - """ - Initialize the formatter with the given pattern. - - Args: - pattern (str): The log message format pattern. - """ - super().__init__(pattern) - self.disable_color = os.environ.get('PRE_COMMIT_COLOR') == 'never' - - def format(self, record: logging.LogRecord) -> str: - """ - Format the log record and add color to the levelname. - - Args: - record (logging.LogRecord): The log record to format. - - Returns: - str: The formatted log message. - """ - if self.disable_color: - return super().format(record) - - color_mapping = { - 'DEBUG': 37, # white - 'INFO': 36, # cyan - 'WARNING': 33, # yellow - 'ERROR': 31, # red - 'CRITICAL': 41, # white on red background - } - - prefix = '\033[' - suffix = '\033[0m' - - colored_record = copy(record) - levelname = colored_record.levelname - seq = color_mapping.get(levelname, 37) # default white # noqa: WPS432 - colored_levelname = f'{prefix}{seq}m{levelname}{suffix}' - colored_record.levelname = colored_levelname - - return super().format(colored_record) - - -def setup_logging() -> None: - """ - Set up the logging configuration based on the value of the 'PCT_LOG' environment variable. - - The 'PCT_LOG' environment variable determines the logging level to be used. - The available levels are: - - 'error': Only log error messages. - - 'warn' or 'warning': Log warning messages and above. - - 'info': Log informational messages and above. - - 'debug': Log debug messages and above. - - If the 'PCT_LOG' environment variable is not set or has an invalid value, - the default logging level is 'warning'. - """ - log_level = { - 'error': logging.ERROR, - 'warn': logging.WARNING, - 'warning': logging.WARNING, - 'info': logging.INFO, - 'debug': logging.DEBUG, - }[os.environ.get('PCT_LOG', 'warning').lower()] - - log_format = '%(levelname)s:%(funcName)s:%(message)s' - if log_level == logging.DEBUG: - log_format = ( - '\n%(levelname)s:\t%(asctime)s.%(msecs)03d %(filename)s:%(lineno)s -> %(funcName)s()' - + '\n%(message)s' - ) - - formatter = ColoredFormatter(log_format) - log_handler = logging.StreamHandler() - log_handler.setFormatter(formatter) - - log = logging.getLogger() - log.setLevel(log_level) - log.addHandler(log_handler) - - def parse_env_vars(env_var_strs: list[str]) -> dict[str, str]: """ Expand environment variables definition into their values in '--args'. diff --git a/hooks/logger.py b/hooks/logger.py new file mode 100644 index 000000000..c00d4df7e --- /dev/null +++ b/hooks/logger.py @@ -0,0 +1,89 @@ +"""Here located logs-related functions.""" + +import logging +import os +from copy import copy + + +class ColoredFormatter(logging.Formatter): + """A logging formatter that adds color to the log messages.""" + + def __init__(self, pattern: str) -> None: + """ + Initialize the formatter with the given pattern. + + Args: + pattern (str): The log message format pattern. + """ + super().__init__(pattern) + self.disable_color = os.environ.get('PRE_COMMIT_COLOR') == 'never' + + def format(self, record: logging.LogRecord) -> str: + """ + Format the log record and add color to the levelname. + + Args: + record (logging.LogRecord): The log record to format. + + Returns: + str: The formatted log message. + """ + if self.disable_color: + return super().format(record) + + color_mapping = { + 'DEBUG': 37, # white + 'INFO': 36, # cyan + 'WARNING': 33, # yellow + 'ERROR': 31, # red + 'CRITICAL': 41, # white on red background + } + + prefix = '\033[' + suffix = '\033[0m' + + colored_record = copy(record) + levelname = colored_record.levelname + seq = color_mapping.get(levelname, 37) # default white # noqa: WPS432 + colored_levelname = f'{prefix}{seq}m{levelname}{suffix}' + colored_record.levelname = colored_levelname + + return super().format(colored_record) + + +def setup_logging() -> None: + """ + Set up the logging configuration based on the value of the 'PCT_LOG' environment variable. + + The 'PCT_LOG' environment variable determines the logging level to be used. + The available levels are: + - 'error': Only log error messages. + - 'warn' or 'warning': Log warning messages and above. + - 'info': Log informational messages and above. + - 'debug': Log debug messages and above. + + If the 'PCT_LOG' environment variable is not set or has an invalid value, + the default logging level is 'warning'. + """ + log_level = { + 'error': logging.ERROR, + 'warn': logging.WARNING, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG, + }[os.environ.get('PCT_LOG', 'warning').lower()] + + log_format = '%(levelname)s:%(funcName)s:%(message)s' + if log_level == logging.DEBUG: + log_format = ( + '\n%(levelname)s:\t%(asctime)s.%(msecs)03d %(filename)s:%(lineno)s -> %(funcName)s()' + + '\n%(message)s' + ) + + formatter = ColoredFormatter(log_format) + log_handler = logging.StreamHandler() + log_handler.setFormatter(formatter) + + log = logging.getLogger() + log.setLevel(log_level) + log.addHandler(log_handler) diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index d523c4507..ab79f2e0b 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -11,6 +11,7 @@ from typing import Sequence from . import common +from .logger import setup_logging logger = logging.getLogger(__name__) @@ -22,7 +23,7 @@ def main(argv: Sequence[str] | None = None) -> int: Parses args and calls `terraform fmt` on list of files provided by pre-commit. """ - common.setup_logging() + setup_logging() logger.debug(sys.version_info) args, hook_config, files, _tf_init_args, env_vars_strs = common.parse_cmdline(argv) From 4c459dfaaf8996898cf84d01e738b7ba0532fdd2 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 27 Dec 2024 23:49:01 +0200 Subject: [PATCH 26/67] Add tests to terraform_fmt --- hooks/test_terraform_fmt.py | 153 ++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 hooks/test_terraform_fmt.py diff --git a/hooks/test_terraform_fmt.py b/hooks/test_terraform_fmt.py new file mode 100644 index 000000000..ff3abb209 --- /dev/null +++ b/hooks/test_terraform_fmt.py @@ -0,0 +1,153 @@ +# pylint: skip-file +import os +import sys +from subprocess import PIPE + +import pytest + +from hooks.terraform_fmt import main +from hooks.terraform_fmt import per_dir_hook_unique_part + + +@pytest.fixture +def mock_setup_logging(mocker): + return mocker.patch('hooks.terraform_fmt.setup_logging') + + +@pytest.fixture +def mock_parse_cmdline(mocker): + return mocker.patch('hooks.terraform_fmt.common.parse_cmdline') + + +@pytest.fixture +def mock_parse_env_vars(mocker): + return mocker.patch('hooks.terraform_fmt.common.parse_env_vars') + + +@pytest.fixture +def mock_expand_env_vars(mocker): + return mocker.patch('hooks.terraform_fmt.common.expand_env_vars') + + +@pytest.fixture +def mock_per_dir_hook(mocker): + return mocker.patch('hooks.terraform_fmt.common.per_dir_hook') + + +@pytest.fixture +def mock_run(mocker): + return mocker.patch('hooks.terraform_fmt.run') + + +def test_main( + mocker, + mock_setup_logging, + mock_parse_cmdline, + mock_parse_env_vars, + mock_expand_env_vars, + mock_per_dir_hook, +): + mock_parse_cmdline.return_value = (['arg1'], ['hook1'], ['file1'], [], ['VAR1=value1']) + mock_parse_env_vars.return_value = {'VAR1': 'value1'} + mock_expand_env_vars.return_value = ['expanded_arg1'] + mock_per_dir_hook.return_value = 0 + + mocker.patch.object(sys, 'argv', ['terraform_fmt.py']) + exit_code = main(sys.argv) + assert exit_code == 0 + + mock_setup_logging.assert_called_once() + mock_parse_cmdline.assert_called_once_with(['terraform_fmt.py']) + mock_parse_env_vars.assert_called_once_with(['VAR1=value1']) + mock_expand_env_vars.assert_called_once_with(['arg1'], {**os.environ, 'VAR1': 'value1'}) + mock_per_dir_hook.assert_called_once_with( + ['hook1'], + ['file1'], + ['expanded_arg1'], + {**os.environ, 'VAR1': 'value1'}, + mocker.ANY, + ) + + +def test_main_with_no_color( + mocker, + mock_setup_logging, + mock_parse_cmdline, + mock_parse_env_vars, + mock_expand_env_vars, + mock_per_dir_hook, +): + mock_parse_cmdline.return_value = (['arg1'], ['hook1'], ['file1'], [], ['VAR1=value1']) + mock_parse_env_vars.return_value = {'VAR1': 'value1'} + mock_expand_env_vars.return_value = ['expanded_arg1'] + mock_per_dir_hook.return_value = 0 + + mocker.patch.dict(os.environ, {'PRE_COMMIT_COLOR': 'never'}) + mocker.patch.object(sys, 'argv', ['terraform_fmt.py']) + exit_code = main(sys.argv) + assert exit_code == 0 + + mock_setup_logging.assert_called_once() + mock_parse_cmdline.assert_called_once_with(['terraform_fmt.py']) + mock_parse_env_vars.assert_called_once_with(['VAR1=value1']) + mock_expand_env_vars.assert_called_once_with( + ['arg1', '-no-color'], + {**os.environ, 'VAR1': 'value1'}, + ) + mock_per_dir_hook.assert_called_once_with( + ['hook1'], + ['file1'], + ['expanded_arg1'], + {**os.environ, 'VAR1': 'value1'}, + mocker.ANY, + ) + + +def test_per_dir_hook_unique_part(mocker, mock_run): + tf_path = '/path/to/terraform' + dir_path = '/path/to/dir' + args = ['arg1', 'arg2'] + env_vars = {'VAR1': 'value1'} + + mock_completed_process = mocker.MagicMock() + mock_completed_process.stdout = 'output' + mock_completed_process.returncode = 0 + mock_run.return_value = mock_completed_process + + exit_code = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) + assert exit_code == 0 + + mock_run.assert_called_once_with( + ['/path/to/terraform', 'fmt', 'arg1', 'arg2', '/path/to/dir'], + env=env_vars, + text=True, + stdout=PIPE, + check=False, + ) + + +def test_per_dir_hook_unique_part_with_error(mocker, mock_run): + tf_path = '/path/to/terraform' + dir_path = '/path/to/dir' + args = ['arg1', 'arg2'] + env_vars = {'VAR1': 'value1'} + + mock_completed_process = mocker.MagicMock() + mock_completed_process.stdout = 'error output' + mock_completed_process.returncode = 1 + mock_run.return_value = mock_completed_process + + exit_code = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) + assert exit_code == 1 + + mock_run.assert_called_once_with( + ['/path/to/terraform', 'fmt', 'arg1', 'arg2', '/path/to/dir'], + env=env_vars, + text=True, + stdout=PIPE, + check=False, + ) + + +if __name__ == '__main__': + pytest.main() From 33e15479baaf742929c12b1464dec873a5004bfe Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Sat, 28 Dec 2024 00:42:56 +0200 Subject: [PATCH 27/67] Itint terraform_checkov hook --- .pre-commit-config.yaml | 2 + .pre-commit-hooks.yaml | 10 ++ hooks/terraform_checkov.py | 102 ++++++++++++++++++ hooks/terraform_fmt.py | 7 +- hooks/test_terraform_checkov.py | 42 ++++++++ setup.py | 3 + {hooks => tests/pytest}/test_common.py | 0 {hooks => tests/pytest}/test_terraform_fmt.py | 0 8 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 hooks/terraform_checkov.py create mode 100644 hooks/test_terraform_checkov.py rename {hooks => tests/pytest}/test_common.py (100%) rename {hooks => tests/pytest}/test_terraform_fmt.py (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f56d5968..4b69ad0a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -105,6 +105,8 @@ repos: - --disable=fixme # W0511. 'TODO' notations. - --disable=logging-fstring-interpolation # Conflict with "use a single formatting" WPS323 - --disable=ungrouped-imports # ignore `if TYPE_CHECKING` case. Other do reorder-python-imports + - --disable=R0801 # Similar lines in 2 files. Currently I don't think that it possible to DRY hooks unique files boilerplate + exclude: test_.+\.py$ - repo: https://github.com/pre-commit/mirrors-mypy diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 3244db675..00ebc073f 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -147,6 +147,16 @@ exclude: \.terraform/.*$ require_serial: true +- id: terraform_checkov_py + name: Checkov + description: Runs checkov on Terraform templates. + entry: terraform_checkov + language: python + always_run: false + files: \.tf$ + exclude: \.terraform/.*$ + require_serial: true + - id: terraform_wrapper_module_for_each name: Terraform wrapper with for_each in module description: Generate Terraform wrappers with for_each in module. diff --git a/hooks/terraform_checkov.py b/hooks/terraform_checkov.py new file mode 100644 index 000000000..ff65b2965 --- /dev/null +++ b/hooks/terraform_checkov.py @@ -0,0 +1,102 @@ +"""Pre-commit hook for terraform fmt.""" + +from __future__ import annotations + +import logging +import os +import shlex +import sys +from subprocess import PIPE +from subprocess import run +from typing import Sequence + +from . import common +from .logger import setup_logging + +logger = logging.getLogger(__name__) + + +def replace_git_working_dir_to_repo_root(args: list[str]) -> list[str]: + """ + Support for setting PATH to repo root. + + Replace '__GIT_WORKING_DIR__' with the current working directory in each argument. + + Args: + args: List of arguments to process. + + Returns: + List of arguments with '__GIT_WORKING_DIR__' replaced. + """ + return [arg.replace('__GIT_WORKING_DIR__', os.getcwd()) for arg in args] + + +def main(argv: Sequence[str] | None = None) -> int: + # noqa: DAR101, DAR201 # TODO: Add docstrings when will end up with final implementation + """ + Execute terraform_fmt_py pre-commit hook. + + Parses args and calls `terraform fmt` on list of files provided by pre-commit. + """ + setup_logging() + logger.debug(sys.version_info) + + args, hook_config, files, _tf_init_args, env_vars_strs = common.parse_cmdline(argv) + + all_env_vars = {**os.environ, **common.parse_env_vars(env_vars_strs)} + expanded_args = common.expand_env_vars(args, all_env_vars) + expanded_args = replace_git_working_dir_to_repo_root(expanded_args) + # Just in case is someone somehow will add something like "; rm -rf" in the args + safe_args = [shlex.quote(arg) for arg in expanded_args] + + if os.environ.get('PRE_COMMIT_COLOR') == 'never': + all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # TODO: Check is it works as expected + + return common.per_dir_hook( + hook_config, + files, + safe_args, + all_env_vars, + per_dir_hook_unique_part, + ) + + +def per_dir_hook_unique_part( + tf_path: str, # pylint: disable=unused-argument + dir_path: str, + args: list[str], + env_vars: dict[str, str], +) -> int: + """ + Run the hook against a single directory. + + Args: + tf_path: The path to the terraform binary. + dir_path: The directory to run the hook against. + args: The arguments to pass to the hook + env_vars: All environment variables provided to hook from system and + defined by user in hook config. + + Returns: + int: The exit code of the hook. + """ + cmd = ['checkov', '-d', dir_path, *args] + + logger.info('calling %s', shlex.join(cmd)) + + completed_process = run( + cmd, + env=env_vars, + text=True, + stdout=PIPE, + check=False, + ) + + if completed_process.stdout: + sys.stdout.write(completed_process.stdout) + + return completed_process.returncode + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index ab79f2e0b..811026b4e 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -30,6 +30,8 @@ def main(argv: Sequence[str] | None = None) -> int: all_env_vars = {**os.environ, **common.parse_env_vars(env_vars_strs)} expanded_args = common.expand_env_vars(args, all_env_vars) + # Just in case is someone somehow will add something like "; rm -rf" in the args + safe_args = [shlex.quote(arg) for arg in expanded_args] if os.environ.get('PRE_COMMIT_COLOR') == 'never': args.append('-no-color') @@ -37,7 +39,7 @@ def main(argv: Sequence[str] | None = None) -> int: return common.per_dir_hook( hook_config, files, - expanded_args, + safe_args, all_env_vars, per_dir_hook_unique_part, ) @@ -63,8 +65,7 @@ def per_dir_hook_unique_part( int: The exit code of the hook. """ # Just in case is someone somehow will add something like "; rm -rf" in the args - quoted_args = [shlex.quote(arg) for arg in args] - cmd = [tf_path, 'fmt', *quoted_args, dir_path] + cmd = [tf_path, 'fmt', *args, dir_path] logger.info('calling %s', shlex.join(cmd)) diff --git a/hooks/test_terraform_checkov.py b/hooks/test_terraform_checkov.py new file mode 100644 index 000000000..5c364f6b3 --- /dev/null +++ b/hooks/test_terraform_checkov.py @@ -0,0 +1,42 @@ +import pytest + +from hooks.terraform_checkov import replace_git_working_dir_to_repo_root + +# FILE: hooks/test_terraform_checkov.py + + +def test_replace_git_working_dir_to_repo_root_empty(): + args = [] + result = replace_git_working_dir_to_repo_root(args) + assert result == [] + + +def test_replace_git_working_dir_to_repo_root_no_replacement(): + args = ['arg1', 'arg2'] + result = replace_git_working_dir_to_repo_root(args) + assert result == ['arg1', 'arg2'] + + +def test_replace_git_working_dir_to_repo_root_single_replacement(mocker): + mocker.patch('os.getcwd', return_value='/current/working/dir') + args = ['arg1', '__GIT_WORKING_DIR__/arg2'] + result = replace_git_working_dir_to_repo_root(args) + assert result == ['arg1', '/current/working/dir/arg2'] + + +def test_replace_git_working_dir_to_repo_root_multiple_replacements(mocker): + mocker.patch('os.getcwd', return_value='/current/working/dir') + args = ['__GIT_WORKING_DIR__/arg1', 'arg2', '__GIT_WORKING_DIR__/arg3'] + result = replace_git_working_dir_to_repo_root(args) + assert result == ['/current/working/dir/arg1', 'arg2', '/current/working/dir/arg3'] + + +def test_replace_git_working_dir_to_repo_root_partial_replacement(mocker): + mocker.patch('os.getcwd', return_value='/current/working/dir') + args = ['arg1', '__GIT_WORKING_DIR__/arg2', 'arg3'] + result = replace_git_working_dir_to_repo_root(args) + assert result == ['arg1', '/current/working/dir/arg2', 'arg3'] + + +if __name__ == '__main__': + pytest.main() diff --git a/setup.py b/setup.py index d47288cf5..11e843cee 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,6 @@ +"""setup.py for pre-commit-terraform.""" # pylint: skip-file + from setuptools import find_packages from setuptools import setup @@ -23,6 +25,7 @@ 'console_scripts': [ 'terraform_docs_replace = hooks.terraform_docs_replace:main', 'terraform_fmt = hooks.terraform_fmt:main', + 'terraform_checkov = hooks.terraform_checkov:main', ], }, ) diff --git a/hooks/test_common.py b/tests/pytest/test_common.py similarity index 100% rename from hooks/test_common.py rename to tests/pytest/test_common.py diff --git a/hooks/test_terraform_fmt.py b/tests/pytest/test_terraform_fmt.py similarity index 100% rename from hooks/test_terraform_fmt.py rename to tests/pytest/test_terraform_fmt.py From 7d9ba2fca0e4bca71033b78641ff8bef3528da26 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Mon, 30 Dec 2024 18:53:26 +0200 Subject: [PATCH 28/67] Init part of run_hook_on_whole_repo functionality --- hooks/common.py | 42 ++++++++++++++++++++++++++++++++++++- hooks/terraform_checkov.py | 39 ++++++++++++++++++++++++++++++++++ tests/pytest/test_common.py | 42 +++++++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) diff --git a/hooks/common.py b/hooks/common.py index 714e8a6b4..0e0b477bb 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -65,7 +65,7 @@ def parse_cmdline( parser.add_argument('FILES', nargs='*', help='Files') parsed_args = parser.parse_args(argv) - + # TODO: move to defaults args = parsed_args.args or [] hook_config = parsed_args.hook_config or [] files = parsed_args.FILES or [] @@ -235,3 +235,43 @@ def get_tf_binary_path(hook_config: list[str]) -> str: + ' hook configuration argument, or set the "PCT_TFPATH" environment variable, or set the' + ' "TERRAGRUNT_TFPATH" environment variable, or install Terraform or OpenTofu globally.', ) + + +# ? +# ? Related to run_hook_on_whole_repo functions +# ? +def is_function_defined(func_name: str, scope: dict) -> bool: + """ + Check if a function is defined in the global scope. + + Args: + func_name (str): The name of the function to check. + scope (dict): The scope (usually globals()) to check in. + + Returns: + bool: True if the function is defined, False otherwise. + """ + is_defined = func_name in scope + is_callable = callable(scope[func_name]) if is_defined else False + + logger.debug( + 'Checking if "%s":\n1. Defined in hook: %s\n2. Is it callable: %s', + func_name, + is_defined, + is_callable, + ) + + return is_defined and is_callable + + +def is_hook_run_on_whole_repo(files: list[str]) -> bool: + """ + Check if the hook is run on the whole repository. + + Args: + files: The list of files. + + Returns: + bool: True if the hook is run on the whole repository, False otherwise. + """ + return True diff --git a/hooks/terraform_checkov.py b/hooks/terraform_checkov.py index ff65b2965..a3b45dfef 100644 --- a/hooks/terraform_checkov.py +++ b/hooks/terraform_checkov.py @@ -51,6 +51,10 @@ def main(argv: Sequence[str] | None = None) -> int: if os.environ.get('PRE_COMMIT_COLOR') == 'never': all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # TODO: Check is it works as expected + # WPS421 - IDK how to check is function exist w/o passing globals() + if common.is_function_defined('run_hook_on_whole_repo', globals()): # noqa: WPS421 + if common.is_hook_run_on_whole_repo(files): + return run_hook_on_whole_repo(safe_args, all_env_vars) return common.per_dir_hook( hook_config, @@ -61,6 +65,41 @@ def main(argv: Sequence[str] | None = None) -> int: ) +def run_hook_on_whole_repo(args: list[str], env_vars: dict[str, str]) -> int: + """ + Run the hook on the whole repository. + + Args: + args: The arguments to pass to the hook + env_vars: All environment variables provided to hook from system and + defined by user in hook config. + + Returns: + int: The exit code of the hook. + """ + cmd = ['checkov', '-d', '.', *args] + + logger.debug( + 'Running hook on the whole repository with values:\nargs: %s \nenv_vars: %r', + args, + env_vars, + ) + logger.info('calling %s', shlex.join(cmd)) + + completed_process = run( + cmd, + env=env_vars, + text=True, + stdout=PIPE, + check=False, + ) + + if completed_process.stdout: + sys.stdout.write(completed_process.stdout) + + return completed_process.returncode + + def per_dir_hook_unique_part( tf_path: str, # pylint: disable=unused-argument dir_path: str, diff --git a/tests/pytest/test_common.py b/tests/pytest/test_common.py index afe5fb882..d60918f25 100644 --- a/tests/pytest/test_common.py +++ b/tests/pytest/test_common.py @@ -8,6 +8,7 @@ from hooks.common import _get_unique_dirs from hooks.common import expand_env_vars from hooks.common import get_tf_binary_path +from hooks.common import is_function_defined from hooks.common import parse_cmdline from hooks.common import parse_env_vars from hooks.common import per_dir_hook @@ -337,5 +338,46 @@ def test_get_tf_binary_path_not_found(mocker): get_tf_binary_path(hook_config) +# ? +# ? is_function_defined +# ? + + +def test_is_function_defined_existing_function(): + def sample_function(): + pass + + scope = globals() + scope['sample_function'] = sample_function + + assert is_function_defined('sample_function', scope) is True + + +def test_is_function_defined_non_existing_function(): + scope = globals() + + assert is_function_defined('non_existing_function', scope) is False + + +def test_is_function_defined_non_callable(): + non_callable = 'I am not a function' + scope = globals() + scope['non_callable'] = non_callable + + assert is_function_defined('non_callable', scope) is False + + +def test_is_function_defined_callable_object(): + class CallableObject: + def __call__(self): + pass + + callable_object = CallableObject() + scope = globals() + scope['callable_object'] = callable_object + + assert is_function_defined('callable_object', scope) is True + + if __name__ == '__main__': pytest.main() From 7d96f1bb61018697f7d7ee9f6334307fc73c1017 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Mon, 30 Dec 2024 19:40:13 +0200 Subject: [PATCH 29/67] Fix structure to actual in master branch. Fix pytest config --- .vscode/settings.json | 3 +++ pytest.ini | 5 +++++ {hooks => src/pre_commit_terraform}/common.py | 0 {hooks => src/pre_commit_terraform}/logger.py | 0 .../pre_commit_terraform}/terraform_checkov.py | 4 ++-- .../pre_commit_terraform}/terraform_fmt.py | 4 ++-- tests/pytest/test_common.py | 18 +++++++++--------- .../pytest}/test_terraform_checkov.py | 2 +- tests/pytest/test_terraform_fmt.py | 16 ++++++++-------- 9 files changed, 30 insertions(+), 22 deletions(-) create mode 100644 pytest.ini rename {hooks => src/pre_commit_terraform}/common.py (100%) rename {hooks => src/pre_commit_terraform}/logger.py (100%) rename {hooks => src/pre_commit_terraform}/terraform_checkov.py (97%) rename {hooks => src/pre_commit_terraform}/terraform_fmt.py (95%) rename {hooks => tests/pytest}/test_terraform_checkov.py (94%) diff --git a/.vscode/settings.json b/.vscode/settings.json index c1dc51901..caa4801a9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,4 +7,7 @@ "code-block-style": false }, "markdown.validate.enabled": true, + "python.analysis.extraPaths": [ + "./src" + ], } diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..5c1b90ba2 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +pythonpath = src + +testpaths = + tests/pytest diff --git a/hooks/common.py b/src/pre_commit_terraform/common.py similarity index 100% rename from hooks/common.py rename to src/pre_commit_terraform/common.py diff --git a/hooks/logger.py b/src/pre_commit_terraform/logger.py similarity index 100% rename from hooks/logger.py rename to src/pre_commit_terraform/logger.py diff --git a/hooks/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py similarity index 97% rename from hooks/terraform_checkov.py rename to src/pre_commit_terraform/terraform_checkov.py index a3b45dfef..6e93b1933 100644 --- a/hooks/terraform_checkov.py +++ b/src/pre_commit_terraform/terraform_checkov.py @@ -10,8 +10,8 @@ from subprocess import run from typing import Sequence -from . import common -from .logger import setup_logging +from pre_commit_terraform import common +from pre_commit_terraform.logger import setup_logging logger = logging.getLogger(__name__) diff --git a/hooks/terraform_fmt.py b/src/pre_commit_terraform/terraform_fmt.py similarity index 95% rename from hooks/terraform_fmt.py rename to src/pre_commit_terraform/terraform_fmt.py index 811026b4e..c8bc7d097 100644 --- a/hooks/terraform_fmt.py +++ b/src/pre_commit_terraform/terraform_fmt.py @@ -10,8 +10,8 @@ from subprocess import run from typing import Sequence -from . import common -from .logger import setup_logging +from pre_commit_terraform import common +from pre_commit_terraform.logger import setup_logging logger = logging.getLogger(__name__) diff --git a/tests/pytest/test_common.py b/tests/pytest/test_common.py index d60918f25..ae711e51a 100644 --- a/tests/pytest/test_common.py +++ b/tests/pytest/test_common.py @@ -4,14 +4,14 @@ import pytest -from hooks.common import BinaryNotFoundError -from hooks.common import _get_unique_dirs -from hooks.common import expand_env_vars -from hooks.common import get_tf_binary_path -from hooks.common import is_function_defined -from hooks.common import parse_cmdline -from hooks.common import parse_env_vars -from hooks.common import per_dir_hook +from pre_commit_terraform.common import BinaryNotFoundError +from pre_commit_terraform.common import _get_unique_dirs +from pre_commit_terraform.common import expand_env_vars +from pre_commit_terraform.common import get_tf_binary_path +from pre_commit_terraform.common import is_function_defined +from pre_commit_terraform.common import parse_cmdline +from pre_commit_terraform.common import parse_env_vars +from pre_commit_terraform.common import per_dir_hook # ? @@ -52,7 +52,7 @@ def test_get_unique_dirs_nested_dirs(): # ? @pytest.fixture def mock_per_dir_hook_unique_part(mocker): - return mocker.patch('hooks.terraform_fmt.per_dir_hook_unique_part') + return mocker.patch('pre_commit_terraform.terraform_fmt.per_dir_hook_unique_part') def test_per_dir_hook_empty_files(mock_per_dir_hook_unique_part): diff --git a/hooks/test_terraform_checkov.py b/tests/pytest/test_terraform_checkov.py similarity index 94% rename from hooks/test_terraform_checkov.py rename to tests/pytest/test_terraform_checkov.py index 5c364f6b3..4e5916c1a 100644 --- a/hooks/test_terraform_checkov.py +++ b/tests/pytest/test_terraform_checkov.py @@ -1,6 +1,6 @@ import pytest -from hooks.terraform_checkov import replace_git_working_dir_to_repo_root +from pre_commit_terraform.terraform_checkov import replace_git_working_dir_to_repo_root # FILE: hooks/test_terraform_checkov.py diff --git a/tests/pytest/test_terraform_fmt.py b/tests/pytest/test_terraform_fmt.py index ff3abb209..286d1d326 100644 --- a/tests/pytest/test_terraform_fmt.py +++ b/tests/pytest/test_terraform_fmt.py @@ -5,38 +5,38 @@ import pytest -from hooks.terraform_fmt import main -from hooks.terraform_fmt import per_dir_hook_unique_part +from pre_commit_terraform.terraform_fmt import main +from pre_commit_terraform.terraform_fmt import per_dir_hook_unique_part @pytest.fixture def mock_setup_logging(mocker): - return mocker.patch('hooks.terraform_fmt.setup_logging') + return mocker.patch('pre_commit_terraform.terraform_fmt.setup_logging') @pytest.fixture def mock_parse_cmdline(mocker): - return mocker.patch('hooks.terraform_fmt.common.parse_cmdline') + return mocker.patch('pre_commit_terraform.terraform_fmt.common.parse_cmdline') @pytest.fixture def mock_parse_env_vars(mocker): - return mocker.patch('hooks.terraform_fmt.common.parse_env_vars') + return mocker.patch('pre_commit_terraform.terraform_fmt.common.parse_env_vars') @pytest.fixture def mock_expand_env_vars(mocker): - return mocker.patch('hooks.terraform_fmt.common.expand_env_vars') + return mocker.patch('pre_commit_terraform.terraform_fmt.common.expand_env_vars') @pytest.fixture def mock_per_dir_hook(mocker): - return mocker.patch('hooks.terraform_fmt.common.per_dir_hook') + return mocker.patch('pre_commit_terraform.terraform_fmt.common.per_dir_hook') @pytest.fixture def mock_run(mocker): - return mocker.patch('hooks.terraform_fmt.run') + return mocker.patch('pre_commit_terraform.terraform_fmt.run') def test_main( From 0c267eebcee859882d4dd0152124484f3646afe0 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Mon, 30 Dec 2024 22:03:58 +0200 Subject: [PATCH 30/67] Fully init run_hook_on_whole_repo functional --- .pre-commit-config.yaml | 2 + pyproject.toml | 4 +- src/pre_commit_terraform/common.py | 66 ++++++++++++++++++- src/pre_commit_terraform/terraform_checkov.py | 3 +- tests/pytest/test_common.py | 65 ++++++++++++++++++ 5 files changed, 135 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b69ad0a7..bc213d953 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -113,6 +113,8 @@ repos: rev: v1.10.0 hooks: - id: mypy + additional_dependencies: + - types-PyYAML args: [ --ignore-missing-imports, --disallow-untyped-calls, diff --git a/pyproject.toml b/pyproject.toml index f9137f8fb..85c4e02ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,9 @@ classifiers = [ 'Programming Language :: Python :: Implementation :: PyPy', ] description='Pre-commit hooks for terraform' -dependencies = [] +dependencies = [ + 'pyyaml', +] dynamic = [ 'urls', 'version', diff --git a/src/pre_commit_terraform/common.py b/src/pre_commit_terraform/common.py index 0e0b477bb..d053c3971 100644 --- a/src/pre_commit_terraform/common.py +++ b/src/pre_commit_terraform/common.py @@ -9,10 +9,15 @@ import argparse import logging import os +import re import shutil +import subprocess from collections.abc import Sequence +from pathlib import Path from typing import Callable +import yaml + logger = logging.getLogger(__name__) @@ -264,14 +269,69 @@ def is_function_defined(func_name: str, scope: dict) -> bool: return is_defined and is_callable -def is_hook_run_on_whole_repo(files: list[str]) -> bool: +def is_hook_run_on_whole_repo(hook_id: str, file_paths: list[str]) -> bool: """ Check if the hook is run on the whole repository. Args: - files: The list of files. + hook_id (str): The ID of the hook. + file_paths: The list of files paths. Returns: bool: True if the hook is run on the whole repository, False otherwise. + + Raises: + ValueError: If the hook ID is not found in the .pre-commit-hooks.yaml file. """ - return True + logger.debug('Hook ID: %s', hook_id) + + # Get the directory containing `.pre-commit-hooks.yaml` file + git_repo_root = Path(__file__).resolve().parents[5] + hook_config_path = os.path.join(git_repo_root, '.pre-commit-hooks.yaml') + + logger.debug('Hook config path: %s', hook_config_path) + + # Read the .pre-commit-hooks.yaml file + with open(hook_config_path, 'r', encoding='utf-8') as pre_commit_hooks_yaml: + hooks_config = yaml.safe_load(pre_commit_hooks_yaml) + + # Get the included and excluded file patterns for the given hook_id + for hook in hooks_config: + if hook['id'] == hook_id: + included_pattern = re.compile(hook.get('files', '')) + excluded_pattern = re.compile(hook.get('exclude', '')) + break + else: + raise ValueError(f'Hook ID "{hook_id}" not found in .pre-commit-hooks.yaml') + + logger.debug( + 'Included files pattern: %s\nExcluded files pattern: %s', + included_pattern, + excluded_pattern, + ) + # S607 disabled as we need to maintain ability to call git command no matter where it located. + git_ls_files_cmd = ['git', 'ls-files'] # noqa: S607 + # Get the sorted list of all files that can be checked using `git ls-files` + git_ls_file_paths = subprocess.check_output(git_ls_files_cmd, text=True).splitlines() + + if excluded_pattern: + all_file_paths_that_can_be_checked = [ + file_path + for file_path in git_ls_file_paths + if included_pattern.search(file_path) and not excluded_pattern.search(file_path) + ] + else: + all_file_paths_that_can_be_checked = [ + file_path for file_path in git_ls_file_paths if included_pattern.search(file_path) + ] + + # Get the sorted list of files passed to the hook + file_paths_to_check = sorted(file_paths) + logger.debug( + 'Files to check:\n%s\n\nAll files that can be checked:\n%s\n\nIdentical lists: %s', + file_paths_to_check, + all_file_paths_that_can_be_checked, + file_paths_to_check == all_file_paths_that_can_be_checked, + ) + # Compare the sorted lists of file + return file_paths_to_check == all_file_paths_that_can_be_checked diff --git a/src/pre_commit_terraform/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py index 6e93b1933..00bddd485 100644 --- a/src/pre_commit_terraform/terraform_checkov.py +++ b/src/pre_commit_terraform/terraform_checkov.py @@ -53,7 +53,8 @@ def main(argv: Sequence[str] | None = None) -> int: all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # TODO: Check is it works as expected # WPS421 - IDK how to check is function exist w/o passing globals() if common.is_function_defined('run_hook_on_whole_repo', globals()): # noqa: WPS421 - if common.is_hook_run_on_whole_repo(files): + hook_id = os.path.basename(__file__).replace('.py', '_py') + if common.is_hook_run_on_whole_repo(hook_id, files): return run_hook_on_whole_repo(safe_args, all_env_vars) return common.per_dir_hook( diff --git a/tests/pytest/test_common.py b/tests/pytest/test_common.py index ae711e51a..40bbd3814 100644 --- a/tests/pytest/test_common.py +++ b/tests/pytest/test_common.py @@ -1,14 +1,17 @@ # pylint: skip-file import os from os.path import join +from pathlib import Path import pytest +import yaml from pre_commit_terraform.common import BinaryNotFoundError from pre_commit_terraform.common import _get_unique_dirs from pre_commit_terraform.common import expand_env_vars from pre_commit_terraform.common import get_tf_binary_path from pre_commit_terraform.common import is_function_defined +from pre_commit_terraform.common import is_hook_run_on_whole_repo from pre_commit_terraform.common import parse_cmdline from pre_commit_terraform.common import parse_env_vars from pre_commit_terraform.common import per_dir_hook @@ -379,5 +382,67 @@ def __call__(self): assert is_function_defined('callable_object', scope) is True +# ? +# ? is_hook_run_on_whole_repo +# ? + + +@pytest.fixture +def mock_git_ls_files(): + return [ + 'environment/prd/backends.tf', + 'environment/prd/data.tf', + 'environment/prd/main.tf', + 'environment/prd/outputs.tf', + 'environment/prd/providers.tf', + 'environment/prd/variables.tf', + 'environment/prd/versions.tf', + 'environment/qa/backends.tf', + ] + + +@pytest.fixture +def mock_hooks_config(): + return [{'id': 'example_hook_id', 'files': r'\.tf$', 'exclude': r'\.terraform/.*$'}] + + +def test_is_hook_run_on_whole_repo(mocker, mock_git_ls_files, mock_hooks_config): + # Mock the return value of git ls-files + mocker.patch('subprocess.check_output', return_value='\n'.join(mock_git_ls_files)) + # Mock the return value of reading the .pre-commit-hooks.yaml file + mocker.patch('builtins.open', mocker.mock_open(read_data=yaml.dump(mock_hooks_config))) + # Mock the Path object to return a specific path + mock_path = mocker.patch('pathlib.Path.resolve') + mock_path.return_value.parents.__getitem__.return_value = Path('/mocked/path') + + # Test case where files match the included pattern and do not match the excluded pattern + files = [ + 'environment/prd/backends.tf', + 'environment/prd/data.tf', + 'environment/prd/main.tf', + 'environment/prd/outputs.tf', + 'environment/prd/providers.tf', + 'environment/prd/variables.tf', + 'environment/prd/versions.tf', + 'environment/qa/backends.tf', + ] + assert is_hook_run_on_whole_repo('example_hook_id', files) is True + + # Test case where files do not match the included pattern + files = ['environment/prd/README.md'] + assert is_hook_run_on_whole_repo('example_hook_id', files) is False + + # Test case where files match the excluded pattern + files = ['environment/prd/.terraform/config.tf'] + assert is_hook_run_on_whole_repo('example_hook_id', files) is False + + # Test case where hook_id is not found + with pytest.raises( + ValueError, + match='Hook ID "non_existing_hook_id" not found in .pre-commit-hooks.yaml', + ): + is_hook_run_on_whole_repo('non_existing_hook_id', files) + + if __name__ == '__main__': pytest.main() From a3e425205b674c617cfd71e249e46df72fa85dd7 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Mon, 30 Dec 2024 22:38:10 +0200 Subject: [PATCH 31/67] Polish code and clarify TODOs --- src/pre_commit_terraform/common.py | 54 +++++++++++++------ src/pre_commit_terraform/terraform_checkov.py | 2 +- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/pre_commit_terraform/common.py b/src/pre_commit_terraform/common.py index d053c3971..05a793287 100644 --- a/src/pre_commit_terraform/common.py +++ b/src/pre_commit_terraform/common.py @@ -59,34 +59,54 @@ def parse_cmdline( - files (list[str]): File paths on which we should run the hook. - tf_init_args (list[str]): Arguments for `terraform init` command. - env_vars (list[str]): Custom environment variable strings in the format "name=value". + + Raises: + ValueError: If no files are provided. """ parser = argparse.ArgumentParser( add_help=False, # Allow the use of `-h` for compatibility with the Bash version of the hook ) - parser.add_argument('-a', '--args', action='append', help='Arguments') - parser.add_argument('-h', '--hook-config', action='append', help='Hook Config') - parser.add_argument('-i', '--tf-init-args', '--init-args', action='append', help='Init Args') - parser.add_argument('-e', '--env-vars', '--envs', action='append', help='Environment Variables') - parser.add_argument('FILES', nargs='*', help='Files') + parser.add_argument('-a', '--args', action='append', help='Arguments', default=[]) + parser.add_argument('-h', '--hook-config', action='append', help='Hook Config', default=[]) + parser.add_argument( + '-i', + '--tf-init-args', + '--init-args', + action='append', + help='TF Init Args', + default=[], + ) + parser.add_argument( + '-e', + '--env-vars', + '--envs', + action='append', + help='Environment Variables', + default=[], + ) + parser.add_argument('files', nargs='*', help='Files') parsed_args = parser.parse_args(argv) - # TODO: move to defaults - args = parsed_args.args or [] - hook_config = parsed_args.hook_config or [] - files = parsed_args.FILES or [] - tf_init_args = parsed_args.tf_init_args or [] - env_vars = parsed_args.env_vars or [] + + if parsed_args.files is None: + raise ValueError('No files provided') logger.debug( 'Parsed values:\nargs: %r\nhook_config: %r\nfiles: %r\ntf_init_args: %r\nenv_vars: %r', - args, - hook_config, - files, - tf_init_args, - env_vars, + parsed_args.args, + parsed_args.hook_config, + parsed_args.files, + parsed_args.tf_init_args, + parsed_args.env_vars, ) - return args, hook_config, files, tf_init_args, env_vars + return ( + parsed_args.args, + parsed_args.hook_config, + parsed_args.files, + parsed_args.tf_init_args, + parsed_args.env_vars, + ) def _get_unique_dirs(files: list[str]) -> set[str]: diff --git a/src/pre_commit_terraform/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py index 00bddd485..3b156d2eb 100644 --- a/src/pre_commit_terraform/terraform_checkov.py +++ b/src/pre_commit_terraform/terraform_checkov.py @@ -50,7 +50,7 @@ def main(argv: Sequence[str] | None = None) -> int: safe_args = [shlex.quote(arg) for arg in expanded_args] if os.environ.get('PRE_COMMIT_COLOR') == 'never': - all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # TODO: Check is it works as expected + all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # TODO: subprocess.run ignore colors # WPS421 - IDK how to check is function exist w/o passing globals() if common.is_function_defined('run_hook_on_whole_repo', globals()): # noqa: WPS421 hook_id = os.path.basename(__file__).replace('.py', '_py') From 00674ef13e781ebbf22ddc4ca3aedd3f52049a40 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 30 Dec 2024 06:24:44 +0100 Subject: [PATCH 32/67] =?UTF-8?q?=F0=9F=92=85=20Introduce=20a=20Python=20C?= =?UTF-8?q?LI=20app=20layout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This includes a structure with purpose-based modules and a standard mechanism for adding more subcommands. When adding a new subcommand, one has to wire the `invoke_cli_app()` and `populate_argument_parser()` from their new module into the mappings defined in `_cli_subcommands.py` and `_cli_parsing.py` respectively. This is the only integration point necessary. `populate_argument_parser()` accepts a subparser instance of `argparse.ArgumentParser()` that a new subcommand would need to attach new arguments into. It does not need to return anything. And the `invoke_cli_app()` hook is called with an instance of `argparse.Namespace()` with all the arguments parsed and pre-processed. This function is supposed to have the main check logic and return an instance of `._structs.ReturnCode()` or `int`. --- .pre-commit-hooks.yaml | 2 +- pyproject.toml | 3 - src/pre_commit_terraform/__main__.py | 10 ++++ src/pre_commit_terraform/_cli.py | 56 +++++++++++++++++++ src/pre_commit_terraform/_cli_parsing.py | 43 ++++++++++++++ src/pre_commit_terraform/_cli_subcommands.py | 31 ++++++++++ src/pre_commit_terraform/_errors.py | 16 ++++++ src/pre_commit_terraform/_structs.py | 16 ++++++ src/pre_commit_terraform/_types.py | 6 ++ .../terraform_docs_replace.py | 54 ++++++++++-------- 10 files changed, 210 insertions(+), 27 deletions(-) create mode 100644 src/pre_commit_terraform/__main__.py create mode 100644 src/pre_commit_terraform/_cli.py create mode 100644 src/pre_commit_terraform/_cli_parsing.py create mode 100644 src/pre_commit_terraform/_cli_subcommands.py create mode 100644 src/pre_commit_terraform/_errors.py create mode 100644 src/pre_commit_terraform/_structs.py create mode 100644 src/pre_commit_terraform/_types.py diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 520c3f08a..9b8a0cf19 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -37,7 +37,7 @@ name: Terraform docs (overwrite README.md) description: Overwrite content of README.md with terraform-docs. require_serial: true - entry: terraform_docs_replace + entry: python -Im pre_commit_terraform replace-docs language: python files: (\.tf)$ exclude: \.terraform/.*$ diff --git a/pyproject.toml b/pyproject.toml index 05559d39a..e5ff0159a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,3 @@ email = 'yz@yz.kiev.ua' [project.readme] file = 'README.md' content-type = 'text/markdown' - -[project.scripts] -terraform_docs_replace = 'pre_commit_terraform.terraform_docs_replace:main' diff --git a/src/pre_commit_terraform/__main__.py b/src/pre_commit_terraform/__main__.py new file mode 100644 index 000000000..3b50a0896 --- /dev/null +++ b/src/pre_commit_terraform/__main__.py @@ -0,0 +1,10 @@ +"""A runpy-style CLI entry-point module.""" + +from sys import argv, exit as exit_with_return_code + +from ._cli import invoke_cli_app + + +if __name__ == '__main__': + return_code = invoke_cli_app(argv[1:]) + exit_with_return_code(return_code) diff --git a/src/pre_commit_terraform/_cli.py b/src/pre_commit_terraform/_cli.py new file mode 100644 index 000000000..6f7c2a60e --- /dev/null +++ b/src/pre_commit_terraform/_cli.py @@ -0,0 +1,56 @@ +"""Outer CLI layer of the app interface.""" + +from sys import stderr + +from ._cli_subcommands import choose_cli_app +from ._cli_parsing import initialize_argument_parser +from ._errors import ( + PreCommitTerraformBaseError, + PreCommitTerraformExit, + PreCommitTerraformRuntimeError, +) +from ._structs import ReturnCode +from ._types import ReturnCodeType + + +def invoke_cli_app(cli_args: list[str]) -> ReturnCodeType: + """Run the entry-point of the CLI app. + + Includes initializing parsers of all the sub-apps and + choosing what to execute. + """ + root_cli_parser = initialize_argument_parser() + parsed_cli_args = root_cli_parser.parse_args(cli_args) + try: + invoke_chosen_app = choose_cli_app(parsed_cli_args.check_name) + except LookupError as lookup_err: + print(f'Sourcing subcommand failed: {lookup_err !s}', file=stderr) + return ReturnCode.ERROR + + try: + return invoke_chosen_app(parsed_cli_args) + except PreCommitTerraformExit as exit_err: + print(f'App exiting: {exit_err !s}', file=stderr) + raise + except PreCommitTerraformRuntimeError as unhandled_exc: + print( + f'App execution took an unexpected turn: {unhandled_exc !s}. ' + 'Exiting...', + file=stderr, + ) + return ReturnCode.ERROR + except PreCommitTerraformBaseError as unhandled_exc: + print( + f'A surprising exception happened: {unhandled_exc !s}. Exiting...', + file=stderr, + ) + return ReturnCode.ERROR + except KeyboardInterrupt as ctrl_c_exc: + print( + f'User-initiated interrupt: {ctrl_c_exc !s}. Exiting...', + file=stderr, + ) + return ReturnCode.ERROR + + +__all__ = ('invoke_cli_app',) diff --git a/src/pre_commit_terraform/_cli_parsing.py b/src/pre_commit_terraform/_cli_parsing.py new file mode 100644 index 000000000..4f78b4f10 --- /dev/null +++ b/src/pre_commit_terraform/_cli_parsing.py @@ -0,0 +1,43 @@ +"""Argument parser initialization logic. + +This defines helpers for setting up both the root parser and the parsers +of all the sub-commands. +""" + +from argparse import ArgumentParser + +from .terraform_docs_replace import ( + populate_argument_parser as populate_replace_docs_argument_parser, +) + + +PARSER_MAP = { + 'replace-docs': populate_replace_docs_argument_parser, +} + + +def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: + """Connect all sub-command parsers to the given one. + + This functions iterates over a mapping of subcommands to their + respective population functions, executing them to augment the + main parser. + """ + subcommand_parsers = root_cli_parser.add_subparsers( + dest='check_name', + help='A check to be performed.', + required=True, + ) + for subcommand_name, initialize_subcommand_parser in PARSER_MAP.items(): + replace_docs_parser = subcommand_parsers.add_parser(subcommand_name) + initialize_subcommand_parser(replace_docs_parser) + + +def initialize_argument_parser() -> ArgumentParser: + """Return the root argument parser with sub-commands.""" + root_cli_parser = ArgumentParser(prog=f'python -m {__package__ !s}') + attach_subcommand_parsers_to(root_cli_parser) + return root_cli_parser + + +__all__ = ('initialize_argument_parser',) diff --git a/src/pre_commit_terraform/_cli_subcommands.py b/src/pre_commit_terraform/_cli_subcommands.py new file mode 100644 index 000000000..239a8e8f1 --- /dev/null +++ b/src/pre_commit_terraform/_cli_subcommands.py @@ -0,0 +1,31 @@ +"""A CLI sub-commands organization module.""" + +from argparse import Namespace +from typing import Callable + +from ._structs import ReturnCode +from .terraform_docs_replace import ( + invoke_cli_app as invoke_replace_docs_cli_app, +) + + +SUBCOMMAND_MAP = { + 'replace-docs': invoke_replace_docs_cli_app, +} + + +def choose_cli_app( + check_name: str, + /, +) -> Callable[[Namespace], ReturnCode | int]: + """Return a subcommand callable by CLI argument name.""" + try: + return SUBCOMMAND_MAP[check_name] + except KeyError as key_err: + raise LookupError( + f'{key_err !s}: Unable to find a callable for ' + f'the `{check_name !s}` subcommand', + ) from key_err + + +__all__ = ('choose_cli_app',) diff --git a/src/pre_commit_terraform/_errors.py b/src/pre_commit_terraform/_errors.py new file mode 100644 index 000000000..c0f973acc --- /dev/null +++ b/src/pre_commit_terraform/_errors.py @@ -0,0 +1,16 @@ +"""App-specific exceptions.""" + + +class PreCommitTerraformBaseError(Exception): + """Base exception for all the in-app errors.""" + + +class PreCommitTerraformRuntimeError( + PreCommitTerraformBaseError, + RuntimeError, +): + """An exception representing a runtime error condition.""" + + +class PreCommitTerraformExit(PreCommitTerraformBaseError, SystemExit): + """An exception for terminating execution from deep app layers.""" diff --git a/src/pre_commit_terraform/_structs.py b/src/pre_commit_terraform/_structs.py new file mode 100644 index 000000000..12cf0dad3 --- /dev/null +++ b/src/pre_commit_terraform/_structs.py @@ -0,0 +1,16 @@ +"""Data structures to be reused across the app.""" + +from enum import IntEnum + + +class ReturnCode(IntEnum): + """POSIX-style return code values. + + To be used in check callable implementations. + """ + + OK = 0 + ERROR = 1 + + +__all__ = ('ReturnCode',) diff --git a/src/pre_commit_terraform/_types.py b/src/pre_commit_terraform/_types.py new file mode 100644 index 000000000..979e4f858 --- /dev/null +++ b/src/pre_commit_terraform/_types.py @@ -0,0 +1,6 @@ +"""Composite types for annotating in-project code.""" + +from ._structs import ReturnCode + + +ReturnCodeType = ReturnCode | int diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py index 3e10913df..20318432a 100644 --- a/src/pre_commit_terraform/terraform_docs_replace.py +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -1,33 +1,42 @@ -import argparse import os import subprocess -import sys import warnings +from argparse import ArgumentParser, Namespace +from ._structs import ReturnCode +from ._types import ReturnCodeType -def main(argv=None): - parser = argparse.ArgumentParser( - description="""Run terraform-docs on a set of files. Follows the standard convention of - pulling the documentation from main.tf in order to replace the entire - README.md file each time.""" + +def populate_argument_parser(subcommand_parser: ArgumentParser) -> None: + subcommand_parser.description = ( + 'Run terraform-docs on a set of files. Follows the standard ' + 'convention of pulling the documentation from main.tf in order to ' + 'replace the entire README.md file each time.' ) - parser.add_argument( + subcommand_parser.add_argument( '--dest', dest='dest', default='README.md', ) - parser.add_argument( + subcommand_parser.add_argument( '--sort-inputs-by-required', dest='sort', action='store_true', help='[deprecated] use --sort-by-required instead', ) - parser.add_argument( + subcommand_parser.add_argument( '--sort-by-required', dest='sort', action='store_true', ) - parser.add_argument( - '--with-aggregate-type-defaults', dest='aggregate', action='store_true', + subcommand_parser.add_argument( + '--with-aggregate-type-defaults', + dest='aggregate', + action='store_true', help='[deprecated]', ) - parser.add_argument('filenames', nargs='*', help='Filenames to check.') - args = parser.parse_args(argv) + subcommand_parser.add_argument( + 'filenames', + nargs='*', + help='Filenames to check.', + ) + +def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: warnings.warn( '`terraform_docs_replace` hook is DEPRECATED.' 'For migration instructions see ' @@ -37,29 +46,28 @@ def main(argv=None): ) dirs = [] - for filename in args.filenames: + for filename in parsed_cli_args.filenames: if (os.path.realpath(filename) not in dirs and (filename.endswith(".tf") or filename.endswith(".tfvars"))): dirs.append(os.path.dirname(filename)) - retval = 0 + retval = ReturnCode.OK for dir in dirs: try: procArgs = [] procArgs.append('terraform-docs') - if args.sort: + if parsed_cli_args.sort: procArgs.append('--sort-by-required') procArgs.append('md') procArgs.append("./{dir}".format(dir=dir)) procArgs.append('>') - procArgs.append("./{dir}/{dest}".format(dir=dir, dest=args.dest)) + procArgs.append( + './{dir}/{dest}'. + format(dir=dir, dest=parsed_cli_args.dest), + ) subprocess.check_call(" ".join(procArgs), shell=True) except subprocess.CalledProcessError as e: print(e) - retval = 1 + retval = ReturnCode.ERROR return retval - - -if __name__ == '__main__': - sys.exit(main()) From b6aa20b95df04b90a7ebef7e0e5ac813406f724d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=87=BA=F0=9F=87=A6=20Sviatoslav=20Sydorenko=20=28?= =?UTF-8?q?=D0=A1=D0=B2=D1=8F=D1=82=D0=BE=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1?= =?UTF-8?q?=D0=B8=D0=B4=D0=BE=D1=80=D0=B5=D0=BD=D0=BA=D0=BE=29?= Date: Mon, 30 Dec 2024 20:19:48 +0100 Subject: [PATCH 33/67] Add a maintainer's manual into the importable package dir --- src/pre_commit_terraform/README.md | 66 ++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/pre_commit_terraform/README.md diff --git a/src/pre_commit_terraform/README.md b/src/pre_commit_terraform/README.md new file mode 100644 index 000000000..fb5005190 --- /dev/null +++ b/src/pre_commit_terraform/README.md @@ -0,0 +1,66 @@ +# Maintainer's manual + +## Structure + +This folder is what's called an [importable package]. It's a top-level folder +that ends up being installed into `site-packages/` of virtualenvs. + +When the Git repository is `pip install`ed, this [import package] becomes +available for use within respective Python interpreter instance. It can be +imported and sub-modules can be imported through the dot-syntax. +Additionally, the modules within are able to import the neighboring ones +using relative imports that have a leading dot in them. + +It additionally implements a [runpy interface], meaning that its name can +be passed to `python -m` in order to invoke the CLI. This is the primary method +of integration with the [`pre-commit` framework] and local development/testing. + +The layout allows for having several Python modules wrapping third-party tools, +each having an argument parser and being a subcommand for the main CLI +interface. + +## Control flow + +When `python -m pre_commit_terraform` is executed, it imports `__main__.py`. +Which in turn, performs the initialization of the main argument parser and the +parsers of subcommands, followed by executing the logic defined in dedicated +subcommand modules. + +## Integrating a new subcommand + +1. Create a new module called `subcommand_x.py`. +2. Within that module, define two functions — + `invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType | int` and + `populate_argument_parser(subcommand_parser: ArgumentParser) -> None`. +3. Edit [`_cli_parsing.py`], importing `populate_argument_parser` from + `subcommand_x` and adding it into `PARSER_MAP` with `subcommand-x` as + a key. +5. Edit [`_cli_subcommands.py`] `invoke_cli_app` from `subcommand_x` and + adding it into `SUBCOMMAND_MAP` with `subcommand-x` as a key. +6. Edit [`.pre-commit-hooks.yaml`], adding a new hook that invokes + `python -m pre_commit_terraform subcommand-x`. + +## Manual testing + +Usually, having a development virtualenv where you `pip install -e .` is enough +to make it possible to invoke the CLI app. Do so first. Most source code +updates do not require running it again. But sometimes, it's needed. + +Once done, you can run `python -m pre_commit_terraform` and/or +`python -m pre_commit_terraform subcommand-x` to see how it behaves. There's +`--help` and all other typical conventions one would usually expect from a +POSIX-inspired CLI app. + +## DX/UX considerations + +Since it's an app that can be executed outside of the [`pre-commit` framework], +it is useful to check out and follow these [CLI guidelines][clig]. + +[`.pre-commit-hooks.yaml`]: ../../.pre-commit-hooks.yaml +[`_cli_parsing.py`]: ./_cli_parsing.py +[`_cli_subcommands.py`]: ./_cli_subcommands.py +[clig]: https://clig.dev +[importable package]: https://docs.python.org/3/tutorial/modules.html#packages +[import package]: https://packaging.python.org/en/latest/glossary/#term-Import-Package +[`pre-commit` framework]: https://pre-commit.com +[runpy interface]: https://docs.python.org/3/library/__main__.html From 60f129289586f50d6768892c2eee386b1d96b2d8 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 30 Dec 2024 20:57:12 +0100 Subject: [PATCH 34/67] =?UTF-8?q?=F3=B0=91=8C=20Simplify=20subcommand=20in?= =?UTF-8?q?tegration=20to=20one=20place?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, adding a new subcommand required importing it into two separate Python modules, adding them into two mappings, maintaining the same dictionary key string. This is prone to human error and is underintegrated. This patch makes it easier by defining a required subcommand module shape on the typing level. Now, one just need to implement two hook- functions and a constant with specific signatures and import it in a single place. --- src/pre_commit_terraform/README.md | 23 +++++++------- src/pre_commit_terraform/_cli.py | 8 +---- src/pre_commit_terraform/_cli_parsing.py | 18 +++++------ src/pre_commit_terraform/_cli_subcommands.py | 31 ++++--------------- src/pre_commit_terraform/_types.py | 24 ++++++++++++++ .../terraform_docs_replace.py | 4 +++ 6 files changed, 53 insertions(+), 55 deletions(-) diff --git a/src/pre_commit_terraform/README.md b/src/pre_commit_terraform/README.md index fb5005190..2a0e740f7 100644 --- a/src/pre_commit_terraform/README.md +++ b/src/pre_commit_terraform/README.md @@ -7,13 +7,13 @@ that ends up being installed into `site-packages/` of virtualenvs. When the Git repository is `pip install`ed, this [import package] becomes available for use within respective Python interpreter instance. It can be -imported and sub-modules can be imported through the dot-syntax. -Additionally, the modules within are able to import the neighboring ones -using relative imports that have a leading dot in them. +imported and sub-modules can be imported through the dot-syntax. Additionally, +the modules within can import the neighboring ones using relative imports that +have a leading dot in them. It additionally implements a [runpy interface], meaning that its name can -be passed to `python -m` in order to invoke the CLI. This is the primary method -of integration with the [`pre-commit` framework] and local development/testing. +be passed to `python -m` to invoke the CLI. This is the primary method of +integration with the [`pre-commit` framework] and local development/testing. The layout allows for having several Python modules wrapping third-party tools, each having an argument parser and being a subcommand for the main CLI @@ -32,12 +32,11 @@ subcommand modules. 2. Within that module, define two functions — `invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType | int` and `populate_argument_parser(subcommand_parser: ArgumentParser) -> None`. -3. Edit [`_cli_parsing.py`], importing `populate_argument_parser` from - `subcommand_x` and adding it into `PARSER_MAP` with `subcommand-x` as - a key. -5. Edit [`_cli_subcommands.py`] `invoke_cli_app` from `subcommand_x` and - adding it into `SUBCOMMAND_MAP` with `subcommand-x` as a key. -6. Edit [`.pre-commit-hooks.yaml`], adding a new hook that invokes + Additionally, define a module-level constant + `CLI_SUBCOMMAND_NAME: Final[str] = 'subcommand-x'`. +3. Edit [`_cli_subcommands.py`], importing `subcommand_x` as a relative module + and add it into the `SUBCOMMAND_MODULES` list. +4. Edit [`.pre-commit-hooks.yaml`], adding a new hook that invokes `python -m pre_commit_terraform subcommand-x`. ## Manual testing @@ -53,7 +52,7 @@ POSIX-inspired CLI app. ## DX/UX considerations -Since it's an app that can be executed outside of the [`pre-commit` framework], +Since it's an app that can be executed outside the [`pre-commit` framework], it is useful to check out and follow these [CLI guidelines][clig]. [`.pre-commit-hooks.yaml`]: ../../.pre-commit-hooks.yaml diff --git a/src/pre_commit_terraform/_cli.py b/src/pre_commit_terraform/_cli.py index 6f7c2a60e..f52a50b0b 100644 --- a/src/pre_commit_terraform/_cli.py +++ b/src/pre_commit_terraform/_cli.py @@ -2,7 +2,6 @@ from sys import stderr -from ._cli_subcommands import choose_cli_app from ._cli_parsing import initialize_argument_parser from ._errors import ( PreCommitTerraformBaseError, @@ -21,14 +20,9 @@ def invoke_cli_app(cli_args: list[str]) -> ReturnCodeType: """ root_cli_parser = initialize_argument_parser() parsed_cli_args = root_cli_parser.parse_args(cli_args) - try: - invoke_chosen_app = choose_cli_app(parsed_cli_args.check_name) - except LookupError as lookup_err: - print(f'Sourcing subcommand failed: {lookup_err !s}', file=stderr) - return ReturnCode.ERROR try: - return invoke_chosen_app(parsed_cli_args) + return parsed_cli_args.invoke_cli_app(parsed_cli_args) except PreCommitTerraformExit as exit_err: print(f'App exiting: {exit_err !s}', file=stderr) raise diff --git a/src/pre_commit_terraform/_cli_parsing.py b/src/pre_commit_terraform/_cli_parsing.py index 4f78b4f10..1cd1dd0fc 100644 --- a/src/pre_commit_terraform/_cli_parsing.py +++ b/src/pre_commit_terraform/_cli_parsing.py @@ -6,14 +6,7 @@ from argparse import ArgumentParser -from .terraform_docs_replace import ( - populate_argument_parser as populate_replace_docs_argument_parser, -) - - -PARSER_MAP = { - 'replace-docs': populate_replace_docs_argument_parser, -} +from ._cli_subcommands import SUBCOMMAND_MODULES def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: @@ -28,9 +21,12 @@ def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: help='A check to be performed.', required=True, ) - for subcommand_name, initialize_subcommand_parser in PARSER_MAP.items(): - replace_docs_parser = subcommand_parsers.add_parser(subcommand_name) - initialize_subcommand_parser(replace_docs_parser) + for subcommand_module in SUBCOMMAND_MODULES: + replace_docs_parser = subcommand_parsers.add_parser(subcommand_module.CLI_SUBCOMMAND_NAME) + replace_docs_parser.set_defaults( + invoke_cli_app=subcommand_module.invoke_cli_app, + ) + subcommand_module.populate_argument_parser(replace_docs_parser) def initialize_argument_parser() -> ArgumentParser: diff --git a/src/pre_commit_terraform/_cli_subcommands.py b/src/pre_commit_terraform/_cli_subcommands.py index 239a8e8f1..fc268e552 100644 --- a/src/pre_commit_terraform/_cli_subcommands.py +++ b/src/pre_commit_terraform/_cli_subcommands.py @@ -1,31 +1,12 @@ """A CLI sub-commands organization module.""" -from argparse import Namespace -from typing import Callable +from . import terraform_docs_replace +from ._types import CLISubcommandModuleProtocol -from ._structs import ReturnCode -from .terraform_docs_replace import ( - invoke_cli_app as invoke_replace_docs_cli_app, -) +SUBCOMMAND_MODULES: list[CLISubcommandModuleProtocol] = [ + terraform_docs_replace, +] -SUBCOMMAND_MAP = { - 'replace-docs': invoke_replace_docs_cli_app, -} - -def choose_cli_app( - check_name: str, - /, -) -> Callable[[Namespace], ReturnCode | int]: - """Return a subcommand callable by CLI argument name.""" - try: - return SUBCOMMAND_MAP[check_name] - except KeyError as key_err: - raise LookupError( - f'{key_err !s}: Unable to find a callable for ' - f'the `{check_name !s}` subcommand', - ) from key_err - - -__all__ = ('choose_cli_app',) +__all__ = ('SUBCOMMAND_MODULES',) diff --git a/src/pre_commit_terraform/_types.py b/src/pre_commit_terraform/_types.py index 979e4f858..99402b447 100644 --- a/src/pre_commit_terraform/_types.py +++ b/src/pre_commit_terraform/_types.py @@ -1,6 +1,30 @@ """Composite types for annotating in-project code.""" +from argparse import ArgumentParser, Namespace +from typing import Final, Protocol + from ._structs import ReturnCode ReturnCodeType = ReturnCode | int + + +class CLISubcommandModuleProtocol(Protocol): + """A protocol for the subcommand-implementing module shape.""" + + CLI_SUBCOMMAND_NAME: Final[str] + """This constant contains a CLI.""" + + def populate_argument_parser( + self, subcommand_parser: ArgumentParser, + ) -> None: + """Run a module hook for populating the subcommand parser.""" + + def invoke_cli_app( + self, parsed_cli_args: Namespace, + ) -> ReturnCodeType | int: + """Run a module hook implementing the subcommand logic.""" + ... # pylint: disable=unnecessary-ellipsis + + +__all__ = ('CLISubcommandModuleProtocol', 'ReturnCodeType') diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py index 20318432a..b79ba479e 100644 --- a/src/pre_commit_terraform/terraform_docs_replace.py +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -2,11 +2,15 @@ import subprocess import warnings from argparse import ArgumentParser, Namespace +from typing import Final from ._structs import ReturnCode from ._types import ReturnCodeType +CLI_SUBCOMMAND_NAME: Final[str] = 'replace-docs' + + def populate_argument_parser(subcommand_parser: ArgumentParser) -> None: subcommand_parser.description = ( 'Run terraform-docs on a set of files. Follows the standard ' From 7dde1dfd1d37d3e69b525ec4c7a1550e01467323 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 30 Dec 2024 21:16:58 +0100 Subject: [PATCH 35/67] =?UTF-8?q?=F3=B0=91=8C=20Drop=20the=20`=5F=5Fmain?= =?UTF-8?q?=5F=5F`=20check=20per=20Python=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://docs.python.org/3/library/__main__.html#id1. --- src/pre_commit_terraform/__main__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pre_commit_terraform/__main__.py b/src/pre_commit_terraform/__main__.py index 3b50a0896..18a63dfd0 100644 --- a/src/pre_commit_terraform/__main__.py +++ b/src/pre_commit_terraform/__main__.py @@ -5,6 +5,5 @@ from ._cli import invoke_cli_app -if __name__ == '__main__': - return_code = invoke_cli_app(argv[1:]) - exit_with_return_code(return_code) +return_code = invoke_cli_app(argv[1:]) +exit_with_return_code(return_code) From 52cf580131241c916e3e37bf748bfe1ce94321aa Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 30 Dec 2024 21:39:19 +0100 Subject: [PATCH 36/67] =?UTF-8?q?=F0=9F=93=9D=20Extend=20the=20manual=20w/?= =?UTF-8?q?=20subcommand=20guide?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pre_commit_terraform/README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/pre_commit_terraform/README.md b/src/pre_commit_terraform/README.md index 2a0e740f7..392d4221b 100644 --- a/src/pre_commit_terraform/README.md +++ b/src/pre_commit_terraform/README.md @@ -55,9 +55,37 @@ POSIX-inspired CLI app. Since it's an app that can be executed outside the [`pre-commit` framework], it is useful to check out and follow these [CLI guidelines][clig]. +## Subcommand development + +`populate_argument_parser()` accepts a regular instance of +[`argparse.ArgumentParser`]. Call its methods to extend the CLI arguments that +would be specific for the subcommand you are creating. Those arguments will be +available later, as an argument to the `invoke_cli_app()` function — through an +instance of [`argparse.Namespace`]. For the `CLI_SUBCOMMAND_NAME` constant, +choose `kebab-space-sub-command-style`, it does not need to be `snake_case`. + +Make sure to return a `ReturnCode` instance or an integer from +`invoke_cli_app()`. Returning a non-zero value will result in the CLI app +exiting with a return code typically interpreted as an error while zero means +success. You can `import errno` to use typical POSIX error codes through their +human-readable identifiers. + +Another way to interrupt the CLI app control flow is by raising an instance of +one of the in-app errors. `raise PreCommitTerraformExit` for a successful exit, +but it can be turned into an error outcome via +`raise PreCommitTerraformExit(1)`. +`raise PreCommitTerraformRuntimeError('The world is broken')` to indicate +problems within the runtime. The framework will intercept any exceptions +inheriting `PreCommitTerraformBaseError`, so they won't be presented to the +end-users. + [`.pre-commit-hooks.yaml`]: ../../.pre-commit-hooks.yaml [`_cli_parsing.py`]: ./_cli_parsing.py [`_cli_subcommands.py`]: ./_cli_subcommands.py +[`argparse.ArgumentParser`]: +https://docs.python.org/3/library/argparse.html#argparse.ArgumentParser +[`argparse.Namespace`]: +https://docs.python.org/3/library/argparse.html#argparse.Namespace [clig]: https://clig.dev [importable package]: https://docs.python.org/3/tutorial/modules.html#packages [import package]: https://packaging.python.org/en/latest/glossary/#term-Import-Package From 328543126a9592ffadbb3a9197b52b89f88cea66 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 14:32:59 +0200 Subject: [PATCH 37/67] Fix linter violations --- src/pre_commit_terraform/__main__.py | 4 ++-- src/pre_commit_terraform/_cli.py | 11 ++++------- src/pre_commit_terraform/_cli_subcommands.py | 1 - src/pre_commit_terraform/_errors.py | 5 +---- src/pre_commit_terraform/_types.py | 15 ++++++--------- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/pre_commit_terraform/__main__.py b/src/pre_commit_terraform/__main__.py index 18a63dfd0..34ebfdbc0 100644 --- a/src/pre_commit_terraform/__main__.py +++ b/src/pre_commit_terraform/__main__.py @@ -1,9 +1,9 @@ """A runpy-style CLI entry-point module.""" -from sys import argv, exit as exit_with_return_code +from sys import argv +from sys import exit as exit_with_return_code from ._cli import invoke_cli_app - return_code = invoke_cli_app(argv[1:]) exit_with_return_code(return_code) diff --git a/src/pre_commit_terraform/_cli.py b/src/pre_commit_terraform/_cli.py index f52a50b0b..7cbcf129d 100644 --- a/src/pre_commit_terraform/_cli.py +++ b/src/pre_commit_terraform/_cli.py @@ -3,11 +3,9 @@ from sys import stderr from ._cli_parsing import initialize_argument_parser -from ._errors import ( - PreCommitTerraformBaseError, - PreCommitTerraformExit, - PreCommitTerraformRuntimeError, -) +from ._errors import PreCommitTerraformBaseError +from ._errors import PreCommitTerraformExit +from ._errors import PreCommitTerraformRuntimeError from ._structs import ReturnCode from ._types import ReturnCodeType @@ -28,8 +26,7 @@ def invoke_cli_app(cli_args: list[str]) -> ReturnCodeType: raise except PreCommitTerraformRuntimeError as unhandled_exc: print( - f'App execution took an unexpected turn: {unhandled_exc !s}. ' - 'Exiting...', + f'App execution took an unexpected turn: {unhandled_exc !s}. Exiting...', file=stderr, ) return ReturnCode.ERROR diff --git a/src/pre_commit_terraform/_cli_subcommands.py b/src/pre_commit_terraform/_cli_subcommands.py index fc268e552..2923ea927 100644 --- a/src/pre_commit_terraform/_cli_subcommands.py +++ b/src/pre_commit_terraform/_cli_subcommands.py @@ -3,7 +3,6 @@ from . import terraform_docs_replace from ._types import CLISubcommandModuleProtocol - SUBCOMMAND_MODULES: list[CLISubcommandModuleProtocol] = [ terraform_docs_replace, ] diff --git a/src/pre_commit_terraform/_errors.py b/src/pre_commit_terraform/_errors.py index c0f973acc..6f0f20e99 100644 --- a/src/pre_commit_terraform/_errors.py +++ b/src/pre_commit_terraform/_errors.py @@ -5,10 +5,7 @@ class PreCommitTerraformBaseError(Exception): """Base exception for all the in-app errors.""" -class PreCommitTerraformRuntimeError( - PreCommitTerraformBaseError, - RuntimeError, -): +class PreCommitTerraformRuntimeError(PreCommitTerraformBaseError, RuntimeError): """An exception representing a runtime error condition.""" diff --git a/src/pre_commit_terraform/_types.py b/src/pre_commit_terraform/_types.py index 99402b447..8c7fa4bdd 100644 --- a/src/pre_commit_terraform/_types.py +++ b/src/pre_commit_terraform/_types.py @@ -1,11 +1,12 @@ """Composite types for annotating in-project code.""" -from argparse import ArgumentParser, Namespace -from typing import Final, Protocol +from argparse import ArgumentParser +from argparse import Namespace +from typing import Final +from typing import Protocol from ._structs import ReturnCode - ReturnCodeType = ReturnCode | int @@ -15,14 +16,10 @@ class CLISubcommandModuleProtocol(Protocol): CLI_SUBCOMMAND_NAME: Final[str] """This constant contains a CLI.""" - def populate_argument_parser( - self, subcommand_parser: ArgumentParser, - ) -> None: + def populate_argument_parser(self, subcommand_parser: ArgumentParser) -> None: """Run a module hook for populating the subcommand parser.""" - def invoke_cli_app( - self, parsed_cli_args: Namespace, - ) -> ReturnCodeType | int: + def invoke_cli_app(self, parsed_cli_args: Namespace) -> ReturnCodeType | int: """Run a module hook implementing the subcommand logic.""" ... # pylint: disable=unnecessary-ellipsis From 96e44ef42e944576d17f9e0ce815f04e9f966413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=87=BA=F0=9F=87=A6=20Sviatoslav=20Sydorenko=20=28?= =?UTF-8?q?=D0=A1=D0=B2=D1=8F=D1=82=D0=BE=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1?= =?UTF-8?q?=D0=B8=D0=B4=D0=BE=D1=80=D0=B5=D0=BD=D0=BA=D0=BE=29?= Date: Tue, 31 Dec 2024 15:46:15 +0100 Subject: [PATCH 38/67] fix(wip): Access `.pre-commit-hooks.yaml` as artifact (#740) Previously, the path lookup logic was unstable due to a flawed heuristic which meant guessing the artifact path incorrectly in some cases. This patch improves that by employing a reliable approach of putting the artifact into a predictable location within the installed import package. It then uses a standard `importlib.resources` module to retrive it from the always-existing place. The only possibly unobvious quirk is that during development, whenever this file changes in the Git repository, the `pip install` command must be executed again since this is what refreshes the file contents in the installed location. --- hatch.toml | 4 ++++ src/pre_commit_terraform/common.py | 17 ++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/hatch.toml b/hatch.toml index 053ff6bfb..4aa700eed 100644 --- a/hatch.toml +++ b/hatch.toml @@ -1,5 +1,6 @@ [build.targets.sdist] include = [ + '.pre-commit-hooks.yaml', 'src/', ] @@ -8,6 +9,9 @@ packages = [ 'src/pre_commit_terraform/', ] +[build.targets.wheel.force-include] +'.pre-commit-hooks.yaml' = 'pre_commit_terraform/_artifacts/.pre-commit-hooks.yaml' + [metadata.hooks.vcs.urls] 'Source Archive' = 'https://github.com/antonbabenko/pre-commit-terraform/archive/{commit_hash}.tar.gz' 'GitHub: repo' = 'https://github.com/antonbabenko/pre-commit-terraform' diff --git a/src/pre_commit_terraform/common.py b/src/pre_commit_terraform/common.py index 05a793287..af15cac28 100644 --- a/src/pre_commit_terraform/common.py +++ b/src/pre_commit_terraform/common.py @@ -13,7 +13,7 @@ import shutil import subprocess from collections.abc import Sequence -from pathlib import Path +from importlib.resources import files as access_artifacts_of from typing import Callable import yaml @@ -305,15 +305,18 @@ def is_hook_run_on_whole_repo(hook_id: str, file_paths: list[str]) -> bool: """ logger.debug('Hook ID: %s', hook_id) - # Get the directory containing `.pre-commit-hooks.yaml` file - git_repo_root = Path(__file__).resolve().parents[5] - hook_config_path = os.path.join(git_repo_root, '.pre-commit-hooks.yaml') + # Get the directory containing the packaged `.pre-commit-hooks.yaml` copy + artifacts_root_path = access_artifacts_of('pre_commit_terraform') / '_artifacts' + pre_commit_hooks_yaml_path = artifacts_root_path / '.pre-commit-hooks.yaml' + pre_commit_hooks_yaml_path.read_text(encoding='utf-8') - logger.debug('Hook config path: %s', hook_config_path) + logger.debug('Hook config path: %s', pre_commit_hooks_yaml_path) # Read the .pre-commit-hooks.yaml file - with open(hook_config_path, 'r', encoding='utf-8') as pre_commit_hooks_yaml: - hooks_config = yaml.safe_load(pre_commit_hooks_yaml) + pre_commit_hooks_yaml_txt = pre_commit_hooks_yaml_path.read_text( + encoding='utf-8', + ) + hooks_config = yaml.safe_load(pre_commit_hooks_yaml_txt) # Get the included and excluded file patterns for the given hook_id for hook in hooks_config: From 2c8b44b557140c95870e5294b791d8681ee9bc68 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 17:37:14 +0200 Subject: [PATCH 39/67] Clarify and consolidate args parser code --- src/pre_commit_terraform/_cli_parsing.py | 43 +++++++++++- src/pre_commit_terraform/common.py | 67 ------------------- .../terraform_docs_replace.py | 2 +- 3 files changed, 43 insertions(+), 69 deletions(-) diff --git a/src/pre_commit_terraform/_cli_parsing.py b/src/pre_commit_terraform/_cli_parsing.py index 969b0ae17..b710a9fce 100644 --- a/src/pre_commit_terraform/_cli_parsing.py +++ b/src/pre_commit_terraform/_cli_parsing.py @@ -9,6 +9,46 @@ from ._cli_subcommands import SUBCOMMAND_MODULES +def populate_common_argument_parser(parser: ArgumentParser) -> None: + """ + Populate the argument parser with the common arguments. + + Args: + parser (argparse.ArgumentParser): The argument parser to populate. + """ + parser.add_argument( + '-a', + '--args', + action='append', + help='Arguments that configure wrapped tool behavior', + default=[], + ) + parser.add_argument( + '-h', + '--hook-config', + action='append', + help='Arguments that configure hook behavior', + default=[], + ) + parser.add_argument( + '-i', + '--tf-init-args', + '--init-args', + action='append', + help='Arguments for `tf init` command', + default=[], + ) + parser.add_argument( + '-e', + '--env-vars', + '--envs', + action='append', + help='Setup additional Environment Variables during hook execution', + default=[], + ) + parser.add_argument('files', nargs='*', help='Changed files paths') + + def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: """Connect all sub-command parsers to the given one. @@ -26,7 +66,8 @@ def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: subcommand_parser.set_defaults( invoke_cli_app=subcommand_module.invoke_cli_app, ) - subcommand_module.populate_argument_parser(subcommand_parser) + populate_common_argument_parser(subcommand_parser) + subcommand_module.populate_hook_specific_argument_parser(subcommand_parser) def initialize_argument_parser() -> ArgumentParser: diff --git a/src/pre_commit_terraform/common.py b/src/pre_commit_terraform/common.py index af15cac28..b84779652 100644 --- a/src/pre_commit_terraform/common.py +++ b/src/pre_commit_terraform/common.py @@ -42,73 +42,6 @@ def parse_env_vars(env_var_strs: list[str]) -> dict[str, str]: return env_var_dict -def parse_cmdline( - argv: Sequence[str] | None = None, -) -> tuple[list[str], list[str], list[str], list[str], list[str]]: # noqa: WPS221 - """ - Parse the command line arguments and return a tuple containing the parsed values. - - Args: - argv (Sequence[str] | None): The command line arguments to parse. - If None, the arguments from sys.argv will be used. - - Returns: - A tuple containing the parsed values: - - args (list[str]): Arguments that configure wrapped tool behavior. - - hook_config (list[str]): Arguments that configure hook behavior. - - files (list[str]): File paths on which we should run the hook. - - tf_init_args (list[str]): Arguments for `terraform init` command. - - env_vars (list[str]): Custom environment variable strings in the format "name=value". - - Raises: - ValueError: If no files are provided. - """ - parser = argparse.ArgumentParser( - add_help=False, # Allow the use of `-h` for compatibility with the Bash version of the hook - ) - parser.add_argument('-a', '--args', action='append', help='Arguments', default=[]) - parser.add_argument('-h', '--hook-config', action='append', help='Hook Config', default=[]) - parser.add_argument( - '-i', - '--tf-init-args', - '--init-args', - action='append', - help='TF Init Args', - default=[], - ) - parser.add_argument( - '-e', - '--env-vars', - '--envs', - action='append', - help='Environment Variables', - default=[], - ) - parser.add_argument('files', nargs='*', help='Files') - - parsed_args = parser.parse_args(argv) - - if parsed_args.files is None: - raise ValueError('No files provided') - - logger.debug( - 'Parsed values:\nargs: %r\nhook_config: %r\nfiles: %r\ntf_init_args: %r\nenv_vars: %r', - parsed_args.args, - parsed_args.hook_config, - parsed_args.files, - parsed_args.tf_init_args, - parsed_args.env_vars, - ) - - return ( - parsed_args.args, - parsed_args.hook_config, - parsed_args.files, - parsed_args.tf_init_args, - parsed_args.env_vars, - ) - - def _get_unique_dirs(files: list[str]) -> set[str]: """ Get unique directories from a list of files. diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py index 63284d11a..22981b3ee 100644 --- a/src/pre_commit_terraform/terraform_docs_replace.py +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -11,7 +11,7 @@ CLI_SUBCOMMAND_NAME: Final[str] = 'replace-docs' -def populate_argument_parser(subcommand_parser: ArgumentParser) -> None: +def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: subcommand_parser.description = ( 'Run terraform-docs on a set of files. Follows the standard ' 'convention of pulling the documentation from main.tf in order to ' From 934202f0ed18b891af9b992a398a2019709029cb Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 17:38:53 +0200 Subject: [PATCH 40/67] Disable help as it conflicts with --hook-config --- src/pre_commit_terraform/_cli_parsing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pre_commit_terraform/_cli_parsing.py b/src/pre_commit_terraform/_cli_parsing.py index b710a9fce..9034e5eb8 100644 --- a/src/pre_commit_terraform/_cli_parsing.py +++ b/src/pre_commit_terraform/_cli_parsing.py @@ -58,11 +58,10 @@ def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: """ subcommand_parsers = root_cli_parser.add_subparsers( dest='check_name', - help='A check to be performed.', required=True, ) for subcommand_module in SUBCOMMAND_MODULES: - subcommand_parser = subcommand_parsers.add_parser(subcommand_module.CLI_SUBCOMMAND_NAME) + subcommand_parser = subcommand_parsers.add_parser(subcommand_module.CLI_SUBCOMMAND_NAME, add_help=False,) subcommand_parser.set_defaults( invoke_cli_app=subcommand_module.invoke_cli_app, ) From 3cbc22038d70580cfbd37dc6cbb814eea0813295 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 17:48:44 +0200 Subject: [PATCH 41/67] Rewrite checkov hook to new structure and set CLI to hook_id --- .pre-commit-hooks.yaml | 4 +- src/pre_commit_terraform/_cli_subcommands.py | 2 + src/pre_commit_terraform/terraform_checkov.py | 37 ++++++++++++------- .../terraform_docs_replace.py | 2 +- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 203091c3f..eb5e84590 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -46,7 +46,7 @@ name: Terraform docs (overwrite README.md) description: Overwrite content of README.md with terraform-docs. require_serial: true - entry: python -Im pre_commit_terraform replace-docs + entry: python -Im pre_commit_terraform terraform_docs_replace language: python files: (\.tf)$ exclude: \.terraform/.*$ @@ -150,7 +150,7 @@ - id: terraform_checkov_py name: Checkov description: Runs checkov on Terraform templates. - entry: terraform_checkov + entry: python -Im pre_commit_terraform terraform_checkov_py language: python always_run: false files: \.tf$ diff --git a/src/pre_commit_terraform/_cli_subcommands.py b/src/pre_commit_terraform/_cli_subcommands.py index 2923ea927..0f9b99c89 100644 --- a/src/pre_commit_terraform/_cli_subcommands.py +++ b/src/pre_commit_terraform/_cli_subcommands.py @@ -1,10 +1,12 @@ """A CLI sub-commands organization module.""" from . import terraform_docs_replace +from . import terraform_checkov from ._types import CLISubcommandModuleProtocol SUBCOMMAND_MODULES: list[CLISubcommandModuleProtocol] = [ terraform_docs_replace, + terraform_checkov, ] diff --git a/src/pre_commit_terraform/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py index 3b156d2eb..24cd04336 100644 --- a/src/pre_commit_terraform/terraform_checkov.py +++ b/src/pre_commit_terraform/terraform_checkov.py @@ -8,7 +8,11 @@ import sys from subprocess import PIPE from subprocess import run -from typing import Sequence +from typing import Final +from argparse import ArgumentParser +from argparse import Namespace +from ._types import ReturnCodeType + from pre_commit_terraform import common from pre_commit_terraform.logger import setup_logging @@ -31,20 +35,30 @@ def replace_git_working_dir_to_repo_root(args: list[str]) -> list[str]: return [arg.replace('__GIT_WORKING_DIR__', os.getcwd()) for arg in args] -def main(argv: Sequence[str] | None = None) -> int: +CLI_SUBCOMMAND_NAME: Final[str] = 'terraform_checkov_py' + +def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: + pass + + +def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: # noqa: DAR101, DAR201 # TODO: Add docstrings when will end up with final implementation """ Execute terraform_fmt_py pre-commit hook. Parses args and calls `terraform fmt` on list of files provided by pre-commit. + + Args: + parsed_cli_args: Parsed arguments from CLI. + + Returns: + int: The exit code of the hook. """ setup_logging() logger.debug(sys.version_info) - args, hook_config, files, _tf_init_args, env_vars_strs = common.parse_cmdline(argv) - - all_env_vars = {**os.environ, **common.parse_env_vars(env_vars_strs)} - expanded_args = common.expand_env_vars(args, all_env_vars) + all_env_vars = {**os.environ, **common.parse_env_vars(parsed_cli_args.env_vars)} + expanded_args = common.expand_env_vars(parsed_cli_args.args, all_env_vars) expanded_args = replace_git_working_dir_to_repo_root(expanded_args) # Just in case is someone somehow will add something like "; rm -rf" in the args safe_args = [shlex.quote(arg) for arg in expanded_args] @@ -53,13 +67,12 @@ def main(argv: Sequence[str] | None = None) -> int: all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # TODO: subprocess.run ignore colors # WPS421 - IDK how to check is function exist w/o passing globals() if common.is_function_defined('run_hook_on_whole_repo', globals()): # noqa: WPS421 - hook_id = os.path.basename(__file__).replace('.py', '_py') - if common.is_hook_run_on_whole_repo(hook_id, files): + if common.is_hook_run_on_whole_repo(CLI_SUBCOMMAND_NAME, parsed_cli_args.files): return run_hook_on_whole_repo(safe_args, all_env_vars) return common.per_dir_hook( - hook_config, - files, + parsed_cli_args.hook_config, + parsed_cli_args.files, safe_args, all_env_vars, per_dir_hook_unique_part, @@ -136,7 +149,3 @@ def per_dir_hook_unique_part( sys.stdout.write(completed_process.stdout) return completed_process.returncode - - -if __name__ == '__main__': - raise SystemExit(main()) diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py index 22981b3ee..ca3e868de 100644 --- a/src/pre_commit_terraform/terraform_docs_replace.py +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -8,7 +8,7 @@ from ._structs import ReturnCode from ._types import ReturnCodeType -CLI_SUBCOMMAND_NAME: Final[str] = 'replace-docs' +CLI_SUBCOMMAND_NAME: Final[str] = 'terraform_docs_replace' def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: From 7e647ad160e7ee330ad510fb13d5db0c03485c17 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 17:58:04 +0200 Subject: [PATCH 42/67] Rename CLI_SUBCOMMAND_NAME to HOOK_ID as it make more sense and make it calculable from file name --- src/pre_commit_terraform/_cli_parsing.py | 2 +- src/pre_commit_terraform/_types.py | 2 +- src/pre_commit_terraform/terraform_checkov.py | 4 ++-- src/pre_commit_terraform/terraform_docs_replace.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pre_commit_terraform/_cli_parsing.py b/src/pre_commit_terraform/_cli_parsing.py index 9034e5eb8..f8fe5bd2f 100644 --- a/src/pre_commit_terraform/_cli_parsing.py +++ b/src/pre_commit_terraform/_cli_parsing.py @@ -61,7 +61,7 @@ def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: required=True, ) for subcommand_module in SUBCOMMAND_MODULES: - subcommand_parser = subcommand_parsers.add_parser(subcommand_module.CLI_SUBCOMMAND_NAME, add_help=False,) + subcommand_parser = subcommand_parsers.add_parser(subcommand_module.HOOK_ID, add_help=False,) subcommand_parser.set_defaults( invoke_cli_app=subcommand_module.invoke_cli_app, ) diff --git a/src/pre_commit_terraform/_types.py b/src/pre_commit_terraform/_types.py index 8c7fa4bdd..cbbe2a048 100644 --- a/src/pre_commit_terraform/_types.py +++ b/src/pre_commit_terraform/_types.py @@ -13,7 +13,7 @@ class CLISubcommandModuleProtocol(Protocol): """A protocol for the subcommand-implementing module shape.""" - CLI_SUBCOMMAND_NAME: Final[str] + HOOK_ID: Final[str] """This constant contains a CLI.""" def populate_argument_parser(self, subcommand_parser: ArgumentParser) -> None: diff --git a/src/pre_commit_terraform/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py index 24cd04336..7bd6b7d95 100644 --- a/src/pre_commit_terraform/terraform_checkov.py +++ b/src/pre_commit_terraform/terraform_checkov.py @@ -35,7 +35,7 @@ def replace_git_working_dir_to_repo_root(args: list[str]) -> list[str]: return [arg.replace('__GIT_WORKING_DIR__', os.getcwd()) for arg in args] -CLI_SUBCOMMAND_NAME: Final[str] = 'terraform_checkov_py' +HOOK_ID: Final[str] = "%s_py" % __name__.rpartition('.')[-1] def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: pass @@ -67,7 +67,7 @@ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # TODO: subprocess.run ignore colors # WPS421 - IDK how to check is function exist w/o passing globals() if common.is_function_defined('run_hook_on_whole_repo', globals()): # noqa: WPS421 - if common.is_hook_run_on_whole_repo(CLI_SUBCOMMAND_NAME, parsed_cli_args.files): + if common.is_hook_run_on_whole_repo(HOOK_ID, parsed_cli_args.files): return run_hook_on_whole_repo(safe_args, all_env_vars) return common.per_dir_hook( diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py index ca3e868de..4faad0ff9 100644 --- a/src/pre_commit_terraform/terraform_docs_replace.py +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -8,7 +8,7 @@ from ._structs import ReturnCode from ._types import ReturnCodeType -CLI_SUBCOMMAND_NAME: Final[str] = 'terraform_docs_replace' +HOOK_ID: Final[str] = __name__.rpartition('.')[-1] def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: From 0defc96ff2e99002b878066e47d0231e3c59b73e Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 18:28:48 +0200 Subject: [PATCH 43/67] Rewrite terraform_fmt_py to new structure --- .pre-commit-hooks.yaml | 2 +- src/pre_commit_terraform/_cli_subcommands.py | 4 ++- src/pre_commit_terraform/terraform_fmt.py | 37 +++++++++++++------- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index eb5e84590..e464f6ba4 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -19,7 +19,7 @@ name: Terraform fmt description: Rewrites all Terraform configuration files to a canonical format. require_serial: true - entry: terraform_fmt + entry: python -Im pre_commit_terraform terraform_fmt_py language: python files: \.tf(vars)?$ exclude: \.terraform/.*$ diff --git a/src/pre_commit_terraform/_cli_subcommands.py b/src/pre_commit_terraform/_cli_subcommands.py index 0f9b99c89..87b4c0465 100644 --- a/src/pre_commit_terraform/_cli_subcommands.py +++ b/src/pre_commit_terraform/_cli_subcommands.py @@ -1,11 +1,13 @@ """A CLI sub-commands organization module.""" -from . import terraform_docs_replace from . import terraform_checkov +from . import terraform_docs_replace +from . import terraform_fmt from ._types import CLISubcommandModuleProtocol SUBCOMMAND_MODULES: list[CLISubcommandModuleProtocol] = [ terraform_docs_replace, + terraform_fmt, terraform_checkov, ] diff --git a/src/pre_commit_terraform/terraform_fmt.py b/src/pre_commit_terraform/terraform_fmt.py index c8bc7d097..a45145e09 100644 --- a/src/pre_commit_terraform/terraform_fmt.py +++ b/src/pre_commit_terraform/terraform_fmt.py @@ -6,39 +6,54 @@ import os import shlex import sys +from argparse import ArgumentParser +from argparse import Namespace from subprocess import PIPE from subprocess import run -from typing import Sequence +from typing import Final from pre_commit_terraform import common from pre_commit_terraform.logger import setup_logging +from ._types import ReturnCodeType + logger = logging.getLogger(__name__) -def main(argv: Sequence[str] | None = None) -> int: +HOOK_ID: Final[str] = '%s_py' % __name__.rpartition('.')[-1] + + +def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: + pass + + +def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: # noqa: DAR101, DAR201 # TODO: Add docstrings when will end up with final implementation """ Execute terraform_fmt_py pre-commit hook. Parses args and calls `terraform fmt` on list of files provided by pre-commit. + + Args: + parsed_cli_args: Parsed arguments from CLI. + + Returns: + int: The exit code of the hook. """ setup_logging() logger.debug(sys.version_info) - args, hook_config, files, _tf_init_args, env_vars_strs = common.parse_cmdline(argv) - - all_env_vars = {**os.environ, **common.parse_env_vars(env_vars_strs)} - expanded_args = common.expand_env_vars(args, all_env_vars) + all_env_vars = {**os.environ, **common.parse_env_vars(parsed_cli_args.env_vars)} + expanded_args = common.expand_env_vars(parsed_cli_args.args, all_env_vars) # Just in case is someone somehow will add something like "; rm -rf" in the args safe_args = [shlex.quote(arg) for arg in expanded_args] if os.environ.get('PRE_COMMIT_COLOR') == 'never': - args.append('-no-color') + safe_args.append('-no-color') return common.per_dir_hook( - hook_config, - files, + parsed_cli_args.hook_config, + parsed_cli_args.files, safe_args, all_env_vars, per_dir_hook_unique_part, @@ -81,7 +96,3 @@ def per_dir_hook_unique_part( sys.stdout.write(completed_process.stdout) return completed_process.returncode - - -if __name__ == '__main__': - raise SystemExit(main()) From 7b3cc6c5489ee681fdaf0193712e0be892d6a996 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 19:01:40 +0200 Subject: [PATCH 44/67] Fix pylint violations --- src/pre_commit_terraform/__main__.py | 2 +- src/pre_commit_terraform/_cli.py | 12 +++--- src/pre_commit_terraform/_cli_parsing.py | 14 ++++-- src/pre_commit_terraform/_cli_subcommands.py | 8 ++-- src/pre_commit_terraform/_types.py | 2 +- src/pre_commit_terraform/common.py | 2 - src/pre_commit_terraform/terraform_checkov.py | 26 ++++++----- .../terraform_docs_replace.py | 43 +++++++++++++------ src/pre_commit_terraform/terraform_fmt.py | 18 ++++---- 9 files changed, 78 insertions(+), 49 deletions(-) diff --git a/src/pre_commit_terraform/__main__.py b/src/pre_commit_terraform/__main__.py index 34ebfdbc0..da387f5bd 100644 --- a/src/pre_commit_terraform/__main__.py +++ b/src/pre_commit_terraform/__main__.py @@ -3,7 +3,7 @@ from sys import argv from sys import exit as exit_with_return_code -from ._cli import invoke_cli_app +from pre_commit_terraform._cli import invoke_cli_app return_code = invoke_cli_app(argv[1:]) exit_with_return_code(return_code) diff --git a/src/pre_commit_terraform/_cli.py b/src/pre_commit_terraform/_cli.py index 7cbcf129d..805fa9d36 100644 --- a/src/pre_commit_terraform/_cli.py +++ b/src/pre_commit_terraform/_cli.py @@ -2,12 +2,12 @@ from sys import stderr -from ._cli_parsing import initialize_argument_parser -from ._errors import PreCommitTerraformBaseError -from ._errors import PreCommitTerraformExit -from ._errors import PreCommitTerraformRuntimeError -from ._structs import ReturnCode -from ._types import ReturnCodeType +from pre_commit_terraform._cli_parsing import initialize_argument_parser +from pre_commit_terraform._errors import PreCommitTerraformBaseError +from pre_commit_terraform._errors import PreCommitTerraformExit +from pre_commit_terraform._errors import PreCommitTerraformRuntimeError +from pre_commit_terraform._structs import ReturnCode +from pre_commit_terraform._types import ReturnCodeType def invoke_cli_app(cli_args: list[str]) -> ReturnCodeType: diff --git a/src/pre_commit_terraform/_cli_parsing.py b/src/pre_commit_terraform/_cli_parsing.py index f8fe5bd2f..2b18c6508 100644 --- a/src/pre_commit_terraform/_cli_parsing.py +++ b/src/pre_commit_terraform/_cli_parsing.py @@ -6,7 +6,7 @@ from argparse import ArgumentParser -from ._cli_subcommands import SUBCOMMAND_MODULES +from pre_commit_terraform._cli_subcommands import SUBCOMMAND_MODULES def populate_common_argument_parser(parser: ArgumentParser) -> None: @@ -61,7 +61,10 @@ def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: required=True, ) for subcommand_module in SUBCOMMAND_MODULES: - subcommand_parser = subcommand_parsers.add_parser(subcommand_module.HOOK_ID, add_help=False,) + subcommand_parser = subcommand_parsers.add_parser( + subcommand_module.HOOK_ID, + add_help=False, + ) subcommand_parser.set_defaults( invoke_cli_app=subcommand_module.invoke_cli_app, ) @@ -70,7 +73,12 @@ def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: def initialize_argument_parser() -> ArgumentParser: - """Return the root argument parser with sub-commands.""" + """ + Parse the command line arguments and return the parsed arguments. + + Return the root argument parser with sub-commands. + + """ root_cli_parser = ArgumentParser(prog=f'python -m {__package__ !s}') attach_subcommand_parsers_to(root_cli_parser) return root_cli_parser diff --git a/src/pre_commit_terraform/_cli_subcommands.py b/src/pre_commit_terraform/_cli_subcommands.py index 87b4c0465..2703587ca 100644 --- a/src/pre_commit_terraform/_cli_subcommands.py +++ b/src/pre_commit_terraform/_cli_subcommands.py @@ -1,9 +1,9 @@ """A CLI sub-commands organization module.""" -from . import terraform_checkov -from . import terraform_docs_replace -from . import terraform_fmt -from ._types import CLISubcommandModuleProtocol +from pre_commit_terraform import terraform_checkov +from pre_commit_terraform import terraform_docs_replace +from pre_commit_terraform import terraform_fmt +from pre_commit_terraform._types import CLISubcommandModuleProtocol SUBCOMMAND_MODULES: list[CLISubcommandModuleProtocol] = [ terraform_docs_replace, diff --git a/src/pre_commit_terraform/_types.py b/src/pre_commit_terraform/_types.py index cbbe2a048..c5020d248 100644 --- a/src/pre_commit_terraform/_types.py +++ b/src/pre_commit_terraform/_types.py @@ -5,7 +5,7 @@ from typing import Final from typing import Protocol -from ._structs import ReturnCode +from pre_commit_terraform._structs import ReturnCode ReturnCodeType = ReturnCode | int diff --git a/src/pre_commit_terraform/common.py b/src/pre_commit_terraform/common.py index b84779652..e1496882b 100644 --- a/src/pre_commit_terraform/common.py +++ b/src/pre_commit_terraform/common.py @@ -6,13 +6,11 @@ from __future__ import annotations -import argparse import logging import os import re import shutil import subprocess -from collections.abc import Sequence from importlib.resources import files as access_artifacts_of from typing import Callable diff --git a/src/pre_commit_terraform/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py index 7bd6b7d95..57cb1a6e4 100644 --- a/src/pre_commit_terraform/terraform_checkov.py +++ b/src/pre_commit_terraform/terraform_checkov.py @@ -6,15 +6,14 @@ import os import shlex import sys +from argparse import ArgumentParser +from argparse import Namespace from subprocess import PIPE from subprocess import run from typing import Final -from argparse import ArgumentParser -from argparse import Namespace -from ._types import ReturnCodeType - from pre_commit_terraform import common +from pre_commit_terraform._types import ReturnCodeType from pre_commit_terraform.logger import setup_logging logger = logging.getLogger(__name__) @@ -35,18 +34,22 @@ def replace_git_working_dir_to_repo_root(args: list[str]) -> list[str]: return [arg.replace('__GIT_WORKING_DIR__', os.getcwd()) for arg in args] -HOOK_ID: Final[str] = "%s_py" % __name__.rpartition('.')[-1] +HOOK_ID: Final[str] = f"{__name__.rpartition('.')[-1]}_py" + +# pylint: disable=unused-argument def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: - pass + """ + Populate the argument parser with the hook-specific arguments. + + Args: + subcommand_parser: The argument parser to populate. + """ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: - # noqa: DAR101, DAR201 # TODO: Add docstrings when will end up with final implementation """ - Execute terraform_fmt_py pre-commit hook. - - Parses args and calls `terraform fmt` on list of files provided by pre-commit. + Execute main pre-commit hook logic. Args: parsed_cli_args: Parsed arguments from CLI. @@ -64,7 +67,8 @@ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: safe_args = [shlex.quote(arg) for arg in expanded_args] if os.environ.get('PRE_COMMIT_COLOR') == 'never': - all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # TODO: subprocess.run ignore colors + # TODO: subprocess.run ignore colors. Try `rich` lib + all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # WPS421 - IDK how to check is function exist w/o passing globals() if common.is_function_defined('run_hook_on_whole_repo', globals()): # noqa: WPS421 if common.is_hook_run_on_whole_repo(HOOK_ID, parsed_cli_args.files): diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py index 4faad0ff9..7c8d51393 100644 --- a/src/pre_commit_terraform/terraform_docs_replace.py +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -1,3 +1,5 @@ +"""Deprecated hook. Don't use it.""" + import os import subprocess import warnings @@ -5,13 +7,20 @@ from argparse import Namespace from typing import Final -from ._structs import ReturnCode -from ._types import ReturnCodeType +from pre_commit_terraform._structs import ReturnCode +from pre_commit_terraform._types import ReturnCodeType HOOK_ID: Final[str] = __name__.rpartition('.')[-1] def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: + """ + Populate the argument parser with the hook-specific arguments. + + Args: + subcommand_parser: The argument parser to populate. + """ + subcommand_parser.description = ( 'Run terraform-docs on a set of files. Follows the standard ' 'convention of pulling the documentation from main.tf in order to ' @@ -47,6 +56,16 @@ def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: + """ + Execute main pre-commit hook logic. + + Args: + parsed_cli_args: Parsed arguments from CLI. + + Returns: + int: The exit code of the hook. + """ + warnings.warn( '`terraform_docs_replace` hook is DEPRECATED.' + 'For migration instructions see ' @@ -64,19 +83,17 @@ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: retval = ReturnCode.OK - for dir in dirs: + for directory in dirs: try: - procArgs = [] - procArgs.append('terraform-docs') + proc_args = [] + proc_args.append('terraform-docs') if parsed_cli_args.sort: - procArgs.append('--sort-by-required') - procArgs.append('md') - procArgs.append('./{dir}'.format(dir=dir)) - procArgs.append('>') - procArgs.append( - './{dir}/{dest}'.format(dir=dir, dest=parsed_cli_args.dest), - ) - subprocess.check_call(' '.join(procArgs), shell=True) + proc_args.append('--sort-by-required') + proc_args.append('md') + proc_args.append(f'./{directory}') + proc_args.append('>') + proc_args.append(f'./{directory}/{parsed_cli_args.dest}') + subprocess.check_call(' '.join(proc_args), shell=True) except subprocess.CalledProcessError as e: print(e) retval = ReturnCode.ERROR diff --git a/src/pre_commit_terraform/terraform_fmt.py b/src/pre_commit_terraform/terraform_fmt.py index a45145e09..7458778bb 100644 --- a/src/pre_commit_terraform/terraform_fmt.py +++ b/src/pre_commit_terraform/terraform_fmt.py @@ -13,26 +13,28 @@ from typing import Final from pre_commit_terraform import common +from pre_commit_terraform._types import ReturnCodeType from pre_commit_terraform.logger import setup_logging -from ._types import ReturnCodeType - logger = logging.getLogger(__name__) -HOOK_ID: Final[str] = '%s_py' % __name__.rpartition('.')[-1] +HOOK_ID: Final[str] = f"{__name__.rpartition('.')[-1]}_py" +# pylint: disable=unused-argument def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: - pass + """ + Populate the argument parser with the hook-specific arguments. + + Args: + subcommand_parser: The argument parser to populate. + """ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: - # noqa: DAR101, DAR201 # TODO: Add docstrings when will end up with final implementation """ - Execute terraform_fmt_py pre-commit hook. - - Parses args and calls `terraform fmt` on list of files provided by pre-commit. + Execute main pre-commit hook logic. Args: parsed_cli_args: Parsed arguments from CLI. From f1d8faf3594cddb33a0cc2e64d85a6b9d606ab2b Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 19:07:23 +0200 Subject: [PATCH 45/67] Fix mypy violations --- src/pre_commit_terraform/_types.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pre_commit_terraform/_types.py b/src/pre_commit_terraform/_types.py index c5020d248..1052c9175 100644 --- a/src/pre_commit_terraform/_types.py +++ b/src/pre_commit_terraform/_types.py @@ -2,24 +2,24 @@ from argparse import ArgumentParser from argparse import Namespace -from typing import Final from typing import Protocol +from typing import TypeAlias from pre_commit_terraform._structs import ReturnCode -ReturnCodeType = ReturnCode | int +ReturnCodeType: TypeAlias = ReturnCode | int class CLISubcommandModuleProtocol(Protocol): """A protocol for the subcommand-implementing module shape.""" - HOOK_ID: Final[str] + HOOK_ID: str """This constant contains a CLI.""" def populate_argument_parser(self, subcommand_parser: ArgumentParser) -> None: """Run a module hook for populating the subcommand parser.""" - def invoke_cli_app(self, parsed_cli_args: Namespace) -> ReturnCodeType | int: + def invoke_cli_app(self, parsed_cli_args: Namespace) -> ReturnCodeType: """Run a module hook implementing the subcommand logic.""" ... # pylint: disable=unnecessary-ellipsis From 2e54190366b948f3707885b7c15362308b6e2878 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 20:23:07 +0200 Subject: [PATCH 46/67] Fix most of wemake-python-styleguide violations --- .pre-commit-config.yaml | 55 ++++++++----------- src/pre_commit_terraform/_cli.py | 2 +- src/pre_commit_terraform/_cli_subcommands.py | 4 +- src/pre_commit_terraform/common.py | 8 +-- src/pre_commit_terraform/logger.py | 10 ++-- src/pre_commit_terraform/terraform_checkov.py | 2 +- .../terraform_docs_replace.py | 4 +- src/pre_commit_terraform/terraform_fmt.py | 2 +- 8 files changed, 36 insertions(+), 51 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 467d61ce5..1ae7da76d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -128,37 +128,26 @@ repos: --warn-redundant-casts, ] -- repo: https://github.com/pycqa/flake8.git - rev: 7.0.0 +- repo: https://github.com/wemake-services/wemake-python-styleguide + rev: 1.0.0 hooks: - - id: flake8 - additional_dependencies: - - flake8-2020 - - flake8-docstrings - - flake8-pytest-style - - wemake-python-styleguide - args: - - --max-returns=5 # Default to 2 - - --max-arguments=5 # Default to 4 - # https://www.flake8rules.com/ - # https://wemake-python-stylegui.de/en/latest/pages/usage/violations/index.html - - --extend-ignore= - WPS210 - WPS226 - WPS236 - WPS300 - WPS305 - WPS323 - I - S404 - S603 - - - RST201 RST203 RST301 - - - # WPS211 - # WPS420 - # RST, - exclude: test_.+\.py$ + - id: wemake-python-styleguide + args: + - --allowed-module-metadata=__all__ # Default to '' + - --max-local-variables=6 # Default to 5 + - --max-returns=6 # Default to 5 + - --max-imports=13 # Default to 12 + # https://wemake-python-stylegui.de/en/latest/pages/usage/violations/index.html + - --extend-ignore= + E501 + WPS115 + WPS226 + + + exclude: | + (?x) + # Ignore tests + (^tests/pytest/test_.+\.py$ + # Ignore deprecated hook + |^src/pre_commit_terraform/terraform_docs_replace.py$ + ) diff --git a/src/pre_commit_terraform/_cli.py b/src/pre_commit_terraform/_cli.py index 805fa9d36..8ab55af8b 100644 --- a/src/pre_commit_terraform/_cli.py +++ b/src/pre_commit_terraform/_cli.py @@ -19,7 +19,7 @@ def invoke_cli_app(cli_args: list[str]) -> ReturnCodeType: root_cli_parser = initialize_argument_parser() parsed_cli_args = root_cli_parser.parse_args(cli_args) - try: + try: # noqa: WPS225 - Found too many `except` cases: 4 > 3 return parsed_cli_args.invoke_cli_app(parsed_cli_args) except PreCommitTerraformExit as exit_err: print(f'App exiting: {exit_err !s}', file=stderr) diff --git a/src/pre_commit_terraform/_cli_subcommands.py b/src/pre_commit_terraform/_cli_subcommands.py index 2703587ca..cac2f408e 100644 --- a/src/pre_commit_terraform/_cli_subcommands.py +++ b/src/pre_commit_terraform/_cli_subcommands.py @@ -5,11 +5,11 @@ from pre_commit_terraform import terraform_fmt from pre_commit_terraform._types import CLISubcommandModuleProtocol -SUBCOMMAND_MODULES: list[CLISubcommandModuleProtocol] = [ +SUBCOMMAND_MODULES: tuple[CLISubcommandModuleProtocol, ...] = ( terraform_docs_replace, terraform_fmt, terraform_checkov, -] +) __all__ = ('SUBCOMMAND_MODULES',) diff --git a/src/pre_commit_terraform/common.py b/src/pre_commit_terraform/common.py index e1496882b..4bf27678d 100644 --- a/src/pre_commit_terraform/common.py +++ b/src/pre_commit_terraform/common.py @@ -154,12 +154,12 @@ def get_tf_binary_path(hook_config: list[str]) -> str: BinaryNotFoundError: If neither Terraform nor OpenTofu binary could be found. """ - hook_config_tf_path = None + # direct hook config, has the highest precedence for config in hook_config: if config.startswith('--tf-path='): hook_config_tf_path = config.split('=', 1)[1].rstrip(';') - break + return hook_config_tf_path # direct hook config, has the highest precedence if hook_config_tf_path: @@ -244,9 +244,7 @@ def is_hook_run_on_whole_repo(hook_id: str, file_paths: list[str]) -> bool: logger.debug('Hook config path: %s', pre_commit_hooks_yaml_path) # Read the .pre-commit-hooks.yaml file - pre_commit_hooks_yaml_txt = pre_commit_hooks_yaml_path.read_text( - encoding='utf-8', - ) + pre_commit_hooks_yaml_txt = pre_commit_hooks_yaml_path.read_text(encoding='utf-8') hooks_config = yaml.safe_load(pre_commit_hooks_yaml_txt) # Get the included and excluded file patterns for the given hook_id diff --git a/src/pre_commit_terraform/logger.py b/src/pre_commit_terraform/logger.py index c00d4df7e..816df29f6 100644 --- a/src/pre_commit_terraform/logger.py +++ b/src/pre_commit_terraform/logger.py @@ -39,14 +39,12 @@ def format(self, record: logging.LogRecord) -> str: 'CRITICAL': 41, # white on red background } - prefix = '\033[' - suffix = '\033[0m' - colored_record = copy(record) levelname = colored_record.levelname - seq = color_mapping.get(levelname, 37) # default white # noqa: WPS432 - colored_levelname = f'{prefix}{seq}m{levelname}{suffix}' - colored_record.levelname = colored_levelname + + set_color = f'\033[{color_mapping.get(levelname, 37)}' # default white # noqa: WPS432 + reset_color = '\033[0m' + colored_record.levelname = f'{set_color}m{levelname}{reset_color}' return super().format(colored_record) diff --git a/src/pre_commit_terraform/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py index 57cb1a6e4..6afe0a805 100644 --- a/src/pre_commit_terraform/terraform_checkov.py +++ b/src/pre_commit_terraform/terraform_checkov.py @@ -34,7 +34,7 @@ def replace_git_working_dir_to_repo_root(args: list[str]) -> list[str]: return [arg.replace('__GIT_WORKING_DIR__', os.getcwd()) for arg in args] -HOOK_ID: Final[str] = f"{__name__.rpartition('.')[-1]}_py" +HOOK_ID: Final[str] = __name__.rpartition('.')[-1] + '_py' # noqa: WPS336 # pylint: disable=unused-argument diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py index 7c8d51393..b8770efaf 100644 --- a/src/pre_commit_terraform/terraform_docs_replace.py +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -94,7 +94,7 @@ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: proc_args.append('>') proc_args.append(f'./{directory}/{parsed_cli_args.dest}') subprocess.check_call(' '.join(proc_args), shell=True) - except subprocess.CalledProcessError as e: - print(e) + except subprocess.CalledProcessError as exception: + print(exception) retval = ReturnCode.ERROR return retval diff --git a/src/pre_commit_terraform/terraform_fmt.py b/src/pre_commit_terraform/terraform_fmt.py index 7458778bb..c6f43ef54 100644 --- a/src/pre_commit_terraform/terraform_fmt.py +++ b/src/pre_commit_terraform/terraform_fmt.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) -HOOK_ID: Final[str] = f"{__name__.rpartition('.')[-1]}_py" +HOOK_ID: Final[str] = __name__.rpartition('.')[-1] + '_py' # noqa: WPS336 # pylint: disable=unused-argument From 1f525fcd450e6ce5cab4fedaf408580b80238933 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 21:09:25 +0200 Subject: [PATCH 47/67] Split run_on_whole_repo functions to separate file and mark as internal --- .pre-commit-config.yaml | 2 +- .../{common.py => _common.py} | 101 ---------- .../_run_on_whole_repo.py | 103 ++++++++++ src/pre_commit_terraform/terraform_checkov.py | 8 +- src/pre_commit_terraform/terraform_fmt.py | 2 +- tests/pytest/test_common.py | 180 +----------------- tests/pytest/test_run_on_whole_repo.py | 112 +++++++++++ 7 files changed, 228 insertions(+), 280 deletions(-) rename src/pre_commit_terraform/{common.py => _common.py} (61%) create mode 100644 src/pre_commit_terraform/_run_on_whole_repo.py create mode 100644 tests/pytest/test_run_on_whole_repo.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ae7da76d..003ceb3b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,7 +136,7 @@ repos: - --allowed-module-metadata=__all__ # Default to '' - --max-local-variables=6 # Default to 5 - --max-returns=6 # Default to 5 - - --max-imports=13 # Default to 12 + - --max-imports=15 # Default to 12 # https://wemake-python-stylegui.de/en/latest/pages/usage/violations/index.html - --extend-ignore= E501 diff --git a/src/pre_commit_terraform/common.py b/src/pre_commit_terraform/_common.py similarity index 61% rename from src/pre_commit_terraform/common.py rename to src/pre_commit_terraform/_common.py index 4bf27678d..177bdab0f 100644 --- a/src/pre_commit_terraform/common.py +++ b/src/pre_commit_terraform/_common.py @@ -8,14 +8,9 @@ import logging import os -import re import shutil -import subprocess -from importlib.resources import files as access_artifacts_of from typing import Callable -import yaml - logger = logging.getLogger(__name__) @@ -191,99 +186,3 @@ def get_tf_binary_path(hook_config: list[str]) -> str: + ' hook configuration argument, or set the "PCT_TFPATH" environment variable, or set the' + ' "TERRAGRUNT_TFPATH" environment variable, or install Terraform or OpenTofu globally.', ) - - -# ? -# ? Related to run_hook_on_whole_repo functions -# ? -def is_function_defined(func_name: str, scope: dict) -> bool: - """ - Check if a function is defined in the global scope. - - Args: - func_name (str): The name of the function to check. - scope (dict): The scope (usually globals()) to check in. - - Returns: - bool: True if the function is defined, False otherwise. - """ - is_defined = func_name in scope - is_callable = callable(scope[func_name]) if is_defined else False - - logger.debug( - 'Checking if "%s":\n1. Defined in hook: %s\n2. Is it callable: %s', - func_name, - is_defined, - is_callable, - ) - - return is_defined and is_callable - - -def is_hook_run_on_whole_repo(hook_id: str, file_paths: list[str]) -> bool: - """ - Check if the hook is run on the whole repository. - - Args: - hook_id (str): The ID of the hook. - file_paths: The list of files paths. - - Returns: - bool: True if the hook is run on the whole repository, False otherwise. - - Raises: - ValueError: If the hook ID is not found in the .pre-commit-hooks.yaml file. - """ - logger.debug('Hook ID: %s', hook_id) - - # Get the directory containing the packaged `.pre-commit-hooks.yaml` copy - artifacts_root_path = access_artifacts_of('pre_commit_terraform') / '_artifacts' - pre_commit_hooks_yaml_path = artifacts_root_path / '.pre-commit-hooks.yaml' - pre_commit_hooks_yaml_path.read_text(encoding='utf-8') - - logger.debug('Hook config path: %s', pre_commit_hooks_yaml_path) - - # Read the .pre-commit-hooks.yaml file - pre_commit_hooks_yaml_txt = pre_commit_hooks_yaml_path.read_text(encoding='utf-8') - hooks_config = yaml.safe_load(pre_commit_hooks_yaml_txt) - - # Get the included and excluded file patterns for the given hook_id - for hook in hooks_config: - if hook['id'] == hook_id: - included_pattern = re.compile(hook.get('files', '')) - excluded_pattern = re.compile(hook.get('exclude', '')) - break - else: - raise ValueError(f'Hook ID "{hook_id}" not found in .pre-commit-hooks.yaml') - - logger.debug( - 'Included files pattern: %s\nExcluded files pattern: %s', - included_pattern, - excluded_pattern, - ) - # S607 disabled as we need to maintain ability to call git command no matter where it located. - git_ls_files_cmd = ['git', 'ls-files'] # noqa: S607 - # Get the sorted list of all files that can be checked using `git ls-files` - git_ls_file_paths = subprocess.check_output(git_ls_files_cmd, text=True).splitlines() - - if excluded_pattern: - all_file_paths_that_can_be_checked = [ - file_path - for file_path in git_ls_file_paths - if included_pattern.search(file_path) and not excluded_pattern.search(file_path) - ] - else: - all_file_paths_that_can_be_checked = [ - file_path for file_path in git_ls_file_paths if included_pattern.search(file_path) - ] - - # Get the sorted list of files passed to the hook - file_paths_to_check = sorted(file_paths) - logger.debug( - 'Files to check:\n%s\n\nAll files that can be checked:\n%s\n\nIdentical lists: %s', - file_paths_to_check, - all_file_paths_that_can_be_checked, - file_paths_to_check == all_file_paths_that_can_be_checked, - ) - # Compare the sorted lists of file - return file_paths_to_check == all_file_paths_that_can_be_checked diff --git a/src/pre_commit_terraform/_run_on_whole_repo.py b/src/pre_commit_terraform/_run_on_whole_repo.py new file mode 100644 index 000000000..878050dbe --- /dev/null +++ b/src/pre_commit_terraform/_run_on_whole_repo.py @@ -0,0 +1,103 @@ +"""Common functions to check if a hook is run on the whole repository.""" + +import logging +import re +import subprocess +from importlib.resources import files as access_artifacts_of + +import yaml + +logger = logging.getLogger(__name__) + + +def is_function_defined(func_name: str, scope: dict) -> bool: + """ + Check if a function is defined in the global scope. + + Args: + scope (dict): The scope (usually globals()) to check in. + func_name (str): The name of the function to check. + + Returns: + bool: True if the function is defined, False otherwise. + """ + is_defined = func_name in scope + is_callable = callable(scope[func_name]) if is_defined else False + + logger.debug( + 'Checking if "%s":\n1. Defined in hook: %s\n2. Is it callable: %s', + func_name, + is_defined, + is_callable, + ) + + return is_defined and is_callable + + +def is_hook_run_on_whole_repo(hook_id: str, file_paths: list[str]) -> bool: + """ + Check if the hook is run on the whole repository. + + Args: + hook_id (str): The ID of the hook. + file_paths: The list of files paths. + + Returns: + bool: True if the hook is run on the whole repository, False otherwise. + + Raises: + ValueError: If the hook ID is not found in the .pre-commit-hooks.yaml file. + """ + logger.debug('Hook ID: %s', hook_id) + + # Get the directory containing the packaged `.pre-commit-hooks.yaml` copy + artifacts_root_path = access_artifacts_of('pre_commit_terraform') / '_artifacts' + pre_commit_hooks_yaml_path = artifacts_root_path / '.pre-commit-hooks.yaml' + pre_commit_hooks_yaml_path.read_text(encoding='utf-8') + + logger.debug('Hook config path: %s', pre_commit_hooks_yaml_path) + + # Read the .pre-commit-hooks.yaml file + pre_commit_hooks_yaml_txt = pre_commit_hooks_yaml_path.read_text(encoding='utf-8') + hooks_config = yaml.safe_load(pre_commit_hooks_yaml_txt) + + # Get the included and excluded file patterns for the given hook_id + for hook in hooks_config: + if hook['id'] == hook_id: + included_pattern = re.compile(hook.get('files', '')) + excluded_pattern = re.compile(hook.get('exclude', '')) + break + else: + raise ValueError(f'Hook ID "{hook_id}" not found in .pre-commit-hooks.yaml') + + logger.debug( + 'Included files pattern: %s\nExcluded files pattern: %s', + included_pattern, + excluded_pattern, + ) + # S607 disabled as we need to maintain ability to call git command no matter where it located. + git_ls_files_cmd = ['git', 'ls-files'] # noqa: S607 + # Get the sorted list of all files that can be checked using `git ls-files` + git_ls_file_paths = subprocess.check_output(git_ls_files_cmd, text=True).splitlines() + + if excluded_pattern: + all_file_paths_that_can_be_checked = [ + file_path + for file_path in git_ls_file_paths + if included_pattern.search(file_path) and not excluded_pattern.search(file_path) + ] + else: + all_file_paths_that_can_be_checked = [ + file_path for file_path in git_ls_file_paths if included_pattern.search(file_path) + ] + + # Get the sorted list of files passed to the hook + file_paths_to_check = sorted(file_paths) + logger.debug( + 'Files to check:\n%s\n\nAll files that can be checked:\n%s\n\nIdentical lists: %s', + file_paths_to_check, + all_file_paths_that_can_be_checked, + file_paths_to_check == all_file_paths_that_can_be_checked, + ) + # Compare the sorted lists of file + return file_paths_to_check == all_file_paths_that_can_be_checked diff --git a/src/pre_commit_terraform/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py index 6afe0a805..023fa623c 100644 --- a/src/pre_commit_terraform/terraform_checkov.py +++ b/src/pre_commit_terraform/terraform_checkov.py @@ -12,7 +12,9 @@ from subprocess import run from typing import Final -from pre_commit_terraform import common +from pre_commit_terraform import _common as common +from pre_commit_terraform._run_on_whole_repo import is_function_defined +from pre_commit_terraform._run_on_whole_repo import is_hook_run_on_whole_repo from pre_commit_terraform._types import ReturnCodeType from pre_commit_terraform.logger import setup_logging @@ -70,8 +72,8 @@ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: # TODO: subprocess.run ignore colors. Try `rich` lib all_env_vars['ANSI_COLORS_DISABLED'] = 'true' # WPS421 - IDK how to check is function exist w/o passing globals() - if common.is_function_defined('run_hook_on_whole_repo', globals()): # noqa: WPS421 - if common.is_hook_run_on_whole_repo(HOOK_ID, parsed_cli_args.files): + if is_function_defined('run_hook_on_whole_repo', globals()): # noqa: WPS421 + if is_hook_run_on_whole_repo(HOOK_ID, parsed_cli_args.files): return run_hook_on_whole_repo(safe_args, all_env_vars) return common.per_dir_hook( diff --git a/src/pre_commit_terraform/terraform_fmt.py b/src/pre_commit_terraform/terraform_fmt.py index c6f43ef54..9007f28d3 100644 --- a/src/pre_commit_terraform/terraform_fmt.py +++ b/src/pre_commit_terraform/terraform_fmt.py @@ -12,7 +12,7 @@ from subprocess import run from typing import Final -from pre_commit_terraform import common +from pre_commit_terraform import _common as common from pre_commit_terraform._types import ReturnCodeType from pre_commit_terraform.logger import setup_logging diff --git a/tests/pytest/test_common.py b/tests/pytest/test_common.py index 40bbd3814..e9858e58d 100644 --- a/tests/pytest/test_common.py +++ b/tests/pytest/test_common.py @@ -1,20 +1,15 @@ # pylint: skip-file import os from os.path import join -from pathlib import Path import pytest -import yaml -from pre_commit_terraform.common import BinaryNotFoundError -from pre_commit_terraform.common import _get_unique_dirs -from pre_commit_terraform.common import expand_env_vars -from pre_commit_terraform.common import get_tf_binary_path -from pre_commit_terraform.common import is_function_defined -from pre_commit_terraform.common import is_hook_run_on_whole_repo -from pre_commit_terraform.common import parse_cmdline -from pre_commit_terraform.common import parse_env_vars -from pre_commit_terraform.common import per_dir_hook +from pre_commit_terraform._common import BinaryNotFoundError +from pre_commit_terraform._common import _get_unique_dirs +from pre_commit_terraform._common import expand_env_vars +from pre_commit_terraform._common import get_tf_binary_path +from pre_commit_terraform._common import parse_env_vars +from pre_commit_terraform._common import per_dir_hook # ? @@ -187,66 +182,6 @@ def test_parse_env_vars_with_empty_value(): assert result == {'VAR1': '', 'VAR2': ''} -def test_parse_cmdline_no_arguments(): - argv = [] - args, hook_config, files, tf_init_args, env_vars_strs = parse_cmdline(argv) - assert args == [] - assert hook_config == [] - assert files == [] - assert tf_init_args == [] - assert env_vars_strs == [] - - -def test_parse_cmdline_with_arguments(): - argv = ['-a', 'arg1', '-a', 'arg2', '-h', 'hook1', 'file1', 'file2'] - args, hook_config, files, tf_init_args, env_vars_strs = parse_cmdline(argv) - assert args == ['arg1', 'arg2'] - assert hook_config == ['hook1'] - assert files == ['file1', 'file2'] - assert tf_init_args == [] - assert env_vars_strs == [] - - -def test_parse_cmdline_with_env_vars(): - argv = ['-e', 'VAR1=value1', '-e', 'VAR2=value2'] - args, hook_config, files, tf_init_args, env_vars_strs = parse_cmdline(argv) - assert args == [] - assert hook_config == [] - assert files == [] - assert tf_init_args == [] - assert env_vars_strs == ['VAR1=value1', 'VAR2=value2'] - - -def test_parse_cmdline_with_tf_init_args(): - argv = ['-i', 'init1', '-i', 'init2'] - args, hook_config, files, tf_init_args, env_vars_strs = parse_cmdline(argv) - assert args == [] - assert hook_config == [] - assert files == [] - assert tf_init_args == ['init1', 'init2'] - assert env_vars_strs == [] - - -def test_parse_cmdline_with_files(): - argv = ['file1', 'file2'] - args, hook_config, files, tf_init_args, env_vars_strs = parse_cmdline(argv) - assert args == [] - assert hook_config == [] - assert files == ['file1', 'file2'] - assert tf_init_args == [] - assert env_vars_strs == [] - - -def test_parse_cmdline_with_hook_config(): - argv = ['-h', 'hook1', '-h', 'hook2'] - args, hook_config, files, tf_init_args, env_vars_strs = parse_cmdline(argv) - assert args == [] - assert hook_config == ['hook1', 'hook2'] - assert files == [] - assert tf_init_args == [] - assert env_vars_strs == [] - - # ? # ? expand_env_vars # ? @@ -341,108 +276,5 @@ def test_get_tf_binary_path_not_found(mocker): get_tf_binary_path(hook_config) -# ? -# ? is_function_defined -# ? - - -def test_is_function_defined_existing_function(): - def sample_function(): - pass - - scope = globals() - scope['sample_function'] = sample_function - - assert is_function_defined('sample_function', scope) is True - - -def test_is_function_defined_non_existing_function(): - scope = globals() - - assert is_function_defined('non_existing_function', scope) is False - - -def test_is_function_defined_non_callable(): - non_callable = 'I am not a function' - scope = globals() - scope['non_callable'] = non_callable - - assert is_function_defined('non_callable', scope) is False - - -def test_is_function_defined_callable_object(): - class CallableObject: - def __call__(self): - pass - - callable_object = CallableObject() - scope = globals() - scope['callable_object'] = callable_object - - assert is_function_defined('callable_object', scope) is True - - -# ? -# ? is_hook_run_on_whole_repo -# ? - - -@pytest.fixture -def mock_git_ls_files(): - return [ - 'environment/prd/backends.tf', - 'environment/prd/data.tf', - 'environment/prd/main.tf', - 'environment/prd/outputs.tf', - 'environment/prd/providers.tf', - 'environment/prd/variables.tf', - 'environment/prd/versions.tf', - 'environment/qa/backends.tf', - ] - - -@pytest.fixture -def mock_hooks_config(): - return [{'id': 'example_hook_id', 'files': r'\.tf$', 'exclude': r'\.terraform/.*$'}] - - -def test_is_hook_run_on_whole_repo(mocker, mock_git_ls_files, mock_hooks_config): - # Mock the return value of git ls-files - mocker.patch('subprocess.check_output', return_value='\n'.join(mock_git_ls_files)) - # Mock the return value of reading the .pre-commit-hooks.yaml file - mocker.patch('builtins.open', mocker.mock_open(read_data=yaml.dump(mock_hooks_config))) - # Mock the Path object to return a specific path - mock_path = mocker.patch('pathlib.Path.resolve') - mock_path.return_value.parents.__getitem__.return_value = Path('/mocked/path') - - # Test case where files match the included pattern and do not match the excluded pattern - files = [ - 'environment/prd/backends.tf', - 'environment/prd/data.tf', - 'environment/prd/main.tf', - 'environment/prd/outputs.tf', - 'environment/prd/providers.tf', - 'environment/prd/variables.tf', - 'environment/prd/versions.tf', - 'environment/qa/backends.tf', - ] - assert is_hook_run_on_whole_repo('example_hook_id', files) is True - - # Test case where files do not match the included pattern - files = ['environment/prd/README.md'] - assert is_hook_run_on_whole_repo('example_hook_id', files) is False - - # Test case where files match the excluded pattern - files = ['environment/prd/.terraform/config.tf'] - assert is_hook_run_on_whole_repo('example_hook_id', files) is False - - # Test case where hook_id is not found - with pytest.raises( - ValueError, - match='Hook ID "non_existing_hook_id" not found in .pre-commit-hooks.yaml', - ): - is_hook_run_on_whole_repo('non_existing_hook_id', files) - - if __name__ == '__main__': pytest.main() diff --git a/tests/pytest/test_run_on_whole_repo.py b/tests/pytest/test_run_on_whole_repo.py new file mode 100644 index 000000000..1a35e9cac --- /dev/null +++ b/tests/pytest/test_run_on_whole_repo.py @@ -0,0 +1,112 @@ +from pathlib import Path + +import pytest +import yaml + +from pre_commit_terraform._run_on_whole_repo import is_function_defined +from pre_commit_terraform._run_on_whole_repo import is_hook_run_on_whole_repo + + +# ? +# ? is_function_defined +# ? +def test_is_function_defined_existing_function(): + def sample_function(): + pass + + scope = globals() + scope['sample_function'] = sample_function + + assert is_function_defined('sample_function', scope) is True + + +def test_is_function_defined_non_existing_function(): + scope = globals() + + assert is_function_defined('non_existing_function', scope) is False + + +def test_is_function_defined_non_callable(): + non_callable = 'I am not a function' + scope = globals() + scope['non_callable'] = non_callable + + assert is_function_defined('non_callable', scope) is False + + +def test_is_function_defined_callable_object(): + class CallableObject: + def __call__(self): + pass + + callable_object = CallableObject() + scope = globals() + scope['callable_object'] = callable_object + + assert is_function_defined('callable_object', scope) is True + + +# ? +# ? is_hook_run_on_whole_repo +# ? + + +@pytest.fixture +def mock_git_ls_files(): + return [ + 'environment/prd/backends.tf', + 'environment/prd/data.tf', + 'environment/prd/main.tf', + 'environment/prd/outputs.tf', + 'environment/prd/providers.tf', + 'environment/prd/variables.tf', + 'environment/prd/versions.tf', + 'environment/qa/backends.tf', + ] + + +@pytest.fixture +def mock_hooks_config(): + return [{'id': 'example_hook_id', 'files': r'\.tf$', 'exclude': r'\.terraform/.*$'}] + + +def test_is_hook_run_on_whole_repo(mocker, mock_git_ls_files, mock_hooks_config): + # Mock the return value of git ls-files + mocker.patch('subprocess.check_output', return_value='\n'.join(mock_git_ls_files)) + # Mock the return value of reading the .pre-commit-hooks.yaml file + mocker.patch('builtins.open', mocker.mock_open(read_data=yaml.dump(mock_hooks_config))) + # Mock the Path object to return a specific path + mock_path = mocker.patch('pathlib.Path.resolve') + mock_path.return_value.parents.__getitem__.return_value = Path('/mocked/path') + + # Test case where files match the included pattern and do not match the excluded pattern + files = [ + 'environment/prd/backends.tf', + 'environment/prd/data.tf', + 'environment/prd/main.tf', + 'environment/prd/outputs.tf', + 'environment/prd/providers.tf', + 'environment/prd/variables.tf', + 'environment/prd/versions.tf', + 'environment/qa/backends.tf', + ] + assert is_hook_run_on_whole_repo('example_hook_id', files) is True + + # Test case where files do not match the included pattern + files = ['environment/prd/README.md'] + assert is_hook_run_on_whole_repo('example_hook_id', files) is False + + # Test case where files match the excluded pattern + files = ['environment/prd/.terraform/config.tf'] + assert is_hook_run_on_whole_repo('example_hook_id', files) is False + + # Test case where hook_id is not found + with pytest.raises( + ValueError, + match='Hook ID "non_existing_hook_id" not found in .pre-commit-hooks.yaml', + ): + is_hook_run_on_whole_repo('non_existing_hook_id', files) + + +if __name__ == '__main__': + pytest.main() From 79b633ecc611e726a526337527cfa39cbea23611 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 21:13:04 +0200 Subject: [PATCH 48/67] Mark logger as internal --- src/pre_commit_terraform/{logger.py => _logger.py} | 0 src/pre_commit_terraform/terraform_checkov.py | 2 +- src/pre_commit_terraform/terraform_fmt.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/pre_commit_terraform/{logger.py => _logger.py} (100%) diff --git a/src/pre_commit_terraform/logger.py b/src/pre_commit_terraform/_logger.py similarity index 100% rename from src/pre_commit_terraform/logger.py rename to src/pre_commit_terraform/_logger.py diff --git a/src/pre_commit_terraform/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py index 023fa623c..a8e3855e3 100644 --- a/src/pre_commit_terraform/terraform_checkov.py +++ b/src/pre_commit_terraform/terraform_checkov.py @@ -13,10 +13,10 @@ from typing import Final from pre_commit_terraform import _common as common +from pre_commit_terraform._logger import setup_logging from pre_commit_terraform._run_on_whole_repo import is_function_defined from pre_commit_terraform._run_on_whole_repo import is_hook_run_on_whole_repo from pre_commit_terraform._types import ReturnCodeType -from pre_commit_terraform.logger import setup_logging logger = logging.getLogger(__name__) diff --git a/src/pre_commit_terraform/terraform_fmt.py b/src/pre_commit_terraform/terraform_fmt.py index 9007f28d3..ec68fe9f8 100644 --- a/src/pre_commit_terraform/terraform_fmt.py +++ b/src/pre_commit_terraform/terraform_fmt.py @@ -13,8 +13,8 @@ from typing import Final from pre_commit_terraform import _common as common +from pre_commit_terraform._logger import setup_logging from pre_commit_terraform._types import ReturnCodeType -from pre_commit_terraform.logger import setup_logging logger = logging.getLogger(__name__) From 294cf79270be711fe5abe261c858b9cbf5b12aeb Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 21:20:12 +0200 Subject: [PATCH 49/67] Fix acidental refactoring artifact --- src/pre_commit_terraform/_common.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/pre_commit_terraform/_common.py b/src/pre_commit_terraform/_common.py index 177bdab0f..3c356855d 100644 --- a/src/pre_commit_terraform/_common.py +++ b/src/pre_commit_terraform/_common.py @@ -156,10 +156,6 @@ def get_tf_binary_path(hook_config: list[str]) -> str: hook_config_tf_path = config.split('=', 1)[1].rstrip(';') return hook_config_tf_path - # direct hook config, has the highest precedence - if hook_config_tf_path: - return hook_config_tf_path - # environment variable pct_tfpath = os.getenv('PCT_TFPATH') if pct_tfpath: From f1673ce9b7e14c213301d6d9ec62f8a0c4de5940 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 21:50:05 +0200 Subject: [PATCH 50/67] Fixing and extend tests for terraform_fmt and terraform_checkov --- tests/pytest/test_terraform_checkov.py | 269 ++++++++++++++++++++++++- tests/pytest/test_terraform_fmt.py | 211 +++++++++---------- 2 files changed, 365 insertions(+), 115 deletions(-) diff --git a/tests/pytest/test_terraform_checkov.py b/tests/pytest/test_terraform_checkov.py index 4e5916c1a..f633beb8d 100644 --- a/tests/pytest/test_terraform_checkov.py +++ b/tests/pytest/test_terraform_checkov.py @@ -1,10 +1,18 @@ +import os +from argparse import Namespace +from subprocess import PIPE + import pytest +from pre_commit_terraform.terraform_checkov import invoke_cli_app +from pre_commit_terraform.terraform_checkov import per_dir_hook_unique_part from pre_commit_terraform.terraform_checkov import replace_git_working_dir_to_repo_root - -# FILE: hooks/test_terraform_checkov.py +from pre_commit_terraform.terraform_checkov import run_hook_on_whole_repo +# ? +# ? replace_git_working_dir_to_repo_root +# ? def test_replace_git_working_dir_to_repo_root_empty(): args = [] result = replace_git_working_dir_to_repo_root(args) @@ -38,5 +46,262 @@ def test_replace_git_working_dir_to_repo_root_partial_replacement(mocker): assert result == ['arg1', '/current/working/dir/arg2', 'arg3'] +# ? +# ? invoke_cli_app +# ? +def test_invoke_cli_app_no_color(mocker): + mock_parsed_cli_args = Namespace( + hook_config=[], + files=['file1.tf', 'file2.tf'], + args=['-d', '.'], + env_vars=['ENV_VAR=value'], + ) + mock_env_vars = {'ENV_VAR': 'value', 'PRE_COMMIT_COLOR': 'never'} + + mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_checkov.setup_logging') + mock_expand_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_checkov.common.expand_env_vars', + return_value=['-d', '.'], + ) + mock_parse_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_checkov.common.parse_env_vars', + return_value=mock_env_vars, + ) + mock_per_dir_hook = mocker.patch( + 'pre_commit_terraform.terraform_checkov.common.per_dir_hook', + return_value=0, + ) + mock_run_hook_on_whole_repo = mocker.patch( + 'pre_commit_terraform.terraform_checkov.run_hook_on_whole_repo', + return_value=0, + ) + mocker.patch( + 'pre_commit_terraform.terraform_checkov.is_function_defined', + return_value=True, + ) + mocker.patch( + 'pre_commit_terraform.terraform_checkov.is_hook_run_on_whole_repo', + return_value=False, + ) + + result = invoke_cli_app(mock_parsed_cli_args) + + mock_setup_logging.assert_called_once() + mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) + mock_expand_env_vars.assert_called_once_with( + mock_parsed_cli_args.args, + {**os.environ, **mock_env_vars}, + ) + mock_per_dir_hook.assert_called_once() + mock_run_hook_on_whole_repo.assert_not_called() + assert result == 0 + + +def test_invoke_cli_app_run_on_whole_repo(mocker): + mock_parsed_cli_args = Namespace( + hook_config=[], + files=['file1.tf', 'file2.tf'], + args=['-d', '.'], + env_vars=['ENV_VAR=value'], + ) + mock_env_vars = {'ENV_VAR': 'value'} + + mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_checkov.setup_logging') + mock_expand_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_checkov.common.expand_env_vars', + return_value=['-d', '.'], + ) + mock_parse_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_checkov.common.parse_env_vars', + return_value=mock_env_vars, + ) + mock_per_dir_hook = mocker.patch( + 'pre_commit_terraform.terraform_checkov.common.per_dir_hook', + return_value=0, + ) + mock_run_hook_on_whole_repo = mocker.patch( + 'pre_commit_terraform.terraform_checkov.run_hook_on_whole_repo', + return_value=0, + ) + mocker.patch( + 'pre_commit_terraform.terraform_checkov.is_function_defined', + return_value=True, + ) + mocker.patch( + 'pre_commit_terraform.terraform_checkov.is_hook_run_on_whole_repo', + return_value=True, + ) + + result = invoke_cli_app(mock_parsed_cli_args) + + mock_setup_logging.assert_called_once() + mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) + mock_expand_env_vars.assert_called_once_with( + mock_parsed_cli_args.args, + {**os.environ, **mock_env_vars}, + ) + mock_run_hook_on_whole_repo.assert_called_once() + mock_per_dir_hook.assert_not_called() + assert result == 0 + + +def test_invoke_cli_app_per_dir_hook(mocker): + mock_parsed_cli_args = Namespace( + hook_config=[], + files=['file1.tf', 'file2.tf'], + args=['-d', '.'], + env_vars=['ENV_VAR=value'], + ) + mock_env_vars = {'ENV_VAR': 'value'} + + mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_checkov.setup_logging') + mock_expand_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_checkov.common.expand_env_vars', + return_value=['-d', '.'], + ) + mock_parse_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_checkov.common.parse_env_vars', + return_value=mock_env_vars, + ) + mock_per_dir_hook = mocker.patch( + 'pre_commit_terraform.terraform_checkov.common.per_dir_hook', + return_value=0, + ) + mock_run_hook_on_whole_repo = mocker.patch( + 'pre_commit_terraform.terraform_checkov.run_hook_on_whole_repo', + return_value=0, + ) + mocker.patch( + 'pre_commit_terraform.terraform_checkov.is_function_defined', + return_value=False, + ) + + result = invoke_cli_app(mock_parsed_cli_args) + + mock_setup_logging.assert_called_once() + mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) + mock_expand_env_vars.assert_called_once_with( + mock_parsed_cli_args.args, + {**os.environ, **mock_env_vars}, + ) + mock_per_dir_hook.assert_called_once() + mock_run_hook_on_whole_repo.assert_not_called() + assert result == 0 + + +# ? +# ? run_hook_on_whole_repo +# ? +def test_run_hook_on_whole_repo_success(mocker): + mock_args = ['-d', '.'] + mock_env_vars = {'ENV_VAR': 'value'} + mock_completed_process = mocker.MagicMock() + mock_completed_process.returncode = 0 + mock_completed_process.stdout = 'Checkov output' + + mock_run = mocker.patch( + 'pre_commit_terraform.terraform_checkov.run', + return_value=mock_completed_process, + ) + mock_sys_stdout_write = mocker.patch('sys.stdout.write') + + result = run_hook_on_whole_repo(mock_args, mock_env_vars) + + mock_run.assert_called_once_with( + ['checkov', '-d', '.', *mock_args], + env=mock_env_vars, + text=True, + stdout=PIPE, + check=False, + ) + mock_sys_stdout_write.assert_called_once_with('Checkov output') + assert result == 0 + + +def test_run_hook_on_whole_repo_failure(mocker): + mock_args = ['-d', '.'] + mock_env_vars = {'ENV_VAR': 'value'} + mock_completed_process = mocker.MagicMock() + mock_completed_process.returncode = 1 + mock_completed_process.stdout = 'Checkov error output' + + mock_run = mocker.patch( + 'pre_commit_terraform.terraform_checkov.run', + return_value=mock_completed_process, + ) + mock_sys_stdout_write = mocker.patch('sys.stdout.write') + + result = run_hook_on_whole_repo(mock_args, mock_env_vars) + + mock_run.assert_called_once_with( + ['checkov', '-d', '.', *mock_args], + env=mock_env_vars, + text=True, + stdout=PIPE, + check=False, + ) + mock_sys_stdout_write.assert_called_once_with('Checkov error output') + assert result == 1 + + +# ? +# ? per_dir_hook_unique_part +# ? +def test_per_dir_hook_unique_part_success(mocker): + tf_path = '/usr/local/bin/terraform' + dir_path = 'test_dir' + args = ['-d', '.'] + env_vars = {'ENV_VAR': 'value'} + mock_completed_process = mocker.MagicMock() + mock_completed_process.returncode = 0 + mock_completed_process.stdout = 'Checkov output' + + mock_run = mocker.patch( + 'pre_commit_terraform.terraform_checkov.run', + return_value=mock_completed_process, + ) + mock_sys_stdout_write = mocker.patch('sys.stdout.write') + + result = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) + + mock_run.assert_called_once_with( + ['checkov', '-d', dir_path, *args], + env=env_vars, + text=True, + stdout=PIPE, + check=False, + ) + mock_sys_stdout_write.assert_called_once_with('Checkov output') + assert result == 0 + + +def test_per_dir_hook_unique_part_failure(mocker): + tf_path = '/usr/local/bin/terraform' + dir_path = 'test_dir' + args = ['-d', '.'] + env_vars = {'ENV_VAR': 'value'} + mock_completed_process = mocker.MagicMock() + mock_completed_process.returncode = 1 + mock_completed_process.stdout = 'Checkov error output' + + mock_run = mocker.patch( + 'pre_commit_terraform.terraform_checkov.run', + return_value=mock_completed_process, + ) + mock_sys_stdout_write = mocker.patch('sys.stdout.write') + + result = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) + + mock_run.assert_called_once_with( + ['checkov', '-d', dir_path, *args], + env=env_vars, + text=True, + stdout=PIPE, + check=False, + ) + mock_sys_stdout_write.assert_called_once_with('Checkov error output') + assert result == 1 + + if __name__ == '__main__': pytest.main() diff --git a/tests/pytest/test_terraform_fmt.py b/tests/pytest/test_terraform_fmt.py index 286d1d326..cbc8a189a 100644 --- a/tests/pytest/test_terraform_fmt.py +++ b/tests/pytest/test_terraform_fmt.py @@ -1,153 +1,138 @@ -# pylint: skip-file import os -import sys -from subprocess import PIPE +import subprocess +from argparse import Namespace import pytest -from pre_commit_terraform.terraform_fmt import main +from pre_commit_terraform.terraform_fmt import invoke_cli_app from pre_commit_terraform.terraform_fmt import per_dir_hook_unique_part @pytest.fixture -def mock_setup_logging(mocker): - return mocker.patch('pre_commit_terraform.terraform_fmt.setup_logging') - - -@pytest.fixture -def mock_parse_cmdline(mocker): - return mocker.patch('pre_commit_terraform.terraform_fmt.common.parse_cmdline') - - -@pytest.fixture -def mock_parse_env_vars(mocker): - return mocker.patch('pre_commit_terraform.terraform_fmt.common.parse_env_vars') - - -@pytest.fixture -def mock_expand_env_vars(mocker): - return mocker.patch('pre_commit_terraform.terraform_fmt.common.expand_env_vars') +def mock_parsed_cli_args(): + return Namespace( + hook_config=[], + files=['file1.tf', 'file2.tf'], + args=['-diff'], + env_vars=['ENV_VAR=value'], + ) @pytest.fixture -def mock_per_dir_hook(mocker): - return mocker.patch('pre_commit_terraform.terraform_fmt.common.per_dir_hook') - +def mock_env_vars(): + return {'ENV_VAR': 'value', 'PRE_COMMIT_COLOR': 'always'} -@pytest.fixture -def mock_run(mocker): - return mocker.patch('pre_commit_terraform.terraform_fmt.run') - - -def test_main( - mocker, - mock_setup_logging, - mock_parse_cmdline, - mock_parse_env_vars, - mock_expand_env_vars, - mock_per_dir_hook, -): - mock_parse_cmdline.return_value = (['arg1'], ['hook1'], ['file1'], [], ['VAR1=value1']) - mock_parse_env_vars.return_value = {'VAR1': 'value1'} - mock_expand_env_vars.return_value = ['expanded_arg1'] - mock_per_dir_hook.return_value = 0 - - mocker.patch.object(sys, 'argv', ['terraform_fmt.py']) - exit_code = main(sys.argv) - assert exit_code == 0 - mock_setup_logging.assert_called_once() - mock_parse_cmdline.assert_called_once_with(['terraform_fmt.py']) - mock_parse_env_vars.assert_called_once_with(['VAR1=value1']) - mock_expand_env_vars.assert_called_once_with(['arg1'], {**os.environ, 'VAR1': 'value1'}) - mock_per_dir_hook.assert_called_once_with( - ['hook1'], - ['file1'], - ['expanded_arg1'], - {**os.environ, 'VAR1': 'value1'}, - mocker.ANY, +def test_invoke_cli_app(mocker, mock_parsed_cli_args, mock_env_vars): + mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_fmt.setup_logging') + mock_expand_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_fmt.common.expand_env_vars', + return_value=['-diff'], + ) + mock_parse_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_fmt.common.parse_env_vars', + return_value=mock_env_vars, + ) + mock_run = mocker.patch( + 'pre_commit_terraform.terraform_fmt.run', + return_value=subprocess.CompletedProcess( + args=['terraform', 'fmt'], + returncode=0, + stdout='Formatted output', + ), ) - -def test_main_with_no_color( - mocker, - mock_setup_logging, - mock_parse_cmdline, - mock_parse_env_vars, - mock_expand_env_vars, - mock_per_dir_hook, -): - mock_parse_cmdline.return_value = (['arg1'], ['hook1'], ['file1'], [], ['VAR1=value1']) - mock_parse_env_vars.return_value = {'VAR1': 'value1'} - mock_expand_env_vars.return_value = ['expanded_arg1'] - mock_per_dir_hook.return_value = 0 - - mocker.patch.dict(os.environ, {'PRE_COMMIT_COLOR': 'never'}) - mocker.patch.object(sys, 'argv', ['terraform_fmt.py']) - exit_code = main(sys.argv) - assert exit_code == 0 + result = invoke_cli_app(mock_parsed_cli_args) mock_setup_logging.assert_called_once() - mock_parse_cmdline.assert_called_once_with(['terraform_fmt.py']) - mock_parse_env_vars.assert_called_once_with(['VAR1=value1']) + mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) mock_expand_env_vars.assert_called_once_with( - ['arg1', '-no-color'], - {**os.environ, 'VAR1': 'value1'}, - ) - mock_per_dir_hook.assert_called_once_with( - ['hook1'], - ['file1'], - ['expanded_arg1'], - {**os.environ, 'VAR1': 'value1'}, - mocker.ANY, + mock_parsed_cli_args.args, + {**os.environ, **mock_env_vars}, ) + mock_run.assert_called_once() + assert result == 0 -def test_per_dir_hook_unique_part(mocker, mock_run): - tf_path = '/path/to/terraform' - dir_path = '/path/to/dir' - args = ['arg1', 'arg2'] - env_vars = {'VAR1': 'value1'} - mock_completed_process = mocker.MagicMock() - mock_completed_process.stdout = 'output' - mock_completed_process.returncode = 0 - mock_run.return_value = mock_completed_process +def test_per_dir_hook_unique_part(mocker): + tf_path = '/usr/local/bin/terraform' + dir_path = 'test_dir' + args = ['-diff'] + env_vars = {'ENV_VAR': 'value'} - exit_code = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) - assert exit_code == 0 + mock_run = mocker.patch( + 'pre_commit_terraform.terraform_fmt.run', + return_value=subprocess.CompletedProcess(args, 0, stdout='Formatted output'), + ) + + result = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) + expected_cmd = [tf_path, 'fmt', *args, dir_path] mock_run.assert_called_once_with( - ['/path/to/terraform', 'fmt', 'arg1', 'arg2', '/path/to/dir'], + expected_cmd, env=env_vars, text=True, - stdout=PIPE, + stdout=subprocess.PIPE, check=False, ) + assert result == 0 + + +def test_invoke_cli_app_no_color(mocker, mock_parsed_cli_args, mock_env_vars): + mock_env_vars['PRE_COMMIT_COLOR'] = 'never' + mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_fmt.setup_logging') + mock_expand_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_fmt.common.expand_env_vars', + return_value=['-diff'], + ) + mock_parse_env_vars = mocker.patch( + 'pre_commit_terraform.terraform_fmt.common.parse_env_vars', + return_value=mock_env_vars, + ) + mock_run = mocker.patch( + 'pre_commit_terraform.terraform_fmt.run', + return_value=subprocess.CompletedProcess( + args=['terraform', 'fmt'], + returncode=0, + stdout='Formatted output', + ), + ) + + result = invoke_cli_app(mock_parsed_cli_args) -def test_per_dir_hook_unique_part_with_error(mocker, mock_run): - tf_path = '/path/to/terraform' - dir_path = '/path/to/dir' - args = ['arg1', 'arg2'] - env_vars = {'VAR1': 'value1'} + mock_setup_logging.assert_called_once() + mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) + mock_expand_env_vars.assert_called_once_with( + mock_parsed_cli_args.args, + {**os.environ, **mock_env_vars}, + ) + mock_run.assert_called_once() + + assert result == 0 - mock_completed_process = mocker.MagicMock() - mock_completed_process.stdout = 'error output' - mock_completed_process.returncode = 1 - mock_run.return_value = mock_completed_process - exit_code = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) - assert exit_code == 1 +def test_per_dir_hook_unique_part_failure(mocker): + tf_path = '/usr/local/bin/terraform' + dir_path = 'test_dir' + args = ['-diff'] + env_vars = {'ENV_VAR': 'value'} + mock_run = mocker.patch( + 'pre_commit_terraform.terraform_fmt.run', + return_value=subprocess.CompletedProcess(args, 1, stdout='Error output'), + ) + + result = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) + + expected_cmd = [tf_path, 'fmt', *args, dir_path] mock_run.assert_called_once_with( - ['/path/to/terraform', 'fmt', 'arg1', 'arg2', '/path/to/dir'], + expected_cmd, env=env_vars, text=True, - stdout=PIPE, + stdout=subprocess.PIPE, check=False, ) - -if __name__ == '__main__': - pytest.main() + assert result == 1 From 0e03e6884056f549aa8eea0afab640234c46a763 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 21:55:09 +0200 Subject: [PATCH 51/67] Fix all tests --- tests/pytest/test_run_on_whole_repo.py | 4 ++-- tests/pytest/test_terraform_fmt.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/pytest/test_run_on_whole_repo.py b/tests/pytest/test_run_on_whole_repo.py index 1a35e9cac..aa5d22e63 100644 --- a/tests/pytest/test_run_on_whole_repo.py +++ b/tests/pytest/test_run_on_whole_repo.py @@ -49,8 +49,6 @@ def __call__(self): # ? # ? is_hook_run_on_whole_repo # ? - - @pytest.fixture def mock_git_ls_files(): return [ @@ -78,6 +76,8 @@ def test_is_hook_run_on_whole_repo(mocker, mock_git_ls_files, mock_hooks_config) # Mock the Path object to return a specific path mock_path = mocker.patch('pathlib.Path.resolve') mock_path.return_value.parents.__getitem__.return_value = Path('/mocked/path') + # Mock the read_text method of Path to return the hooks config + mocker.patch('pathlib.Path.read_text', return_value=yaml.dump(mock_hooks_config)) # Test case where files match the included pattern and do not match the excluded pattern files = [ diff --git a/tests/pytest/test_terraform_fmt.py b/tests/pytest/test_terraform_fmt.py index cbc8a189a..60cba7e7a 100644 --- a/tests/pytest/test_terraform_fmt.py +++ b/tests/pytest/test_terraform_fmt.py @@ -8,6 +8,9 @@ from pre_commit_terraform.terraform_fmt import per_dir_hook_unique_part +# ? +# ? invoke_cli_app +# ? @pytest.fixture def mock_parsed_cli_args(): return Namespace( @@ -55,6 +58,9 @@ def test_invoke_cli_app(mocker, mock_parsed_cli_args, mock_env_vars): assert result == 0 +# ? +# ? per_dir_hook_unique_part +# ? def test_per_dir_hook_unique_part(mocker): tf_path = '/usr/local/bin/terraform' dir_path = 'test_dir' From 83c7a29da89c298969a351e93c993b8c18da0750 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 22:11:37 +0200 Subject: [PATCH 52/67] Remove not needed pytests imports and add main.py tests --- tests/pytest/test_common.py | 4 --- tests/pytest/test_main.py | 44 ++++++++++++++++++++++++++ tests/pytest/test_run_on_whole_repo.py | 4 --- tests/pytest/test_terraform_checkov.py | 6 ---- 4 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 tests/pytest/test_main.py diff --git a/tests/pytest/test_common.py b/tests/pytest/test_common.py index e9858e58d..efe36a1ef 100644 --- a/tests/pytest/test_common.py +++ b/tests/pytest/test_common.py @@ -274,7 +274,3 @@ def test_get_tf_binary_path_not_found(mocker): + ' "TERRAGRUNT_TFPATH" environment variable, or install Terraform or OpenTofu globally.', ): get_tf_binary_path(hook_config) - - -if __name__ == '__main__': - pytest.main() diff --git a/tests/pytest/test_main.py b/tests/pytest/test_main.py new file mode 100644 index 000000000..3c7dffa55 --- /dev/null +++ b/tests/pytest/test_main.py @@ -0,0 +1,44 @@ +import importlib +import sys + + +def test_main_success(mocker): + mock_argv = ['__main__.py', 'arg1', 'arg2'] + mock_return_code = 0 + + mocker.patch('sys.argv', mock_argv) + mock_invoke_cli_app = mocker.patch( + 'pre_commit_terraform._cli.invoke_cli_app', + return_value=mock_return_code, + ) + mock_exit = mocker.patch('sys.exit') + + # Reload the module to trigger the main logic + if 'pre_commit_terraform.__main__' in sys.modules: + importlib.reload(sys.modules['pre_commit_terraform.__main__']) + else: + import pre_commit_terraform.__main__ # noqa: F401 + + mock_invoke_cli_app.assert_called_once_with(mock_argv[1:]) + mock_exit.assert_called_once_with(mock_return_code) + + +def test_main_failure(mocker): + mock_argv = ['__main__.py', 'arg1', 'arg2'] + mock_return_code = 1 + + mocker.patch('sys.argv', mock_argv) + mock_invoke_cli_app = mocker.patch( + 'pre_commit_terraform._cli.invoke_cli_app', + return_value=mock_return_code, + ) + mock_exit = mocker.patch('sys.exit') + + # Reload the module to trigger the main logic + if 'pre_commit_terraform.__main__' in sys.modules: + importlib.reload(sys.modules['pre_commit_terraform.__main__']) + else: + import pre_commit_terraform.__main__ # noqa: F401 + + mock_invoke_cli_app.assert_called_once_with(mock_argv[1:]) + mock_exit.assert_called_once_with(mock_return_code) diff --git a/tests/pytest/test_run_on_whole_repo.py b/tests/pytest/test_run_on_whole_repo.py index aa5d22e63..96e2b6ace 100644 --- a/tests/pytest/test_run_on_whole_repo.py +++ b/tests/pytest/test_run_on_whole_repo.py @@ -106,7 +106,3 @@ def test_is_hook_run_on_whole_repo(mocker, mock_git_ls_files, mock_hooks_config) match='Hook ID "non_existing_hook_id" not found in .pre-commit-hooks.yaml', ): is_hook_run_on_whole_repo('non_existing_hook_id', files) - - -if __name__ == '__main__': - pytest.main() diff --git a/tests/pytest/test_terraform_checkov.py b/tests/pytest/test_terraform_checkov.py index f633beb8d..3fc2ce453 100644 --- a/tests/pytest/test_terraform_checkov.py +++ b/tests/pytest/test_terraform_checkov.py @@ -2,8 +2,6 @@ from argparse import Namespace from subprocess import PIPE -import pytest - from pre_commit_terraform.terraform_checkov import invoke_cli_app from pre_commit_terraform.terraform_checkov import per_dir_hook_unique_part from pre_commit_terraform.terraform_checkov import replace_git_working_dir_to_repo_root @@ -301,7 +299,3 @@ def test_per_dir_hook_unique_part_failure(mocker): ) mock_sys_stdout_write.assert_called_once_with('Checkov error output') assert result == 1 - - -if __name__ == '__main__': - pytest.main() From 650d90c964ee2093795dbd90f10351589e680e69 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 22:19:28 +0200 Subject: [PATCH 53/67] Add tests for cli_parsing --- tests/pytest/test_cli_parsing.py | 223 +++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 tests/pytest/test_cli_parsing.py diff --git a/tests/pytest/test_cli_parsing.py b/tests/pytest/test_cli_parsing.py new file mode 100644 index 000000000..61b8ebcf5 --- /dev/null +++ b/tests/pytest/test_cli_parsing.py @@ -0,0 +1,223 @@ +from argparse import ArgumentParser + +import pytest + +from pre_commit_terraform._cli_parsing import attach_subcommand_parsers_to +from pre_commit_terraform._cli_parsing import initialize_argument_parser +from pre_commit_terraform._cli_parsing import populate_common_argument_parser + + +# ? +# ? populate_common_argument_parser +# ? +def test_populate_common_argument_parser(mocker): + parser = ArgumentParser(add_help=False) + populate_common_argument_parser(parser) + args = parser.parse_args( + ['-a', 'arg1', '-h', 'hook1', '-i', 'init1', '-e', 'env1', 'file1', 'file2'], + ) + + assert args.args == ['arg1'] + assert args.hook_config == ['hook1'] + assert args.tf_init_args == ['init1'] + assert args.env_vars == ['env1'] + assert args.files == ['file1', 'file2'] + + +def test_populate_common_argument_parser_defaults(mocker): + parser = ArgumentParser(add_help=False) + populate_common_argument_parser(parser) + args = parser.parse_args([]) + + assert args.args == [] + assert args.hook_config == [] + assert args.tf_init_args == [] + assert args.env_vars == [] + assert args.files == [] + + +def test_populate_common_argument_parser_multiple_values(mocker): + parser = ArgumentParser(add_help=False) + populate_common_argument_parser(parser) + args = parser.parse_args( + [ + '-a', + 'arg1', + '-a', + 'arg2', + '-h', + 'hook1', + '-h', + 'hook2', + '-i', + 'init1', + '-i', + 'init2', + '-e', + 'env1', + '-e', + 'env2', + 'file1', + 'file2', + ], + ) + + assert args.args == ['arg1', 'arg2'] + assert args.hook_config == ['hook1', 'hook2'] + assert args.tf_init_args == ['init1', 'init2'] + assert args.env_vars == ['env1', 'env2'] + assert args.files == ['file1', 'file2'] + + +# ? +# ? attach_subcommand_parsers_to +# ? +def test_attach_subcommand_parsers_to(mocker): + parser = ArgumentParser(add_help=False) + mock_subcommand_module = mocker.MagicMock() + mock_subcommand_module.HOOK_ID = 'mock_hook' + mock_subcommand_module.invoke_cli_app = mocker.Mock() + mock_subcommand_module.populate_hook_specific_argument_parser = mocker.Mock() + + mocker.patch('pre_commit_terraform._cli_parsing.SUBCOMMAND_MODULES', [mock_subcommand_module]) + + attach_subcommand_parsers_to(parser) + + args = parser.parse_args( + ['mock_hook', '-a', 'arg1', '-h', 'hook1', '-i', 'init1', '-e', 'env1', 'file1', 'file2'], + ) + + assert args.check_name == 'mock_hook' + assert args.args == ['arg1'] + assert args.hook_config == ['hook1'] + assert args.tf_init_args == ['init1'] + assert args.env_vars == ['env1'] + assert args.files == ['file1', 'file2'] + assert args.invoke_cli_app == mock_subcommand_module.invoke_cli_app + + mock_subcommand_module.populate_hook_specific_argument_parser.assert_called_once() + + +def test_attach_subcommand_parsers_to_no_args(mocker): + parser = ArgumentParser(add_help=False) + mock_subcommand_module = mocker.MagicMock() + mock_subcommand_module.HOOK_ID = 'mock_hook' + mock_subcommand_module.invoke_cli_app = mocker.Mock() + mock_subcommand_module.populate_hook_specific_argument_parser = mocker.Mock() + + mocker.patch('pre_commit_terraform._cli_parsing.SUBCOMMAND_MODULES', [mock_subcommand_module]) + + attach_subcommand_parsers_to(parser) + + with pytest.raises(SystemExit): + parser.parse_args([]) + + +def test_attach_subcommand_parsers_to_multiple_subcommands(mocker): + parser = ArgumentParser(add_help=False) + mock_subcommand_module1 = mocker.MagicMock() + mock_subcommand_module1.HOOK_ID = 'mock_hook1' + mock_subcommand_module1.invoke_cli_app = mocker.Mock() + mock_subcommand_module1.populate_hook_specific_argument_parser = mocker.Mock() + + mock_subcommand_module2 = mocker.MagicMock() + mock_subcommand_module2.HOOK_ID = 'mock_hook2' + mock_subcommand_module2.invoke_cli_app = mocker.Mock() + mock_subcommand_module2.populate_hook_specific_argument_parser = mocker.Mock() + + mocker.patch( + 'pre_commit_terraform._cli_parsing.SUBCOMMAND_MODULES', + [mock_subcommand_module1, mock_subcommand_module2], + ) + + attach_subcommand_parsers_to(parser) + + args1 = parser.parse_args(['mock_hook1', '-a', 'arg1']) + assert args1.check_name == 'mock_hook1' + assert args1.args == ['arg1'] + assert args1.invoke_cli_app == mock_subcommand_module1.invoke_cli_app + + args2 = parser.parse_args(['mock_hook2', '-a', 'arg2']) + assert args2.check_name == 'mock_hook2' + assert args2.args == ['arg2'] + assert args2.invoke_cli_app == mock_subcommand_module2.invoke_cli_app + + mock_subcommand_module1.populate_hook_specific_argument_parser.assert_called_once() + mock_subcommand_module2.populate_hook_specific_argument_parser.assert_called_once() + + +# ? +# ? initialize_argument_parser +# ? +def test_initialize_argument_parser(mocker): + mock_subcommand_module = mocker.MagicMock() + mock_subcommand_module.HOOK_ID = 'mock_hook' + mock_subcommand_module.invoke_cli_app = mocker.Mock() + mock_subcommand_module.populate_hook_specific_argument_parser = mocker.Mock() + + mocker.patch('pre_commit_terraform._cli_parsing.SUBCOMMAND_MODULES', [mock_subcommand_module]) + + parser = initialize_argument_parser() + assert isinstance(parser, ArgumentParser) + + args = parser.parse_args( + ['mock_hook', '-a', 'arg1', '-h', 'hook1', '-i', 'init1', '-e', 'env1', 'file1', 'file2'], + ) + + assert args.check_name == 'mock_hook' + assert args.args == ['arg1'] + assert args.hook_config == ['hook1'] + assert args.tf_init_args == ['init1'] + assert args.env_vars == ['env1'] + assert args.files == ['file1', 'file2'] + assert args.invoke_cli_app == mock_subcommand_module.invoke_cli_app + + mock_subcommand_module.populate_hook_specific_argument_parser.assert_called_once() + + +def test_initialize_argument_parser_no_args(mocker): + mock_subcommand_module = mocker.MagicMock() + mock_subcommand_module.HOOK_ID = 'mock_hook' + mock_subcommand_module.invoke_cli_app = mocker.Mock() + mock_subcommand_module.populate_hook_specific_argument_parser = mocker.Mock() + + mocker.patch('pre_commit_terraform._cli_parsing.SUBCOMMAND_MODULES', [mock_subcommand_module]) + + parser = initialize_argument_parser() + assert isinstance(parser, ArgumentParser) + + with pytest.raises(SystemExit): + parser.parse_args([]) + + +def test_initialize_argument_parser_multiple_subcommands(mocker): + mock_subcommand_module1 = mocker.MagicMock() + mock_subcommand_module1.HOOK_ID = 'mock_hook1' + mock_subcommand_module1.invoke_cli_app = mocker.Mock() + mock_subcommand_module1.populate_hook_specific_argument_parser = mocker.Mock() + + mock_subcommand_module2 = mocker.MagicMock() + mock_subcommand_module2.HOOK_ID = 'mock_hook2' + mock_subcommand_module2.invoke_cli_app = mocker.Mock() + mock_subcommand_module2.populate_hook_specific_argument_parser = mocker.Mock() + + mocker.patch( + 'pre_commit_terraform._cli_parsing.SUBCOMMAND_MODULES', + [mock_subcommand_module1, mock_subcommand_module2], + ) + + parser = initialize_argument_parser() + assert isinstance(parser, ArgumentParser) + + args1 = parser.parse_args(['mock_hook1', '-a', 'arg1']) + assert args1.check_name == 'mock_hook1' + assert args1.args == ['arg1'] + assert args1.invoke_cli_app == mock_subcommand_module1.invoke_cli_app + + args2 = parser.parse_args(['mock_hook2', '-a', 'arg2']) + assert args2.check_name == 'mock_hook2' + assert args2.args == ['arg2'] + assert args2.invoke_cli_app == mock_subcommand_module2.invoke_cli_app + + mock_subcommand_module1.populate_hook_specific_argument_parser.assert_called_once() + mock_subcommand_module2.populate_hook_specific_argument_parser.assert_called_once() From 8953dd9c51b7a39e2d8adb7b469d76b89e658d21 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 31 Dec 2024 22:48:48 +0200 Subject: [PATCH 54/67] Add tests for cli_subcommands --- .pre-commit-config.yaml | 14 ++++++++---- tests/pytest/test_cli_subcommands.py | 33 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 tests/pytest/test_cli_subcommands.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 003ceb3b9..f0ad6c374 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,7 +45,7 @@ repos: # Dockerfile linter - repo: https://github.com/hadolint/hadolint - rev: v2.13.0-beta + rev: v2.13.1-beta hooks: - id: hadolint args: [ @@ -61,7 +61,7 @@ repos: # JSON5 Linter - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.1.0 + rev: v4.0.0-alpha.8 hooks: - id: prettier # https://prettier.io/docs/en/options.html#parser @@ -88,6 +88,12 @@ repos: - id: isort name: isort args: [--force-single-line, --profile=black] + exclude: | + (?x) + # It set wrong indent in imports in place where it shouldn't + # https://github.com/PyCQA/isort/issues/2315#issuecomment-2566703698 + (^tests/pytest/test_cli_subcommands.py$ + ) - repo: https://github.com/asottile/add-trailing-comma rev: v3.1.0 @@ -104,7 +110,7 @@ repos: # Usage: http://pylint.pycqa.org/en/latest/user_guide/message-control.html - repo: https://github.com/PyCQA/pylint - rev: v3.1.0 + rev: v3.3.3 hooks: - id: pylint args: @@ -117,7 +123,7 @@ repos: exclude: test_.+\.py$ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v1.14.1 hooks: - id: mypy additional_dependencies: diff --git a/tests/pytest/test_cli_subcommands.py b/tests/pytest/test_cli_subcommands.py new file mode 100644 index 000000000..78a9d56cc --- /dev/null +++ b/tests/pytest/test_cli_subcommands.py @@ -0,0 +1,33 @@ +from pre_commit_terraform import terraform_checkov +from pre_commit_terraform import terraform_docs_replace +from pre_commit_terraform import terraform_fmt +from pre_commit_terraform._cli_subcommands import SUBCOMMAND_MODULES + + +def test_subcommand_modules(mocker): + mock_terraform_checkov = mocker.patch('pre_commit_terraform.terraform_checkov') + mock_terraform_docs_replace = mocker.patch('pre_commit_terraform.terraform_docs_replace') + mock_terraform_fmt = mocker.patch('pre_commit_terraform.terraform_fmt') + + mock_subcommand_modules = ( + mock_terraform_docs_replace, + mock_terraform_fmt, + mock_terraform_checkov, + ) + + mocker.patch( + 'pre_commit_terraform._cli_subcommands.SUBCOMMAND_MODULES', + mock_subcommand_modules, + ) + + from pre_commit_terraform._cli_subcommands import ( + SUBCOMMAND_MODULES as patched_subcommand_modules, + ) + + assert patched_subcommand_modules == mock_subcommand_modules + + +def test_subcommand_modules_content(): + assert terraform_docs_replace in SUBCOMMAND_MODULES + assert terraform_fmt in SUBCOMMAND_MODULES + assert terraform_checkov in SUBCOMMAND_MODULES From 53e2dd0c125b98f3e3c063447ce396c1a066c7a6 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 3 Jan 2025 12:00:35 +0200 Subject: [PATCH 55/67] Add more tests, rename test files to `test_` pattern --- .pre-commit-config.yaml | 2 +- .../pytest/{test_main.py => test___main__.py} | 0 tests/pytest/test__cli.py | 82 +++++++++++++++++++ ...st_cli_parsing.py => test__cli_parsing.py} | 0 ...ubcommands.py => test__cli_subcommands.py} | 0 .../{test_common.py => test__common.py} | 0 tests/pytest/test__errors.py | 20 +++++ ...ole_repo.py => test__run_on_whole_repo.py} | 0 tests/pytest/test__structs.py | 18 ++++ 9 files changed, 121 insertions(+), 1 deletion(-) rename tests/pytest/{test_main.py => test___main__.py} (100%) create mode 100644 tests/pytest/test__cli.py rename tests/pytest/{test_cli_parsing.py => test__cli_parsing.py} (100%) rename tests/pytest/{test_cli_subcommands.py => test__cli_subcommands.py} (100%) rename tests/pytest/{test_common.py => test__common.py} (100%) create mode 100644 tests/pytest/test__errors.py rename tests/pytest/{test_run_on_whole_repo.py => test__run_on_whole_repo.py} (100%) create mode 100644 tests/pytest/test__structs.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0ad6c374..73bb6e5f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -92,7 +92,7 @@ repos: (?x) # It set wrong indent in imports in place where it shouldn't # https://github.com/PyCQA/isort/issues/2315#issuecomment-2566703698 - (^tests/pytest/test_cli_subcommands.py$ + (^tests/pytest/test__cli_subcommands.py$ ) - repo: https://github.com/asottile/add-trailing-comma diff --git a/tests/pytest/test_main.py b/tests/pytest/test___main__.py similarity index 100% rename from tests/pytest/test_main.py rename to tests/pytest/test___main__.py diff --git a/tests/pytest/test__cli.py b/tests/pytest/test__cli.py new file mode 100644 index 000000000..58b303b60 --- /dev/null +++ b/tests/pytest/test__cli.py @@ -0,0 +1,82 @@ +import pytest + +from pre_commit_terraform._cli import invoke_cli_app +from pre_commit_terraform._errors import PreCommitTerraformBaseError +from pre_commit_terraform._errors import PreCommitTerraformExit +from pre_commit_terraform._errors import PreCommitTerraformRuntimeError +from pre_commit_terraform._structs import ReturnCode + + +def test_invoke_cli_app_success(mocker): + mock_parsed_args = mocker.MagicMock() + mock_parsed_args.invoke_cli_app.return_value = ReturnCode.OK + + mock_initialize_argument_parser = mocker.patch( + 'pre_commit_terraform._cli.initialize_argument_parser', + ) + mock_initialize_argument_parser.return_value.parse_args.return_value = mock_parsed_args + + result = invoke_cli_app(['mock_arg']) + + assert result == ReturnCode.OK + mock_parsed_args.invoke_cli_app.assert_called_once_with(mock_parsed_args) + + +def test_invoke_cli_app_pre_commit_terraform_exit(mocker): + mock_parsed_args = mocker.MagicMock() + mock_parsed_args.invoke_cli_app.side_effect = PreCommitTerraformExit('Exit error') + + mock_initialize_argument_parser = mocker.patch( + 'pre_commit_terraform._cli.initialize_argument_parser', + ) + mock_initialize_argument_parser.return_value.parse_args.return_value = mock_parsed_args + + with pytest.raises(PreCommitTerraformExit): + invoke_cli_app(['mock_arg']) + + mock_parsed_args.invoke_cli_app.assert_called_once_with(mock_parsed_args) + + +def test_invoke_cli_app_pre_commit_terraform_runtime_error(mocker): + mock_parsed_args = mocker.MagicMock() + mock_parsed_args.invoke_cli_app.side_effect = PreCommitTerraformRuntimeError('Runtime error') + + mock_initialize_argument_parser = mocker.patch( + 'pre_commit_terraform._cli.initialize_argument_parser', + ) + mock_initialize_argument_parser.return_value.parse_args.return_value = mock_parsed_args + + result = invoke_cli_app(['mock_arg']) + + assert result == ReturnCode.ERROR + mock_parsed_args.invoke_cli_app.assert_called_once_with(mock_parsed_args) + + +def test_invoke_cli_app_pre_commit_terraform_base_error(mocker): + mock_parsed_args = mocker.MagicMock() + mock_parsed_args.invoke_cli_app.side_effect = PreCommitTerraformBaseError('Base error') + + mock_initialize_argument_parser = mocker.patch( + 'pre_commit_terraform._cli.initialize_argument_parser', + ) + mock_initialize_argument_parser.return_value.parse_args.return_value = mock_parsed_args + + result = invoke_cli_app(['mock_arg']) + + assert result == ReturnCode.ERROR + mock_parsed_args.invoke_cli_app.assert_called_once_with(mock_parsed_args) + + +def test_invoke_cli_app_keyboard_interrupt(mocker): + mock_parsed_args = mocker.MagicMock() + mock_parsed_args.invoke_cli_app.side_effect = KeyboardInterrupt('Interrupt') + + mock_initialize_argument_parser = mocker.patch( + 'pre_commit_terraform._cli.initialize_argument_parser', + ) + mock_initialize_argument_parser.return_value.parse_args.return_value = mock_parsed_args + + result = invoke_cli_app(['mock_arg']) + + assert result == ReturnCode.ERROR + mock_parsed_args.invoke_cli_app.assert_called_once_with(mock_parsed_args) diff --git a/tests/pytest/test_cli_parsing.py b/tests/pytest/test__cli_parsing.py similarity index 100% rename from tests/pytest/test_cli_parsing.py rename to tests/pytest/test__cli_parsing.py diff --git a/tests/pytest/test_cli_subcommands.py b/tests/pytest/test__cli_subcommands.py similarity index 100% rename from tests/pytest/test_cli_subcommands.py rename to tests/pytest/test__cli_subcommands.py diff --git a/tests/pytest/test_common.py b/tests/pytest/test__common.py similarity index 100% rename from tests/pytest/test_common.py rename to tests/pytest/test__common.py diff --git a/tests/pytest/test__errors.py b/tests/pytest/test__errors.py new file mode 100644 index 000000000..8b8ac3328 --- /dev/null +++ b/tests/pytest/test__errors.py @@ -0,0 +1,20 @@ +import pytest + +from pre_commit_terraform._errors import PreCommitTerraformBaseError +from pre_commit_terraform._errors import PreCommitTerraformExit +from pre_commit_terraform._errors import PreCommitTerraformRuntimeError + + +def test_pre_commit_terraform_base_error(): + with pytest.raises(PreCommitTerraformBaseError): + raise PreCommitTerraformBaseError('Base error occurred') + + +def test_pre_commit_terraform_runtime_error(): + with pytest.raises(PreCommitTerraformRuntimeError): + raise PreCommitTerraformRuntimeError('Runtime error occurred') + + +def test_pre_commit_terraform_exit(): + with pytest.raises(PreCommitTerraformExit): + raise PreCommitTerraformExit('Exit error occurred') diff --git a/tests/pytest/test_run_on_whole_repo.py b/tests/pytest/test__run_on_whole_repo.py similarity index 100% rename from tests/pytest/test_run_on_whole_repo.py rename to tests/pytest/test__run_on_whole_repo.py diff --git a/tests/pytest/test__structs.py b/tests/pytest/test__structs.py new file mode 100644 index 000000000..b0285c475 --- /dev/null +++ b/tests/pytest/test__structs.py @@ -0,0 +1,18 @@ +import pytest + +from pre_commit_terraform._structs import ReturnCode + + +def test_return_code_values(): + assert ReturnCode.OK == 0 + assert ReturnCode.ERROR == 1 + + +def test_return_code_names(): + assert ReturnCode(0).name == 'OK' + assert ReturnCode(1).name == 'ERROR' + + +def test_return_code_invalid_value(): + with pytest.raises(ValueError): + ReturnCode(2) From 3cf99fd497454af6c82cad5ac24dc387c9d6f29c Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 3 Jan 2025 12:02:07 +0200 Subject: [PATCH 56/67] Add types tests. Deal with pytest TypeError TypeError: Instance and class checks can only be used with @runtime_checkable protocols --- src/pre_commit_terraform/_types.py | 2 ++ tests/pytest/test__types.py | 36 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/pytest/test__types.py diff --git a/src/pre_commit_terraform/_types.py b/src/pre_commit_terraform/_types.py index 1052c9175..edc15ea7c 100644 --- a/src/pre_commit_terraform/_types.py +++ b/src/pre_commit_terraform/_types.py @@ -4,12 +4,14 @@ from argparse import Namespace from typing import Protocol from typing import TypeAlias +from typing import runtime_checkable from pre_commit_terraform._structs import ReturnCode ReturnCodeType: TypeAlias = ReturnCode | int +@runtime_checkable class CLISubcommandModuleProtocol(Protocol): """A protocol for the subcommand-implementing module shape.""" diff --git a/tests/pytest/test__types.py b/tests/pytest/test__types.py new file mode 100644 index 000000000..9dbcfc2e8 --- /dev/null +++ b/tests/pytest/test__types.py @@ -0,0 +1,36 @@ +from argparse import ArgumentParser +from argparse import Namespace + +from pre_commit_terraform._structs import ReturnCode +from pre_commit_terraform._types import CLISubcommandModuleProtocol +from pre_commit_terraform._types import ReturnCodeType + + +def test_return_code_type(): + assert isinstance(ReturnCode.OK, ReturnCodeType) + assert isinstance(ReturnCode.ERROR, ReturnCodeType) + assert isinstance(0, ReturnCodeType) + assert isinstance(1, ReturnCodeType) + assert not isinstance(2.5, ReturnCodeType) + assert not isinstance('string', ReturnCodeType) + + +def test_cli_subcommand_module_protocol(): + class MockSubcommandModule: + HOOK_ID = 'mock_hook' + + def populate_argument_parser(self, subcommand_parser: ArgumentParser) -> None: + pass + + def invoke_cli_app(self, parsed_cli_args: Namespace) -> ReturnCodeType: + return ReturnCode.OK + + assert isinstance(MockSubcommandModule(), CLISubcommandModuleProtocol) + + class InvalidSubcommandModule: + HOOK_ID = 'invalid_hook' + + def populate_argument_parser(self, subcommand_parser: ArgumentParser) -> None: + pass + + assert not isinstance(InvalidSubcommandModule(), CLISubcommandModuleProtocol) From 89e5a45d027a754101c8bf7839fce82de44cb8dd Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 3 Jan 2025 12:06:45 +0200 Subject: [PATCH 57/67] Use `files` from root args parser to deduplicate args --- src/pre_commit_terraform/terraform_docs_replace.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py index b8770efaf..131abcbbe 100644 --- a/src/pre_commit_terraform/terraform_docs_replace.py +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -48,11 +48,6 @@ def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> action='store_true', help='[deprecated]', ) - subcommand_parser.add_argument( - 'filenames', - nargs='*', - help='Filenames to check.', - ) def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: @@ -75,7 +70,7 @@ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: ) dirs = [] - for filename in parsed_cli_args.filenames: + for filename in parsed_cli_args.files: if os.path.realpath(filename) not in dirs and ( filename.endswith('.tf') or filename.endswith('.tfvars') ): From c2aab9255287ae13571e71237c9d4d50f2202371 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 3 Jan 2025 12:33:57 +0200 Subject: [PATCH 58/67] Drop parts (new hooks) which will requires new release --- src/pre_commit_terraform/_cli_subcommands.py | 8 +- src/pre_commit_terraform/terraform_checkov.py | 157 --------- src/pre_commit_terraform/terraform_fmt.py | 100 ------ tests/pytest/test__cli_subcommands.py | 12 +- tests/pytest/test__common.py | 189 ++++++----- tests/pytest/test_terraform_checkov.py | 301 ------------------ tests/pytest/test_terraform_fmt.py | 144 --------- 7 files changed, 96 insertions(+), 815 deletions(-) delete mode 100644 src/pre_commit_terraform/terraform_checkov.py delete mode 100644 src/pre_commit_terraform/terraform_fmt.py delete mode 100644 tests/pytest/test_terraform_checkov.py delete mode 100644 tests/pytest/test_terraform_fmt.py diff --git a/src/pre_commit_terraform/_cli_subcommands.py b/src/pre_commit_terraform/_cli_subcommands.py index cac2f408e..3fe5e9404 100644 --- a/src/pre_commit_terraform/_cli_subcommands.py +++ b/src/pre_commit_terraform/_cli_subcommands.py @@ -1,15 +1,9 @@ """A CLI sub-commands organization module.""" -from pre_commit_terraform import terraform_checkov from pre_commit_terraform import terraform_docs_replace -from pre_commit_terraform import terraform_fmt from pre_commit_terraform._types import CLISubcommandModuleProtocol -SUBCOMMAND_MODULES: tuple[CLISubcommandModuleProtocol, ...] = ( - terraform_docs_replace, - terraform_fmt, - terraform_checkov, -) +SUBCOMMAND_MODULES: tuple[CLISubcommandModuleProtocol, ...] = (terraform_docs_replace,) __all__ = ('SUBCOMMAND_MODULES',) diff --git a/src/pre_commit_terraform/terraform_checkov.py b/src/pre_commit_terraform/terraform_checkov.py deleted file mode 100644 index a8e3855e3..000000000 --- a/src/pre_commit_terraform/terraform_checkov.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Pre-commit hook for terraform fmt.""" - -from __future__ import annotations - -import logging -import os -import shlex -import sys -from argparse import ArgumentParser -from argparse import Namespace -from subprocess import PIPE -from subprocess import run -from typing import Final - -from pre_commit_terraform import _common as common -from pre_commit_terraform._logger import setup_logging -from pre_commit_terraform._run_on_whole_repo import is_function_defined -from pre_commit_terraform._run_on_whole_repo import is_hook_run_on_whole_repo -from pre_commit_terraform._types import ReturnCodeType - -logger = logging.getLogger(__name__) - - -def replace_git_working_dir_to_repo_root(args: list[str]) -> list[str]: - """ - Support for setting PATH to repo root. - - Replace '__GIT_WORKING_DIR__' with the current working directory in each argument. - - Args: - args: List of arguments to process. - - Returns: - List of arguments with '__GIT_WORKING_DIR__' replaced. - """ - return [arg.replace('__GIT_WORKING_DIR__', os.getcwd()) for arg in args] - - -HOOK_ID: Final[str] = __name__.rpartition('.')[-1] + '_py' # noqa: WPS336 - - -# pylint: disable=unused-argument -def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: - """ - Populate the argument parser with the hook-specific arguments. - - Args: - subcommand_parser: The argument parser to populate. - """ - - -def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: - """ - Execute main pre-commit hook logic. - - Args: - parsed_cli_args: Parsed arguments from CLI. - - Returns: - int: The exit code of the hook. - """ - setup_logging() - logger.debug(sys.version_info) - - all_env_vars = {**os.environ, **common.parse_env_vars(parsed_cli_args.env_vars)} - expanded_args = common.expand_env_vars(parsed_cli_args.args, all_env_vars) - expanded_args = replace_git_working_dir_to_repo_root(expanded_args) - # Just in case is someone somehow will add something like "; rm -rf" in the args - safe_args = [shlex.quote(arg) for arg in expanded_args] - - if os.environ.get('PRE_COMMIT_COLOR') == 'never': - # TODO: subprocess.run ignore colors. Try `rich` lib - all_env_vars['ANSI_COLORS_DISABLED'] = 'true' - # WPS421 - IDK how to check is function exist w/o passing globals() - if is_function_defined('run_hook_on_whole_repo', globals()): # noqa: WPS421 - if is_hook_run_on_whole_repo(HOOK_ID, parsed_cli_args.files): - return run_hook_on_whole_repo(safe_args, all_env_vars) - - return common.per_dir_hook( - parsed_cli_args.hook_config, - parsed_cli_args.files, - safe_args, - all_env_vars, - per_dir_hook_unique_part, - ) - - -def run_hook_on_whole_repo(args: list[str], env_vars: dict[str, str]) -> int: - """ - Run the hook on the whole repository. - - Args: - args: The arguments to pass to the hook - env_vars: All environment variables provided to hook from system and - defined by user in hook config. - - Returns: - int: The exit code of the hook. - """ - cmd = ['checkov', '-d', '.', *args] - - logger.debug( - 'Running hook on the whole repository with values:\nargs: %s \nenv_vars: %r', - args, - env_vars, - ) - logger.info('calling %s', shlex.join(cmd)) - - completed_process = run( - cmd, - env=env_vars, - text=True, - stdout=PIPE, - check=False, - ) - - if completed_process.stdout: - sys.stdout.write(completed_process.stdout) - - return completed_process.returncode - - -def per_dir_hook_unique_part( - tf_path: str, # pylint: disable=unused-argument - dir_path: str, - args: list[str], - env_vars: dict[str, str], -) -> int: - """ - Run the hook against a single directory. - - Args: - tf_path: The path to the terraform binary. - dir_path: The directory to run the hook against. - args: The arguments to pass to the hook - env_vars: All environment variables provided to hook from system and - defined by user in hook config. - - Returns: - int: The exit code of the hook. - """ - cmd = ['checkov', '-d', dir_path, *args] - - logger.info('calling %s', shlex.join(cmd)) - - completed_process = run( - cmd, - env=env_vars, - text=True, - stdout=PIPE, - check=False, - ) - - if completed_process.stdout: - sys.stdout.write(completed_process.stdout) - - return completed_process.returncode diff --git a/src/pre_commit_terraform/terraform_fmt.py b/src/pre_commit_terraform/terraform_fmt.py deleted file mode 100644 index ec68fe9f8..000000000 --- a/src/pre_commit_terraform/terraform_fmt.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Pre-commit hook for terraform fmt.""" - -from __future__ import annotations - -import logging -import os -import shlex -import sys -from argparse import ArgumentParser -from argparse import Namespace -from subprocess import PIPE -from subprocess import run -from typing import Final - -from pre_commit_terraform import _common as common -from pre_commit_terraform._logger import setup_logging -from pre_commit_terraform._types import ReturnCodeType - -logger = logging.getLogger(__name__) - - -HOOK_ID: Final[str] = __name__.rpartition('.')[-1] + '_py' # noqa: WPS336 - - -# pylint: disable=unused-argument -def populate_hook_specific_argument_parser(subcommand_parser: ArgumentParser) -> None: - """ - Populate the argument parser with the hook-specific arguments. - - Args: - subcommand_parser: The argument parser to populate. - """ - - -def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: - """ - Execute main pre-commit hook logic. - - Args: - parsed_cli_args: Parsed arguments from CLI. - - Returns: - int: The exit code of the hook. - """ - setup_logging() - logger.debug(sys.version_info) - - all_env_vars = {**os.environ, **common.parse_env_vars(parsed_cli_args.env_vars)} - expanded_args = common.expand_env_vars(parsed_cli_args.args, all_env_vars) - # Just in case is someone somehow will add something like "; rm -rf" in the args - safe_args = [shlex.quote(arg) for arg in expanded_args] - - if os.environ.get('PRE_COMMIT_COLOR') == 'never': - safe_args.append('-no-color') - - return common.per_dir_hook( - parsed_cli_args.hook_config, - parsed_cli_args.files, - safe_args, - all_env_vars, - per_dir_hook_unique_part, - ) - - -def per_dir_hook_unique_part( - tf_path: str, - dir_path: str, - args: list[str], - env_vars: dict[str, str], -) -> int: - """ - Run the hook against a single directory. - - Args: - tf_path: The path to the terraform binary. - dir_path: The directory to run the hook against. - args: The arguments to pass to the hook - env_vars: All environment variables provided to hook from system and - defined by user in hook config. - - Returns: - int: The exit code of the hook. - """ - # Just in case is someone somehow will add something like "; rm -rf" in the args - cmd = [tf_path, 'fmt', *args, dir_path] - - logger.info('calling %s', shlex.join(cmd)) - - completed_process = run( - cmd, - env=env_vars, - text=True, - stdout=PIPE, - check=False, - ) - - if completed_process.stdout: - sys.stdout.write(completed_process.stdout) - - return completed_process.returncode diff --git a/tests/pytest/test__cli_subcommands.py b/tests/pytest/test__cli_subcommands.py index 78a9d56cc..c4771afd7 100644 --- a/tests/pytest/test__cli_subcommands.py +++ b/tests/pytest/test__cli_subcommands.py @@ -1,19 +1,11 @@ -from pre_commit_terraform import terraform_checkov from pre_commit_terraform import terraform_docs_replace -from pre_commit_terraform import terraform_fmt from pre_commit_terraform._cli_subcommands import SUBCOMMAND_MODULES def test_subcommand_modules(mocker): - mock_terraform_checkov = mocker.patch('pre_commit_terraform.terraform_checkov') mock_terraform_docs_replace = mocker.patch('pre_commit_terraform.terraform_docs_replace') - mock_terraform_fmt = mocker.patch('pre_commit_terraform.terraform_fmt') - mock_subcommand_modules = ( - mock_terraform_docs_replace, - mock_terraform_fmt, - mock_terraform_checkov, - ) + mock_subcommand_modules = (mock_terraform_docs_replace,) mocker.patch( 'pre_commit_terraform._cli_subcommands.SUBCOMMAND_MODULES', @@ -29,5 +21,3 @@ def test_subcommand_modules(mocker): def test_subcommand_modules_content(): assert terraform_docs_replace in SUBCOMMAND_MODULES - assert terraform_fmt in SUBCOMMAND_MODULES - assert terraform_checkov in SUBCOMMAND_MODULES diff --git a/tests/pytest/test__common.py b/tests/pytest/test__common.py index efe36a1ef..aab34a33d 100644 --- a/tests/pytest/test__common.py +++ b/tests/pytest/test__common.py @@ -9,7 +9,6 @@ from pre_commit_terraform._common import expand_env_vars from pre_commit_terraform._common import get_tf_binary_path from pre_commit_terraform._common import parse_env_vars -from pre_commit_terraform._common import per_dir_hook # ? @@ -46,101 +45,101 @@ def test_get_unique_dirs_nested_dirs(): # ? -# ? per_dir_hook +# ? per_dir_hook # TODO: Requires `terraform_fmt` to be implemented # ? -@pytest.fixture -def mock_per_dir_hook_unique_part(mocker): - return mocker.patch('pre_commit_terraform.terraform_fmt.per_dir_hook_unique_part') - - -def test_per_dir_hook_empty_files(mock_per_dir_hook_unique_part): - hook_config = [] - files = [] - args = [] - env_vars = {} - result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) - assert result == 0 - mock_per_dir_hook_unique_part.assert_not_called() - - -def test_per_dir_hook_single_file(mocker, mock_per_dir_hook_unique_part): - hook_config = [] - files = [os.path.join('path', 'to', 'file1.tf')] - args = [] - env_vars = {} - mock_per_dir_hook_unique_part.return_value = 0 - result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) - assert result == 0 - mock_per_dir_hook_unique_part.assert_called_once_with( - mocker.ANY, # Terraform binary path - os.path.join('path', 'to'), - args, - env_vars, - ) - - -def test_per_dir_hook_multiple_files_same_dir(mocker, mock_per_dir_hook_unique_part): - hook_config = [] - files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('path', 'to', 'file2.tf')] - args = [] - env_vars = {} - mock_per_dir_hook_unique_part.return_value = 0 - result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) - assert result == 0 - mock_per_dir_hook_unique_part.assert_called_once_with( - mocker.ANY, # Terraform binary path - os.path.join('path', 'to'), - args, - env_vars, - ) - - -def test_per_dir_hook_multiple_files_different_dirs(mocker, mock_per_dir_hook_unique_part): - hook_config = [] - files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('another', 'path', 'file2.tf')] - args = [] - env_vars = {} - mock_per_dir_hook_unique_part.return_value = 0 - result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) - assert result == 0 - expected_calls = [ - mocker.call(mocker.ANY, os.path.join('path', 'to'), args, env_vars), - mocker.call(mocker.ANY, os.path.join('another', 'path'), args, env_vars), - ] - mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) - - -def test_per_dir_hook_nested_dirs(mocker, mock_per_dir_hook_unique_part): - hook_config = [] - files = [ - os.path.join('path', 'to', 'file1.tf'), - os.path.join('path', 'to', 'nested', 'file2.tf'), - ] - args = [] - env_vars = {} - mock_per_dir_hook_unique_part.return_value = 0 - result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) - assert result == 0 - expected_calls = [ - mocker.call(mocker.ANY, os.path.join('path', 'to'), args, env_vars), - mocker.call(mocker.ANY, os.path.join('path', 'to', 'nested'), args, env_vars), - ] - mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) - - -def test_per_dir_hook_with_errors(mocker, mock_per_dir_hook_unique_part): - hook_config = [] - files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('another', 'path', 'file2.tf')] - args = [] - env_vars = {} - mock_per_dir_hook_unique_part.side_effect = [0, 1] - result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) - assert result == 1 - expected_calls = [ - mocker.call(mocker.ANY, os.path.join('path', 'to'), args, env_vars), - mocker.call(mocker.ANY, os.path.join('another', 'path'), args, env_vars), - ] - mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) +# @pytest.fixture +# def mock_per_dir_hook_unique_part(mocker): +# return mocker.patch('pre_commit_terraform.terraform_fmt.per_dir_hook_unique_part') + + +# def test_per_dir_hook_empty_files(mock_per_dir_hook_unique_part): +# hook_config = [] +# files = [] +# args = [] +# env_vars = {} +# result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) +# assert result == 0 +# mock_per_dir_hook_unique_part.assert_not_called() + + +# def test_per_dir_hook_single_file(mocker, mock_per_dir_hook_unique_part): +# hook_config = [] +# files = [os.path.join('path', 'to', 'file1.tf')] +# args = [] +# env_vars = {} +# mock_per_dir_hook_unique_part.return_value = 0 +# result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) +# assert result == 0 +# mock_per_dir_hook_unique_part.assert_called_once_with( +# mocker.ANY, # Terraform binary path +# os.path.join('path', 'to'), +# args, +# env_vars, +# ) + + +# def test_per_dir_hook_multiple_files_same_dir(mocker, mock_per_dir_hook_unique_part): +# hook_config = [] +# files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('path', 'to', 'file2.tf')] +# args = [] +# env_vars = {} +# mock_per_dir_hook_unique_part.return_value = 0 +# result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) +# assert result == 0 +# mock_per_dir_hook_unique_part.assert_called_once_with( +# mocker.ANY, # Terraform binary path +# os.path.join('path', 'to'), +# args, +# env_vars, +# ) + + +# def test_per_dir_hook_multiple_files_different_dirs(mocker, mock_per_dir_hook_unique_part): +# hook_config = [] +# files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('another', 'path', 'file2.tf')] +# args = [] +# env_vars = {} +# mock_per_dir_hook_unique_part.return_value = 0 +# result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) +# assert result == 0 +# expected_calls = [ +# mocker.call(mocker.ANY, os.path.join('path', 'to'), args, env_vars), +# mocker.call(mocker.ANY, os.path.join('another', 'path'), args, env_vars), +# ] +# mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) + + +# def test_per_dir_hook_nested_dirs(mocker, mock_per_dir_hook_unique_part): +# hook_config = [] +# files = [ +# os.path.join('path', 'to', 'file1.tf'), +# os.path.join('path', 'to', 'nested', 'file2.tf'), +# ] +# args = [] +# env_vars = {} +# mock_per_dir_hook_unique_part.return_value = 0 +# result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) +# assert result == 0 +# expected_calls = [ +# mocker.call(mocker.ANY, os.path.join('path', 'to'), args, env_vars), +# mocker.call(mocker.ANY, os.path.join('path', 'to', 'nested'), args, env_vars), +# ] +# mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) + + +# def test_per_dir_hook_with_errors(mocker, mock_per_dir_hook_unique_part): +# hook_config = [] +# files = [os.path.join('path', 'to', 'file1.tf'), os.path.join('another', 'path', 'file2.tf')] +# args = [] +# env_vars = {} +# mock_per_dir_hook_unique_part.side_effect = [0, 1] +# result = per_dir_hook(hook_config, files, args, env_vars, mock_per_dir_hook_unique_part) +# assert result == 1 +# expected_calls = [ +# mocker.call(mocker.ANY, os.path.join('path', 'to'), args, env_vars), +# mocker.call(mocker.ANY, os.path.join('another', 'path'), args, env_vars), +# ] +# mock_per_dir_hook_unique_part.assert_has_calls(expected_calls, any_order=True) # ? diff --git a/tests/pytest/test_terraform_checkov.py b/tests/pytest/test_terraform_checkov.py deleted file mode 100644 index 3fc2ce453..000000000 --- a/tests/pytest/test_terraform_checkov.py +++ /dev/null @@ -1,301 +0,0 @@ -import os -from argparse import Namespace -from subprocess import PIPE - -from pre_commit_terraform.terraform_checkov import invoke_cli_app -from pre_commit_terraform.terraform_checkov import per_dir_hook_unique_part -from pre_commit_terraform.terraform_checkov import replace_git_working_dir_to_repo_root -from pre_commit_terraform.terraform_checkov import run_hook_on_whole_repo - - -# ? -# ? replace_git_working_dir_to_repo_root -# ? -def test_replace_git_working_dir_to_repo_root_empty(): - args = [] - result = replace_git_working_dir_to_repo_root(args) - assert result == [] - - -def test_replace_git_working_dir_to_repo_root_no_replacement(): - args = ['arg1', 'arg2'] - result = replace_git_working_dir_to_repo_root(args) - assert result == ['arg1', 'arg2'] - - -def test_replace_git_working_dir_to_repo_root_single_replacement(mocker): - mocker.patch('os.getcwd', return_value='/current/working/dir') - args = ['arg1', '__GIT_WORKING_DIR__/arg2'] - result = replace_git_working_dir_to_repo_root(args) - assert result == ['arg1', '/current/working/dir/arg2'] - - -def test_replace_git_working_dir_to_repo_root_multiple_replacements(mocker): - mocker.patch('os.getcwd', return_value='/current/working/dir') - args = ['__GIT_WORKING_DIR__/arg1', 'arg2', '__GIT_WORKING_DIR__/arg3'] - result = replace_git_working_dir_to_repo_root(args) - assert result == ['/current/working/dir/arg1', 'arg2', '/current/working/dir/arg3'] - - -def test_replace_git_working_dir_to_repo_root_partial_replacement(mocker): - mocker.patch('os.getcwd', return_value='/current/working/dir') - args = ['arg1', '__GIT_WORKING_DIR__/arg2', 'arg3'] - result = replace_git_working_dir_to_repo_root(args) - assert result == ['arg1', '/current/working/dir/arg2', 'arg3'] - - -# ? -# ? invoke_cli_app -# ? -def test_invoke_cli_app_no_color(mocker): - mock_parsed_cli_args = Namespace( - hook_config=[], - files=['file1.tf', 'file2.tf'], - args=['-d', '.'], - env_vars=['ENV_VAR=value'], - ) - mock_env_vars = {'ENV_VAR': 'value', 'PRE_COMMIT_COLOR': 'never'} - - mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_checkov.setup_logging') - mock_expand_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_checkov.common.expand_env_vars', - return_value=['-d', '.'], - ) - mock_parse_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_checkov.common.parse_env_vars', - return_value=mock_env_vars, - ) - mock_per_dir_hook = mocker.patch( - 'pre_commit_terraform.terraform_checkov.common.per_dir_hook', - return_value=0, - ) - mock_run_hook_on_whole_repo = mocker.patch( - 'pre_commit_terraform.terraform_checkov.run_hook_on_whole_repo', - return_value=0, - ) - mocker.patch( - 'pre_commit_terraform.terraform_checkov.is_function_defined', - return_value=True, - ) - mocker.patch( - 'pre_commit_terraform.terraform_checkov.is_hook_run_on_whole_repo', - return_value=False, - ) - - result = invoke_cli_app(mock_parsed_cli_args) - - mock_setup_logging.assert_called_once() - mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) - mock_expand_env_vars.assert_called_once_with( - mock_parsed_cli_args.args, - {**os.environ, **mock_env_vars}, - ) - mock_per_dir_hook.assert_called_once() - mock_run_hook_on_whole_repo.assert_not_called() - assert result == 0 - - -def test_invoke_cli_app_run_on_whole_repo(mocker): - mock_parsed_cli_args = Namespace( - hook_config=[], - files=['file1.tf', 'file2.tf'], - args=['-d', '.'], - env_vars=['ENV_VAR=value'], - ) - mock_env_vars = {'ENV_VAR': 'value'} - - mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_checkov.setup_logging') - mock_expand_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_checkov.common.expand_env_vars', - return_value=['-d', '.'], - ) - mock_parse_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_checkov.common.parse_env_vars', - return_value=mock_env_vars, - ) - mock_per_dir_hook = mocker.patch( - 'pre_commit_terraform.terraform_checkov.common.per_dir_hook', - return_value=0, - ) - mock_run_hook_on_whole_repo = mocker.patch( - 'pre_commit_terraform.terraform_checkov.run_hook_on_whole_repo', - return_value=0, - ) - mocker.patch( - 'pre_commit_terraform.terraform_checkov.is_function_defined', - return_value=True, - ) - mocker.patch( - 'pre_commit_terraform.terraform_checkov.is_hook_run_on_whole_repo', - return_value=True, - ) - - result = invoke_cli_app(mock_parsed_cli_args) - - mock_setup_logging.assert_called_once() - mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) - mock_expand_env_vars.assert_called_once_with( - mock_parsed_cli_args.args, - {**os.environ, **mock_env_vars}, - ) - mock_run_hook_on_whole_repo.assert_called_once() - mock_per_dir_hook.assert_not_called() - assert result == 0 - - -def test_invoke_cli_app_per_dir_hook(mocker): - mock_parsed_cli_args = Namespace( - hook_config=[], - files=['file1.tf', 'file2.tf'], - args=['-d', '.'], - env_vars=['ENV_VAR=value'], - ) - mock_env_vars = {'ENV_VAR': 'value'} - - mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_checkov.setup_logging') - mock_expand_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_checkov.common.expand_env_vars', - return_value=['-d', '.'], - ) - mock_parse_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_checkov.common.parse_env_vars', - return_value=mock_env_vars, - ) - mock_per_dir_hook = mocker.patch( - 'pre_commit_terraform.terraform_checkov.common.per_dir_hook', - return_value=0, - ) - mock_run_hook_on_whole_repo = mocker.patch( - 'pre_commit_terraform.terraform_checkov.run_hook_on_whole_repo', - return_value=0, - ) - mocker.patch( - 'pre_commit_terraform.terraform_checkov.is_function_defined', - return_value=False, - ) - - result = invoke_cli_app(mock_parsed_cli_args) - - mock_setup_logging.assert_called_once() - mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) - mock_expand_env_vars.assert_called_once_with( - mock_parsed_cli_args.args, - {**os.environ, **mock_env_vars}, - ) - mock_per_dir_hook.assert_called_once() - mock_run_hook_on_whole_repo.assert_not_called() - assert result == 0 - - -# ? -# ? run_hook_on_whole_repo -# ? -def test_run_hook_on_whole_repo_success(mocker): - mock_args = ['-d', '.'] - mock_env_vars = {'ENV_VAR': 'value'} - mock_completed_process = mocker.MagicMock() - mock_completed_process.returncode = 0 - mock_completed_process.stdout = 'Checkov output' - - mock_run = mocker.patch( - 'pre_commit_terraform.terraform_checkov.run', - return_value=mock_completed_process, - ) - mock_sys_stdout_write = mocker.patch('sys.stdout.write') - - result = run_hook_on_whole_repo(mock_args, mock_env_vars) - - mock_run.assert_called_once_with( - ['checkov', '-d', '.', *mock_args], - env=mock_env_vars, - text=True, - stdout=PIPE, - check=False, - ) - mock_sys_stdout_write.assert_called_once_with('Checkov output') - assert result == 0 - - -def test_run_hook_on_whole_repo_failure(mocker): - mock_args = ['-d', '.'] - mock_env_vars = {'ENV_VAR': 'value'} - mock_completed_process = mocker.MagicMock() - mock_completed_process.returncode = 1 - mock_completed_process.stdout = 'Checkov error output' - - mock_run = mocker.patch( - 'pre_commit_terraform.terraform_checkov.run', - return_value=mock_completed_process, - ) - mock_sys_stdout_write = mocker.patch('sys.stdout.write') - - result = run_hook_on_whole_repo(mock_args, mock_env_vars) - - mock_run.assert_called_once_with( - ['checkov', '-d', '.', *mock_args], - env=mock_env_vars, - text=True, - stdout=PIPE, - check=False, - ) - mock_sys_stdout_write.assert_called_once_with('Checkov error output') - assert result == 1 - - -# ? -# ? per_dir_hook_unique_part -# ? -def test_per_dir_hook_unique_part_success(mocker): - tf_path = '/usr/local/bin/terraform' - dir_path = 'test_dir' - args = ['-d', '.'] - env_vars = {'ENV_VAR': 'value'} - mock_completed_process = mocker.MagicMock() - mock_completed_process.returncode = 0 - mock_completed_process.stdout = 'Checkov output' - - mock_run = mocker.patch( - 'pre_commit_terraform.terraform_checkov.run', - return_value=mock_completed_process, - ) - mock_sys_stdout_write = mocker.patch('sys.stdout.write') - - result = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) - - mock_run.assert_called_once_with( - ['checkov', '-d', dir_path, *args], - env=env_vars, - text=True, - stdout=PIPE, - check=False, - ) - mock_sys_stdout_write.assert_called_once_with('Checkov output') - assert result == 0 - - -def test_per_dir_hook_unique_part_failure(mocker): - tf_path = '/usr/local/bin/terraform' - dir_path = 'test_dir' - args = ['-d', '.'] - env_vars = {'ENV_VAR': 'value'} - mock_completed_process = mocker.MagicMock() - mock_completed_process.returncode = 1 - mock_completed_process.stdout = 'Checkov error output' - - mock_run = mocker.patch( - 'pre_commit_terraform.terraform_checkov.run', - return_value=mock_completed_process, - ) - mock_sys_stdout_write = mocker.patch('sys.stdout.write') - - result = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) - - mock_run.assert_called_once_with( - ['checkov', '-d', dir_path, *args], - env=env_vars, - text=True, - stdout=PIPE, - check=False, - ) - mock_sys_stdout_write.assert_called_once_with('Checkov error output') - assert result == 1 diff --git a/tests/pytest/test_terraform_fmt.py b/tests/pytest/test_terraform_fmt.py deleted file mode 100644 index 60cba7e7a..000000000 --- a/tests/pytest/test_terraform_fmt.py +++ /dev/null @@ -1,144 +0,0 @@ -import os -import subprocess -from argparse import Namespace - -import pytest - -from pre_commit_terraform.terraform_fmt import invoke_cli_app -from pre_commit_terraform.terraform_fmt import per_dir_hook_unique_part - - -# ? -# ? invoke_cli_app -# ? -@pytest.fixture -def mock_parsed_cli_args(): - return Namespace( - hook_config=[], - files=['file1.tf', 'file2.tf'], - args=['-diff'], - env_vars=['ENV_VAR=value'], - ) - - -@pytest.fixture -def mock_env_vars(): - return {'ENV_VAR': 'value', 'PRE_COMMIT_COLOR': 'always'} - - -def test_invoke_cli_app(mocker, mock_parsed_cli_args, mock_env_vars): - mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_fmt.setup_logging') - mock_expand_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_fmt.common.expand_env_vars', - return_value=['-diff'], - ) - mock_parse_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_fmt.common.parse_env_vars', - return_value=mock_env_vars, - ) - mock_run = mocker.patch( - 'pre_commit_terraform.terraform_fmt.run', - return_value=subprocess.CompletedProcess( - args=['terraform', 'fmt'], - returncode=0, - stdout='Formatted output', - ), - ) - - result = invoke_cli_app(mock_parsed_cli_args) - - mock_setup_logging.assert_called_once() - mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) - mock_expand_env_vars.assert_called_once_with( - mock_parsed_cli_args.args, - {**os.environ, **mock_env_vars}, - ) - mock_run.assert_called_once() - - assert result == 0 - - -# ? -# ? per_dir_hook_unique_part -# ? -def test_per_dir_hook_unique_part(mocker): - tf_path = '/usr/local/bin/terraform' - dir_path = 'test_dir' - args = ['-diff'] - env_vars = {'ENV_VAR': 'value'} - - mock_run = mocker.patch( - 'pre_commit_terraform.terraform_fmt.run', - return_value=subprocess.CompletedProcess(args, 0, stdout='Formatted output'), - ) - - result = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) - - expected_cmd = [tf_path, 'fmt', *args, dir_path] - mock_run.assert_called_once_with( - expected_cmd, - env=env_vars, - text=True, - stdout=subprocess.PIPE, - check=False, - ) - - assert result == 0 - - -def test_invoke_cli_app_no_color(mocker, mock_parsed_cli_args, mock_env_vars): - mock_env_vars['PRE_COMMIT_COLOR'] = 'never' - mock_setup_logging = mocker.patch('pre_commit_terraform.terraform_fmt.setup_logging') - mock_expand_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_fmt.common.expand_env_vars', - return_value=['-diff'], - ) - mock_parse_env_vars = mocker.patch( - 'pre_commit_terraform.terraform_fmt.common.parse_env_vars', - return_value=mock_env_vars, - ) - mock_run = mocker.patch( - 'pre_commit_terraform.terraform_fmt.run', - return_value=subprocess.CompletedProcess( - args=['terraform', 'fmt'], - returncode=0, - stdout='Formatted output', - ), - ) - - result = invoke_cli_app(mock_parsed_cli_args) - - mock_setup_logging.assert_called_once() - mock_parse_env_vars.assert_called_once_with(mock_parsed_cli_args.env_vars) - mock_expand_env_vars.assert_called_once_with( - mock_parsed_cli_args.args, - {**os.environ, **mock_env_vars}, - ) - mock_run.assert_called_once() - - assert result == 0 - - -def test_per_dir_hook_unique_part_failure(mocker): - tf_path = '/usr/local/bin/terraform' - dir_path = 'test_dir' - args = ['-diff'] - env_vars = {'ENV_VAR': 'value'} - - mock_run = mocker.patch( - 'pre_commit_terraform.terraform_fmt.run', - return_value=subprocess.CompletedProcess(args, 1, stdout='Error output'), - ) - - result = per_dir_hook_unique_part(tf_path, dir_path, args, env_vars) - - expected_cmd = [tf_path, 'fmt', *args, dir_path] - mock_run.assert_called_once_with( - expected_cmd, - env=env_vars, - text=True, - stdout=subprocess.PIPE, - check=False, - ) - - assert result == 1 From 449d448d05ccce4150f61048c2dc2005608aea15 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 3 Jan 2025 12:52:00 +0200 Subject: [PATCH 59/67] Improve naming and help output --- src/pre_commit_terraform/_cli_parsing.py | 3 +++ tests/pytest/test__cli_parsing.py | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pre_commit_terraform/_cli_parsing.py b/src/pre_commit_terraform/_cli_parsing.py index 2b18c6508..3beb32f47 100644 --- a/src/pre_commit_terraform/_cli_parsing.py +++ b/src/pre_commit_terraform/_cli_parsing.py @@ -27,6 +27,7 @@ def populate_common_argument_parser(parser: ArgumentParser) -> None: '-h', '--hook-config', action='append', + metavar='KEY=VALUE', help='Arguments that configure hook behavior', default=[], ) @@ -42,6 +43,8 @@ def populate_common_argument_parser(parser: ArgumentParser) -> None: '-e', '--env-vars', '--envs', + dest='env_vars_strs', + metavar='KEY=VALUE', action='append', help='Setup additional Environment Variables during hook execution', default=[], diff --git a/tests/pytest/test__cli_parsing.py b/tests/pytest/test__cli_parsing.py index 61b8ebcf5..58ba1ea5e 100644 --- a/tests/pytest/test__cli_parsing.py +++ b/tests/pytest/test__cli_parsing.py @@ -20,7 +20,7 @@ def test_populate_common_argument_parser(mocker): assert args.args == ['arg1'] assert args.hook_config == ['hook1'] assert args.tf_init_args == ['init1'] - assert args.env_vars == ['env1'] + assert args.env_vars_strs == ['env1'] assert args.files == ['file1', 'file2'] @@ -32,7 +32,7 @@ def test_populate_common_argument_parser_defaults(mocker): assert args.args == [] assert args.hook_config == [] assert args.tf_init_args == [] - assert args.env_vars == [] + assert args.env_vars_strs == [] assert args.files == [] @@ -65,7 +65,7 @@ def test_populate_common_argument_parser_multiple_values(mocker): assert args.args == ['arg1', 'arg2'] assert args.hook_config == ['hook1', 'hook2'] assert args.tf_init_args == ['init1', 'init2'] - assert args.env_vars == ['env1', 'env2'] + assert args.env_vars_strs == ['env1', 'env2'] assert args.files == ['file1', 'file2'] @@ -91,7 +91,7 @@ def test_attach_subcommand_parsers_to(mocker): assert args.args == ['arg1'] assert args.hook_config == ['hook1'] assert args.tf_init_args == ['init1'] - assert args.env_vars == ['env1'] + assert args.env_vars_strs == ['env1'] assert args.files == ['file1', 'file2'] assert args.invoke_cli_app == mock_subcommand_module.invoke_cli_app @@ -168,7 +168,7 @@ def test_initialize_argument_parser(mocker): assert args.args == ['arg1'] assert args.hook_config == ['hook1'] assert args.tf_init_args == ['init1'] - assert args.env_vars == ['env1'] + assert args.env_vars_strs == ['env1'] assert args.files == ['file1', 'file2'] assert args.invoke_cli_app == mock_subcommand_module.invoke_cli_app From ac42c5169bbb15b7b5a2e6e418ba0f23277fb464 Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Fri, 3 Jan 2025 13:19:06 +0200 Subject: [PATCH 60/67] Apply suggestions from code review --- .github/CONTRIBUTING.md | 21 ++++++++++++++++++--- .pre-commit-hooks.yaml | 19 ------------------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 3d1fe9d50..051bbf45f 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -18,7 +18,7 @@ Enjoy the clean, valid, and documented code! * [Prepare basic documentation](#prepare-basic-documentation) * [Add code](#add-code) * [Finish with the documentation](#finish-with-the-documentation) -* [Testing](#testing) +* [Testing](#testing-python-hooks) ## Run and debug hooks locally @@ -157,10 +157,25 @@ You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/ 2. Create and populate a new hook section in [Hooks usage notes and examples](../README.md#hooks-usage-notes-and-examples). -## Testing +## Testing python hooks -``` +Tu run tests, you need: + +```bash pip install pytest pytest-mock pytest -vv ``` + +To run and debug hooks as CLI: + +1. Create [`venv`](https://docs.python.org/3/library/venv.html) and activate it, IE by `virtualenv venv` +2. Inside virtual env install pre-commit-terraform as package: + ```bash + pip install --editable . + ``` + +3. Run next to show basic help: + ```bash + python -Im pre_commit_terraform --help + ``` diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index e464f6ba4..8c4877a6e 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -15,15 +15,6 @@ files: (\.tf|\.tfvars)$ exclude: \.terraform/.*$ -- id: terraform_fmt_py - name: Terraform fmt - description: Rewrites all Terraform configuration files to a canonical format. - require_serial: true - entry: python -Im pre_commit_terraform terraform_fmt_py - language: python - files: \.tf(vars)?$ - exclude: \.terraform/.*$ - - id: terraform_docs name: Terraform docs description: Inserts input and output documentation into README.md (using terraform-docs). @@ -147,16 +138,6 @@ exclude: \.terraform/.*$ require_serial: true -- id: terraform_checkov_py - name: Checkov - description: Runs checkov on Terraform templates. - entry: python -Im pre_commit_terraform terraform_checkov_py - language: python - always_run: false - files: \.tf$ - exclude: \.terraform/.*$ - require_serial: true - - id: terraform_wrapper_module_for_each name: Terraform wrapper with for_each in module description: Generate Terraform wrappers with for_each in module. From a278a04b257f82ee86c84410e7f462fb357a9810 Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Fri, 3 Jan 2025 13:21:27 +0200 Subject: [PATCH 61/67] Apply suggestions from code review --- .github/CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 051bbf45f..c24c87d81 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -18,7 +18,7 @@ Enjoy the clean, valid, and documented code! * [Prepare basic documentation](#prepare-basic-documentation) * [Add code](#add-code) * [Finish with the documentation](#finish-with-the-documentation) -* [Testing](#testing-python-hooks) +* [Testing python hooks](#testing-python-hooks) ## Run and debug hooks locally @@ -159,7 +159,7 @@ You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/ ## Testing python hooks -Tu run tests, you need: +To run tests, you need: ```bash pip install pytest pytest-mock From 07998171f46b71bb2abc8f03677e8d59efdbafcc Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 3 Jan 2025 14:56:37 +0200 Subject: [PATCH 62/67] we-make-styleguide requires python 3.10 --- .github/workflows/pre-commit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 99e246818..9ecfabe4c 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -38,7 +38,7 @@ jobs: # Skip terraform_tflint which interferes to commit pre-commit auto-fixes - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: - python-version: '3.9' + python-version: '3.10' - name: Execute pre-commit uses: pre-commit/action@9b88afc9cd57fd75b655d5c71bd38146d07135fe # v2.0.3 env: From d3df63bf8bec3fa3e540c97f27144c13c85fe6ba Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Mon, 6 Jan 2025 22:36:08 +0200 Subject: [PATCH 63/67] Discard changes to .github/CONTRIBUTING.md --- .github/CONTRIBUTING.md | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index c24c87d81..8066f6f30 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -18,7 +18,6 @@ Enjoy the clean, valid, and documented code! * [Prepare basic documentation](#prepare-basic-documentation) * [Add code](#add-code) * [Finish with the documentation](#finish-with-the-documentation) -* [Testing python hooks](#testing-python-hooks) ## Run and debug hooks locally @@ -155,27 +154,3 @@ You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/ 1. Add hook description to [Available Hooks](../README.md#available-hooks). 2. Create and populate a new hook section in [Hooks usage notes and examples](../README.md#hooks-usage-notes-and-examples). - - -## Testing python hooks - -To run tests, you need: - -```bash -pip install pytest pytest-mock - -pytest -vv -``` - -To run and debug hooks as CLI: - -1. Create [`venv`](https://docs.python.org/3/library/venv.html) and activate it, IE by `virtualenv venv` -2. Inside virtual env install pre-commit-terraform as package: - ```bash - pip install --editable . - ``` - -3. Run next to show basic help: - ```bash - python -Im pre_commit_terraform --help - ``` From 56ab01e22e48409bbeb5c431f61f638b69ea48f3 Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Mon, 6 Jan 2025 23:08:25 +0200 Subject: [PATCH 64/67] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: George L. Yermulnik Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- .pre-commit-config.yaml | 8 ++++---- src/pre_commit_terraform/_common.py | 14 ++++---------- src/pre_commit_terraform/_logger.py | 2 +- src/pre_commit_terraform/_run_on_whole_repo.py | 9 ++++----- 4 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 73bb6e5f3..04722166c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,7 +61,7 @@ repos: # JSON5 Linter - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 + rev: v3.1.0 hooks: - id: prettier # https://prettier.io/docs/en/options.html#parser @@ -90,9 +90,9 @@ repos: args: [--force-single-line, --profile=black] exclude: | (?x) - # It set wrong indent in imports in place where it shouldn't + # Uses incorrect indentation in imports in places where it shouldn't # https://github.com/PyCQA/isort/issues/2315#issuecomment-2566703698 - (^tests/pytest/test__cli_subcommands.py$ + (^tests/pytest/test__cli_subcommands\.py$ ) - repo: https://github.com/asottile/add-trailing-comma @@ -127,7 +127,7 @@ repos: hooks: - id: mypy additional_dependencies: - - types-PyYAML + - types-PyYAML args: [ --ignore-missing-imports, --disallow-untyped-calls, diff --git a/src/pre_commit_terraform/_common.py b/src/pre_commit_terraform/_common.py index 3c356855d..3b443e6af 100644 --- a/src/pre_commit_terraform/_common.py +++ b/src/pre_commit_terraform/_common.py @@ -1,7 +1,7 @@ """ -Here located common functions for hooks. +Common functions for hooks. -It not executed directly, but imported by other hooks. +These are not executed directly, but imported by other hooks. """ from __future__ import annotations @@ -45,20 +45,14 @@ def _get_unique_dirs(files: list[str]) -> set[str]: Returns: Set of unique directories. """ - unique_dirs = set() - - for file_path in files: - dir_path = os.path.dirname(file_path) - unique_dirs.add(dir_path) - - return unique_dirs + return set(os.path.dirname(path) for path in files) def expand_env_vars(args: list[str], env_vars: dict[str, str]) -> list[str]: """ Expand environment variables definition into their values in '--args'. - Support expansion only for ${ENV_VAR} vars, not $ENV_VAR. + Supports expansion only for ${ENV_VAR} vars, not $ENV_VAR. Args: args: The arguments to expand environment variables in. diff --git a/src/pre_commit_terraform/_logger.py b/src/pre_commit_terraform/_logger.py index 816df29f6..9944770a7 100644 --- a/src/pre_commit_terraform/_logger.py +++ b/src/pre_commit_terraform/_logger.py @@ -1,4 +1,4 @@ -"""Here located logs-related functions.""" +"""Logs-related functions.""" import logging import os diff --git a/src/pre_commit_terraform/_run_on_whole_repo.py b/src/pre_commit_terraform/_run_on_whole_repo.py index 878050dbe..397a0d7f2 100644 --- a/src/pre_commit_terraform/_run_on_whole_repo.py +++ b/src/pre_commit_terraform/_run_on_whole_repo.py @@ -25,7 +25,7 @@ def is_function_defined(func_name: str, scope: dict) -> bool: is_callable = callable(scope[func_name]) if is_defined else False logger.debug( - 'Checking if "%s":\n1. Defined in hook: %s\n2. Is it callable: %s', + 'Checking if "%s":\n1. Defined in hook: %s\n2. Is callable: %s', func_name, is_defined, is_callable, @@ -53,7 +53,6 @@ def is_hook_run_on_whole_repo(hook_id: str, file_paths: list[str]) -> bool: # Get the directory containing the packaged `.pre-commit-hooks.yaml` copy artifacts_root_path = access_artifacts_of('pre_commit_terraform') / '_artifacts' pre_commit_hooks_yaml_path = artifacts_root_path / '.pre-commit-hooks.yaml' - pre_commit_hooks_yaml_path.read_text(encoding='utf-8') logger.debug('Hook config path: %s', pre_commit_hooks_yaml_path) @@ -75,7 +74,7 @@ def is_hook_run_on_whole_repo(hook_id: str, file_paths: list[str]) -> bool: included_pattern, excluded_pattern, ) - # S607 disabled as we need to maintain ability to call git command no matter where it located. + # S607 disabled as we need to maintain ability to call git command no matter where it is located. git_ls_files_cmd = ['git', 'ls-files'] # noqa: S607 # Get the sorted list of all files that can be checked using `git ls-files` git_ls_file_paths = subprocess.check_output(git_ls_files_cmd, text=True).splitlines() @@ -94,10 +93,10 @@ def is_hook_run_on_whole_repo(hook_id: str, file_paths: list[str]) -> bool: # Get the sorted list of files passed to the hook file_paths_to_check = sorted(file_paths) logger.debug( - 'Files to check:\n%s\n\nAll files that can be checked:\n%s\n\nIdentical lists: %s', + 'Files to check:\n%s\n\nAll files that can be checked:\n%s\n\nAre these lists identical: %s', file_paths_to_check, all_file_paths_that_can_be_checked, file_paths_to_check == all_file_paths_that_can_be_checked, ) - # Compare the sorted lists of file + # Compare the sorted lists of files return file_paths_to_check == all_file_paths_that_can_be_checked From f622b100f37cb8ac2a4c54721815df69ed125dfe Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Mon, 6 Jan 2025 23:11:49 +0200 Subject: [PATCH 65/67] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04722166c..58387760c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -155,5 +155,5 @@ repos: # Ignore tests (^tests/pytest/test_.+\.py$ # Ignore deprecated hook - |^src/pre_commit_terraform/terraform_docs_replace.py$ + |^src/pre_commit_terraform/terraform_docs_replace\.py$ ) From 14f9d2a3a07f674851aff7bbf4e8b63575a82787 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:30:35 +0000 Subject: [PATCH 66/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pre_commit_terraform/_cli.py | 14 ++++---- src/pre_commit_terraform/_types.py | 7 ++-- .../terraform_docs_replace.py | 15 +++++---- tests/pytest/_cli_test.py | 33 ++++++++++--------- tests/pytest/terraform_docs_replace_test.py | 33 ++++++++----------- 5 files changed, 50 insertions(+), 52 deletions(-) diff --git a/src/pre_commit_terraform/_cli.py b/src/pre_commit_terraform/_cli.py index edcb2ea30..e05539fd1 100644 --- a/src/pre_commit_terraform/_cli.py +++ b/src/pre_commit_terraform/_cli.py @@ -4,13 +4,12 @@ from typing import cast as cast_to from ._cli_parsing import initialize_argument_parser -from ._errors import ( - PreCommitTerraformBaseError, - PreCommitTerraformExit, - PreCommitTerraformRuntimeError, -) +from ._errors import PreCommitTerraformBaseError +from ._errors import PreCommitTerraformExit +from ._errors import PreCommitTerraformRuntimeError from ._structs import ReturnCode -from ._types import CLIAppEntryPointCallableType, ReturnCodeType +from ._types import CLIAppEntryPointCallableType +from ._types import ReturnCodeType def invoke_cli_app(cli_args: list[str]) -> ReturnCodeType: @@ -34,8 +33,7 @@ def invoke_cli_app(cli_args: list[str]) -> ReturnCodeType: raise except PreCommitTerraformRuntimeError as unhandled_exc: print( - f'App execution took an unexpected turn: {unhandled_exc !s}. ' - 'Exiting...', + f'App execution took an unexpected turn: {unhandled_exc !s}. ' 'Exiting...', file=sys.stderr, ) return ReturnCode.ERROR diff --git a/src/pre_commit_terraform/_types.py b/src/pre_commit_terraform/_types.py index dac66d75b..60613ea70 100644 --- a/src/pre_commit_terraform/_types.py +++ b/src/pre_commit_terraform/_types.py @@ -1,12 +1,13 @@ """Composite types for annotating in-project code.""" -from argparse import ArgumentParser, Namespace +from argparse import ArgumentParser +from argparse import Namespace from collections.abc import Callable -from typing import Protocol, Union +from typing import Protocol +from typing import Union from pre_commit_terraform._structs import ReturnCode - ReturnCodeType = Union[ReturnCode, int] # Union instead of pipe for Python 3.9 CLIAppEntryPointCallableType = Callable[[Namespace], ReturnCodeType] diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py index 7f0cbd13f..aebb05443 100644 --- a/src/pre_commit_terraform/terraform_docs_replace.py +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -3,7 +3,8 @@ import os import subprocess import warnings -from argparse import ArgumentParser, Namespace +from argparse import ArgumentParser +from argparse import Namespace from typing import Final from typing import cast as cast_to @@ -71,8 +72,9 @@ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: dirs = [] for filename in cast_to(list[str], parsed_cli_args.filenames): - if (os.path.realpath(filename) not in dirs and - (filename.endswith(".tf") or filename.endswith(".tfvars"))): + if os.path.realpath(filename) not in dirs and ( + filename.endswith('.tf') or filename.endswith('.tfvars') + ): dirs.append(os.path.dirname(filename)) retval = ReturnCode.OK @@ -84,13 +86,12 @@ def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: if cast_to(bool, parsed_cli_args.sort): procArgs.append('--sort-by-required') procArgs.append('md') - procArgs.append("./{dir}".format(dir=dir)) + procArgs.append('./{dir}'.format(dir=dir)) procArgs.append('>') procArgs.append( - './{dir}/{dest}'. - format(dir=dir, dest=cast_to(bool, parsed_cli_args.dest)), + './{dir}/{dest}'.format(dir=dir, dest=cast_to(bool, parsed_cli_args.dest)), ) - subprocess.check_call(" ".join(procArgs), shell=True) + subprocess.check_call(' '.join(procArgs), shell=True) except subprocess.CalledProcessError as e: print(e) retval = ReturnCode.ERROR diff --git a/tests/pytest/_cli_test.py b/tests/pytest/_cli_test.py index 52ea82ab6..c7f85bf46 100644 --- a/tests/pytest/_cli_test.py +++ b/tests/pytest/_cli_test.py @@ -1,19 +1,18 @@ """Tests for the high-level CLI entry point.""" -from argparse import ArgumentParser, Namespace +from argparse import ArgumentParser +from argparse import Namespace + import pytest from pre_commit_terraform import _cli_parsing as _cli_parsing_mod from pre_commit_terraform._cli import invoke_cli_app -from pre_commit_terraform._errors import ( - PreCommitTerraformExit, - PreCommitTerraformBaseError, - PreCommitTerraformRuntimeError, -) +from pre_commit_terraform._errors import PreCommitTerraformBaseError +from pre_commit_terraform._errors import PreCommitTerraformExit +from pre_commit_terraform._errors import PreCommitTerraformRuntimeError from pre_commit_terraform._structs import ReturnCode from pre_commit_terraform._types import ReturnCodeType - pytestmark = pytest.mark.filterwarnings( 'ignore:`terraform_docs_replace` hook is DEPRECATED.:UserWarning:' 'pre_commit_terraform.terraform_docs_replace', @@ -42,17 +41,19 @@ ), ) def test_known_interrupts( - capsys: pytest.CaptureFixture[str], - expected_stderr: str, - monkeypatch: pytest.MonkeyPatch, - raised_error: BaseException, + capsys: pytest.CaptureFixture[str], + expected_stderr: str, + monkeypatch: pytest.MonkeyPatch, + raised_error: BaseException, ) -> None: """Check that known interrupts are turned into return code 1.""" + class CustomCmdStub: CLI_SUBCOMMAND_NAME = 'sentinel' def populate_argument_parser( - self, subcommand_parser: ArgumentParser, + self, + subcommand_parser: ArgumentParser, ) -> None: return None @@ -72,15 +73,17 @@ def invoke_cli_app(self, parsed_cli_args: Namespace) -> ReturnCodeType: def test_app_exit( - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: """Check that an exit exception is re-raised.""" + class CustomCmdStub: CLI_SUBCOMMAND_NAME = 'sentinel' def populate_argument_parser( - self, subcommand_parser: ArgumentParser, + self, + subcommand_parser: ArgumentParser, ) -> None: return None diff --git a/tests/pytest/terraform_docs_replace_test.py b/tests/pytest/terraform_docs_replace_test.py index 87989d965..0a66d8e81 100644 --- a/tests/pytest/terraform_docs_replace_test.py +++ b/tests/pytest/terraform_docs_replace_test.py @@ -1,16 +1,17 @@ """Tests for the `replace-docs` subcommand.""" -from argparse import ArgumentParser, Namespace +from argparse import ArgumentParser +from argparse import Namespace from subprocess import CalledProcessError import pytest import pytest_mock from pre_commit_terraform._structs import ReturnCode +from pre_commit_terraform.terraform_docs_replace import invoke_cli_app +from pre_commit_terraform.terraform_docs_replace import populate_argument_parser from pre_commit_terraform.terraform_docs_replace import ( - invoke_cli_app, - populate_argument_parser, - subprocess as replace_docs_subprocess_mod, + subprocess as replace_docs_subprocess_mod, ) @@ -23,10 +24,7 @@ def test_arg_parser_populated() -> None: def test_check_is_deprecated() -> None: """Verify that `replace-docs` shows a deprecation warning.""" - deprecation_msg_regex = ( - r'^`terraform_docs_replace` hook is DEPRECATED\.' - 'For migration.*$' - ) + deprecation_msg_regex = r'^`terraform_docs_replace` hook is DEPRECATED\.' 'For migration.*$' with pytest.warns(UserWarning, match=deprecation_msg_regex): # not `pytest.deprecated_call()` due to this being a user warning invoke_cli_app(Namespace(filenames=[])) @@ -53,8 +51,7 @@ def test_check_is_deprecated() -> None: ), [ 'terraform-docs --sort-by-required md ./ > .//SENTINEL.md', - 'terraform-docs --sort-by-required md ./thing ' - '> ./thing/SENTINEL.md', + 'terraform-docs --sort-by-required md ./thing ' '> ./thing/SENTINEL.md', ], id='two-sorted-files', ), @@ -70,10 +67,10 @@ def test_check_is_deprecated() -> None: 'pre_commit_terraform.terraform_docs_replace', ) def test_control_flow_positive( - expected_cmds: list[str], - mocker: pytest_mock.MockerFixture, - monkeypatch: pytest.MonkeyPatch, - parsed_cli_args: Namespace, + expected_cmds: list[str], + mocker: pytest_mock.MockerFixture, + monkeypatch: pytest.MonkeyPatch, + parsed_cli_args: Namespace, ) -> None: """Check that the subcommand's happy path works.""" check_call_mock = mocker.Mock() @@ -85,9 +82,7 @@ def test_control_flow_positive( assert ReturnCode.OK == invoke_cli_app(parsed_cli_args) - executed_commands = [ - cmd for ((cmd, ), _shell) in check_call_mock.call_args_list - ] + executed_commands = [cmd for ((cmd,), _shell) in check_call_mock.call_args_list] assert len(expected_cmds) == check_call_mock.call_count assert expected_cmds == executed_commands @@ -98,8 +93,8 @@ def test_control_flow_positive( 'pre_commit_terraform.terraform_docs_replace', ) def test_control_flow_negative( - mocker: pytest_mock.MockerFixture, - monkeypatch: pytest.MonkeyPatch, + mocker: pytest_mock.MockerFixture, + monkeypatch: pytest.MonkeyPatch, ) -> None: """Check that the subcommand's error processing works.""" parsed_cli_args = Namespace( From 1bdb83e4ac2523719cd8d464234acc32b50fb416 Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Wed, 8 Jan 2025 19:31:20 +0200 Subject: [PATCH 67/67] Discard changes to .gitignore --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index b637bfb4b..10bbad3e8 100644 --- a/.gitignore +++ b/.gitignore @@ -177,6 +177,3 @@ pyrightconfig.json # End of https://www.toptal.com/developers/gitignore/api/python tests/results/* -__pycache__/ -*.py[cod] -node_modules/