From 6b74adffdd26227d25d21ce096a4cced26ce009e Mon Sep 17 00:00:00 2001 From: Marek Goldmann Date: Tue, 2 May 2017 16:35:51 -0400 Subject: [PATCH 1/3] Initial support for parametrization of descriptor files --- dogen/cli.py | 14 ++++++++++++++ dogen/generator.py | 14 +++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/dogen/cli.py b/dogen/cli.py index 2e8930d..3e6b16e 100644 --- a/dogen/cli.py +++ b/dogen/cli.py @@ -56,6 +56,7 @@ def run(self): parser.add_argument('--skip-ssl-verification', action='store_true', help='Should we skip SSL verification when retrieving data?') parser.add_argument('--scripts-path', help='Location of the scripts directory containing script packages.') parser.add_argument('--additional-script', action='append', help='Location of additional script (can be url). Can be specified multiple times.') + parser.add_argument('--param', action='append', dest='params', help='List of key and value pairs (format: KEY=value) used for substitution in the image descriptor. Can be specified multiple times.') parser.add_argument('--template', help='Path to custom template (can be url)') parser.add_argument('path', help="Path to yaml descriptor to process") @@ -69,6 +70,19 @@ def run(self): parser.epilog = epilog args = parser.parse_args() + params = {} + + if args.params: + for param in args.params: + if '=' not in param: + self.log.error("The --param argument with value '%s' could not be parsed. Please make sure you use the 'KEY=value' format" % param) + sys.exit(1) + + k, v = param.split("=", 1) + params[k] = v + + args.params = params + if args.verbose: self.log.setLevel(logging.DEBUG) else: diff --git a/dogen/generator.py b/dogen/generator.py index 1a8dd65..d310a8f 100644 --- a/dogen/generator.py +++ b/dogen/generator.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- import hashlib +import re import os import shutil +import six import requests import yaml import tempfile @@ -29,6 +31,7 @@ def __init__(self, log, args, plugins=[]): self.template = args.template self.scripts_path = args.scripts_path self.additional_scripts = args.additional_script + self.params = args.params ssl_verify = None if args.skip_ssl_verification: @@ -203,7 +206,16 @@ def _validate_cfg(self): plugin.extend_schema(schema) with open(self.descriptor, 'r') as stream: - self.cfg = yaml.safe_load(stream) + content = stream.read() + + for k, v in six.iteritems(self.params): + self.log.debug("Substituting '%s' with '%s' value..." % (k, v)) + content = re.sub(r"{{%s.*}}" % k, v, content) + + # See if there any params without substitutions. If yes, use specified default values. + content = re.sub(r"{{\w+:(.*)}}", '\g<1>', content) + + self.cfg = yaml.safe_load(content) c = Core(source_data=self.cfg, schema_data=schema) try: From a09be91caa5bc2c3f4ff560a23af146a96cc7e0c Mon Sep 17 00:00:00 2001 From: Marek Goldmann Date: Tue, 2 May 2017 17:02:59 -0400 Subject: [PATCH 2/3] Fix tests after adding 'params' argument --- tests/test_dockerfile.py | 2 +- tests/test_package.py | 2 +- tests/test_plugin_repo.py | 2 +- tests/test_schema.py | 2 +- tests/test_unit_generate_configuration.py | 2 +- tests/test_unit_generate_handle_files.py | 8 ++++---- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_dockerfile.py b/tests/test_dockerfile.py index 63033dc..a07dd7a 100644 --- a/tests/test_dockerfile.py +++ b/tests/test_dockerfile.py @@ -31,7 +31,7 @@ def setUp(self): os.mkdir(self.target) self.args = argparse.Namespace(path=self.yaml, output=self.target, without_sources=None, template=None, scripts_path=None, additional_script=None, - skip_ssl_verification=None) + skip_ssl_verification=None, params={}) with open(self.yaml, 'wb') as f: f.write(self.basic_config.encode()) diff --git a/tests/test_package.py b/tests/test_package.py index 3b961bc..1a429a3 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -26,7 +26,7 @@ def write_config(self, config): def prepare_dogen(self, repo_files_dir=None): args = argparse.Namespace(path=self.descriptor.name, output=self.target_dir, without_sources=None, template=None, scripts_path=None, additional_script=None, - skip_ssl_verification=None, repo_files_dir=repo_files_dir) + skip_ssl_verification=None, repo_files_dir=repo_files_dir, params={}) self.dogen = Generator(self.log, args, [Repo]) def test_custom_repo_files_should_add_two(self): diff --git a/tests/test_plugin_repo.py b/tests/test_plugin_repo.py index 99ba860..1950f05 100644 --- a/tests/test_plugin_repo.py +++ b/tests/test_plugin_repo.py @@ -32,7 +32,7 @@ def write_config(self, config): def prepare_dogen(self, repo_files_dir=None): args = argparse.Namespace(path=self.descriptor.name, output=self.target_dir, without_sources=None, template=None, scripts_path=None, additional_script=None, - skip_ssl_verification=None, repo_files_dir=repo_files_dir) + skip_ssl_verification=None, repo_files_dir=repo_files_dir, params={}) self.dogen = Generator(self.log, args, [Repo]) def test_should_skip_plugin_if_no_path_to_repo_is_provided(self): diff --git a/tests/test_schema.py b/tests/test_schema.py index 25abf46..eede138 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -13,7 +13,7 @@ def gen_test(path, good): def test(self): args = argparse.Namespace(path=path, output="target", without_sources=None, template=None, scripts_path=None, additional_script=None, - skip_ssl_verification=None) + skip_ssl_verification=None, params={}) generator = Generator(self.log, args) if good: generator.configure() diff --git a/tests/test_unit_generate_configuration.py b/tests/test_unit_generate_configuration.py index 4572a7d..4f6238c 100644 --- a/tests/test_unit_generate_configuration.py +++ b/tests/test_unit_generate_configuration.py @@ -20,7 +20,7 @@ def setUp(self): self.descriptor.write(self.basic_config.encode()) self.args = argparse.Namespace(path=self.descriptor.name, output="target", without_sources=False, template=None, scripts_path=None, additional_script=None, - skip_ssl_verification=None) + skip_ssl_verification=None, params={}) def tearDown(self): os.remove(self.descriptor.name) diff --git a/tests/test_unit_generate_handle_files.py b/tests/test_unit_generate_handle_files.py index eb85ab9..44adc9b 100644 --- a/tests/test_unit_generate_handle_files.py +++ b/tests/test_unit_generate_handle_files.py @@ -12,7 +12,7 @@ def setUp(self): self.log = mock.Mock() args = argparse.Namespace(path="image.yaml", output="target", without_sources=None, template=None, scripts_path=None, additional_script=None, - skip_ssl_verification=None) + skip_ssl_verification=None, params={}) self.generator = Generator(self.log, args) def test_local_file(self): @@ -29,7 +29,7 @@ def setUp(self): self.log = mock.Mock() args = argparse.Namespace(path="image.yaml", output="target", without_sources=None, template=None, scripts_path=None, additional_script=None, - skip_ssl_verification=None) + skip_ssl_verification=None, params={}) self.generator = Generator(self.log, args) @mock.patch('dogen.generator.requests.get') @@ -72,13 +72,13 @@ def setUp(self): self.log = mock.Mock() args = argparse.Namespace(path="image.yaml", output="target", without_sources=None, template="http://host/custom-template", scripts_path=None, - additional_script=None, skip_ssl_verification=None) + additional_script=None, skip_ssl_verification=None, params={}) self.generator = Generator(self.log, args) def test_do_not_fail_if_no_template_is_provided(self): args = argparse.Namespace(path="image.yaml", output="target", without_sources=None, template=None, scripts_path=None, additional_script=None, - skip_ssl_verification=None) + skip_ssl_verification=None, params={}) self.generator = Generator(self.log, args) fetch_file_mock = mock.Mock() From d9f0178834297d4beb2a14c5990ef2c3f8f4d85c Mon Sep 17 00:00:00 2001 From: Marek Goldmann Date: Tue, 2 May 2017 17:18:49 -0400 Subject: [PATCH 3/3] Add tests for new --param argument --- tests/test_dockerfile.py | 53 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/test_dockerfile.py b/tests/test_dockerfile.py index a07dd7a..e0af423 100644 --- a/tests/test_dockerfile.py +++ b/tests/test_dockerfile.py @@ -112,9 +112,6 @@ def test_set_entrypoint(self): self.assertRegexpMatches(dockerfile, regex) def test_volumes(self): - """ - Test that cmd: is mapped into a CMD instruction - """ with open(self.yaml, 'ab') as f: f.write("volumes:\n - '/var/lib'\n - '/usr/lib'".encode()) @@ -128,3 +125,53 @@ def test_volumes(self): dockerfile = f.read() regex = re.compile(r'.*VOLUME \["/var/lib"\]\nVOLUME \["/usr/lib"\]', re.MULTILINE) self.assertRegexpMatches(dockerfile, regex) + + def test_default_substitution(self): + with open(self.yaml, 'ab') as f: + f.write("from: {{FROM:rhel:7}}\nversion: 1.0".encode()) + + generator = Generator(self.log, self.args) + generator.configure() + generator.render_from_template() + + self.assertEqual(generator.cfg['from'], 'rhel:7') + + with open(os.path.join(self.target, "Dockerfile"), "r") as f: + dockerfile = f.read() + regex = re.compile(r'.*FROM rhel:7', re.MULTILINE) + self.assertRegexpMatches(dockerfile, regex) + + def test_substitution_with_parameters(self): + with open(self.yaml, 'ab') as f: + f.write("from: {{FROM:rhel:7}}\nversion: 1.0".encode()) + + self.args.params = {'FROM': 'centos:7'} + + generator = Generator(self.log, self.args) + generator.configure() + generator.render_from_template() + + self.assertEqual(generator.cfg['from'], 'centos:7') + + with open(os.path.join(self.target, "Dockerfile"), "r") as f: + dockerfile = f.read() + regex = re.compile(r'.*FROM centos:7', re.MULTILINE) + self.assertRegexpMatches(dockerfile, regex) + + def test_substitution_of_multiple_parameters(self): + with open(self.yaml, 'ab') as f: + f.write("from: rhel:7\ndescription: 'Image {{VERSION}}'\nenvs:\n information:\n - name: VERSION\n value: {{VERSION}}\n".encode()) + + self.args.params = {'VERSION': '1.0'} + + generator = Generator(self.log, self.args) + generator.configure() + generator.render_from_template() + + self.assertEqual(generator.cfg['description'], 'Image 1.0') + self.assertEqual(generator.cfg['envs']['information'], [{'name': 'VERSION', 'value': 1.0}]) + + with open(os.path.join(self.target, "Dockerfile"), "r") as f: + dockerfile = f.read() + regex = re.compile(r'.*VERSION="1.0".*', re.MULTILINE) + self.assertRegexpMatches(dockerfile, regex)