From 2f660f71f9d35c3430fbd81377f958d81969304e Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Mon, 2 Oct 2017 14:28:43 -0700 Subject: [PATCH 1/4] Test file creation and properly clean up environments --- test/__init__.py | 0 test/data/ignores.yml | 2 +- test/data/multibase.yml | 6 +- .../relative_path_test_dir/include_target.yml | 2 +- test/helpers.py | 56 +++++++++++++++++++ test/test_features.py | 29 +++++++--- 6 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 test/__init__.py create mode 100644 test/helpers.py diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/data/ignores.yml b/test/data/ignores.yml index 7a4fa83..7295492 100644 --- a/test/data/ignores.yml +++ b/test/data/ignores.yml @@ -1,4 +1,4 @@ -target: +target_ignore_string: FROM: alpine build_directory: ./test_build ignore: | diff --git a/test/data/multibase.yml b/test/data/multibase.yml index 327b022..c20aa15 100644 --- a/test/data/multibase.yml +++ b/test/data/multibase.yml @@ -9,13 +9,15 @@ install_something: build: | RUN mkdir -p /opt/java8 RUN touch /opt/java8/java + RUN echo success2 > /opt/success -target3: +target3_bases: requires: - install_something FROM: continuumio/miniconda3 + build: RUN echo success3 > /opt/success -target2: +target2_bases: requires: - install_something - base2 diff --git a/test/data/relative_path_test_dir/include_target.yml b/test/data/relative_path_test_dir/include_target.yml index bfb2d64..37fe84a 100644 --- a/test/data/relative_path_test_dir/include_target.yml +++ b/test/data/relative_path_test_dir/include_target.yml @@ -1,4 +1,4 @@ -target: +target_include: FROM: debian:jessie build_directory: ../ build: diff --git a/test/helpers.py b/test/helpers.py new file mode 100644 index 0000000..ba6ad5b --- /dev/null +++ b/test/helpers.py @@ -0,0 +1,56 @@ +import os +import io +import tarfile +import pytest +import docker.errors + +__client = None + +def _get_client(): + """ + Returns: + docker.APIClient + """ + global __client + if __client is None: + __client = docker.from_env() + return __client + + +def creates_images(*imgnames): + """ Creates fixtures to make sure to remove named images after (and before, if necessary) + running a test + """ + @pytest.fixture + def fixture(): + client = _get_client() + for name in imgnames: + try: + client.images.remove(name, force=True) + except docker.errors.ImageNotFound: + pass + + yield + + for name in imgnames: # force it to also remove the containers + client.images.remove(name, force=True) + return fixture + + +def assert_file_content(imgname, path, content): + """ Asserts that an image exists with a file at the + specified path containing the specified content + """ + client = _get_client() + try: + image = client.images.get(imgname) + except (docker.errors.ImageNotFound, docker.errors.APIError) as exc: + assert False, "Image %s not found: %s" % (imgname, exc) + + container = client.containers.create(image) + tarstream, stat = container.get_archive(path) + container.remove() + + tf = tarfile.open(fileobj=io.BytesIO(tarstream.read())) + val = tf.extractfile(os.path.basename(path)).read().decode('utf-8') + assert val.strip() == content diff --git a/test/test_features.py b/test/test_features.py index 21c730a..9f936ac 100644 --- a/test/test_features.py +++ b/test/test_features.py @@ -1,14 +1,29 @@ -import subprocess +import pytest +from dockermake.__main__ import _runargs as run_docker_make +from .helpers import assert_file_content, creates_images -def test_multiple_bases(): - subprocess.check_call(['docker-make', '-f', 'data/multibase.yml', 'target2', 'target3']) +# note: these tests MUST be run with CWD REPO_ROOT/tests -def test_paths_relative_interpreted_relative_to_definition_file(): - subprocess.check_call(['docker-make', '-f', 'data/include.yml', 'target']) +img1 = creates_images(*'target2_bases target3_bases'.split()) +def test_multiple_bases(img1): + run_docker_make('-f data/multibase.yml target2_bases target3_bases') + assert_file_content('target2_bases', '/opt/success', 'success2') + assert_file_content('target3_bases', '/opt/success', 'success3') -def test_ignore_string(): - subprocess.check_call(['docker-make', '-f', 'data/ignores.yml', 'target']) +img2 = creates_images('target_include') +def test_paths_relative_interpreted_relative_to_definition_file(img2): + run_docker_make('-f data/include.yml target_include') + assert_file_content('target_include', '/opt/testfile.txt', + 'this is a file used in tests for relative path resolution') + + +img3 = creates_images('target_ignore_string') +def test_ignore_string(img3): + run_docker_make('-f data/ignores.yml target_ignore_string') + assert_file_content('target_ignore_string', '/opt/a', 'a') + assert_file_content('target_ignore_string', '/opt/c', 'c') + From 17720d3efce1943dc6e01a3192ac6586637a6eff Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Mon, 2 Oct 2017 15:58:36 -0700 Subject: [PATCH 2/4] Better ignorefile coverage --- test/data/ignores.yml | 13 +++++++++++++ test/helpers.py | 6 +++++- test/test_features.py | 19 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/test/data/ignores.yml b/test/data/ignores.yml index 7295492..8bb7495 100644 --- a/test/data/ignores.yml +++ b/test/data/ignores.yml @@ -6,3 +6,16 @@ target_ignore_string: build: | ADD . /opt +target_ignorefile: + FROM: alpine + build_directory: ./test_build + ignorefile: test_build/custom_ignore_file.txt + build: | + ADD . /opt + + +target_regular_ignore: + FROM: alpine + build_directory: ./test_build + build: | + ADD . /opt \ No newline at end of file diff --git a/test/helpers.py b/test/helpers.py index ba6ad5b..364fb29 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -48,7 +48,11 @@ def assert_file_content(imgname, path, content): assert False, "Image %s not found: %s" % (imgname, exc) container = client.containers.create(image) - tarstream, stat = container.get_archive(path) + + try: + tarstream, stat = container.get_archive(path) + except docker.errors.NotFound: + assert False, 'File %s not found' % path container.remove() tf = tarfile.open(fileobj=io.BytesIO(tarstream.read())) diff --git a/test/test_features.py b/test/test_features.py index 9f936ac..aba5123 100644 --- a/test/test_features.py +++ b/test/test_features.py @@ -24,6 +24,25 @@ def test_paths_relative_interpreted_relative_to_definition_file(img2): def test_ignore_string(img3): run_docker_make('-f data/ignores.yml target_ignore_string') assert_file_content('target_ignore_string', '/opt/a', 'a') + with pytest.raises(AssertionError): + assert_file_content('target_ignore_string', '/opt/b', 'b') assert_file_content('target_ignore_string', '/opt/c', 'c') +img4 = creates_images('target_ignorefile') +def test_ignorefile(img4): + run_docker_make('-f data/ignores.yml target_ignorefile') + assert_file_content('target_ignorefile', '/opt/a', 'a') + assert_file_content('target_ignorefile', '/opt/b', 'b') + with pytest.raises(AssertionError): + assert_file_content('target_ignorefile', '/opt/c', 'c') + + +img5 = creates_images('target_regular_ignore') +def test_regular_ignore(img5): + run_docker_make('-f data/ignores.yml target_regular_ignore') + with pytest.raises(AssertionError): + assert_file_content('target_regular_ignore', '/opt/a', 'a') + with pytest.raises(AssertionError): + assert_file_content('target_regular_ignore', '/opt/b', 'b') + assert_file_content('target_regular_ignore', '/opt/c', 'c') From aca7b3c528d69950cb051931e26f941bb2fe755b Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Mon, 2 Oct 2017 20:03:43 -0700 Subject: [PATCH 3/4] Fix multi-line ignorefile parsing --- dockermake/step.py | 2 +- test/data/ignores.yml | 11 ++++++ test/data/test_build/custom_ignore_file.txt | 5 +++ test/data/test_build/d/d | 1 + test/test_features.py | 39 ++++++++++++++------- 5 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 test/data/test_build/d/d diff --git a/dockermake/step.py b/dockermake/step.py index 67bb5ed..ee1df23 100644 --- a/dockermake/step.py +++ b/dockermake/step.py @@ -59,7 +59,7 @@ def _get_ignorefile(img_def): elif img_def.get('ignorefile', None) is not None: assert 'ignore' not in img_def with open(img_def['ignorefile'], 'r') as igfile: - lines = list(igfile) + lines = igfile.read().splitlines() else: return None diff --git a/test/data/ignores.yml b/test/data/ignores.yml index 8bb7495..5127c35 100644 --- a/test/data/ignores.yml +++ b/test/data/ignores.yml @@ -2,7 +2,9 @@ target_ignore_string: FROM: alpine build_directory: ./test_build ignore: | + # comment b + unmatched_line build: | ADD . /opt @@ -17,5 +19,14 @@ target_ignorefile: target_regular_ignore: FROM: alpine build_directory: ./test_build + build: | + ADD . /opt + +target_ignore_directory: + FROM: alpine + ignore: | + d + unmatched_line + build_directory: ./test_build build: | ADD . /opt \ No newline at end of file diff --git a/test/data/test_build/custom_ignore_file.txt b/test/data/test_build/custom_ignore_file.txt index 3410062..2d5bb9d 100644 --- a/test/data/test_build/custom_ignore_file.txt +++ b/test/data/test_build/custom_ignore_file.txt @@ -1 +1,6 @@ +unmatched-line-1 +c +**/unmatched-line-2 + +# comment c \ No newline at end of file diff --git a/test/data/test_build/d/d b/test/data/test_build/d/d new file mode 100644 index 0000000..c59d9b6 --- /dev/null +++ b/test/data/test_build/d/d @@ -0,0 +1 @@ +d \ No newline at end of file diff --git a/test/test_features.py b/test/test_features.py index aba5123..79c5646 100644 --- a/test/test_features.py +++ b/test/test_features.py @@ -20,29 +20,42 @@ def test_paths_relative_interpreted_relative_to_definition_file(img2): 'this is a file used in tests for relative path resolution') +_FILES = {'a': {'content': 'a', 'path': '/opt/a'}, + 'b': {'content': 'b', 'path': '/opt/b'}, + 'c': {'content': 'c', 'path': '/opt/c'}, + 'd': {'content': 'd', 'path': '/opt/d/d'}} + + img3 = creates_images('target_ignore_string') def test_ignore_string(img3): run_docker_make('-f data/ignores.yml target_ignore_string') - assert_file_content('target_ignore_string', '/opt/a', 'a') - with pytest.raises(AssertionError): - assert_file_content('target_ignore_string', '/opt/b', 'b') - assert_file_content('target_ignore_string', '/opt/c', 'c') + _check_files('target_ignore_string', b=False) img4 = creates_images('target_ignorefile') def test_ignorefile(img4): run_docker_make('-f data/ignores.yml target_ignorefile') - assert_file_content('target_ignorefile', '/opt/a', 'a') - assert_file_content('target_ignorefile', '/opt/b', 'b') - with pytest.raises(AssertionError): - assert_file_content('target_ignorefile', '/opt/c', 'c') + _check_files('target_ignorefile', c=False) img5 = creates_images('target_regular_ignore') def test_regular_ignore(img5): run_docker_make('-f data/ignores.yml target_regular_ignore') - with pytest.raises(AssertionError): - assert_file_content('target_regular_ignore', '/opt/a', 'a') - with pytest.raises(AssertionError): - assert_file_content('target_regular_ignore', '/opt/b', 'b') - assert_file_content('target_regular_ignore', '/opt/c', 'c') + _check_files('target_regular_ignore', a=False, b=False) + + +img6 = creates_images('target_ignore_directory') +def test_ignore_directory(img6): + run_docker_make('-f data/ignores.yml target_ignore_directory') + _check_files('target_ignore_directory', d=False) + + +def _check_files(img, **present): + for f, record in _FILES.items(): + if not present.get(f, True): + with pytest.raises(AssertionError): + assert_file_content(img, record['path'], record['content']) + else: + assert_file_content(img, record['path'], record['content']) + + From 6569454433c1249910fcf684394bb88709d2bd89 Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Mon, 2 Oct 2017 20:04:35 -0700 Subject: [PATCH 4/4] Handle missing top-level makefile --- dockermake/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dockermake/__main__.py b/dockermake/__main__.py index 3c080ad..0592ef4 100755 --- a/dockermake/__main__.py +++ b/dockermake/__main__.py @@ -63,10 +63,10 @@ def run(args): return if not os.path.exists(args.makefile): - print('No docker makefile found at path "%s"'%args.makefile) + msg = 'No docker makefile found at path "%s"' % args.makefile if args.makefile == 'DockerMake.yml': - print('Type `docker-make --help` to see usage.') - sys.exit(1) + msg += '\nType `docker-make --help` to see usage.' + raise errors.MissingFileError(msg) defs = ImageDefs(args.makefile)