scripts: images containers manifest: Build json from dirs

Related: #1273
Signed-off-by: John Andersen <[email protected]>

Author: pdxjohnny

scripts/images_containers_manifest.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# This file generates a manifest for building container images
+# Usage: JSON_INDENT="    " nodemon -e py,Dockerfile,HEAD --exec 'clear; python scripts/images_containers_manifest.py; test 1'
+import os
+import sys
+import json
+import pathlib
+import itertools
+import traceback
+import subprocess
+import urllib.request
+
+
+try:
+    os.environ.update({
+        "COMMIT": subprocess.check_output(["git", "log", "-n", "1", "--format=%H"]).decode().strip(),
+    })
+except:
+    traceback.print_exc(file=sys.stderr)
+
+try:
+    os.environ.update({
+        "ROOT_PATH": str(pathlib.Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode().strip()).relative_to(os.getcwd())),
+        "SCHEMA": "https://github.com/intel/dffml/raw/c82f7ddd29a00d24217c50370907c281c4b5b54d/schema/github/actions/build/images/containers/0.0.0.schema.json",
+        "OWNER_REPOSITORY": "/".join(subprocess.check_output(["git", "remote", "get-url", "origin"]).decode().strip().replace(".git", "").split("/")[-2:]),
+        "BRANCH": subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode().strip(),
+        "PREFIX": os.environ.get("PREFIX", json.dumps([
+            ".",
+            "scripts",
+            "dffml/skel/operations",
+        ])),
+        "NO_DELTA_PREFIX": os.environ.get("NO_DELTA_PREFIX", json.dumps([
+            ".",
+            "scripts",
+            "dffml/skel/operations",
+        ])),
+    })
+except:
+    traceback.print_exc(file=sys.stderr)
+
+def path_to_image_name(path, root_path):
+    # Stem as image name
+    if path.stem != "Dockerfile":
+        return path.stem
+    # Non-top level no stem as image name (filename is Dockerfile)
+    hyphen_dir_path = str(path.parent.relative_to(root_path)).replace(os.sep, "-")
+    if hyphen_dir_path != ".":
+        return hyphen_dir_path
+    # Top level dir Dockerfile use top level dirname
+    return str(root_path.resolve().name)
+
+# Pull request file change delta filter using GitHub API
+prefixes = json.loads(os.environ["PREFIX"])
+no_delta_prefixes = json.loads(os.environ["NO_DELTA_PREFIX"])
+owner, repository = os.environ["OWNER_REPOSITORY"].split("/", maxsplit=1)
+base = None
+env_vars = ["BASE", "BASE_REF"]
+for env_var in env_vars:
+    if env_var in os.environ and os.environ[env_var].strip():
+        # Set if present and not blank
+        base = os.environ[env_var]
+
+# Empty manifest (list of manifests for each build file) in case not triggered
+# from on file change (workflow changed or dispatched).
+manifest = []
+# Path to root of repo
+root_path = pathlib.Path(os.environ["ROOT_PATH"])
+# Grab commit from git
+commit = os.environ["COMMIT"]
+if base is None:
+    print(f"::notice file={__file__},line=1,endLine=1,title=nobase::None of {env_vars!r} found in os.environ", file=sys.stderr)
+
+else:
+    compare_url = os.environ["COMPARE_URL"]
+    compare_url = compare_url.replace("{base}", base)
+    compare_url = compare_url.replace("{head}", os.environ["HEAD"])
+    with urllib.request.urlopen(
+        urllib.request.Request(
+            compare_url,
+            headers={
+                "Authorization": "bearer " + os.environ["GH_ACCESS_TOKEN"],
+            },
+        )
+    ) as response:
+        response_json = json.load(response)
+    # Print for debug
+    print(json.dumps({
+        "@context": {
+            "@vocab": "github_delta_response_json",
+        },
+        "include": manifest,
+    }, sort_keys=True, indent=4), file=sys.stderr)
+    # Build the most recent commit
+    commit = response_json["commits"][-1]["sha"]
+    manifest = list(itertools.chain(*(
+        [
+            [
+                {
+                    "image_name": path_to_image_name(path, root_path),
+                    "dockerfile": str(path.relative_to(root_path)),
+                    "owner": owner,
+                    "repository": repository,
+                    "branch": os.environ["BRANCH"],
+                    "commit": commit,
+                }
+                for path in [
+                    (print(compare_file) or pathlib.Path(compare_file["filename"]))
+                    for compare_file in response_json["files"]
+                    if (
+                        any([
+                            compare_file["filename"].startswith(prefix_path)
+                            for prefix_path in json.loads(os.environ["PREFIX"])
+                        ]) and compare_file["filename"].endswith("Dockerfile")
+                    )
+                ]
+            ]
+        ] + [
+            [
+                json.loads(path.read_text())
+                for path in [
+                    (print(compare_file) or pathlib.Path(compare_file["filename"]))
+                    for compare_file in response_json["files"]
+                    if (
+                        any([
+                            compare_file["filename"].startswith(prefix_path)
+                            for prefix_path in json.loads(os.environ["PREFIX"])
+                        ]) and compare_file["filename"].endswith("manifest.json")
+                    )
+                ]
+            ]
+        ]
+    )))
+
+# Build everything if we aren't sure why we got here
+if not manifest:
+    manifest = list(itertools.chain(*(
+        [
+            [
+                {
+                    "image_name": path_to_image_name(path, root_path),
+                    "dockerfile": str(path.relative_to(root_path)),
+                    "owner": owner,
+                    "repository": repository,
+                    "branch": os.environ["BRANCH"],
+                    "commit": commit,
+                }
+                for path in prefix_path.glob("*Dockerfile")
+            ]
+            for prefix_path in map(pathlib.Path, prefixes)
+            if any(
+                str(prefix_path.relative_to(root_path)) in no_delta_prefix
+                for no_delta_prefix in no_delta_prefixes
+            )
+        ] + [
+            [
+                json.loads(path.read_text())
+                for path in prefix_path.glob("*manifest.json")
+            ]
+            for prefix_path in map(pathlib.Path, prefixes)
+            if any(
+                str(prefix_path.relative_to(root_path)) in no_delta_prefix
+                for no_delta_prefix in no_delta_prefixes
+            )
+        ]
+    )))
+
+# Add proxies or other runtime args/env vars
+for i in manifest:
+    build_args = {}
+    if "build_args" in i:
+        build_args = dict(json.loads(i["build_args"]))
+    for env_var in [
+        "HTTP_PROXY",
+        "HTTPS_PROXY",
+        "NO_PROXY",
+    ]:
+        if not env_var in os.environ:
+            continue
+        build_args[env_var] = os.environ[env_var]
+    i["build_args"] = json.dumps(list(build_args.items()))
+
+manifest = {
+    "@context": {
+        "@vocab": os.environ["SCHEMA"],
+    },
+    "include": manifest,
+}
+print(json.dumps(manifest, sort_keys=True, indent=os.environ.get("JSON_INDENT", None)))
+
+if "GITHUB_OUTPUT" in os.environ:
+    with open(os.environ["GITHUB_OUTPUT"], "a") as fileobj:
+        fileobj.write(f"manifest={json.dumps(manifest)}\n")