open_pdks patching, remove --build-magic (#88)

+ Added on-the-fly patching for some parts of `open_pdks` so all versions can successfully build with the correct reference commits ~ Building now requires a minimum `open_pdks` version: `c74daac`/`1.0.381` - Delisted `sky130_fd_pr_reram` from default includes; will not be available for all versions of sky130 - Removed `--build-magic` flag, replaced with a simple check of whether `magic` is in PATH or not
efabless · May 8, 2024 · b72ce15 · b72ce15
1 parent 13ccef8
commit b72ce15
Show file tree

Hide file tree

Showing 11 changed files with 143 additions and 269 deletions.
diff --git a/Readme.md b/Readme.md
@@ -125,13 +125,9 @@ PDK version 7519dfb04400f224f140749cda44ee7de6f5e095 enabled.
 What's more is: if you're using a repository with a `tool_metadata.yml` file, such as [OpenLane](https://github.com/The-OpenROAD-Project/OpenLane) or [DFFRAM](https://github.com/Cloud-V/DFFRAM), you can just invoke `volare enable --pdk sky130` without the commit hash and Volare will automatically extract the version required by the utility. Once again, if you omit the `--pdk` argument, `sky130` will be used as a default.
 
 ## Building PDKs
-For special cases, i.e. you require other libraries, you'll have to build the PDK yourself, which Volare does support.
+For special cases, you may have to build the PDK yourself, which Volare does support.
 
-You'll either need Magic installed or you'll need to pass the flag `--build-magic` to build Magic ad-hoc for the PDK build, the latter option of which is only supported on Linux and requires all of Magic's dependencies to be installed. On Ubuntu, that's:
-
-```sh
-sudo apt-get install -y python3 tcsh tcl-dev tk-dev libcairo2-dev m4
-```
+You'll need Magic installed and in PATH. You can either do that manually or, if you have [Nix](https://nixos.org), invoke `nix shell github:efabless/openlane2#magic` before building.
 
 You can invoke `volare build --help` for more options. Be aware, the built PDK won't automatically be enabled and you'll have to `volare enable` the appropriate version.
 

diff --git a/volare/__main__.py b/volare/__main__.py
@@ -336,7 +336,6 @@ def enable_or_build_cmd(
     also_push,
     version,
     use_repo_at,
-    build_magic,
     push_libraries,
     session,
 ):
@@ -369,7 +368,6 @@ def enable_or_build_cmd(
                 "jobs": jobs,
                 "clear_build_artifacts": clear_build_artifacts,
                 "use_repo_at": use_repo_at,
-                "build_magic": build_magic,
             },
             push_kwargs={
                 "owner": owner,

diff --git a/volare/__version__.py b/volare/__version__.py
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = "0.16.3"
+__version__ = "0.17.0"
 
 if __name__ == "__main__":
     print(__version__, end="")
diff --git a/volare/build/__init__.py b/volare/build/__init__.py
@@ -56,7 +56,6 @@ def build(
     clear_build_artifacts: bool = True,
     include_libraries: Optional[List[str]] = None,
     use_repo_at: Optional[List[str]] = None,
-    build_magic: bool = False,
 ):
     use_repos = {}
     if use_repo_at is not None:
@@ -74,7 +73,6 @@ def build(
         "clear_build_artifacts": clear_build_artifacts,
         "include_libraries": include_libraries,
         "using_repos": use_repos,
-        "build_magic": build_magic,
     }
 
     build_module = importlib.import_module(f".{pdk}", package=__name__)
@@ -103,7 +101,6 @@ def build_cmd(
     tool_metadata_file_path,
     version,
     use_repo_at,
-    build_magic,
     session,
 ):
     """
@@ -133,7 +130,6 @@ def build_cmd(
         clear_build_artifacts=clear_build_artifacts,
         include_libraries=include_libraries,
         use_repo_at=use_repo_at,
-        build_magic=build_magic,
     )
 
 

diff --git a/volare/build/asap7.py b/volare/build/asap7.py
@@ -144,7 +144,6 @@ def build(
     clear_build_artifacts: bool = True,
     include_libraries: Optional[List[str]] = None,
     using_repos: Optional[Dict[str, str]] = None,
-    build_magic: bool = False,
 ):
     # family = Family.by_name["asap7"]
     # _ = family.resolve_libraries(include_libraries)

diff --git a/volare/build/common.py b/volare/build/common.py
@@ -11,10 +11,95 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
+import shutil
+import subprocess
+
+from volare.github import GitHubSession
 
 
 class RepoMetadata(object):
     def __init__(self, repo, default_commit, default_branch="main"):
         self.repo = repo
         self.default_commit = default_commit
         self.default_branch = default_branch
+
+
+def open_pdks_fix_makefile(at_path: str):
+    backup_path = f"{at_path}.bak"
+    shutil.move(at_path, backup_path)
+
+    fix_fi = False
+
+    with open(backup_path, "r") as file_in, open(at_path, "w") as file_out:
+        for line in file_in:
+            if "_COMMIT = `" in line:
+                line = line.replace("_COMMIT = ", "_COMMIT=")
+            if fix_fi:
+                file_out.write(line.replace("fi", "fi ; \\"))
+                fix_fi = False
+            else:
+                file_out.write(line)
+            if "_COMMIT=`" in line:
+                fix_fi = True
+
+
+def patch_open_pdks(at_path: str):
+    """
+    This functions applies various patches based on the current version of
+    open_pdks in use.
+    """
+    head = subprocess.check_output(
+        ["git", "rev-parse", "HEAD"], cwd=at_path, encoding="utf8"
+    ).strip()
+
+    def is_ancestor(commit: str):
+        nonlocal head, at_path
+        return (
+            subprocess.call(
+                ["git", "merge-base", "--is-ancestor", commit, head],
+                stdout=open(os.devnull, "w"),
+                stderr=open(os.devnull, "w"),
+                cwd=at_path,
+            )
+            == 0
+        )
+
+    can_build = is_ancestor(
+        "c74daac794c83327e54b91cbaf426f722574665c"
+    )  # First one with --with-reference
+    if not can_build:
+        print(
+            f"Commit {head} cannot be built using Volare: the minimum version of open_pdks buildable with Volare is 1.0.381."
+        )
+        exit(-1)
+
+    gf180mcu_sources_ok = is_ancestor(
+        "c1e2118846fd216b2c065a216950e75d2d67ccb8"
+    )  # gf180mcu sources fix
+    if not gf180mcu_sources_ok:
+        print(
+            "Patching gf180mcu Makefile.in…",
+        )
+        open_pdks_fix_makefile(os.path.join(at_path, "gf180mcu", "Makefile.in"))
+
+    download_script_ok = is_ancestor(
+        "ebffedd16788db327af050ac01c3fb1558ebffd1"
+    )  # download script fix
+    if download_script_ok:
+        print("Replacing download.sh…")
+        session = GitHubSession()
+        r = session.get(
+            "https://raw.githubusercontent.com/RTimothyEdwards/open_pdks/ebffedd16788db327af050ac01c3fb1558ebffd1/scripts/download.sh"
+        )
+        with open(os.path.join(at_path, "scripts", "download.sh"), "wb") as f:
+            f.write(r.content)
+
+    sky130_sources_ok = is_ancestor(
+        "274040274a7dfb5fd2c69e0e9c643f80507df3fe"
+    )  # sky130 sources fix
+    if not sky130_sources_ok:
+        print(
+            "Patching sky130 Makefile.in…",
+        )
+        open_pdks_fix_makefile(os.path.join(at_path, "sky130", "Makefile.in"))
diff --git a/volare/build/gf180mcu.py b/volare/build/gf180mcu.py
@@ -14,6 +14,7 @@
 import os
 import io
 import json
+import shlex
 import shutil
 import subprocess
 from datetime import datetime
@@ -24,9 +25,8 @@
 from rich.console import Console
 from rich.progress import Progress
 
-from .magic import with_magic
-from .common import RepoMetadata
 from .git_multi_clone import GitMultiClone
+from .common import RepoMetadata, patch_open_pdks
 from ..families import Family
 from ..github import OPDKS_REPO_HTTPS
 from ..common import (
@@ -77,8 +77,7 @@ def get_open_pdks(
         else:
             console.log(f"Using open_pdks at {repo_path} unaltered.")
 
-        gf180mcu_tag = None
-        magic_tag = None
+        patch_open_pdks(repo_path)
 
         try:
             json_raw = open(f"{repo_path}/gf180mcu/gf180mcu.json").read()
@@ -91,23 +90,17 @@ def get_open_pdks(
                 json_str = sio.getvalue()
             manifest = json.loads(json_str)
             reference_commits = manifest["reference"]
-            magic_tag = reference_commits["magic"]
-            gf180mcu_tag = reference_commits["gf180mcu_pdk"]
+            print(f"Reference commits: {reference_commits}")
         except FileNotFoundError:
-            console.log(
-                "Cannot find open_pdks/gf180mcu JSON manifest. Default versions for gf180mcu/magic will be used."
-            )
+            console.log("Warning: Couldn't find open_pdks/sky130 JSON manifest.")
         except json.JSONDecodeError:
-            print(json_str)
-            console.log(
-                "Failed to parse open_pdks/gf180mcu JSON manifest. Default versions for gf180mcu/magic will be used."
-            )
+            console.log("Warning: Failed to parse open_pdks/sky130 JSON manifest..")
         except KeyError:
             console.log(
-                "Failed to extract reference commits from open_pdks/gf180mcu JSON manifest. Default versions for gf180mcu/magic will be used."
+                "Warning: Failed to extract reference commits from open_pdks/sky130 JSON manifest."
             )
 
-        return (repo_path, gf180mcu_tag, magic_tag)
+        return repo_path
 
     except subprocess.CalledProcessError as e:
         print(e)
@@ -159,12 +152,17 @@ def run_sh(script, log_to):
         )
         magic_dirname = os.path.dirname(magic_bin)
 
+        configuration_flags = ["--enable-gf180mcu-pdk", "--with-reference"] + list(
+            library_flags.union(library_flags_disable)
+        )
+        console.log(f"Configuring with flags {shlex.join(configuration_flags)}")
+
         with console.status("Configuring open_pdks…"):
             run_sh(
                 f"""
                     set -e
                     export PATH="{magic_dirname}:$PATH"
-                    ./configure --enable-gf180mcu-pdk {' '.join(library_flags)} {' '.join(library_flags_disable)} --with-reference
+                    ./configure {shlex.join(configuration_flags)}
                 """,
                 log_to=os.path.join(log_dir, "config.log"),
             )
@@ -182,17 +180,6 @@ def run_sh(script, log_to):
                 log_to=os.path.join(log_dir, "install.log"),
             )
         console.log("Built PDK variants.")
-
-        with console.status("Fixing file ownership…"):
-            run_sh(
-                f"""
-                set -e
-                OWNERSHIP="$(stat -c "%u:%g" "{pdk_root_abs}")"
-                chown -R $OWNERSHIP "{pdk_root_abs}"
-                """,
-                log_to=os.path.join(log_dir, "ownership.log"),
-            )
-        console.log("Fixed file ownership.")
         with console.status("Cleaning build artifacts…"):
             run_sh(
                 """
@@ -250,7 +237,6 @@ def build(
     clear_build_artifacts: bool = True,
     include_libraries: Optional[List[str]] = None,
     using_repos: Optional[Dict[str, str]] = None,
-    build_magic: bool = False,
 ):
     family = Family.by_name["gf180mcu"]
     library_set = family.resolve_libraries(include_libraries)
@@ -268,22 +254,23 @@ def build(
     console = Console()
     console.log(f"Logging to '{log_dir}'…")
 
-    open_pdks_path, _, magic_tag = get_open_pdks(
+    open_pdks_path = get_open_pdks(
         version, build_directory, jobs, using_repos.get("open_pdks")
     )
 
-    with_magic(
-        magic_tag,
-        lambda magic_bin: build_variants(
-            magic_bin,
-            library_set,
-            build_directory,
-            open_pdks_path,
-            log_dir,
-            jobs,
-        ),
-        build_magic=build_magic,
-    )
+    magic_bin = shutil.which("magic")
+    if magic_bin is None:
+        print("Magic is either not installed or not in PATH.")
+        exit(-1)
+
+    build_variants(
+        magic_bin,
+        build_directory,
+        open_pdks_path,
+        library_set,
+        log_dir,
+        jobs,
+    ),
     install_gf180mcu(build_directory, pdk_root, version)
 
     if clear_build_artifacts: