From d092af328ae62ab35130483e1483621a81a64074 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Wed, 31 Jul 2024 08:34:45 -0700 Subject: [PATCH 01/18] add autopush to mirror option --- uberenv.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/uberenv.py b/uberenv.py index f1143dd..29eafe7 100755 --- a/uberenv.py +++ b/uberenv.py @@ -122,6 +122,12 @@ def parse_args(): default=None, help="spack mirror directory") + parser.add_option("--mirror-autopush", + action="store_true", + dest="mirror_autopush", + default=False, + help="Use spack v0.22+ --autopush functionality for binary cache mirrors") + # flag to create mirror parser.add_argument("--create-mirror", action="store_true", @@ -407,7 +413,7 @@ def setup_paths_and_dirs(self): def set_from_args_or_json(self,setting, optional=True): """ - When optional=False: + When optional=False: If the setting key is not in the json file, error and raise an exception. When optional=True: If the setting key is not in the json file or args, return None. @@ -425,7 +431,7 @@ def set_from_args_or_json(self,setting, optional=True): def set_from_json(self,setting, optional=True): """ - When optional=False: + When optional=False: If the setting key is not in the json file, error and raise an exception. When optional=True: If the setting key is not in the json file or args, return None. @@ -509,7 +515,7 @@ def clone_repo(self): os.chdir(self.dest_dir) - clone_args = ("-c http.sslVerify=false " + clone_args = ("-c http.sslVerify=false " if self.args["ignore_ssl_errors"] else "") clone_cmd = "git {0} clone --single-branch -b {1} {2} vcpkg".format(clone_args, vcpkg_branch,vcpkg_url) @@ -1057,7 +1063,7 @@ def install(self): print("[ERROR: Failure of spack install]") return res - # when using install or uberenv-pkg mode, create a symlink to the host config + # when using install or uberenv-pkg mode, create a symlink to the host config if self.build_mode == "install" or \ self.build_mode == "uberenv-pkg" \ or self.use_install: @@ -1157,6 +1163,7 @@ def use_mirror(self): mirror_name = self.pkg_name mirror_path = self.get_mirror_path() existing_mirror_path = self.find_spack_mirror(mirror_name) + autopush = "--autopush" if self.mirror_autopush else "" if existing_mirror_path and mirror_path != existing_mirror_path: # Existing mirror has different URL, error out @@ -1171,8 +1178,8 @@ def use_mirror(self): existing_mirror_path = None if not existing_mirror_path: # Add if not already there - sexe("{0} mirror add --scope=defaults {1} {2}".format( - self.spack_exe(), mirror_name, mirror_path), echo=True) + sexe("{0} mirror add --scope=defaults {1} {2} {3}".format( + self.spack_exe_path(), autopush, mirror_name, mirror_path), echo=True) print("[using mirror {0}]".format(mirror_path)) def find_spack_upstream(self, upstream_name): From 15d49175f3f23adaeedcf8cc98b63c77930f21a2 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Wed, 31 Jul 2024 10:28:55 -0700 Subject: [PATCH 02/18] fix typo --- uberenv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uberenv.py b/uberenv.py index 29eafe7..55a598d 100755 --- a/uberenv.py +++ b/uberenv.py @@ -122,11 +122,11 @@ def parse_args(): default=None, help="spack mirror directory") - parser.add_option("--mirror-autopush", - action="store_true", - dest="mirror_autopush", - default=False, - help="Use spack v0.22+ --autopush functionality for binary cache mirrors") + parser.add_argument("--mirror-autopush", + action="store_true", + dest="mirror_autopush", + default=False, + help="Use spack v0.22+ --autopush functionality for binary cache mirrors") # flag to create mirror parser.add_argument("--create-mirror", From 9037be8633a0e1a386e4e9d66ab24332512db627 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Wed, 31 Jul 2024 12:51:52 -0700 Subject: [PATCH 03/18] args in args --- uberenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uberenv.py b/uberenv.py index 55a598d..694ceff 100755 --- a/uberenv.py +++ b/uberenv.py @@ -1163,7 +1163,7 @@ def use_mirror(self): mirror_name = self.pkg_name mirror_path = self.get_mirror_path() existing_mirror_path = self.find_spack_mirror(mirror_name) - autopush = "--autopush" if self.mirror_autopush else "" + autopush = "--autopush" if self.args["mirror_autopush"] else "" if existing_mirror_path and mirror_path != existing_mirror_path: # Existing mirror has different URL, error out From 1183e1549d0acc3a11b77fc8bbdfaee029dd1f4a Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Wed, 31 Jul 2024 13:17:29 -0700 Subject: [PATCH 04/18] fix for update --- uberenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uberenv.py b/uberenv.py index 694ceff..0fbf3a8 100755 --- a/uberenv.py +++ b/uberenv.py @@ -1179,7 +1179,7 @@ def use_mirror(self): if not existing_mirror_path: # Add if not already there sexe("{0} mirror add --scope=defaults {1} {2} {3}".format( - self.spack_exe_path(), autopush, mirror_name, mirror_path), echo=True) + self.spack_exe(), autopush, mirror_name, mirror_path), echo=True) print("[using mirror {0}]".format(mirror_path)) def find_spack_upstream(self, upstream_name): From a3711c6ab365b2c84c6bd892f5fc6280511a746e Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 14:43:59 -0700 Subject: [PATCH 05/18] add option to trust gpg keys --- uberenv.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/uberenv.py b/uberenv.py index 0fbf3a8..ae7febe 100755 --- a/uberenv.py +++ b/uberenv.py @@ -299,6 +299,13 @@ def parse_args(): default=None, help="Path to Spack Environment file (e.g. spack.yaml or spack.lock)") + # option to add trusted keys to spack gpg keyring + parser.add_option("--trust-keys", + action="store_true", + dest="key_path", + default=None, + help="Add the gpg keys to the spack gpg keyring") + ############### # parse args ############### @@ -1264,6 +1271,16 @@ def setup_clingo(self): print("[ERROR: Clingo install failed with returncode {0}]".format(res)) sys.exit(-1) + def trust_gpg_keys(self): + """ + Tells spack to trust the gpg keys at keypath. + """ + key_path = self.opts["key_path"] + if not key_path: + print("[--trust-key requires a gpg key path]") + key_path = pabs(key_path) + print("[adding gpg keys to spack gpg keyring]") + sexe("gpg trust {0}".format(key_path)) def find_osx_sdks(): """ @@ -1330,6 +1347,10 @@ def main(): # Setup the necessary paths and directories env.setup_paths_and_dirs() + # Trust keys if the option was provided + if args["key_path"]: + env.trust_gpg_keys() + # Go to package manager's destination os.chdir(env.dest_dir) From 41de4a4e8aa11853872a442ada137c47b689aacf Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 15:14:58 -0700 Subject: [PATCH 06/18] fix typo --- uberenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uberenv.py b/uberenv.py index ae7febe..d8eb20f 100755 --- a/uberenv.py +++ b/uberenv.py @@ -300,7 +300,7 @@ def parse_args(): help="Path to Spack Environment file (e.g. spack.yaml or spack.lock)") # option to add trusted keys to spack gpg keyring - parser.add_option("--trust-keys", + parser.add_argument("--trust-keys", action="store_true", dest="key_path", default=None, From b9404dedc3177694b1435cf112591ae1f69ef820 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 15:18:47 -0700 Subject: [PATCH 07/18] variable name changed --- uberenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uberenv.py b/uberenv.py index d8eb20f..ac1f198 100755 --- a/uberenv.py +++ b/uberenv.py @@ -1275,7 +1275,7 @@ def trust_gpg_keys(self): """ Tells spack to trust the gpg keys at keypath. """ - key_path = self.opts["key_path"] + key_path = self.args["key_path"] if not key_path: print("[--trust-key requires a gpg key path]") key_path = pabs(key_path) From 54b9a4ebe235505dcc03c9290df9973744c5f700 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 15:34:50 -0700 Subject: [PATCH 08/18] fix trust_keys arg --- uberenv.py | 1 - 1 file changed, 1 deletion(-) diff --git a/uberenv.py b/uberenv.py index ac1f198..789c350 100755 --- a/uberenv.py +++ b/uberenv.py @@ -301,7 +301,6 @@ def parse_args(): # option to add trusted keys to spack gpg keyring parser.add_argument("--trust-keys", - action="store_true", dest="key_path", default=None, help="Add the gpg keys to the spack gpg keyring") From 525513318ef197f7833a3d4acc80b0e8b2cf3e65 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 16:23:24 -0700 Subject: [PATCH 09/18] fix spack gpg trust command --- uberenv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uberenv.py b/uberenv.py index 789c350..cf1797b 100755 --- a/uberenv.py +++ b/uberenv.py @@ -300,7 +300,7 @@ def parse_args(): help="Path to Spack Environment file (e.g. spack.yaml or spack.lock)") # option to add trusted keys to spack gpg keyring - parser.add_argument("--trust-keys", + parser.add_argument("--trust-key", dest="key_path", default=None, help="Add the gpg keys to the spack gpg keyring") @@ -1270,7 +1270,7 @@ def setup_clingo(self): print("[ERROR: Clingo install failed with returncode {0}]".format(res)) sys.exit(-1) - def trust_gpg_keys(self): + def trust_gpg_key(self): """ Tells spack to trust the gpg keys at keypath. """ @@ -1279,7 +1279,7 @@ def trust_gpg_keys(self): print("[--trust-key requires a gpg key path]") key_path = pabs(key_path) print("[adding gpg keys to spack gpg keyring]") - sexe("gpg trust {0}".format(key_path)) + sexe("{0} gpg trust {1}".format(self.spack_exe(), key_path), echo=True) def find_osx_sdks(): """ @@ -1348,7 +1348,7 @@ def main(): # Trust keys if the option was provided if args["key_path"]: - env.trust_gpg_keys() + env.trust_gpg_key() # Go to package manager's destination os.chdir(env.dest_dir) From 95a0f8bcd867d23c9e822922ec636d8624fb847e Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 16:33:59 -0700 Subject: [PATCH 10/18] move key trust command to after clone --- uberenv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uberenv.py b/uberenv.py index cf1797b..43bd0cf 100755 --- a/uberenv.py +++ b/uberenv.py @@ -1346,10 +1346,6 @@ def main(): # Setup the necessary paths and directories env.setup_paths_and_dirs() - # Trust keys if the option was provided - if args["key_path"]: - env.trust_gpg_key() - # Go to package manager's destination os.chdir(env.dest_dir) @@ -1361,6 +1357,10 @@ def main(): # Patch the package manager, as necessary env.patch() + # Trust keys if the option was provided + if args["key_path"]: + env.trust_gpg_key() + # Clean the build env.clean_build() From c216831e578d8b34f83966cbe7efb115a6c0af0c Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 16:39:28 -0700 Subject: [PATCH 11/18] dont run gpg trust in spack env --- uberenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uberenv.py b/uberenv.py index 43bd0cf..496e1d8 100755 --- a/uberenv.py +++ b/uberenv.py @@ -1279,7 +1279,7 @@ def trust_gpg_key(self): print("[--trust-key requires a gpg key path]") key_path = pabs(key_path) print("[adding gpg keys to spack gpg keyring]") - sexe("{0} gpg trust {1}".format(self.spack_exe(), key_path), echo=True) + sexe("{0} gpg trust {1}".format(self.spack_exe(use_spack_env=False), key_path), echo=True) def find_osx_sdks(): """ From 7acaff96d4737af5e03f1d4938421e588f249f8c Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 22:47:05 -0700 Subject: [PATCH 12/18] trust multiple keys --- uberenv.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/uberenv.py b/uberenv.py index 496e1d8..30dd8cf 100755 --- a/uberenv.py +++ b/uberenv.py @@ -300,8 +300,9 @@ def parse_args(): help="Path to Spack Environment file (e.g. spack.yaml or spack.lock)") # option to add trusted keys to spack gpg keyring - parser.add_argument("--trust-key", - dest="key_path", + parser.add_argument("--trust-keys", + dest="key_paths", + action="append", default=None, help="Add the gpg keys to the spack gpg keyring") @@ -1270,18 +1271,17 @@ def setup_clingo(self): print("[ERROR: Clingo install failed with returncode {0}]".format(res)) sys.exit(-1) - def trust_gpg_key(self): + def trust_gpg_keys(self): """ Tells spack to trust the gpg keys at keypath. """ - key_path = self.args["key_path"] - if not key_path: - print("[--trust-key requires a gpg key path]") - key_path = pabs(key_path) - print("[adding gpg keys to spack gpg keyring]") - sexe("{0} gpg trust {1}".format(self.spack_exe(use_spack_env=False), key_path), echo=True) - -def find_osx_sdks(): + key_paths = self.args["key_paths"] + for key_path in key_paths: + key_path = pabs(key_path) + print("[adding gpg key to spack gpg keyring]") + sexe("{0} gpg trust {1}".format(self.spack_exe(use_spack_env=False), key_path), echo=True) + + def find_osx_sdks(): """ Finds installed osx sdks, returns dict mapping version to file system path """ @@ -1359,7 +1359,7 @@ def main(): # Trust keys if the option was provided if args["key_path"]: - env.trust_gpg_key() + env.trust_gpg_keys() # Clean the build env.clean_build() From 8a47a1e9cef05800007026ecf3613decfd5ff4e3 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 22:47:11 -0700 Subject: [PATCH 13/18] trust multiple keys --- uberenv.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/uberenv.py b/uberenv.py index 30dd8cf..4551904 100755 --- a/uberenv.py +++ b/uberenv.py @@ -1282,16 +1282,16 @@ def trust_gpg_keys(self): sexe("{0} gpg trust {1}".format(self.spack_exe(use_spack_env=False), key_path), echo=True) def find_osx_sdks(): - """ - Finds installed osx sdks, returns dict mapping version to file system path - """ - res = {} - sdks = glob.glob("/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk") - for sdk in sdks: - sdk_base = os.path.split(sdk)[1] - ver = sdk_base[len("MacOSX"):sdk_base.rfind(".")] - res[ver] = sdk - return res + """ + Finds installed osx sdks, returns dict mapping version to file system path + """ + res = {} + sdks = glob.glob("/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk") + for sdk in sdks: + sdk_base = os.path.split(sdk)[1] + ver = sdk_base[len("MacOSX"):sdk_base.rfind(".")] + res[ver] = sdk + return res def setup_osx_sdk_env_vars(): """ From 1b953a2230b79606d675e848f6c4730e39a3cacb Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 22:47:38 -0700 Subject: [PATCH 14/18] rename --- uberenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uberenv.py b/uberenv.py index 4551904..779678e 100755 --- a/uberenv.py +++ b/uberenv.py @@ -300,7 +300,7 @@ def parse_args(): help="Path to Spack Environment file (e.g. spack.yaml or spack.lock)") # option to add trusted keys to spack gpg keyring - parser.add_argument("--trust-keys", + parser.add_argument("--trust-key", dest="key_paths", action="append", default=None, From 07def1e2c8200c530fe84fb3d913e60bbcc5aed9 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 12 Sep 2024 22:55:30 -0700 Subject: [PATCH 15/18] typo --- uberenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uberenv.py b/uberenv.py index 779678e..384a1ee 100755 --- a/uberenv.py +++ b/uberenv.py @@ -1358,7 +1358,7 @@ def main(): env.patch() # Trust keys if the option was provided - if args["key_path"]: + if args["key_paths"]: env.trust_gpg_keys() # Clean the build From ff79671f8b9b7cd7de9ae761bc76b87168da8e2e Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Fri, 27 Sep 2024 13:41:50 -0700 Subject: [PATCH 16/18] fix comment --- uberenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uberenv.py b/uberenv.py index 384a1ee..8e09133 100755 --- a/uberenv.py +++ b/uberenv.py @@ -1273,7 +1273,7 @@ def setup_clingo(self): def trust_gpg_keys(self): """ - Tells spack to trust the gpg keys at keypath. + Tells spack to trust the gpg keys in key_paths. """ key_paths = self.args["key_paths"] for key_path in key_paths: From 66cfa28945dad9051ba52d0c69d86ea94bb606c2 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Fri, 27 Sep 2024 14:18:56 -0700 Subject: [PATCH 17/18] undo indent --- uberenv.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/uberenv.py b/uberenv.py index 8e09133..f9fda20 100755 --- a/uberenv.py +++ b/uberenv.py @@ -1281,17 +1281,17 @@ def trust_gpg_keys(self): print("[adding gpg key to spack gpg keyring]") sexe("{0} gpg trust {1}".format(self.spack_exe(use_spack_env=False), key_path), echo=True) - def find_osx_sdks(): - """ - Finds installed osx sdks, returns dict mapping version to file system path - """ - res = {} - sdks = glob.glob("/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk") - for sdk in sdks: - sdk_base = os.path.split(sdk)[1] - ver = sdk_base[len("MacOSX"):sdk_base.rfind(".")] - res[ver] = sdk - return res +def find_osx_sdks(): + """ + Finds installed osx sdks, returns dict mapping version to file system path + """ + res = {} + sdks = glob.glob("/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk") + for sdk in sdks: + sdk_base = os.path.split(sdk)[1] + ver = sdk_base[len("MacOSX"):sdk_base.rfind(".")] + res[ver] = sdk + return res def setup_osx_sdk_env_vars(): """ From 0c00f0aaf6fae1dc2e69fedf0c88b139890befd0 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Fri, 27 Sep 2024 14:22:01 -0700 Subject: [PATCH 18/18] add docs --- docs/sphinx/index.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/sphinx/index.rst b/docs/sphinx/index.rst index 75121ca..bcb5e5b 100644 --- a/docs/sphinx/index.rst +++ b/docs/sphinx/index.rst @@ -267,13 +267,15 @@ Optimization Uberenv also features options to optimize the installation - ===================== ============================================== ================================================ - Option Description Default - ===================== ============================================== ================================================ - ``--mirror`` Location of a Spack mirror **None** - ``--create-mirror`` Creates a Spack mirror at specified location **None** - ``--upstream`` Location of a Spack upstream **None** - ===================== ============================================== ================================================ + ====================== ============================================== ================================================ + Option Description Default + ====================== ============================================== ================================================ + ``--mirror`` Location of a Spack mirror **None** + ``--create-mirror`` Creates a Spack mirror at specified location **None** + ``--upstream`` Location of a Spack upstream **None** + ``--mirror-autopush`` Push binaries to mirror as they are built False + ``--trust-key`` Load the gpg key(s) to the Spack gpg keyring **None** + ====================== ============================================== ================================================ .. note:: These options are only currently available for spack.