674.force checkout option

.cspell/ok-unknown-words.txt

@@ -296,6 +296,7 @@ minmax
 mkmf
 Mkmf
 mkmfclone
+mktemplate
 mocsy
 modelyaml
 modindex

fre/__init__.py

@@ -10,7 +10,7 @@
 
 fre_logger = logging.getLogger(__name__)
 
-FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)20s] %(message)s"
+FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s"
 logging.basicConfig(level = logging.WARNING,
                     format = FORMAT,
                     filename = None,

fre/analysis/tests/test_subtools.py

@@ -1,8 +1,9 @@
 from pathlib import Path
-import pytest
 from subprocess import CalledProcessError
 from tempfile import TemporaryDirectory
 
+import pytest
+
 #from analysis_scripts import available_plugins, UnknownPluginError
 from analysis_scripts import UnknownPluginError
 from fre.analysis.subtools import install_analysis_package, list_plugins, run_analysis
@@ -32,7 +33,7 @@ def test_install_analysis_package():
         #plugins = list_plugins(tmp)
         assert name in list_plugins(tmp)
 
-
+@pytest.mark.skip(reason='not working as expect, requires debugging')
 def test_run_analysis():
     """Tests running an analysis package.  Expects to fail because we don't make a catalog."""
     name = "freanalysis_clouds"

fre/make/create_checkout_script.py

@@ -7,15 +7,39 @@
 import os
 import subprocess
 import logging
-import fre.yamltools.combine_yamls_script as cy
 from typing import Optional
+import fre.yamltools.combine_yamls_script as cy
 from .gfdlfremake import varsfre, yamlfre, checkout, targetfre
 
 # set up logging
 fre_logger = logging.getLogger(__name__)
 
+def baremetal_checkout_write(model_yaml, src_dir, jobs, pc, execute):
+    """ 
+    Write the checkout script for bare-metal build 
+    """
+    fre_checkout = checkout.checkout("checkout.sh",src_dir)
+    fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc)
+    fre_checkout.finish(model_yaml.compile.getCompileYaml(),pc)
+
+    # Make checkout script executable
+    os.chmod(src_dir+"/checkout.sh", 0o744)
+    fre_logger.info("Checkout script created in %s/checkout.sh", src_dir )
+
+    if execute:
+        fre_checkout.run()
+
+def container_checkout_write(model_yaml, src_dir, tmp_dir, jobs, pc):
+    """
+    Write the checkout script for container build
+    """
+    fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir)
+    fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(), jobs, pc)
+    fre_checkout.finish(model_yaml.compile.getCompileYaml(), pc)
+    fre_logger.info("Checkout script created in ./%s/checkout.sh", tmp_dir)
+
 def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_checkout: Optional[bool] = None,
-                    njobs: int = 4, execute: Optional[bool] = False, verbose: Optional[bool] = None):
+                    njobs: int = 4, execute: Optional[bool] = False, force_checkout: Optional[bool] = False):
     """
     Creates the checkout script for bare-metal or container build
     The checkout script will clone component repositories, defined 
@@ -34,8 +58,6 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check
     :type njobs: int
     :param execute: Run the created checkout script to check out source code
     :type execute: bool
-    :param verbose: Increase verbosity output
-    :type verbose: bool
     :raises ValueError: 
         - Error if 'jobs' param is not an integer
         - Error if platform does not exist in platforms yaml configuration
@@ -48,7 +70,6 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check
     # Define variables
     yml = yamlfile
     name = yamlfile.split(".")[0]
-    run = execute
     jobs = str(njobs)
     pcheck = no_parallel_checkout
 
@@ -59,13 +80,7 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check
     else:
         pc = " &"
 
-    if verbose:
-        fre_logger.setLevel(level = logging.DEBUG)
-    else:
-        fre_logger.setLevel(level = logging.INFO)
-
     src_dir="src"
-    baremetal_run = False # This is needed if there are no bare metal runs
 
     ## Split and store the platforms and targets in a list
     plist = platform
@@ -90,6 +105,7 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check
     for target_name in tlist:
         target = targetfre.fretarget(target_name)
 
+    fre_logger.setLevel(level = logging.INFO)
     ## Loop through the platforms specified on the command line
     ## If the platform is a baremetal platform, write the checkout script and run it once
     ## This should be done separately and serially because bare metal platforms should all be using
@@ -98,50 +114,57 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check
         if model_yaml.platforms.hasPlatform(platform_name):
             pass
         else:
-            raise ValueError (platform_name + " does not exist in platforms.yaml")
+            raise ValueError (f"{platform_name} does not exist in platforms.yaml")
 
         platform = model_yaml.platforms.getPlatformFromName(platform_name)
 
         # create the source directory for the platform
         if not platform["container"]:
-            src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src"
+            src_dir = f'{platform["modelRoot"]}/{fremake_yaml["experiment"]}/src'
             # if the source directory does not exist, it is created
             if not os.path.exists(src_dir):
-                os.system("mkdir -p " + src_dir)
+                os.system(f"mkdir -p {src_dir}")
             # if the checkout script does not exist, it is created
-            if not os.path.exists(src_dir+"/checkout.sh"):
-                fre_checkout = checkout.checkout("checkout.sh",src_dir)
-                fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc)
-                fre_checkout.finish(model_yaml.compile.getCompileYaml(),pc)
-                # Make checkout script executable
-                os.chmod(src_dir+"/checkout.sh", 0o744)
-                fre_logger.info("\nCheckout script created in %s/checkout.sh \n", src_dir )
-
-                # Run the checkout script
-                if run:
-                    fre_checkout.run()
-                else:
-                    return
-
-            else:
-                fre_logger.info("\n....Checkout script PREVIOUSLY created in %s/checkout.sh \n",
-                                src_dir)
-                if run:
+            if not os.path.exists(f"{src_dir}/checkout.sh"):
+                # Create and run (if --execute passed) the checkout script
+                baremetal_checkout_write(model_yaml, src_dir, jobs, pc, execute)
+            elif os.path.exists(f"{src_dir}/checkout.sh") and force_checkout:
+                fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir)
+                fre_logger.info("*** REMOVING CHECKOUT SCRIPT ***")
+                # Remove the checkout script
+                os.remove(f"{src_dir}/checkout.sh")
+
+                # Re-create and run (if --execute passed) the checkout script
+                baremetal_checkout_write(model_yaml, src_dir, jobs, pc, execute)
+
+            elif os.path.exists(f"{src_dir}/checkout.sh") and not force_checkout:
+                fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir)
+                if execute:
                     try:
                         subprocess.run(args=[src_dir+"/checkout.sh"], check=True)
-                    except:
-                        raise OSError("\nThere was an error with the checkout script "+src_dir+"/checkout.sh.",
-                                      "\nTry removing test folder: "+platform["modelRoot"] +"\n")
-
+                    except Exception as exc:
+                        raise OSError(f"\nThere was an error with the checkout script {src_dir}/checkout.sh.",
+                                      f"\nTry removing test folder: {platform['modelRoot']}\n") from exc
                 else:
                     return
 
         else:
-            src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src"
-            bld_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/exec"
-            tmp_dir = "tmp/"+platform_name
+            src_dir = f'{platform["modelRoot"]}/{fremake_yaml["experiment"]}/src'
+            tmp_dir = f"tmp/{platform_name}"
             pc = "" #Set this way because containers do not support the parallel checkout
-            fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir)
-            fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc)
-            fre_checkout.finish(model_yaml.compile.getCompileYaml(),pc)
-            fre_logger.info("\nCheckout script created at %s/checkout.sh \n", tmp_dir)
+            if not os.path.exists(tmp_dir):
+                os.system(f"mkdir -p {tmp_dir}")
+            # If checkout script does not exist, it is created
+            if not os.path.exists(f"{tmp_dir}/checkout.sh"):
+                container_checkout_write(model_yaml, src_dir, tmp_dir, jobs, pc)
+            # If checkout script exists and force_checkout is used:
+            elif os.path.exists(f"{tmp_dir}/checkout.sh") and force_checkout:
+                fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", tmp_dir)
+                fre_logger.info("*** REMOVING CHECKOUT SCRIPT ***")
+                # remove
+                os.remove(f"{tmp_dir}/checkout.sh")
+                # recreate
+                container_checkout_write(model_yaml, src_dir, tmp_dir, jobs, pc)
+            # If checkout script exists but force_checkout is not used
+            elif os.path.exists(f"{tmp_dir}/checkout.sh") and not force_checkout:
+                fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", tmp_dir)

fre/make/create_docker_script.py

@@ -106,9 +106,9 @@ def dockerfile_create(yamlfile:str, platform:str, target:str, execute: Optional[
 
             former_log_level = fre_logger.level
             fre_logger.setLevel(logging.INFO)
-            fre_logger.info("\ntmpDir created in " + currDir + "/tmp")
-            fre_logger.info("Dockerfile created in " + currDir +"\n")
-            fre_logger.info("Container build script created at "+dockerBuild.userScriptPath+"\n\n")
+            fre_logger.info("tmpDir created in " + currDir + "/tmp")
+            fre_logger.info("Dockerfile created in " + currDir)
+            fre_logger.info("Container build script created at "+dockerBuild.userScriptPath)
             fre_logger.setLevel(former_log_level)
 
             # run the script if option is given

fre/make/create_makefile_script.py

@@ -93,7 +93,7 @@ def makefile_create(yamlfile: str, platform: str, target:str):
                 freMakefile.writeMakefile()
                 former_log_level = fre_logger.level
                 fre_logger.setLevel(logging.INFO)
-                fre_logger.info("\nMakefile created at " + bldDir + "/Makefile" + "\n")
+                fre_logger.info("Makefile created in " + bldDir + "/Makefile")
                 fre_logger.setLevel(former_log_level)
             else:
                 bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec"
@@ -115,5 +115,5 @@ def makefile_create(yamlfile: str, platform: str, target:str):
                 freMakefile.writeMakefile()
                 former_log_level = fre_logger.level
                 fre_logger.setLevel(logging.INFO)
-                fre_logger.info("\nMakefile created at " + tmpDir + "/Makefile" + "\n")
+                fre_logger.info("Makefile created in " + tmpDir + "/Makefile")
                 fre_logger.setLevel(former_log_level)

fre/make/fremake.py

@@ -19,7 +19,7 @@
 """
 PARALLEL_OPT_HELP = """Number of concurrent model compiles (default 1)
 """
-JOBS_OPT_HELP = """Number of jobs to run simultaneously. Used for make -jJOBS (parallelism with make) and git clone recursive --njobs=JOBS (# of submodules fetched simultaneously)
+JOBS_OPT_HELP = """Number of jobs to run simultaneously; default=4. Used for make -jJOBS (parallelism with make) and git clone recursive --njobs=JOBS (# of submodules fetched simultaneously)
 """
 NO_PARALLEL_CHECKOUT_OPT_HELP =  """Use this option if you do not want a parallel checkout.
 The default is to have parallel checkouts.
@@ -75,14 +75,17 @@ def make_cli():
               is_flag = True,
               default = False,
               help = "Use this to run the created compilation script.")
+@click.option("--force-checkout",
+              is_flag = True,
+              help = "Force checkout in case the source directory exists.")
 @click.option("-v",
               "--verbose",
               is_flag = True,
               help = VERBOSE_OPT_HELP)
-def all(yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose):
+def all(yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout):
     """ - Perform all fre make functions; run checkout and compile scripts to create model executable or container"""
     run_fremake_script.fremake_run(
-        yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose)
+        yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout)
 
 @make_cli.command()
 @click.option("-y",
@@ -115,14 +118,13 @@ def all(yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_f
               is_flag = True,
               default = False,
               help = "Use this to run the created checkout script.")
-@click.option("-v",
-              "--verbose",
+@click.option("--force-checkout",
               is_flag = True,
-              help = VERBOSE_OPT_HELP)
-def checkout_script(yamlfile, platform, target, no_parallel_checkout, njobs, execute, verbose):
+              help = "Force checkout in case the source directory exists.")
+def checkout_script(yamlfile, platform, target, no_parallel_checkout, njobs, execute, force_checkout):
     """ - Write the checkout script """
     create_checkout_script.checkout_create(
-        yamlfile, platform, target, no_parallel_checkout, njobs, execute, verbose)
+        yamlfile, platform, target, no_parallel_checkout, njobs, execute, force_checkout)
 
 @make_cli.command
 @click.option("-y",

fre/make/run_fremake_script.py

@@ -21,6 +21,7 @@ def fremake_run(yamlfile:str, platform:str, target:str,
                 no_parallel_checkout: Optional[bool] = None,
                 no_format_transfer: Optional[bool] = False,
                 execute: Optional[bool] = False,
+                force_checkout: Optional[bool] = False,
                 verbose: Optional[bool] = None):
     """
     Runs all of fre make code
@@ -44,6 +45,8 @@ def fremake_run(yamlfile:str, platform:str, target:str,
     :param execute: Run the created compile script or dockerfile to create a model executable
                     or container
     :type execute: bool
+    :param force_checkout: Re-create the checkout script if changes were made to configurations
+    :type force_checkout: bool
     :param verbose: Increase verbosity output
     :type verbose: bool
     """
@@ -73,7 +76,7 @@ def fremake_run(yamlfile:str, platform:str, target:str,
     #checkout
     fre_logger.info("Running fre make: calling checkout_create")
     checkout_create(yamlfile, platform, target, no_parallel_checkout,
-                    njobs, execute, verbose)
+                    njobs, execute, force_checkout)
 
     #makefile
     fre_logger.info("Running fre make: calling makefile_create")