Wagn0033 (#1)

* added unittests for misc and get image

* Add "post" method for RestApiV1 library

* added init with get image and get countries

* Added hashbang line to script

* Added "post" to RestApiV1

* added more GET unit tests

* Adding capability to get images

_get_binary added to RestApiV1

* more unit tests under get_tests

* Updated RestApiV1 "post"

* change line endings to unix-style

* Update .gitignore

updated .gitignore to not include .swp, .pyc

* more unit tests/edits

* Change default REST API url

* Converted RestApiV1 unit tests to use new functions instead of _get

* Preparing for release 1.1

Release 1.1 allows using External ID and allows attaching subcomponents

* Fixed bug with institution list

---------

Co-authored-by: wagn0033 <[email protected]>

.gitignore

@@ -0,0 +1,7 @@
+
+lib/Sisyphus/__pycache__/__init__.cpython-39.pyc
+*.pyc
+Examples/upload-docket/Tutorial/Example01/.~lock.HWItems.xlsx\#
+*.pyc
+*.pyc
+*.swp

.spyproject/config/backups/codestyle.ini.bak

@@ -0,0 +1,8 @@
+[codestyle]
+indentation = True
+edge_line = True
+edge_line_columns = 79
+
+[main]
+version = 0.2.0
+

.spyproject/config/backups/encoding.ini.bak

@@ -0,0 +1,6 @@
+[encoding]
+text_encoding = utf-8
+
+[main]
+version = 0.2.0
+

.spyproject/config/backups/vcs.ini.bak

@@ -0,0 +1,7 @@
+[vcs]
+use_version_control = False
+version_control_system = 
+
+[main]
+version = 0.2.0
+

.spyproject/config/backups/workspace.ini.bak

@@ -0,0 +1,12 @@
+[workspace]
+restore_data_on_startup = True
+save_data_on_exit = True
+save_history = True
+save_non_project_files = False
+project_type = 'empty-project-type'
+recent_files = ['lib/Sisyphus/RestApiV1/__init__.py', 'test/RestApiV1/Test__get_misc.py']
+
+[main]
+version = 0.2.0
+recent_files = []
+

.spyproject/config/codestyle.ini

@@ -0,0 +1,8 @@
+[codestyle]
+indentation = True
+edge_line = True
+edge_line_columns = 79
+
+[main]
+version = 0.2.0
+

.spyproject/config/defaults/defaults-codestyle-0.2.0.ini

@@ -0,0 +1,5 @@
+[codestyle]
+indentation = True
+edge_line = True
+edge_line_columns = 79
+

.spyproject/config/defaults/defaults-encoding-0.2.0.ini

@@ -0,0 +1,3 @@
+[encoding]
+text_encoding = utf-8
+

.spyproject/config/defaults/defaults-vcs-0.2.0.ini

@@ -0,0 +1,4 @@
+[vcs]
+use_version_control = False
+version_control_system = 
+

.spyproject/config/defaults/defaults-workspace-0.2.0.ini

@@ -0,0 +1,6 @@
+[workspace]
+restore_data_on_startup = True
+save_data_on_exit = True
+save_history = True
+save_non_project_files = False
+

.spyproject/config/encoding.ini

@@ -0,0 +1,6 @@
+[encoding]
+text_encoding = utf-8
+
+[main]
+version = 0.2.0
+

.spyproject/config/vcs.ini

@@ -0,0 +1,7 @@
+[vcs]
+use_version_control = False
+version_control_system = 
+
+[main]
+version = 0.2.0
+

.spyproject/config/workspace.ini

@@ -0,0 +1,12 @@
+[workspace]
+restore_data_on_startup = True
+save_data_on_exit = True
+save_history = True
+save_non_project_files = False
+project_type = 'empty-project-type'
+recent_files = ['lib/Sisyphus/RestApiV1/__init__.py', 'test/RestApiV1/Test__get_misc.py', 'lib/Sisyphus/RestApi/__init__.py', 'scratch/scratch-01.py']
+
+[main]
+version = 0.2.0
+recent_files = []
+

bin/advanced-upload-docket.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+bin/advanced-upload-docket.py
+Copyright (c) 2023 Regents of the University of Minnesota
+Author: Alex Wagner <[email protected]>, Dept. of Physics and Astronomy
+"""
+
+import sys
+import argparse
+import json, json5
+from Sisyphus.HWDBUploader import Docket
+
+def parse_args(argv):
+
+    description = "TBD"
+
+    arg_table = [
+        (('docket',), {"metavar": "filename", "nargs": 1}),
+        #(('--docket',), {"dest": "docket", "required": True, "metavar": "filename"}),
+        (('--submit',), {"dest": "submit", "action": "store_true"}),
+        #(('--ignore-warnings',), {"dest": "ignore", "action": "store_true"}),
+    ]
+
+    parser = argparse.ArgumentParser(description=description, add_help=True)
+
+    for args, kwargs in arg_table:
+        parser.add_argument(*args, **kwargs)
+
+    # THIS IS BROKEN... I want to pass argv to it, but it just doesn't behave the same.
+    # but I don't need to fix this right now.
+    args = parser.parse_args()
+    return args
+
+
+def main(argv):
+    args = parse_args(argv)
+
+    docket_file = args.docket[0]
+    ext = docket_file.split('.')[-1]
+
+    try:
+        with open(docket_file, "r") as f:
+            contents = f.read()
+    except Exception as ex:
+        print(f"Exception: {type(ex)} {ex}")
+        raise ex
+
+    if ext.lower() in ["json", "json5"]:
+        docket_def = json5.loads(contents)
+    elif ext.lower() in ["py",]:
+        _locals = {}
+        exec(contents, globals(), _locals)
+        docket_def = _locals["contents"]
+
+    docket = Docket(docket_def)
+
+    docket.process_sources()
+    if args.submit:
+        docket.update_hwdb()
+    else:
+        docket.display_plan()
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

bin/configure.py

@@ -10,27 +10,53 @@
 import argparse
 import Sisyphus.Configuration as Config
 
-
-
 def parse(command_line_args=sys.argv):
     parser = argparse.ArgumentParser(
+        add_help=True,
+        parents=[Config.config.arg_parser],
         description='Saves authentication info in ~/.sisyphus/config.json')
-    parser.add_argument('--reset',
+
+    group = parser.add_argument_group(
+                'Configuration Utility Options',
+                'All configuration options above will be permanent, and the '
+                'following additional options are available.')
+
+    group.add_argument('--reset',
                         dest='reset',
                         action='store_true',
                         required=False,
                         help='resets everything in the configuration '
                              '(i.e., reset the configuration, then add everything '
                              'else provided in this command line')
-    parser.add_argument('--set-active',
+    group.add_argument('--set-active',
                         dest='set_active',
                         action='store_true',
                         required=False,
                         help='set this profile to be the default') 
     args = parser.parse_known_args(command_line_args)
     return args
-
-
+
+def check_server(config):
+    # we wait until here to import because we want to process arguments and 
+    # update the configuration before accessing the HWDB.
+    from Sisyphus.RestApiV1 import session, whoami
+
+    # if the config is invalid or incomplete, "session" will be None
+    if session is None:
+        msg = "Server check not attempted"
+        config.logger.info(msg)
+    else:
+        resp = whoami(timeout=10)
+        if resp['status'] != "OK":
+            msg = "Failed to contact server to validate certificate"
+            config.logger.warning(msg)
+        else:
+            user = f"{(resp['data']['full_name'])} ({resp['data']['username']})"
+            msg = f"REST API 'whoami' returned {user}"
+            config.logger.info(msg)
+    return msg
+
+
 def show_summary(config):
     print()
     print("Configuration Summary:")
@@ -41,22 +67,39 @@ def show_summary(config):
     else:
         default_info_msg = f"(default is {config.default_profile})"
 
-    print(f"profile:     {config.profile_name} {default_info_msg}")
-    print(f"REST API:    {config.rest_api}")
+    print(f"profile:      {config.profile_name} {default_info_msg}")
+
+    if config.rest_api == Config.API_DEV:
+        rest_api_msg = "(development)"
+    elif config.rest_api == Config.API_PROD:
+        rest_api_msg = "(PRODUCTION)"
+    else:
+        rest_api_msg = "(custom)"
+
+    print(f"REST API:     {config.rest_api} {rest_api_msg}")
 
     if config.cert_type is None:
-        print( "certificate: None, all commands will require '--cert <certificate>' to function")
+        print( "certificate:  None, all commands will require '--cert <certificate>' to function")
     elif config.cert_type == Config.KW_P12:
-        print(f"certificate: {Config.KW_P12}, all commands will require '--password <password> to function")
+        print(f"certificate:  {Config.KW_P12}, all commands will require '--password <password> to function")
     else:
-        print(f"certificate: {Config.KW_PEM}")
+        print(f"certificate:  {Config.KW_PEM}")
 
     if config.cert_type == Config.KW_PEM:
-        print(f"cert info:   {config.cert_fullname} ({config.cert_username})")
+        print(f"cert info:    {config.cert_fullname} ({config.cert_username})")
         if config.cert_has_expired:
-            print("cert status: Expired")
+            print("cert status:  Expired")
         else:
-            print(f"cert status: Expires in {config.cert_days_left} days")
+            print(f"cert status:  Expires in {config.cert_days_left} days")
+
+    #print(f"server check: {check_server(config)}")
+    sys.stdout.write("server check: (please wait)")
+    sys.stdout.flush()
+    check_result = check_server(config)
+    sys.stdout.write(f"\rserver check: {check_result}\033[K\n")
+
+
+    print()
 
 def main():
 

bin/list-institutions.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+bin/list-institutions.py
+Copyright (c) 2023 Regents of the University of Minnesota
+Author: Alex Wagner <[email protected]>, Dept. of Physics and Astronomy
+"""
+
+from Sisyphus.Configuration import config
+logger = config.getLogger()
+
+from Sisyphus.RestApiV1 import get_institutions
+from Sisyphus.Utils.Terminal import Style
+import sys
+import json
+import argparse
+
+def parse(command_line_args=sys.argv):
+    parser = argparse.ArgumentParser(
+        add_help=True,
+        parents=[config.arg_parser],
+        description='Displays a list of available institution codes')
+    parser.add_argument('--country',
+                        dest='country',
+                        metavar='<country code or name>',
+                        required=False,
+                        help='display only countries matching (or partially matching) string')
+    parser.add_argument('--inst-name', '--institution-name',
+                        dest='name',
+                        metavar='<institution name>',  
+                        required=False,
+                        help='display only institutions matching (or partially matching) string')
+    parser.add_argument('--inst-id',
+                        dest='id',
+                        metavar='<institution id>',
+                        required=False,
+                        help='find the specific institution ID')
+    args = parser.parse_known_args(command_line_args)
+    return args
+
+def main():
+    logger.info("Listing institutions")
+
+    args, unknowns = parse()
+
+    resp = get_institutions()
+
+    if resp['status'] != "OK":
+        print("There was an error obtaining a list of institutions from the server")
+        return 1
+
+    # Separate by country
+    inst_by_country = {}
+    countries = []
+    for inst in resp['data']:
+        country_code = inst['country']['code']
+        country_name = inst['country']['name']
+        inst_id = inst['id']
+        inst_name = inst['name']
+
+        country_display_name = f"{country_name} ({country_code})"
+
+        if args.country is not None:
+            if args.country.upper() not in country_display_name.upper():
+                # skip this institution
+                continue
+        if args.name is not None:
+            if args.name.upper() not in inst_name.upper():
+                # skip this institution
+                continue
+        if args.id is not None:
+            if args.id != str(inst_id):
+                # skip this institution
+                continue
+
+        if country_code not in inst_by_country.keys():
+            inst_by_country[country_code] = []
+            countries.append((country_code, country_display_name))
+        inst_by_country[country_code].append((inst_id, inst_name))
+
+    countries = [ (a, b) for (b, a) in 
+                sorted([(b, a) for (a, b) in countries]) ]
+
+    if len(countries) == 0:
+        print()
+        print("No institutions found matching criteria")
+        print()
+    else:
+        print()
+        print("Institutions found:")
+        print()
+        for country_code, country_display_name in countries:
+
+            institutions = [ (a, b) for (b, a) in 
+                    sorted([(b, a) for (a, b) in inst_by_country[country_code]]) ]
+
+            if args.country is None:
+                print(country_display_name)
+            else:
+                print(Style.highlight(country_display_name, args.country))
+
+            print("="*len(country_display_name))
+            for inst_id, inst_name in institutions:
+                if args.name is None:
+                    print(f"{inst_id:3d} {inst_name}")
+                else:
+                    print(f"{inst_id:3d} {Style.highlight(inst_name, args.name)}")
+            print()
+
+
+if __name__ == '__main__':    
+    sys.exit(main())
+