Merge pull request #73 from Paperspace/PS-10876-add-missing-parameters-to-the-sdk

Add missing is_preemptible parameter to SDK

Author: mkulaczkowski

gradient/api_sdk/clients/experiment_client.py

@@ -18,6 +18,7 @@ def create_single_node(
             experiment_env=None,
             model_type=None,
             model_path=None,
+            is_preemptible=False,
             container=None,
             container_user=None,
             registry_username=None,
@@ -60,6 +61,7 @@ def create_single_node(
         :param dict experiment_env: Environment variables in a JSON
         :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom
         :param str model_path: Model path
+        :param bool is_preemptible: Is preemptible
         :param str container: Container (dockerfile) [required]
         :param str container_user: Container user for running the specified command in the container. If no containerUser is specified, the user will default to 'root' in the container.
         :param str registry_username: Registry username for accessing private docker registry container if nessesary
@@ -71,6 +73,9 @@ def create_single_node(
         :rtype: str
         """
 
+        if not is_preemptible:
+            is_preemptible = None
+
         experiment = models.SingleNodeExperiment(
             experiment_type_id=constants.ExperimentType.SINGLE_NODE,
             name=name,
@@ -84,6 +89,7 @@ def create_single_node(
             experiment_env=experiment_env,
             model_type=model_type,
             model_path=model_path,
+            is_preemptible=is_preemptible,
             container=container,
             command=command,
             container_user=container_user,
@@ -117,6 +123,7 @@ def create_multi_node(
             experiment_env=None,
             model_type=None,
             model_path=None,
+            is_preemptible=False,
             worker_container_user=None,
             worker_registry_username=None,
             worker_registry_password=None,
@@ -172,6 +179,7 @@ def create_multi_node(
         :param dict experiment_env: Environment variables in a JSON
         :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom
         :param str model_path: Model path
+        :param bool is_preemptible: Is preemptible
         :param str worker_container_user: Worker container user
         :param str worker_registry_username: Registry username for accessing private docker registry container if nessesary
         :param str worker_registry_password: Registry password for accessing private docker registry container if nessesary
@@ -185,6 +193,10 @@ def create_multi_node(
         :returns: experiment handle
         :rtype: str
         """
+
+        if not is_preemptible:
+            is_preemptible = None
+
         experiment = models.MultiNodeExperiment(
             name=name,
             project_id=project_id,
@@ -205,6 +217,7 @@ def create_multi_node(
             experiment_env=experiment_env,
             model_type=model_type,
             model_path=model_path,
+            is_preemptible=is_preemptible,
             worker_container_user=worker_container_user,
             worker_registry_username=worker_registry_username,
             worker_registry_password=worker_registry_password,
@@ -233,6 +246,7 @@ def run_single_node(
             experiment_env=None,
             model_type=None,
             model_path=None,
+            is_preemptible=False,
             container=None,
             container_user=None,
             registry_username=None,
@@ -272,6 +286,7 @@ def run_single_node(
         :param dict experiment_env: Environment variables in a JSON
         :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom
         :param str model_path: Model path
+        :param bool is_preemptible: Is preemptible
         :param str container: Container (dockerfile) [required]
         :param str container_user: Container user for running the specified command in the container. If no containerUser is specified, the user will default to 'root' in the container.
         :param str registry_username: Registry username for accessing private docker registry container if nessesary
@@ -283,6 +298,9 @@ def run_single_node(
         :rtype: str
         """
 
+        if not is_preemptible:
+            is_preemptible = None
+
         experiment = models.SingleNodeExperiment(
             experiment_type_id=constants.ExperimentType.SINGLE_NODE,
             name=name,
@@ -296,6 +314,7 @@ def run_single_node(
             experiment_env=experiment_env,
             model_type=model_type,
             model_path=model_path,
+            is_preemptible=is_preemptible,
             container=container,
             command=command,
             container_user=container_user,
@@ -329,6 +348,7 @@ def run_multi_node(
             experiment_env=None,
             model_type=None,
             model_path=None,
+            is_preemptible=False,
             worker_container_user=None,
             worker_registry_username=None,
             worker_registry_password=None,
@@ -384,6 +404,7 @@ def run_multi_node(
         :param dict experiment_env: Environment variables in a JSON
         :param str model_type: defines the type of model that is being generated by the experiment. Model type must be one of Tensorflow, ONNX, or Custom
         :param str model_path: Model path
+        :param bool is_preemptible: Is preemptible
         :param str worker_container_user: Worker container user
         :param str worker_registry_username: Registry username for accessing private docker registry container if nessesary
         :param str worker_registry_password: Registry password for accessing private docker registry container if nessesary
@@ -397,6 +418,10 @@ def run_multi_node(
         :returns: experiment handle
         :rtype: str
         """
+
+        if not is_preemptible:
+            is_preemptible = None
+
         experiment = models.MultiNodeExperiment(
             name=name,
             project_id=project_id,
@@ -417,6 +442,7 @@ def run_multi_node(
             experiment_env=experiment_env,
             model_type=model_type,
             model_path=model_path,
+            is_preemptible=is_preemptible,
             worker_container_user=worker_container_user,
             worker_registry_username=worker_registry_username,
             worker_registry_password=worker_registry_password,

gradient/api_sdk/clients/hyperparameter_client.py

@@ -15,7 +15,7 @@ def create(
             worker_container_user=None,
             worker_registry_username=None,
             worker_registry_password=None,
-            is_preemptible=True,
+            is_preemptible=False,
             ports=None,
             workspace_url=None,
             artifact_directory=None,
@@ -66,6 +66,9 @@ def create(
         :rtype: str
         """
 
+        if not is_preemptible:
+            is_preemptible = None
+
         hyperparameter = models.Hyperparameter(
             name=name,
             project_id=project_id,
@@ -112,7 +115,7 @@ def run(
             worker_registry_username=None,
             worker_registry_password=None,
             worker_container_user=None,
-            is_preemptible=True,
+            is_preemptible=False,
             ports=None,
             workspace_url=None,
             artifact_directory=None,
@@ -176,6 +179,9 @@ def run(
         :rtype: str
         """
 
+        if not is_preemptible:
+            is_preemptible = None
+
         hyperparameter = models.Hyperparameter(
             name=name,
             project_id=project_id,

gradient/api_sdk/models/experiment.py

@@ -23,6 +23,7 @@ class BaseExperiment(object):
     :param str project_id: Project ID
     :param str model_type: defines the type of model that is being generated by the experiment.
     :param str model_path: Model path
+    :param bool is_preemptible: Is preemptible
     :param str id:
     :param int state: state of experiment can be one of::
 
@@ -50,6 +51,7 @@ class BaseExperiment(object):
     project_id = attr.ib(type=str, default=None)
     model_type = attr.ib(type=str, default=None)
     model_path = attr.ib(type=str, default=None)
+    is_preemptible = attr.ib(type=bool, default=None)
     id = attr.ib(type=str, default=None)
     state = attr.ib(type=int, default=None)
 

gradient/api_sdk/serializers/experiment.py

@@ -16,6 +16,7 @@ class BaseExperimentSchema(BaseSchema):
     project_id = marshmallow.fields.Str(required=True, dump_to="projectHandle", load_from="project_handle")
     model_type = marshmallow.fields.Str(dump_to="modelType", load_from="modelType")
     model_path = marshmallow.fields.Str(dump_to="modelPath", load_from="modelPath")
+    is_preemptible = marshmallow.fields.Bool(dump_to="isPreemptible", load_from="isPreemptible")
     id = marshmallow.fields.Str(load_from="handle")
     state = marshmallow.fields.Int()
 

gradient/commands/experiments.py

@@ -44,6 +44,7 @@ def execute(self, json_, use_vpc=False):
 
     def _handle_workspace(self, instance_dict):
         handler = self.workspace_handler.handle(instance_dict)
+        instance_dict.pop("ignore_files", None)
         instance_dict.pop("workspace", None)
         instance_dict.pop("workspace_archive", None)
         instance_dict.pop("workspace_url", None)

tests/functional/test_experiments.py

@@ -40,6 +40,10 @@ class TestExperimentsCreateSingleNode(object):
         "--registryPassword", "passwd",
         "--registryUrl", "registryUrl",
         "--apiKey", "some_key",
+        "--modelPath", "some-model-path",
+        "--modelType", "some-model-type",
+        "--ignoreFiles", "file1,file2",
+        "--isPreemptible",
     ]
     BASIC_OPTIONS_REQUEST = {
         "name": u"exp1",
@@ -67,6 +71,9 @@ class TestExperimentsCreateSingleNode(object):
         "registryPassword": u"passwd",
         "registryUrl": u"registryUrl",
         "experimentTypeId": constants.ExperimentType.SINGLE_NODE,
+        "modelPath": "some-model-path",
+        "modelType": "some-model-type",
+        "isPreemptible": True,
     }
     BASIC_OPTIONS_COMMAND_WITH_VPC_SWITCH = [
         "experiments", "create", "singlenode",
@@ -129,13 +136,13 @@ def test_should_send_proper_data_and_print_message_when_create_experiment_was_ru
         runner = CliRunner()
         result = runner.invoke(cli.cli, self.FULL_OPTIONS_COMMAND)
 
+        assert self.EXPECTED_STDOUT in result.output
         post_patched.assert_called_once_with(self.URL,
                                              headers=self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY,
                                              json=self.FULL_OPTIONS_REQUEST,
                                              params=None,
                                              files=None,
                                              data=None)
-        assert self.EXPECTED_STDOUT in result.output
         assert result.exit_code == 0
         assert self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] == "some_key"
 
@@ -207,6 +214,10 @@ class TestExperimentsCreateMultiNode(object):
         "--parameterServerRegistryPassword", "psrpass",
         "--parameterServerRegistryUrl", "psrurl",
         "--apiKey", "some_key",
+        "--modelPath", "some-model-path",
+        "--modelType", "some-model-type",
+        "--ignoreFiles", "file1,file2",
+        "--isPreemptible",
     ]
     BASIC_OPTIONS_REQUEST = {
         u"name": u"multinode_mpi",
@@ -249,6 +260,9 @@ class TestExperimentsCreateMultiNode(object):
         "parameterServerRegistryUsername": u"psrcus",
         "parameterServerRegistryPassword": u"psrpass",
         "parameterServerRegistryUrl": u"psrurl",
+        "isPreemptible": True,
+        "modelPath": "some-model-path",
+        "modelType": "some-model-type",
     }
     RESPONSE_JSON_200 = {"handle": "sadkfhlskdjh", "message": "success"}
     RESPONSE_CONTENT_200 = b'{"handle":"sadkfhlskdjh","message":"success"}\n'
@@ -357,6 +371,10 @@ class TestExperimentsCreateAndStartSingleNode(TestExperimentsCreateSingleNode):
         "--registryUrl", "registryUrl",
         "--apiKey", "some_key",
         "--no-logs",
+        "--modelPath", "some-model-path",
+        "--modelType", "some-model-type",
+        "--ignoreFiles", "file1,file2",
+        "--isPreemptible",
     ]
     BASIC_OPTIONS_COMMAND_WITH_VPC_SWITCH = [
         "experiments", "run", "singlenode",
@@ -421,6 +439,10 @@ class TestExperimentsCreateAndStartMultiNode(TestExperimentsCreateMultiNode):
         "--parameterServerRegistryUrl", "psrurl",
         "--apiKey", "some_key",
         "--no-logs",
+        "--modelPath", "some-model-path",
+        "--modelType", "some-model-type",
+        "--ignoreFiles", "file1,file2",
+        "--isPreemptible",
     ]
     BASIC_OPTIONS_COMMAND_WITH_VPC_SWITCH = [
         "experiments", "run", "multinode",

tests/functional/test_hyperparameters.py

@@ -49,7 +49,10 @@ class TestCreateHyperparameters(object):
         "--hyperparameterServerContainerUser", "someHyperparameterServerContainerUser",
         "--hyperparameterServerMachineType", "someHyperparameterServerMachineType",
         "--workspace", "none",
-
+        "--modelPath", "some-model-path",
+        "--modelType", "some-model-type",
+        "--ignoreFiles", "file1,file2",
+        "--isPreemptible",
     ]
     EXPECTED_REQUEST_JSON_WHEN_ALL_PARAMETERS_WERE_USED = {
         "workerContainer": "some_container",
@@ -68,6 +71,9 @@ class TestCreateHyperparameters(object):
         "hyperparameterServerContainerUser": "someHyperparameterServerContainerUser",
         "hyperparameterServerMachineType": "someHyperparameterServerMachineType",
         "experimentTypeId": constants.ExperimentType.HYPERPARAMETER_TUNING,
+        "modelPath": "some-model-path",
+        "modelType": "some-model-type",
+        "isPreemptible": True,
     }
     EXPECTED_HEADERS = http_client.default_headers.copy()
     EXPECTED_HEADERS_WITH_CHANGED_API_KEY = http_client.default_headers.copy()
@@ -256,6 +262,10 @@ class TestCreateAndStartHyperparameters(object):
         "--hyperparameterServerContainerUser", "someHyperparameterServerContainerUser",
         "--hyperparameterServerMachineType", "someHyperparameterServerMachineType",
         "--workspace", "none",
+        "--modelPath", "some-model-path",
+        "--modelType", "some-model-type",
+        "--ignoreFiles", "file1,file2",
+        "--isPreemptible",
     ]
     EXPECTED_REQUEST_JSON_WHEN_ALL_PARAMETERS_WERE_USED = {
         "workerContainer": "some_container",
@@ -274,6 +284,9 @@ class TestCreateAndStartHyperparameters(object):
         "hyperparameterServerContainerUser": "someHyperparameterServerContainerUser",
         "hyperparameterServerMachineType": "someHyperparameterServerMachineType",
         "experimentTypeId": constants.ExperimentType.HYPERPARAMETER_TUNING,
+        "modelPath": "some-model-path",
+        "modelType": "some-model-type",
+        "isPreemptible": True,
     }
     EXPECTED_HEADERS = http_client.default_headers.copy()
     EXPECTED_HEADERS_WITH_CHANGED_API_KEY = http_client.default_headers.copy()