Merge pull request #218 from Paperspace/PS-12882-Remove_--workspaceUrl_and_--workspaceArchive

Remove --workspaceUrl and --workspaceArchive and some unused fields f…

Author: BartoszCki

gradient/api_sdk/clients/hyperparameter_client.py

@@ -147,18 +147,6 @@ def run(
     ):
         """Create and start hyperparameter tuning job
 
-        *EXAMPLE*::
-
-            gradient hyperparameters run
-            --name HyperoptKerasExperimentCLI1
-            --projectId <your-project-id>
-            --tuningCommand 'make run_hyperopt'
-            --workerContainer tensorflow/tensorflow:1.13.1-gpu-py3
-            --workerMachineType K80
-            --workerCommand 'make run_hyperopt_worker'
-            --workerCount 2
-            --workspaceUrl git+https://github.com/Paperspace/hyperopt-keras-sample
-
         :param str name: Name of new experiment  [required]
         :param str project_id: Project ID  [required]
         :param str tuning_command: Tuning command  [required]
@@ -262,23 +250,6 @@ def start(self, id):
     def list(self):
         """Get a list of hyperparameter tuning jobs
 
-        *EXAMPLE*::
-
-            gradient hyperparameters list
-
-        *EXAMPLE RETURN*::
-
-            +--------------------------------+----------------+------------+
-            | Name                           | ID             | Project ID |
-            +--------------------------------+----------------+------------+
-            | name-of-your-experiment-job    | job-id         | project-id |
-            | name-of-your-experiment-job    | job-id         | project-id |
-            | name-of-your-experiment-job    | job-id         | project-id |
-            | name-of-your-experiment-job    | job-id         | project-id |
-            | name-of-your-experiment-job    | job-id         | project-id |
-            +--------------------------------+----------------+------------+
-
-
         :rtype: list[models.Hyperparameter]
         """
         repository = self.build_repository(repositories.ListHyperparameterJobs)

gradient/api_sdk/clients/job_client.py

@@ -38,9 +38,6 @@ def create(
             command=None,
             ports=None,
             is_public=None,
-            workspace=None,
-            workspace_archive=None,
-            workspace_url=None,
             working_directory=None,
             experiment_id=None,
             job_env=None,
@@ -78,7 +75,6 @@ def create(
                 name='Example job',
                 command='pip install -r requirements.txt && python mnist.py',
                 ports='5000:5000',
-                workspace_url='git+https://github.com/Paperspace/mnist-sample.git',
                 job_env={
                     'CUSTOM_ENV'='Some value that will be set as system environment',
                 }
@@ -114,11 +110,6 @@ def create(
             Example value: ``5000:5000,8080:8080``
 
         :param bool is_public: bool flag to select if job should be available by default None
-        :param str workspace: this field is used with CLI to upload folder as your workspace. You can provide here path
-            that you wish to upload. (Soon also will support a path to a workspace archive or git repository URL.)
-        :param str workspace_archive: Path to workspace archive. (Currently being deprecated in an upcoming version.)
-        :param str workspace_url: url to repo with code to run inside of job.
-            (Currently being deprecated in an upcoming version.)
         :param str working_directory: location of code to run. By default ``/paperspace``
         :param str experiment_id: Id of experiment to which job should be connected. If not provided there will be
             created new experiment for this job.
@@ -158,9 +149,6 @@ def create(
             command=command,
             ports=ports,
             is_public=is_public,
-            workspace=workspace,
-            workspace_archive=workspace_archive,
-            workspace_url=workspace_url,
             working_directory=working_directory,
             experiment_id=experiment_id,
             job_env=job_env,

gradient/api_sdk/models/job.py

@@ -6,7 +6,6 @@ class Job(object):
     id = attr.ib(type=str, default=None)
     name = attr.ib(type=str, default=None)
     state = attr.ib(type=str, default=None)
-    workspace_url = attr.ib(type=str, default=None)
     working_directory = attr.ib(type=str, default=None)
     artifacts_directory = attr.ib(type=str, default=None)
     entrypoint = attr.ib(type=str, default=None)
@@ -74,8 +73,6 @@ class Job(object):
     experiment_id = attr.ib(type=str, default=None)
 
     command = attr.ib(type=str, default=None)
-    workspace = attr.ib(type=str, default=None)
-    workspace_archive = attr.ib(type=str, default=None)
     workspace_file_name = attr.ib(type=str, default=None)
     use_dockerfile = attr.ib(type=str, default=None)
     rel_dockerfile_path = attr.ib(type=str, default=None)

gradient/api_sdk/s3_uploader.py

@@ -7,7 +7,6 @@
 import progressbar
 from requests_toolbelt.multipart import encoder
 
-from gradient.cli_constants import CLI_PS_CLIENT_NAME
 from . import sdk_exceptions
 from .clients import http_client
 from .config import config
@@ -157,13 +156,14 @@ def _archive_iterate_callback(self, i):
 class S3FileUploader(object):
     DEFAULT_MULTIPART_ENCODER_CLS = MultipartEncoder
 
-    def __init__(self, multipart_encoder_cls=None, logger=None):
+    def __init__(self, multipart_encoder_cls=None, logger=None, ps_client_name=None):
         """
         :param type(MultipartEncoder) multipart_encoder_cls:
         :param Logger logger:
         """
         self.multipart_encoder_cls = multipart_encoder_cls or self.DEFAULT_MULTIPART_ENCODER_CLS
         self.logger = logger or MuteLogger()
+        self.ps_client_name = ps_client_name
 
     def upload(self, file_path, url, bucket_name, s3_fields):
         """Upload a file to S3
@@ -200,7 +200,7 @@ def _upload(self, url, data):
             raise sdk_exceptions.S3UploadFailedError(response)
 
     def _get_client(self, url):
-        client = http_client.API(url, logger=self.logger, ps_client_name=CLI_PS_CLIENT_NAME)
+        client = http_client.API(url, logger=self.logger, ps_client_name=self.ps_client_name)
         return client
 
     def _get_multipart_encoder_monitor(self, fields):
@@ -225,7 +225,7 @@ def _get_bucket_url(bucket_name, s3_fields):
 
 
 class S3ProjectFileUploader(object):
-    def __init__(self, api_key, s3uploader=None, logger=None):
+    def __init__(self, api_key, s3uploader=None, logger=None, ps_client_name=None):
         """
         :param str api_key:
         :param S3FileUploader s3uploader:
@@ -236,9 +236,9 @@ def __init__(self, api_key, s3uploader=None, logger=None):
             config.CONFIG_EXPERIMENTS_HOST,
             api_key=api_key,
             logger=self.logger,
-            ps_client_name=CLI_PS_CLIENT_NAME,
+            ps_client_name=ps_client_name,
         )
-        self.s3uploader = s3uploader or S3FileUploader(logger=self.logger)
+        self.s3uploader = s3uploader or S3FileUploader(logger=self.logger, ps_client_name=ps_client_name)
 
     def upload(self, file_path, project_id, cluster_id=None):
         """Upload file to S3 bucket for a project
@@ -294,7 +294,7 @@ def _get_upload_data(self, file_path, project_handle, cluster_id=None):
 
 
 class S3WorkspaceDirectoryUploader(object):
-    def __init__(self, api_key, temp_dir=None, archiver=None, project_uploader=None):
+    def __init__(self, api_key, temp_dir=None, archiver=None, project_uploader=None, ps_client_name=None):
         """
         :param str api_key:
         :param str temp_dir:
@@ -303,7 +303,8 @@ def __init__(self, api_key, temp_dir=None, archiver=None, project_uploader=None)
         """
         self.temp_dir = temp_dir or tempfile.gettempdir()
         self.archiver = archiver or ZipArchiver()
-        self.project_uploader = project_uploader or S3ProjectFileUploader(api_key)
+        self.ps_client_name = ps_client_name
+        self.project_uploader = project_uploader or S3ProjectFileUploader(api_key, ps_client_name=ps_client_name)
 
     def upload(self, workspace_dir_path, project_id, exclude=None, temp_file_name="temp.zip"):
         """Archive and upload a workspace directory

gradient/api_sdk/serializers/job.py

@@ -10,7 +10,6 @@ class JobSchema(BaseSchema):
     id = marshmallow.fields.Str(dump_to="id", load_from="id")
     name = marshmallow.fields.Str(required=True)
     state = marshmallow.fields.Str()
-    workspace_url = marshmallow.fields.Str(dump_to="workspaceUrl", load_from="workspaceUrl")
     workspace_file_name = marshmallow.fields.Str(dump_to="workspaceFileName", load_from="workspaceFileName")
     working_directory = marshmallow.fields.Str(dump_to="workingDirectory", load_from="workingDirectory")
     artifacts_directory = marshmallow.fields.Str(dump_to="artifactsDirectory", load_from="artifactsDirectory")
@@ -80,8 +79,6 @@ class JobSchema(BaseSchema):
     experiment_id = marshmallow.fields.Str(dump_to="experimentId", load_from="experimentId")
 
     command = marshmallow.fields.Str()
-    workspace = marshmallow.fields.Str()
-    workspace_archive = marshmallow.fields.Str(dump_to="workspaceArchive", load_from="workspaceArchive")
     use_dockerfile = marshmallow.fields.Bool(dump_to="useDockerfile", load_from="useDockerfile")
     rel_dockerfile_path = marshmallow.fields.Str(dump_to="relDockerfilePath", load_from="relDockerfilePath")
     registry_username = marshmallow.fields.Str(dump_to="registryUsername", load_from="registryUsername")

gradient/cli/experiments.py

@@ -2,7 +2,7 @@
 
 import click
 
-from gradient import cliutils, clilogger, workspace
+from gradient import clilogger, workspace
 from gradient.api_sdk import constants
 from gradient.cli import common, validators
 from gradient.cli.cli import cli
@@ -26,7 +26,7 @@
 
 def get_workspace_handler(api_key):
     logger_ = clilogger.CliLogger()
-    workspace_handler = workspace.S3WorkspaceHandlerWithProgressbar(api_key=api_key, logger_=logger_)
+    workspace_handler = workspace.S3WorkspaceHandlerWithProgressbar(api_key=api_key, logger_=logger_, )
     return workspace_handler
 
 
@@ -76,19 +76,6 @@ def common_experiments_create_options(f):
             help="Git commit hash, branch name or tag",
             cls=common.GradientOption,
         ),
-        click.option(
-            "--workspaceArchive",
-            "workspace_archive",
-            help="Path to workspace .zip archive",
-            cls=common.GradientOption,
-        ),
-        click.option(
-            "--workspaceUrl",
-            "workspace_url",
-            metavar="<workspace URL>",
-            help="Project git repository url",
-            cls=common.GradientOption,
-        ),
         click.option(
             "--workspaceUsername",
             "workspace_username",
@@ -467,18 +454,6 @@ def common_experiments_create_single_node_options(f):
     return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
 
 
-def show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs):
-    if kwargs["workspace_archive"] or kwargs["workspace_url"]:
-        msg = """DeprecatedWarning:
-WARNING: --workspaceUrl and --workspaceArchive options will not be included in version 0.6.0
-
-For more information, please see:
-https://docs.paperspace.com
-If you depend on functionality not listed there, please file an issue."""
-
-        clilogger.CliLogger().error(msg)
-
-
 def tensorboard_option(f):
     options = [
         click.option(
@@ -522,11 +497,9 @@ def parse_tensorboard_options(tensorboard, tensorboard_set):
 @common.options_file
 def create_multi_node(api_key, tensorboard, tensorboard_set, options_file, **kwargs):
     kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"))
-    show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs)
     add_to_tensorboard = parse_tensorboard_options(tensorboard, tensorboard_set)
 
     validators.validate_multi_node(kwargs)
-    cliutils.validate_workspace_input(kwargs)
     common.del_if_value_is_none(kwargs)
     experiment_type = kwargs.get('experiment_type_id')
     command_class = MULTI_NODE_CREATE_EXPERIMENT_COMMANDS.get(experiment_type)
@@ -546,10 +519,8 @@ def create_multi_node(api_key, tensorboard, tensorboard_set, options_file, **kwa
 @common.options_file
 def create_single_node(api_key, tensorboard, tensorboard_set, options_file, **kwargs):
     kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"))
-    show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs)
     add_to_tensorboard = parse_tensorboard_options(tensorboard, tensorboard_set)
 
-    cliutils.validate_workspace_input(kwargs)
     common.del_if_value_is_none(kwargs)
 
     command = experiments_commands.CreateSingleNodeExperimentCommand(
@@ -578,11 +549,9 @@ def create_single_node(api_key, tensorboard, tensorboard_set, options_file, **kw
 @click.pass_context
 def create_and_start_multi_node(ctx, api_key, show_logs, tensorboard, tensorboard_set, options_file, **kwargs):
     kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"))
-    show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs)
     add_to_tensorboard = parse_tensorboard_options(tensorboard, tensorboard_set)
 
     validators.validate_multi_node(kwargs)
-    cliutils.validate_workspace_input(kwargs)
     common.del_if_value_is_none(kwargs)
 
     experiment_type = kwargs.get('experiment_type_id')
@@ -617,10 +586,8 @@ def create_and_start_multi_node(ctx, api_key, show_logs, tensorboard, tensorboar
 def create_and_start_single_node(ctx, api_key, show_logs, tensorboard, tensorboard_set, options_file,
                                  **kwargs):
     kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"))
-    show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs)
     add_to_tensorboard = parse_tensorboard_options(tensorboard, tensorboard_set)
 
-    cliutils.validate_workspace_input(kwargs)
     common.del_if_value_is_none(kwargs)
 
     command = experiments_commands.CreateAndStartSingleNodeExperimentCommand(

gradient/cli/hyperparameters.py

@@ -2,7 +2,6 @@
 
 import click
 
-from gradient import cliutils
 from gradient.cli import common
 from gradient.cli.common import ClickGroup, validate_comma_split_option
 from gradient.cli.experiments import common_experiments_create_options, get_workspace_handler, experiments_group
@@ -134,8 +133,7 @@ def common_hyperparameter_create_options(f):
 @common.options_file
 def create_hyperparameter(api_key, options_file, **hyperparameter):
     hyperparameter["tags"] = validate_comma_split_option(hyperparameter.pop("tags_comma"), hyperparameter.pop("tags"))
-    cliutils.validate_workspace_input(hyperparameter)
-    common.del_if_value_is_none(hyperparameter)
+    common.del_if_value_is_none(hyperparameter, del_all_falsy=True)
 
     command = hyperparameters_commands.CreateHyperparameterCommand(
         api_key=api_key,
@@ -151,8 +149,7 @@ def create_hyperparameter(api_key, options_file, **hyperparameter):
 @common.options_file
 def create_and_start_hyperparameter(api_key, options_file, **hyperparameter):
     hyperparameter["tags"] = validate_comma_split_option(hyperparameter.pop("tags_comma"), hyperparameter.pop("tags"))
-    cliutils.validate_workspace_input(hyperparameter)
-    common.del_if_value_is_none(hyperparameter)
+    common.del_if_value_is_none(hyperparameter, del_all_falsy=True)
 
     command = hyperparameters_commands.CreateAndStartHyperparameterCommand(
         api_key=api_key,

gradient/cli/jobs.py

@@ -2,16 +2,14 @@
 
 import click
 
-from gradient import cliutils, clilogger
+from gradient import clilogger
 from gradient.cli import common
 from gradient.cli.cli import cli
 from gradient.cli.cli_types import json_string
 from gradient.cli.common import (
     api_key_option, del_if_value_is_none, ClickGroup, jsonify_dicts,
     validate_comma_split_option,
 )
-from gradient.cli.experiments import \
-    show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used
 from gradient.commands import jobs as jobs_commands
 from gradient.commands.jobs import JobAddTagsCommand, JobRemoveTagsCommand
 from gradient.workspace import WorkspaceHandler
@@ -142,18 +140,6 @@ def common_jobs_create_options(f):
             help="Path to workspace directory",
             cls=common.GradientOption,
         ),
-        click.option(
-            "--workspaceArchive",
-            "workspace_archive",
-            help="Path to workspace archive",
-            cls=common.GradientOption,
-        ),
-        click.option(
-            "--workspaceUrl",
-            "workspace_url",
-            help="Project git repository url",
-            cls=common.GradientOption,
-        ),
         click.option(
             "--workingDirectory",
             "working_directory",
@@ -298,8 +284,6 @@ def common_jobs_create_options(f):
 @click.pass_context
 def create_job(ctx, api_key, options_file, **kwargs):
     kwargs["tags"] = validate_comma_split_option(kwargs.pop("tags_comma"), kwargs.pop("tags"))
-    cliutils.validate_workspace_input(kwargs)
-    show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs)
 
     del_if_value_is_none(kwargs)
     jsonify_dicts(kwargs)

gradient/cli/run.py

@@ -1,6 +1,5 @@
 import click
 
-from gradient import cliutils, clilogger
 from gradient.api_sdk.constants import RunMode
 from gradient.cli import common
 from gradient.cli.cli import cli
@@ -40,11 +39,6 @@
 @common.api_key_option
 @common.options_file
 def run(api_key, options_file, **kwargs):
-    cliutils.validate_workspace_input(kwargs)
-    if kwargs["workspace_archive"] or kwargs["workspace_url"]:
-        clilogger.CliLogger().error("WARNING: --workspaceUrl and --workspaceArchive options will not be included "
-                                    "in version 0.6.0")
-
     del_if_value_is_none(kwargs)
     jsonify_dicts(kwargs)