moved as much of cli components out of sdk

kossak · kossak · commit dd565964ecff · 2020-05-05T10:38:21.000+02:00
changed absolute imports to relative
diff --git a/gradient/api_sdk/clients/base_client.py b/gradient/api_sdk/clients/base_client.py
@@ -1,7 +1,7 @@
 import copy
 
-from gradient.api_sdk.repositories.common import BaseRepository
 from .. import logger as sdk_logger
+from ..repositories.common import BaseRepository
 from ..repositories.tags import ListTagRepository, UpdateTagRepository
 from ...exceptions import ReceivingDataFailedError
 
diff --git a/gradient/api_sdk/clients/clusters.py b/gradient/api_sdk/clients/clusters.py
@@ -1,6 +1,5 @@
-from gradient.api_sdk import repositories
-
-from gradient.api_sdk.clients.base_client import BaseClient
+from .base_client import BaseClient
+from .. import repositories
 
 
 class ClustersClient(BaseClient):
diff --git a/gradient/api_sdk/clients/http_client.py b/gradient/api_sdk/clients/http_client.py
@@ -3,8 +3,7 @@
 import requests
 
 from gradient import version
-from gradient.api_sdk import utils
-from .. import logger as sdk_logger
+from .. import utils, logger as sdk_logger
 from ..config import config
 
 default_headers = {"X-API-Key": config.PAPERSPACE_API_KEY,
diff --git a/gradient/api_sdk/clients/job_client.py b/gradient/api_sdk/clients/job_client.py
@@ -3,12 +3,10 @@
 
 Remember that in code snippets all highlighted lines are required other lines are optional.
 """
-from gradient.api_sdk import repositories
-
 from .base_client import BaseClient
 from ..models import Artifact, Job
 from ..repositories.jobs import ListJobs, ListJobLogs, ListJobArtifacts, CreateJob, DeleteJob, StopJob, \
-    DeleteJobArtifacts, GetJobArtifacts
+    DeleteJobArtifacts, GetJobArtifacts, GetJobMetrics, StreamJobMetrics
 
 
 class JobsClient(BaseClient):
@@ -386,7 +384,7 @@ def get_metrics(self, job_id, start=None, end=None, interval="30s", built_in_met
         :rtype: dict[str,dict[str,list[dict]]]
         """
 
-        repository = self.build_repository(repositories.GetJobMetrics)
+        repository = self.build_repository(GetJobMetrics)
         metrics = repository.get(
             id=job_id,
             start=start,
@@ -409,7 +407,7 @@ def stream_metrics(self, job_id, interval="30s", built_in_metrics=None):
         :rtype: Iterable[dict]
         """
 
-        repository = self.build_repository(repositories.StreamJobMetrics)
+        repository = self.build_repository(StreamJobMetrics)
         metrics = repository.stream(
             id=job_id,
             interval=interval,
diff --git a/gradient/api_sdk/clients/machine_types_client.py b/gradient/api_sdk/clients/machine_types_client.py
@@ -1,6 +1,6 @@
-from gradient.api_sdk import repositories
+from .. import repositories
 
-from gradient.api_sdk.clients.base_client import BaseClient
+from ..clients.base_client import BaseClient
 
 
 class MachineTypesClient(BaseClient):
diff --git a/gradient/api_sdk/clients/machines_client.py b/gradient/api_sdk/clients/machines_client.py
@@ -1,6 +1,6 @@
-from gradient.api_sdk import repositories, models
-from gradient.api_sdk.repositories.machines import CheckMachineAvailability, DeleteMachine, ListMachines, WaitForState
 from .base_client import BaseClient
+from .. import repositories, models
+from ..repositories.machines import CheckMachineAvailability, DeleteMachine, ListMachines, WaitForState
 
 
 class MachinesClient(BaseClient):
diff --git a/gradient/api_sdk/config.py b/gradient/api_sdk/config.py
@@ -1,8 +1,6 @@
 import json
 import os
 
-from gradient.config import get_help_colors_dict
-
 
 # TODO: this function is copy-pasted from login.py;
 #  there is something weird going one with imports in __init__.py and I'm unable to import apikey now
@@ -29,6 +27,17 @@ def get_api_key(config_dir_path, config_file_name):
 _DEFAULT_USE_CONSOLE_COLORS = True
 
 
+def get_help_colors_dict(use_colors, help_headers_color, help_options_color):
+    if not use_colors:
+        return {}
+
+    d = {
+        "help_headers_color": help_headers_color,
+        "help_options_color": help_options_color,
+    }
+    return d
+
+
 class config(object):
     DEBUG = os.environ.get("PAPERSPACE_CLI_DEBUG") in ("true", "1")
 
diff --git a/gradient/api_sdk/models/notebook.py b/gradient/api_sdk/models/notebook.py
@@ -2,8 +2,8 @@
 
 import attr
 
-from gradient.api_sdk.config import config
-from gradient.api_sdk.utils import concatenate_urls
+from ..config import config
+from ..utils import concatenate_urls
 
 
 @attr.s
diff --git a/gradient/api_sdk/repositories/clusters.py b/gradient/api_sdk/repositories/clusters.py
@@ -1,10 +1,10 @@
 import json
 from collections import OrderedDict
 
-from gradient.api_sdk.config import config
-from gradient.api_sdk.repositories.common import GetResource, ListResources
-from gradient.api_sdk.sdk_exceptions import MalformedResponseError
-from gradient.api_sdk.serializers.cluster import ClusterSchema
+from ..config import config
+from ..repositories.common import GetResource, ListResources
+from ..sdk_exceptions import MalformedResponseError
+from ..serializers.cluster import ClusterSchema
 
 
 class ValidateClusterRepository(GetResource):
diff --git a/gradient/api_sdk/repositories/experiments.py b/gradient/api_sdk/repositories/experiments.py
@@ -3,12 +3,12 @@
 import six
 import websocket
 
-from gradient.api_sdk.repositories.jobs import ListJobs
-from gradient.api_sdk.utils import concatenate_urls
 from .common import ListResources, CreateResource, StartResource, StopResource, DeleteResource, GetResource, GetMetrics, \
     StreamMetrics
 from .. import config, serializers, sdk_exceptions
+from ..repositories.jobs import ListJobs
 from ..serializers import utils as serializers_utils
+from ..utils import concatenate_urls
 
 
 class GetBaseExperimentApiUrlMixin(object):
diff --git a/gradient/api_sdk/repositories/jobs.py b/gradient/api_sdk/repositories/jobs.py
@@ -1,9 +1,9 @@
 import json
 
 import gradient.api_sdk.config
-from gradient.api_sdk import serializers, sdk_exceptions
-from gradient.api_sdk.clients import http_client
 from .common import ListResources, CreateResource, GetResource, DeleteResource, StopResource, GetMetrics, StreamMetrics
+from .. import serializers, sdk_exceptions
+from ..clients import http_client
 from ..serializers import JobSchema, LogRowSchema
 
 
diff --git a/gradient/api_sdk/repositories/machine_types.py b/gradient/api_sdk/repositories/machine_types.py
@@ -1,7 +1,7 @@
 from operator import itemgetter
 
-from gradient.api_sdk import config, serializers
-from gradient.api_sdk.repositories.common import ListResources
+from .. import config, serializers
+from ..repositories.common import ListResources
 
 
 class ListMachineTypes(ListResources):
diff --git a/gradient/api_sdk/repositories/machines.py b/gradient/api_sdk/repositories/machines.py
@@ -1,10 +1,10 @@
 import time
 
-from gradient.api_sdk.config import config
-from gradient.api_sdk.serializers import MachineSchema
 from .common import BaseRepository, CreateResource, DeleteResource, ListResources, StartResource, StopResource, \
     GetResource, AlterResource
 from .. import serializers, models
+from ..config import config
+from ..serializers import MachineSchema
 
 
 class MachinesApiUrlMixin(object):
diff --git a/gradient/api_sdk/repositories/notebooks.py b/gradient/api_sdk/repositories/notebooks.py
@@ -1,13 +1,13 @@
 import json
 
-import gradient.api_sdk.config
 from .common import CreateResource, DeleteResource, ListResources, GetResource, GetMetrics, StreamMetrics
+from .. import config
 from .. import serializers, sdk_exceptions
 
 
 class GetNotebookApiUrlMixin(object):
     def _get_api_url(self, **kwargs):
-        return gradient.api_sdk.config.config.CONFIG_HOST
+        return config.config.CONFIG_HOST
 
 
 class CreateNotebook(GetNotebookApiUrlMixin, CreateResource):
diff --git a/gradient/api_sdk/repositories/projects.py b/gradient/api_sdk/repositories/projects.py
@@ -1,12 +1,12 @@
-import gradient.api_sdk.config
+from .. import config
 from .. import serializers
 from ..repositories.common import CreateResource, ListResources, DeleteResource, GetResource
 from ..sdk_exceptions import ResourceFetchingError
 
 
 class GetBaseProjectsApiUrlMixin(object):
     def _get_api_url(self, **_):
-        return gradient.api_sdk.config.config.CONFIG_HOST
+        return config.config.CONFIG_HOST
 
 
 class CreateProject(GetBaseProjectsApiUrlMixin, CreateResource):
diff --git a/gradient/api_sdk/repositories/tensorboards.py b/gradient/api_sdk/repositories/tensorboards.py
@@ -1,11 +1,11 @@
-from gradient.api_sdk.config import config
 from . import common
+from .. import config
 from .. import serializers, models
 
 
 class GetTensorboardApiUrlMixin(object):
     def _get_api_url(self, **kwargs):
-        return config.CONFIG_SERVICE_HOST
+        return config.config.CONFIG_SERVICE_HOST
 
 
 class ParseTensorboardMixin(object):
diff --git a/gradient/api_sdk/serializers/artifact.py b/gradient/api_sdk/serializers/artifact.py
@@ -1,6 +1,6 @@
 import marshmallow
 
-from gradient.api_sdk import models
+from .. import models
 from .base import BaseSchema
 
 
diff --git a/gradient/api_sdk/serializers/cluster.py b/gradient/api_sdk/serializers/cluster.py
@@ -1,7 +1,7 @@
 import marshmallow as ma
 
-from gradient.api_sdk import models
-from gradient.api_sdk.serializers.base import BaseSchema
+from .. import models
+from ..serializers.base import BaseSchema
 
 
 class ClusterSchema(BaseSchema):
diff --git a/gradient/api_sdk/serializers/deployment.py b/gradient/api_sdk/serializers/deployment.py
@@ -1,7 +1,7 @@
 import marshmallow as ma
 
-from gradient.api_sdk import models
 from .base import BaseSchema
+from .. import models
 
 
 class DeploymentSchema(BaseSchema):
diff --git a/gradient/api_sdk/serializers/job.py b/gradient/api_sdk/serializers/job.py
@@ -1,7 +1,7 @@
 import marshmallow
 
-from gradient.api_sdk import models
 from .base import BaseSchema
+from .. import models
 
 
 class JobSchema(BaseSchema):
diff --git a/gradient/api_sdk/serializers/log.py b/gradient/api_sdk/serializers/log.py
@@ -1,7 +1,7 @@
 import marshmallow
 
-from gradient.api_sdk import models
 from .base import BaseSchema
+from .. import models
 
 
 class LogRowSchema(BaseSchema):
diff --git a/gradient/api_sdk/serializers/tag.py b/gradient/api_sdk/serializers/tag.py
@@ -1,7 +1,7 @@
 import marshmallow as ma
 
-from gradient.api_sdk.serializers.base import BaseSchema
 from .. import models
+from ..serializers.base import BaseSchema
 
 
 class TagSchema(BaseSchema):
diff --git a/gradient/api_sdk/serializers/utils.py b/gradient/api_sdk/serializers/utils.py
@@ -1,7 +1,6 @@
-from gradient.api_sdk.serializers import MpiMultiNodeExperimentSchema
-
 from ..sdk_exceptions import GradientSdkError
-from ..serializers import SingleNodeExperimentSchema, MultiNodeExperimentSchema, HyperparameterSchema
+from ..serializers import SingleNodeExperimentSchema, MultiNodeExperimentSchema, HyperparameterSchema, \
+    MpiMultiNodeExperimentSchema
 
 EXPERIMENT_ID_TO_EXPERIMENT_SERIALIZER_MAPPING = {
     1: SingleNodeExperimentSchema,
diff --git a/gradient/api_sdk/utils.py b/gradient/api_sdk/utils.py
@@ -1,4 +1,5 @@
 import base64
+import os
 import random
 import string
 from collections import OrderedDict
@@ -141,3 +142,42 @@ def callback(monitor):
                 bar.update(monitor.bytes_read)
 
         return callback
+
+
+class PathParser(object):
+    LOCAL_DIR = 0
+    LOCAL_FILE = 1
+    GIT_URL = 2
+    S3_URL = 3
+
+    @classmethod
+    def parse_path(cls, path):
+        if cls.is_local_dir(path):
+            return cls.LOCAL_DIR
+
+        if cls.is_local_zip_file(path):
+            return cls.LOCAL_FILE
+
+        if cls.is_git_url(path):
+            return cls.GIT_URL
+
+        if cls.is_s3_url(path):
+            return cls.S3_URL
+
+        raise sdk_exceptions.WrongPathError("Given path is neither local path, nor valid URL")
+
+    @staticmethod
+    def is_local_dir(path):
+        return os.path.exists(path) and os.path.isdir(path)
+
+    @staticmethod
+    def is_local_zip_file(path):
+        return os.path.exists(path) and os.path.isfile(path) and path.endswith(".zip")
+
+    @staticmethod
+    def is_git_url(path):
+        return not os.path.exists(path) and path.endswith(".git") or path.lower().startswith("git:")
+
+    @staticmethod
+    def is_s3_url(path):
+        return not os.path.exists(path) and path.lower().startswith("s3:")
diff --git a/gradient/api_sdk/workspace.py b/gradient/api_sdk/workspace.py
@@ -1,9 +1,6 @@
 import os
 import tempfile
 
-from gradient import cliutils, cli_constants
-from gradient.api_sdk import s3_uploader, archivers
-
 from . import s3_uploader, archivers, utils
 from .logger import MuteLogger
 
@@ -64,12 +61,12 @@ def _validate_input(input_data):
         workspace_url = None
 
         if workspace_path not in ("none", None):
-            path_type = cliutils.PathParser.parse_path(workspace_path)
+            path_type = utils.PathParser.parse_path(workspace_path)
 
-            if path_type != cliutils.PathParser.LOCAL_DIR:
-                if path_type == cliutils.PathParser.LOCAL_FILE:
+            if path_type != utils.PathParser.LOCAL_DIR:
+                if path_type == utils.PathParser.LOCAL_FILE:
                     workspace_archive = workspace_path
-                elif path_type in (cliutils.PathParser.GIT_URL, cliutils.PathParser.S3_URL):
+                elif path_type in (utils.PathParser.GIT_URL, utils.PathParser.S3_URL):
                     workspace_url = workspace_path
 
                 workspace_path = None
@@ -106,7 +103,7 @@ def _upload(self, archive_path, project_id, cluster_id=None):
         return workspace
 
     def _get_workspace_uploader(self, api_key):
-        workspace_uploader = self.WORKSPACE_UPLOADER_CLS(api_key, logger=self.logger, ps_client_name=CLI_PS_CLIENT_NAME)
+        workspace_uploader = self.WORKSPACE_UPLOADER_CLS(api_key, logger=self.logger, ps_client_name=self.client_name)
         return workspace_uploader
 
 
diff --git a/gradient/cliutils.py b/gradient/cliutils.py
diff --git a/gradient/config.py b/gradient/config.py
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
diff --git a/tests/unit/test_workspace.py b/tests/unit/test_workspace.py
