test: add client mode e2e tests

.github/CODEOWNERS

@@ -1 +1,2 @@
 * @mostly-ai/mostly-developers
+/.github/workflows/* @mostly-ai/mostly-devops

.github/workflows/run-tests-cpu.yaml

@@ -3,40 +3,47 @@ name: "[CPU] mostlyai Tests"
 on: [workflow_call]
 
 env:
-    PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
-    FORCE_COLOR: "1"
+  PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
+  FORCE_COLOR: "1"
 
 jobs:
-    run-test-cpu:
-        runs-on: ubuntu-latest
-        permissions:
-            contents: read
-            packages: write
-        steps:
-          - name: Setup | Checkout
-            uses: actions/checkout@v4
-            with:
-                fetch-depth: 1
-                submodules: 'true'
+  run-test-cpu:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Setup | Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          submodules: "true"
 
-          - name: Setup | uv
-            uses: astral-sh/setup-uv@v5
-            with:
-                enable-cache: false
-                python-version: '3.10'
+      - name: Setup | uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: false
+          python-version: "3.10"
 
-          - name: Setup | Dependencies
-            run: |
-              uv sync --frozen --only-group dev
-              uv pip install --index-strategy unsafe-first-match torch==2.7.0+cpu torchvision==0.22.0+cpu ".[local]" --extra-index-url https://download.pytorch.org/whl/cpu
+      - name: Setup | Dependencies
+        run: |
+          uv sync --frozen --only-group dev
+          uv pip install --index-strategy unsafe-first-match torch==2.7.0+cpu torchvision==0.22.0+cpu ".[local]" --extra-index-url https://download.pytorch.org/whl/cpu
 
-          - name: Test | End-to-End Tests
-            run: |
-                uv run --no-sync pytest -vv tests/_local/end_to_end
+      - name: Test | End-to-End Tests
+        env:
+          MOSTLY_API_KEY: ${{ secrets.E2E_CLIENT_MOSTLY_API_KEY }}
+          MOSTLY_BASE_URL: ${{ secrets.E2E_CLIENT_MOSTLY_BASE_URL }}
+          E2E_CLIENT_S3_ACCESS_KEY: ${{ secrets.E2E_CLIENT_S3_ACCESS_KEY }}
+          E2E_CLIENT_S3_SECRET_KEY: ${{ secrets.E2E_CLIENT_S3_SECRET_KEY }}
+          E2E_CLIENT_S3_BUCKET: ${{ secrets.E2E_CLIENT_S3_BUCKET }}
+        # both local and client mode e2e tests will be run
+        run: |
+          uv run --no-sync pytest -vv tests/_local/end_to_end
 
-          - name: Test | Unit Tests
-            run: |
-              uv run --no-sync pytest -vv tests/client/unit
-              uv run --no-sync pytest -vv tests/_data/unit
-              uv run --no-sync pytest -vv tests/_local/unit
-              uv run --no-sync pytest -vv tests/test_domain.py
+      - name: Test | Unit Tests
+        run: |
+          uv run --no-sync pytest -vv tests/client/unit
+          uv run --no-sync pytest -vv tests/_data/unit
+          uv run --no-sync pytest -vv tests/_local/unit
+          uv run --no-sync pytest -vv tests/test_domain.py

.github/workflows/run-tests-gpu.yaml

@@ -3,47 +3,48 @@ name: "[GPU] mostlyai Tests"
 on: [workflow_call]
 
 env:
-    PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
-    FORCE_COLOR: "1"
+  PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
+  FORCE_COLOR: "1"
 
 jobs:
-    run-test-gpu:
-        runs-on: gha-gpu-public
-        container:
-            image: nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
-            options: --gpus all
-        permissions:
-            contents: read
-            packages: write
-        steps:
-          - name: Setup | Install Git
-            run: |
-              export DEBIAN_FRONTEND=noninteractive
-              ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime
-              echo "Etc/UTC" > /etc/timezone
-              apt-get update -qq
-              apt-get install -y --no-install-recommends git tzdata build-essential
-
-          - name: Setup | Checkout
-            uses: actions/checkout@v4
-            with:
-                fetch-depth: 1
-                submodules: 'true'
-
-          - name: Setup | uv
-            uses: astral-sh/setup-uv@v5
-            with:
-                enable-cache: false
-                python-version: '3.10'
-
-          - name: Setup | Dependencies
-            run: |
-              uv sync --frozen --only-group dev
-              uv pip install ".[local-gpu]"
-
-          - name: Setup | Check for available GPU-s
-            run: nvidia-smi
-
-          - name: Test | End-to-End Tests
-            run: |
-                uv run --no-sync pytest -vv tests/_local/end_to_end
+  run-test-gpu:
+    runs-on: gha-gpu-public
+    container:
+      image: nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
+      options: --gpus all
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Setup | Install Git
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime
+          echo "Etc/UTC" > /etc/timezone
+          apt-get update -qq
+          apt-get install -y --no-install-recommends git tzdata build-essential
+
+      - name: Setup | Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          submodules: "true"
+
+      - name: Setup | uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: false
+          python-version: "3.10"
+
+      - name: Setup | Dependencies
+        run: |
+          uv sync --frozen --only-group dev
+          uv pip install ".[local-gpu]"
+
+      - name: Setup | Check for available GPU-s
+        run: nvidia-smi
+
+      - name: Test | End-to-End Tests
+        # client mode e2e test will be skipped when no extra environment variable is provided
+        run: |
+          uv run --no-sync pytest -vv tests/_local/end_to_end

.gitignore

@@ -7,3 +7,4 @@ __pycache__/
 .pytest_cache/
 .ipynb_checkpoints
 .DS_Store
+.env

mostlyai/sdk/_local/routes.py

@@ -369,6 +369,7 @@ async def clone_generator(id: str, config: GeneratorCloneConfig = Body(...)) ->
             if any(read_connector_from_json(c).type == ConnectorType.file_upload for c in connector_dirs):
                 raise HTTPException(status_code=400, detail="Cannot clone a generator with uploaded files.")
             generator_config = generators.get_generator_config(self.home_dir, id)
+            generator_config.name = f"Clone - {generator_config.name}"
             new_generator = create_generator_model(home_dir=self.home_dir, config=generator_config)
             new_generator_dir = self.home_dir / "generators" / new_generator.id
             if config.training_status == GeneratorCloneTrainingStatus.continue_:

mostlyai/sdk/client/api.py

@@ -594,10 +594,12 @@ def train(
             # map config to data, in case user incorrectly provided data as first argument
             data = config
         if isinstance(data, (str, Path)):
-            name, df = read_table_from_path(data)
+            table_name, df = read_table_from_path(data)
+            if name is None:
+                name = table_name
             config = GeneratorConfig(
                 name=name,
-                tables=[SourceTableConfig(data=convert_to_base64(df), name=name)],
+                tables=[SourceTableConfig(data=convert_to_base64(df), name=table_name)],
             )
         elif isinstance(data, pd.DataFrame) or (
             data.__class__.__name__ == "DataFrame" and data.__class__.__module__.startswith("pyspark.sql")

pyproject.toml

@@ -137,6 +137,7 @@ dev = [
     "respx>=0.20",
     "poetry-bumpversion>=0.3.2",
     "httpx>=0.27",
+    "python-dotenv>=1.0.0",
 ]
 docs = [
     "mkdocs>=1.6",