chore: bump engine 2.5.0 / qa 1.10.4, drop Python 3.10, fix pandas 3 compat

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "Client Mode (light)",
-  "image": "mcr.microsoft.com/devcontainers/python:1-3.10-bullseye",
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
   "postCreateCommand": "/bin/bash -c 'source .devcontainer/setup.sh'",
   "containerEnv": {
     "UV_LINK_MODE": "copy",

.devcontainer/local/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "Local Mode (full bundle)",
-  "image": "mcr.microsoft.com/devcontainers/python:1-3.10-bullseye",
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
   "hostRequirements": {
     "cpu": 8,
     "memory": "32gb"

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -19,7 +19,7 @@ body:
     attributes:
       label: Python Version
       description: The version of Python you're using.
-      placeholder: "e.g., 3.8, 3.9, 3.10"
+      placeholder: "e.g., 3.11, 3.12, 3.13"
     validations:
       required: true
   - type: textarea

.github/workflows/pre-commit-check.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/run-tests-cpu.yaml

@@ -23,7 +23,7 @@ jobs:
         uses: astral-sh/setup-uv@ed21f2f24f8dd64503750218de024bcf64c7250a # v7.1.5
         with:
           enable-cache: false
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Setup | Dependencies
         run: |
@@ -58,7 +58,7 @@ jobs:
         uses: astral-sh/setup-uv@ed21f2f24f8dd64503750218de024bcf64c7250a # v7.1.5
         with:
           enable-cache: false
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Setup | Dependencies
         run: |

.github/workflows/run-tests-gpu.yaml

@@ -34,7 +34,7 @@ jobs:
         uses: astral-sh/setup-uv@ed21f2f24f8dd64503750218de024bcf64c7250a # v7.1.5
         with:
           enable-cache: false
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Setup | Dependencies
         run: |

.pre-commit-config.yaml

@@ -46,7 +46,7 @@ repos:
     rev: v3.19.1
     hooks:
     -   id: pyupgrade
-        args: [--py310-plus]
+        args: [--py311-plus]
 - repo: https://github.com/astral-sh/ruff-pre-commit
   rev: v0.11.6
   hooks:

CONTRIBUTING.md

@@ -23,12 +23,12 @@ Thanks for your interest in contributing to Synthetic Data SDK! Follow these gui
 
 3. **Create a virtual environment and install dependencies**:
    ```bash
-   uv sync --frozen --extra local --python=3.10  # For CPU
+   uv sync --frozen --extra local --python=3.11  # For CPU
    source .venv/bin/activate
    ```
    If using GPU, run:
    ```bash
-   uv sync --frozen --extra local-gpu --python=3.10  # For GPU support
+   uv sync --frozen --extra local-gpu --python=3.11  # For GPU support
    source .venv/bin/activate
    ```
 

Makefile

@@ -23,7 +23,7 @@ COMMON_OPTIONS = \
 	--input-file-type openapi \
 	--output $(PUBLIC_API_OUTPUT_PATH) \
 	--snake-case-field \
-	--target-python-version 3.10 \
+	--target-python-version 3.11 \
 	--use-schema-description \
 	--use-union-operator \
 	--use-standard-collections \

README.md

@@ -140,7 +140,7 @@ In either case, a modern GPU is highly recommended when working with language mo
 
 ## Installation
 
-Use `pip` (or better `uv pip`) to install the official `mostlyai` package via PyPI. Python 3.10 or higher is required.
+Use `pip` (or better `uv pip`) to install the official `mostlyai` package via PyPI. Python 3.11 or higher is required.
 
 It is highly recommended to install the package within a dedicated virtual environment using `uv` (see [here](https://docs.astral.sh/uv/)):
 

mostlyai/sdk/_data/dtype.py

@@ -184,6 +184,16 @@ def coerce(self, data: pd.Series) -> pd.Series:
         return bool_coerce(data)
 
 
+def _to_numeric_or_same(data: pd.Series, downcast: str) -> pd.Series:
+    # pandas 3.0 dropped errors="ignore" for pd.to_numeric; emulate the
+    # previous "return input unchanged on failure" contract that our cast()
+    # callers relied on.
+    try:
+        return pd.to_numeric(data, errors="raise", downcast=downcast, dtype_backend="pyarrow")
+    except (ValueError, TypeError):
+        return data
+
+
 class VirtualInteger(VirtualDType):
     def get_encompass_data_fallback_dtypes(self) -> list["DType"]:
         return [VirtualFloat, VirtualVarchar]
@@ -192,15 +202,15 @@ def coerce(self, data: pd.Series) -> pd.Series:
         return int_coerce(data)
 
     def cast(self, data: pd.Series) -> pd.Series:
-        return pd.to_numeric(data, errors="ignore", downcast="integer", dtype_backend="pyarrow")
+        return _to_numeric_or_same(data, downcast="integer")
 
 
 class VirtualFloat(VirtualDType):
     def coerce(self, data: pd.Series) -> pd.Series:
         return float_coerce(data)
 
     def cast(self, data: pd.Series) -> pd.Series:
-        return pd.to_numeric(data, errors="ignore", downcast="float", dtype_backend="pyarrow")
+        return _to_numeric_or_same(data, downcast="float")
 
 
 class VirtualDate(VirtualDType):
@@ -468,8 +478,11 @@ def time_coerce(s: pd.Series) -> pd.Series:
     n_coerced = n_na_1 - n_na_0
     if n_coerced > 0:
         _LOG.warning(f"{n_coerced} values coerced during datetime_coerce")
-    # map to "object"
-    return s.astype("object")
+    # map to "object", and normalize pd.NaT to pd.NA so the result is uniform
+    # regardless of whether any row survived coercion (pandas 3 is stricter
+    # about distinguishing NaT vs NA under the object dtype)
+    s = s.astype("object")
+    return s.where(~s.isna(), pd.NA)
 
 
 class PandasDType(WrappedDType):

mostlyai/sdk/_data/file/table/json.py

@@ -125,13 +125,17 @@ def write_data(self, df: pd.DataFrame, if_exists: str = "append", **kwargs):
             if is_date_dtype(df[c]):
                 df[c] = df[c].dt.strftime("%Y-%m-%d")
             elif is_timestamp_dtype(df[c]):
-                # we need to strip off any milliseconds
+                # we need to strip off any microseconds (pyarrow-backed
+                # timestamps include them in strftime output regardless of
+                # format string; use str.slice rather than str[:-7] because
+                # pandas 3.0 removed StringMethods.__getitem__ for pyarrow
+                # extension arrays)
                 df[c] = (
                     df[c]
                     .dt.tz_localize(None)
                     .astype("timestamp[us][pyarrow]")
                     .dt.strftime("%Y-%m-%d %H:%M:%S")
-                    .str[:-7]
+                    .str.slice(stop=-7)
                 )
         mode = self.handle_if_exists(if_exists)
         df.to_json(self.container.path_str, orient="records", lines=True, mode=mode)

mostlyai/sdk/_data/pull_utils.py

@@ -473,14 +473,14 @@ def fetch_table_data(
             key_fraction_df["n_rows"] = int_part + (random_draws < frac_part).astype(int)
 
             ctx_key = key_fraction_df.columns[0]
+            # shuffle, then keep the first n rows per ctx_key group via cumcount; this preserves
+            # the grouping column in the output across pandas versions (pandas 3.0 drops the
+            # grouping column from groupby().apply() results by default)
             chunk_df = chunk_df.sample(frac=1)
-
-            def sample_n_rows(group):
-                n = key_fraction_df.loc[key_fraction_df[ctx_key] == group.name, "n_rows"].values[0]
-                return group.sample(n=min(n, len(group)))  # don't exceed available rows
-
-            # group by ctx_key and apply the sampling function
-            chunk_df = chunk_df.groupby(ctx_key, group_keys=False).apply(sample_n_rows).reset_index(drop=True)
+            n_per_group = key_fraction_df.set_index(ctx_key)["n_rows"]
+            cumcount = chunk_df.groupby(ctx_key).cumcount()
+            limit = chunk_df[ctx_key].map(n_per_group).fillna(0).astype(int)
+            chunk_df = chunk_df.loc[cumcount < limit].reset_index(drop=True)
 
             _LOG.info(f"drop {chunk_size - len(chunk_df)} ctx_keys to protect privacy")
         if sample_fraction is not None:

mostlyai/sdk/_local/progress.py

@@ -24,7 +24,7 @@
 
 def get_current_utc_time() -> datetime.datetime:
     # always use UTC time for platform compatibility
-    return datetime.datetime.now(datetime.timezone.utc)
+    return datetime.datetime.now(datetime.UTC)
 
 
 class LocalProgressCallback:

pyproject.toml

@@ -3,7 +3,7 @@ name = "mostlyai"
 version = "5.10.1"
 description = "Synthetic Data SDK"
 authors = [{ name = "MOSTLY AI", email = "dev@mostly.ai" }]
-requires-python = ">=3.10,<3.14"
+requires-python = ">=3.11,<3.14"
 readme = "README.md"
 license = "Apache-2.0"
 classifiers = [
@@ -14,7 +14,6 @@ classifiers = [
     "Intended Audience :: Financial and Insurance Industry",
     "Intended Audience :: Healthcare Industry",
     "Intended Audience :: Telecommunications Industry",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -50,8 +49,10 @@ dependencies = [
 local = [
     # "mostlyai-engine @ git+https://github.com/mostly-ai/mostlyai-engine.git@main",  # for development
     # "mostlyai-qa @ git+https://github.com/mostly-ai/mostlyai-qa.git@main",  # for development
-    "mostlyai-engine==2.4.0",  # for release
-    "mostlyai-qa==1.9.8",  # for release
+    # "mostlyai-engine==2.5.0",  # for release (PyPI)
+    # "mostlyai-qa==1.10.4",  # for release (PyPI)
+    "mostlyai-engine @ git+https://github.com/mostly-ai/mostlyai-engine.git@2.5.0",  # for release
+    "mostlyai-qa @ git+https://github.com/mostly-ai/mostlyai-qa.git@1.10.4",  # for release
     "fastapi>=0.116.0,<0.125",
     "uvicorn>=0.34.0",
     "python-multipart>=0.0.20",
@@ -76,8 +77,10 @@ local = [
 local-gpu = [
     # "mostlyai-engine[gpu] @ git+https://github.com/mostly-ai/mostlyai-engine.git@main",  # for development
     # "mostlyai-qa @ git+https://github.com/mostly-ai/mostlyai-qa.git@main",  # for development
-    "mostlyai-engine[gpu]==2.4.0",  # for release
-    "mostlyai-qa==1.9.8",  # for release
+    # "mostlyai-engine[gpu]==2.5.0",  # for release (PyPI)
+    # "mostlyai-qa==1.10.4",  # for release (PyPI)
+    "mostlyai-engine[gpu] @ git+https://github.com/mostly-ai/mostlyai-engine.git@2.5.0",  # for release
+    "mostlyai-qa @ git+https://github.com/mostly-ai/mostlyai-qa.git@1.10.4",  # for release
     "fastapi>=0.116.0,<0.125",
     "uvicorn>=0.34.0",
     "python-multipart>=0.0.20",
@@ -189,7 +192,7 @@ search = '__version__ = "{current_version}"'
 replace = '__version__ = "{new_version}"'
 
 [tool.ruff]
-target-version = "py310"
+target-version = "py311"
 line-length = 120
 
 [tool.ruff.lint]

tests/_data/unit/test_finalize_generation.py

@@ -48,8 +48,10 @@ def test_finalize_table_generation(tmp_path, tgt_data):
     gen_data = tgt_table.read_data()
     gen_data_path = tmp_path / "gen"
     gen_data_path.mkdir(exist_ok=True, parents=True)
-    # make 3 partitions
-    get_data_partitioned = np.array_split(gen_data, 3)
+    # make 3 partitions (np.array_split returns ndarrays for DataFrames on
+    # numpy 2 / pandas 3, so split via positional indices to keep DataFrames)
+    split_points = np.array_split(np.arange(len(gen_data)), 3)
+    get_data_partitioned = [gen_data.iloc[idx] for idx in split_points]
     # save each part into a separate parquet file
     for i, df_part in enumerate(get_data_partitioned):
         part_fn = gen_data_path / f"part.{i:06}.parquet"
@@ -103,7 +105,7 @@ def test_export_data_to_excel(tmp_path):
     excel_file_path = tmp_path / "synthetic-samples.xlsx"
     xls = pd.ExcelFile(excel_file_path)
 
-    expected_values = pd.DataFrame({"A": [0, 1, 2, 3, 4, 5, 6, 7, 8], "X": ["x"] * 9, "N": [None] * 9})
+    expected_values = pd.DataFrame({"A": [0, 1, 2, 3, 4, 5, 6, 7, 8], "X": ["x"] * 9})
     # Iterate over each sheet
     for sheet_name in xls.sheet_names:
         df = pd.read_excel(xls, sheet_name=sheet_name)
@@ -115,7 +117,11 @@ def test_export_data_to_excel(tmp_path):
             # Check the expected values
             df.sort_values(by="A", ascending=True, inplace=True)
             df.reset_index(drop=True, inplace=True)
-            pd.testing.assert_frame_equal(df, expected_values, check_dtype=False)
+            # the "N" column is all-missing; compare via isna() to stay
+            # agnostic of how read_excel materializes nulls (NaN vs None vs NA)
+            # across pandas versions
+            assert df["N"].isna().all()
+            pd.testing.assert_frame_equal(df.drop(columns=["N"]), expected_values, check_dtype=False)
 
     # Check for the sheet names
     expected_sheets = ["_TOC_", "A", "B", "C"]

tests/_data/unit/test_push.py

@@ -55,7 +55,10 @@ def source_table_partitioned(tmp_path):
     source_data_path = tmp_path / "source"
     source_data_path.mkdir(exist_ok=True, parents=True)
     n_partitions = 10
-    get_data_partitioned = np.array_split(source_data, n_partitions)
+    # np.array_split returns ndarrays for DataFrames on numpy 2 / pandas 3,
+    # so split via positional indices to keep DataFrames
+    split_points = np.array_split(np.arange(len(source_data)), n_partitions)
+    get_data_partitioned = [source_data.iloc[idx] for idx in split_points]
     # save each part into a separate parquet file
     for i, df_part in enumerate(get_data_partitioned):
         part_fn = source_data_path / f"part.{i:06}.parquet"