From 35a657b15d4f38da38f33f1b6bd8b3671196ce59 Mon Sep 17 00:00:00 2001 From: chitralverma Date: Wed, 8 Jul 2026 13:11:55 +0530 Subject: [PATCH 1/3] feat(bindings/python): typed service configs via from_config --- bindings/python/docs/api/services.md | 20 + bindings/python/justfile | 2 +- bindings/python/mkdocs.yml | 1 + bindings/python/python/opendal/operator.pyi | 60 +- bindings/python/python/opendal/services.pyi | 2100 ++++- bindings/python/ruff.toml | 2 + bindings/python/scripts/postprocess_stubs.py | 32 + bindings/python/src/config_types.rs | 65 + bindings/python/src/lib.rs | 10 +- bindings/python/src/operator.rs | 143 +- bindings/python/src/services.rs | 7307 +++++++++++++++++- bindings/python/tests/test_from_config.py | 211 + dev/src/generate/python.j2 | 106 + dev/src/generate/python.rs | 350 +- 14 files changed, 10365 insertions(+), 44 deletions(-) create mode 100644 bindings/python/docs/api/services.md create mode 100644 bindings/python/src/config_types.rs create mode 100644 bindings/python/tests/test_from_config.py diff --git a/bindings/python/docs/api/services.md b/bindings/python/docs/api/services.md new file mode 100644 index 000000000000..6894d8976ac1 --- /dev/null +++ b/bindings/python/docs/api/services.md @@ -0,0 +1,20 @@ +# Services + +Each service exposes a typed configuration class. Pass an instance to +`Operator.from_config` or `AsyncOperator.from_config`: + +```python +from opendal import Operator +from opendal.services import S3Config + +op = Operator.from_config(S3Config(bucket="my-bucket")) +``` + +::: opendal.services + options: + show_source: false + heading_level: 2 + show_signature: true + show_signature_annotations: true + separate_signature: true + merge_init_into_class: true diff --git a/bindings/python/justfile b/bindings/python/justfile index 17dbd0b8b36a..88c7848031c6 100644 --- a/bindings/python/justfile +++ b/bindings/python/justfile @@ -55,7 +55,7 @@ clean: stub-gen: setup @echo "{{ BOLD }}--- Generating Python type stubs ---{{ NORMAL }}" @cargo run --quiet --manifest-path=../../dev/Cargo.toml -- generate -l python - @uv run maturin develop --generate-stubs + @uv run maturin develop --generate-stubs --features services-all ## This is temporary till the functionalities land in pyo3-introspection. ref: https://github.com/PyO3/pyo3/issues/5137 @echo "{{ BOLD }}--- Post processing generated stubs ---{{ NORMAL }}" @uv run python scripts/postprocess_stubs.py diff --git a/bindings/python/mkdocs.yml b/bindings/python/mkdocs.yml index eda41191a03c..057399d59bc1 100644 --- a/bindings/python/mkdocs.yml +++ b/bindings/python/mkdocs.yml @@ -50,6 +50,7 @@ nav: - File: api/file.md - Layers: api/layers.md - Operator: api/operator.md + - Services: api/services.md - Types: api/types.md markdown_extensions: diff --git a/bindings/python/python/opendal/operator.pyi b/bindings/python/python/opendal/operator.pyi index 40c7731dfc21..2a4717abf893 100644 --- a/bindings/python/python/opendal/operator.pyi +++ b/bindings/python/python/opendal/operator.pyi @@ -23,7 +23,7 @@ from typing import Any, final from .capability import Capability from .file import AsyncFile, File from .layers import Layer -from .services import Scheme +from .services import Scheme, ServiceConfig from .types import Entry, Metadata, PresignedRequest @final @@ -162,6 +162,35 @@ class AsyncOperator: An awaitable that returns True if the path exists, False otherwise. """ @classmethod + def from_config(cls, /, config: ServiceConfig) -> AsyncOperator: + """ + Create a new `AsyncOperator` from a typed service config. + + The config binds its own scheme, so there is no scheme argument and no + way to pair a config with the wrong service. + + Parameters + ---------- + config : ServiceConfig + A service config such as ``opendal.services.S3Config``. + + Returns + ------- + AsyncOperator + The new async operator. + + Examples + -------- + ```python + import opendal + from opendal.services import S3Config + + op = opendal.AsyncOperator.from_config( + S3Config(bucket="my-bucket") + ) + ``` + """ + @classmethod def from_uri(cls, /, uri: str, **kwargs) -> AsyncOperator: """ Create a new `AsyncOperator` from a URI string. @@ -783,6 +812,35 @@ class Operator: True if the path exists, False otherwise. """ @classmethod + def from_config(cls, /, config: ServiceConfig) -> Operator: + """ + Create a new blocking `Operator` from a typed service config. + + The config binds its own scheme, so there is no scheme argument and no + way to pair a config with the wrong service. + + Parameters + ---------- + config : ServiceConfig + A service config such as ``opendal.services.S3Config``. + + Returns + ------- + Operator + The new operator. + + Examples + -------- + ```python + import opendal + from opendal.services import S3Config + + op = opendal.Operator.from_config( + S3Config(bucket="my-bucket") + ) + ``` + """ + @classmethod def from_uri(cls, /, uri: str, **kwargs) -> Operator: """ Create a new blocking `Operator` from a URI string. diff --git a/bindings/python/python/opendal/services.pyi b/bindings/python/python/opendal/services.pyi index 8ec09c36d339..432055c0e117 100644 --- a/bindings/python/python/opendal/services.pyi +++ b/bindings/python/python/opendal/services.pyi @@ -15,8 +15,1667 @@ # specific language governing permissions and limitations # under the License. +from os import PathLike from typing import Final, final +@final +class AliyunDriveConfig(ServiceConfig): + """Configuration for the `aliyun-drive` service.""" + + def __init__( + self, + /, + drive_type: str, + access_token: str | None = None, + client_id: str | None = None, + client_secret: str | None = None, + refresh_token: str | None = None, + root: str | PathLike[str] | None = None, + ) -> None: ... + @property + def access_token(self, /) -> str | None: + """ + The access_token of this backend. + Solution for client-only purpose. + #4733 Required if no client_id, client_secret and refresh_token are + provided. + """ + @property + def client_id(self, /) -> str | None: + """ + The client_id of this backend. + Required if no access_token is provided. + """ + @property + def client_secret(self, /) -> str | None: + """ + The client_secret of this backend. + Required if no access_token is provided. + """ + @property + def drive_type(self, /) -> str: + """ + The drive_type of this backend. + All operations will happen under this type of drive. + Available values are `default`, `backup` and `resource`. + Fallback to default if not set or no other drives can be found. + """ + @property + def refresh_token(self, /) -> str | None: + """ + The refresh_token of this backend. + Required if no access_token is provided. + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + The Root of this backend. + All operations will happen under this root. + Default to `/` if not set. + """ + +@final +class AlluxioConfig(ServiceConfig): + """Configuration for the `alluxio` service.""" + + def __init__( + self, /, endpoint: str | None = None, root: str | PathLike[str] | None = None + ) -> None: ... + @property + def endpoint(self, /) -> str | None: + """ + Endpoint of this backend. + Endpoint must be full uri, mostly like `http://127.0.0.1:39999`. + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + All operations will happen under this root. + default to `/` if not set. + """ + +@final +class AzblobConfig(ServiceConfig): + """Configuration for the `azblob` service.""" + + def __init__( + self, + /, + container: str, + account_key: str | None = None, + account_name: str | None = None, + batch_max_operations: int | None = None, + encryption_algorithm: str | None = None, + encryption_key: str | None = None, + encryption_key_sha256: str | None = None, + endpoint: str | None = None, + root: str | PathLike[str] | None = None, + sas_token: str | None = None, + skip_signature: bool | None = None, + ) -> None: ... + @property + def account_key(self, /) -> str | None: + """The account key of Azblob service backend.""" + @property + def account_name(self, /) -> str | None: + """The account name of Azblob service backend.""" + @property + def batch_max_operations(self, /) -> int | None: + """ + Deprecated: Azblob delete batch capability is enabled by default with Azure + Blob's 256-operation batch limit. + [Deprecated since 0.57.0] Azblob delete batch capability is enabled by + default with Azure Blob's 256-operation batch limit. + Use CapabilityOverrideLayer to override delete_max_size for specific + endpoints. + """ + @property + def container(self, /) -> str: + """The container name of Azblob service backend.""" + @property + def encryption_algorithm(self, /) -> str | None: + """The encryption algorithm of Azblob service backend.""" + @property + def encryption_key(self, /) -> str | None: + """The encryption key of Azblob service backend.""" + @property + def encryption_key_sha256(self, /) -> str | None: + """The encryption key sha256 of Azblob service backend.""" + @property + def endpoint(self, /) -> str | None: + """ + The endpoint of Azblob service backend. + Endpoint must be full uri, e.g. + - Azblob: `https://accountname.blob.core.windows.net` - Azurite: + `http://127.0.0.1:10000/devstoreaccount1`. + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + The root of Azblob service backend. + All operations will happen under this root. + """ + @property + def sas_token(self, /) -> str | None: + """The sas token of Azblob service backend.""" + @property + def skip_signature(self, /) -> bool | None: + """Skip signature will skip loading credentials and signing requests.""" + +@final +class AzdlsConfig(ServiceConfig): + """Configuration for the `azdls` service.""" + + def __init__( + self, + /, + filesystem: str, + account_key: str | None = None, + account_name: str | None = None, + authority_host: str | None = None, + client_id: str | None = None, + client_secret: str | None = None, + enable_hns: bool | None = None, + endpoint: str | None = None, + root: str | PathLike[str] | None = None, + sas_token: str | None = None, + tenant_id: str | None = None, + ) -> None: ... + @property + def account_key(self, /) -> str | None: + """ + Account key of this backend. + - required for shared_key authentication. + """ + @property + def account_name(self, /) -> str | None: + """Account name of this backend.""" + @property + def authority_host(self, /) -> str | None: + """ + authority_host The authority host of the service principal. + - required for client_credentials authentication - default value: + `https://login.microsoftonline.com`. + """ + @property + def client_id(self, /) -> str | None: + """ + client_id The client id of the service principal. + - required for client_credentials authentication. + """ + @property + def client_secret(self, /) -> str | None: + """ + client_secret The client secret of the service principal. + - required for client_credentials authentication. + """ + @property + def enable_hns(self, /) -> bool | None: + """ + Whether hierarchical namespace (HNS) is enabled for the storage account. + When enabled, recursive deletion can use pagination to avoid timeouts on + large directories. + - default value: `false`. + """ + @property + def endpoint(self, /) -> str | None: + """Endpoint of this backend.""" + @property + def filesystem(self, /) -> str: + """Filesystem name of this backend.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root of this backend.""" + @property + def sas_token(self, /) -> str | None: + """ + sas_token The shared access signature token. + - required for sas authentication. + """ + @property + def tenant_id(self, /) -> str | None: + """ + tenant_id The tenant id of the service principal. + - required for client_credentials authentication. + """ + +@final +class AzfileConfig(ServiceConfig): + """Configuration for the `azfile` service.""" + + def __init__( + self, + /, + share_name: str, + account_key: str | None = None, + account_name: str | None = None, + endpoint: str | None = None, + root: str | PathLike[str] | None = None, + sas_token: str | None = None, + ) -> None: ... + @property + def account_key(self, /) -> str | None: + """The account key for azfile.""" + @property + def account_name(self, /) -> str | None: + """The account name for azfile.""" + @property + def endpoint(self, /) -> str | None: + """The endpoint for azfile.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """The root path for azfile.""" + @property + def sas_token(self, /) -> str | None: + """The sas token for azfile.""" + @property + def share_name(self, /) -> str: + """The share name for azfile.""" + +@final +class B2Config(ServiceConfig): + """Configuration for the `b2` service.""" + + def __init__( + self, + /, + bucket: str, + bucket_id: str, + application_key: str | None = None, + application_key_id: str | None = None, + root: str | PathLike[str] | None = None, + ) -> None: ... + @property + def application_key(self, /) -> str | None: + """ + ApplicationKey of this backend. + - If application_key is set, we will take user's input first. + - If not, we will try to load it from environment. + """ + @property + def application_key_id(self, /) -> str | None: + """ + KeyID of this backend. + - If application_key_id is set, we will take user's input first. + - If not, we will try to load it from environment. + """ + @property + def bucket(self, /) -> str: + """ + Bucket of this backend. + required. + """ + @property + def bucket_id(self, /) -> str: + """ + Bucket id of this backend. + required. + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + All operations will happen under this root. + """ + +@final +class CacacheConfig(ServiceConfig): + """Configuration for the `cacache` service.""" + + def __init__(self, /, datadir: str | None = None) -> None: ... + @property + def datadir(self, /) -> str | None: + """That path to the cacache data directory.""" + +@final +class CosConfig(ServiceConfig): + """Configuration for the `cos` service.""" + + def __init__( + self, + /, + bucket: str | None = None, + disable_config_load: bool | None = None, + enable_versioning: bool | None = None, + endpoint: str | None = None, + root: str | PathLike[str] | None = None, + secret_id: str | None = None, + secret_key: str | None = None, + security_token: str | None = None, + ) -> None: ... + @property + def bucket(self, /) -> str | None: + """Bucket of this backend.""" + @property + def disable_config_load(self, /) -> bool | None: + """Disable config load so that opendal will not load config from.""" + @property + def enable_versioning(self, /) -> bool | None: + """ + Deprecated: COS versioning capability is enabled by default. + [Deprecated since 0.57.0] COS versioning capability is enabled by default + and this option is no longer needed. + """ + @property + def endpoint(self, /) -> str | None: + """Endpoint of this backend.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root of this backend.""" + @property + def secret_id(self, /) -> str | None: + """Secret ID of this backend.""" + @property + def secret_key(self, /) -> str | None: + """Secret key of this backend.""" + @property + def security_token(self, /) -> str | None: + """ + Security token (a.k.a. + session token) of this backend. + This is used for temporary credentials issued by Tencent Cloud STS (e.g. + `GetFederationToken` / `AssumeRole`). + When `security_token` is provided, it will be used together with `secret_id` + and `secret_key` to sign requests, and the `x-cos-security-token` header + will be attached automatically by the signer. + If this field is not set, OpenDAL will also fall back to reading the token + from environment variables `TENCENTCLOUD_TOKEN`, + `TENCENTCLOUD_SECURITY_TOKEN` or `QCLOUD_SECRET_TOKEN` (unless + `disable_config_load` is enabled). + """ + +@final +class DashmapConfig(ServiceConfig): + """Configuration for the `dashmap` service.""" + + def __init__(self, /, root: str | PathLike[str] | None = None) -> None: ... + @property + def root(self, /) -> str | PathLike[str] | None: + """Root path of this backend.""" + +@final +class DropboxConfig(ServiceConfig): + """Configuration for the `dropbox` service.""" + + def __init__( + self, + /, + access_token: str | None = None, + client_id: str | None = None, + client_secret: str | None = None, + refresh_token: str | None = None, + root: str | PathLike[str] | None = None, + ) -> None: ... + @property + def access_token(self, /) -> str | None: + """Access token for dropbox.""" + @property + def client_id(self, /) -> str | None: + """client_id for dropbox.""" + @property + def client_secret(self, /) -> str | None: + """client_secret for dropbox.""" + @property + def refresh_token(self, /) -> str | None: + """refresh_token for dropbox.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root path for dropbox.""" + +@final +class FsConfig(ServiceConfig): + """Configuration for the `fs` service.""" + + def __init__( + self, + /, + atomic_write_dir: str | PathLike[str] | None = None, + root: str | PathLike[str] | None = None, + ) -> None: ... + @property + def atomic_write_dir(self, /) -> str | PathLike[str] | None: + """Tmp dir for atomic write.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root dir for backend.""" + +@final +class FtpConfig(ServiceConfig): + """Configuration for the `ftp` service.""" + + def __init__( + self, + /, + endpoint: str | None = None, + password: str | None = None, + root: str | PathLike[str] | None = None, + user: str | None = None, + ) -> None: ... + @property + def endpoint(self, /) -> str | None: + """Endpoint of this backend.""" + @property + def password(self, /) -> str | None: + """Password of this backend.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root of this backend.""" + @property + def user(self, /) -> str | None: + """User of this backend.""" + +@final +class GcsConfig(ServiceConfig): + """Configuration for the `gcs` service.""" + + def __init__( + self, + /, + bucket: str, + allow_anonymous: bool | None = None, + credential: str | None = None, + credential_path: str | None = None, + default_storage_class: str | None = None, + disable_config_load: bool | None = None, + disable_vm_metadata: bool | None = None, + endpoint: str | None = None, + predefined_acl: str | None = None, + root: str | PathLike[str] | None = None, + scope: str | None = None, + service_account: str | None = None, + skip_signature: bool | None = None, + token: str | None = None, + ) -> None: ... + @property + def allow_anonymous(self, /) -> bool | None: + """ + Allow opendal to send requests without signing when credentials are not + loaded. + [Deprecated since 0.57.0] Please use `skip_signature` instead of + `allow_anonymous`. + """ + @property + def bucket(self, /) -> str: + """Bucket name.""" + @property + def credential(self, /) -> str | None: + """Credentials string for GCS service OAuth2 authentication.""" + @property + def credential_path(self, /) -> str | None: + """Local path to credentials file for GCS service OAuth2 authentication.""" + @property + def default_storage_class(self, /) -> str | None: + """The default storage class used by gcs.""" + @property + def disable_config_load(self, /) -> bool | None: + """Disable loading configuration from the environment.""" + @property + def disable_vm_metadata(self, /) -> bool | None: + """ + Disable attempting to load credentials from the GCE metadata server when + running within Google Cloud. + """ + @property + def endpoint(self, /) -> str | None: + """Endpoint URI of GCS service, default is `https://storage.googleapis.com`.""" + @property + def predefined_acl(self, /) -> str | None: + """The predefined acl for GCS.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root URI, all operations happens under `root`.""" + @property + def scope(self, /) -> str | None: + """Scope for gcs.""" + @property + def service_account(self, /) -> str | None: + """Service Account for gcs.""" + @property + def skip_signature(self, /) -> bool | None: + """Skip signature will skip loading credentials and signing requests.""" + @property + def token(self, /) -> str | None: + """ + A Google Cloud OAuth2 token. + Takes precedence over `credential` and `credential_path`. + """ + +@final +class GdriveConfig(ServiceConfig): + """Configuration for the `gdrive` service.""" + + def __init__( + self, + /, + access_token: str | None = None, + client_id: str | None = None, + client_secret: str | None = None, + refresh_token: str | None = None, + root: str | PathLike[str] | None = None, + ) -> None: ... + @property + def access_token(self, /) -> str | None: + """Access token for gdrive.""" + @property + def client_id(self, /) -> str | None: + """Client id for gdrive.""" + @property + def client_secret(self, /) -> str | None: + """Client secret for gdrive.""" + @property + def refresh_token(self, /) -> str | None: + """Refresh token for gdrive.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """The root for gdrive.""" + +@final +class GhacConfig(ServiceConfig): + """Configuration for the `ghac` service.""" + + def __init__( + self, + /, + endpoint: str | None = None, + root: str | PathLike[str] | None = None, + runtime_token: str | None = None, + version: str | None = None, + ) -> None: ... + @property + def endpoint(self, /) -> str | None: + """The endpoint for ghac service.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """The root path for ghac.""" + @property + def runtime_token(self, /) -> str | None: + """The runtime token for ghac service.""" + @property + def version(self, /) -> str | None: + """The version that used by cache.""" + +@final +class GoosefsConfig(ServiceConfig): + """Configuration for the `goosefs` service.""" + + def __init__( + self, + /, + auth_type: str | None = None, + auth_username: str | None = None, + block_size: int | None = None, + chunk_size: int | None = None, + master_addr: str | None = None, + root: str | PathLike[str] | None = None, + write_type: str | None = None, + ) -> None: ... + @property + def auth_type(self, /) -> str | None: + """ + Authentication type. + Supported values: `"nosasl"`, `"simple"`. + Default: `"simple"` โ€” PLAIN SASL with usernam + e. `"nosasl"` โ€” skip authentication entirely. + """ + @property + def auth_username(self, /) -> str | None: + """ + Authentication username. + Used in SIMPLE mode as the login identity. + Default: current OS user (`$USER` / `$USERNAME`). + """ + @property + def block_size(self, /) -> int | None: + """Block size in bytes for new files (default: 64 MiB).""" + @property + def chunk_size(self, /) -> int | None: + """Chunk size in bytes for streaming RPCs (default: 1 MiB).""" + @property + def master_addr(self, /) -> str | None: + """ + Master address(es) in `host:port` format. + For single master: `"10.0.0.1:9200"` For HA (comma-separated): + `"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"` When multiple addresses are + provided, the client uses `PollingMasterInquireClient` to discover the + Primary Master automatically. + Resolution precedence at `build()` time (highest โ†’ lowest), following + `goosefs-sdk` `docs/CLIENT_CONFIGURATION.md` ยง1: + 1. This field (when set on the builder / OpenDAL config map) + 2. `GOOSEFS_MASTER_ADDR` environment variable + 3. `goosefs.master.rpc.addresses` / `goosefs.master.hostname` in + `goosefs-site.properties` `build()` fails with `ConfigInvalid` only when + **none** of the above supplies a master address. + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root path of this backend. + All operations will happen under this root. + Default to `/` if not set. + """ + @property + def write_type(self, /) -> str | None: + """ + Default write type for new files. + Supported values: `"must_cache"`, `"cache_through"`, `"through"`, + `"async_through"`. + Default: `"must_cache"`. + """ + +@final +class GridfsConfig(ServiceConfig): + """Configuration for the `gridfs` service.""" + + def __init__( + self, + /, + bucket: str | None = None, + chunk_size: int | None = None, + connection_string: str | None = None, + database: str | None = None, + root: str | PathLike[str] | None = None, + ) -> None: ... + @property + def bucket(self, /) -> str | None: + """The bucket name of the MongoDB GridFs service to read/write.""" + @property + def chunk_size(self, /) -> int | None: + """ + The chunk size of the MongoDB GridFs service used to break the user file + into chunks. + """ + @property + def connection_string(self, /) -> str | None: + """The connection string of the MongoDB service.""" + @property + def database(self, /) -> str | None: + """The database name of the MongoDB GridFs service to read/write.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """The working directory, all operations will be performed under it.""" + +@final +class HfConfig(ServiceConfig): + """Configuration for the `hf` service.""" + + def __init__( + self, + /, + download_mode: str | None = None, + endpoint: str | None = None, + repo_id: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + root: str | PathLike[str] | None = None, + token: str | None = None, + ) -> None: ... + @property + def endpoint(self, /) -> str | None: + """ + Endpoint of the Hugging Face Hub. + Default is "https://huggingface.co". + """ + @property + def repo_id(self, /) -> str | None: + """ + Repo id of this backend. + This is required. + """ + @property + def revision(self, /) -> str | None: + """ + Revision of this backend. + Default is main. + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + Can be "/path/to/dir". + Default is "/". + """ + @property + def token(self, /) -> str | None: + """ + Token of this backend. + This is optional. + """ + +@final +class HttpConfig(ServiceConfig): + """Configuration for the `http` service.""" + + def __init__( + self, + /, + endpoint: str | None = None, + password: str | None = None, + root: str | PathLike[str] | None = None, + token: str | None = None, + username: str | None = None, + ) -> None: ... + @property + def endpoint(self, /) -> str | None: + """Endpoint of this backend.""" + @property + def password(self, /) -> str | None: + """Password of this backend.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root of this backend.""" + @property + def token(self, /) -> str | None: + """Token of this backend.""" + @property + def username(self, /) -> str | None: + """Username of this backend.""" + +@final +class IpfsConfig(ServiceConfig): + """Configuration for the `ipfs` service.""" + + def __init__( + self, /, endpoint: str | None = None, root: str | PathLike[str] | None = None + ) -> None: ... + @property + def endpoint(self, /) -> str | None: + """IPFS gateway endpoint.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """IPFS root.""" + +@final +class IpmfsConfig(ServiceConfig): + """Configuration for the `ipmfs` service.""" + + def __init__( + self, /, endpoint: str | None = None, root: str | PathLike[str] | None = None + ) -> None: ... + @property + def endpoint(self, /) -> str | None: + """Endpoint for ipfs.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root for ipfs.""" + +@final +class KoofrConfig(ServiceConfig): + """Configuration for the `koofr` service.""" + + def __init__( + self, + /, + email: str, + endpoint: str, + password: str | None = None, + root: str | PathLike[str] | None = None, + ) -> None: ... + @property + def email(self, /) -> str: + """Koofr email.""" + @property + def endpoint(self, /) -> str: + """Koofr endpoint.""" + @property + def password(self, /) -> str | None: + """ + Password of this backend. + (Must be the application password). + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + All operations will happen under this root. + """ + +@final +class MemcachedConfig(ServiceConfig): + """Configuration for the `memcached` service.""" + + def __init__( + self, + /, + connection_pool_max_size: int | None = None, + default_ttl: str | None = None, + endpoint: str | None = None, + password: str | None = None, + root: str | PathLike[str] | None = None, + username: str | None = None, + ) -> None: ... + @property + def connection_pool_max_size(self, /) -> int | None: + """ + The maximum number of connections allowed. + default is 10. + """ + @property + def default_ttl(self, /) -> str | None: + """ + The default ttl for put operations. + Accepts a humantime duration string (e.g. + "5s"). + """ + @property + def endpoint(self, /) -> str | None: + """ + Network address of the memcached service. + For example: "tcp://localhost:11211". + """ + @property + def password(self, /) -> str | None: + """Memcached password, optional.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """ + The working directory of the service. + Can be "/path/to/dir" default is "/". + """ + @property + def username(self, /) -> str | None: + """Memcached username, optional.""" + +@final +class MemoryConfig(ServiceConfig): + """Configuration for the `memory` service.""" + + def __init__(self, /, root: str | PathLike[str] | None = None) -> None: ... + @property + def root(self, /) -> str | PathLike[str] | None: + """Root of the backend.""" + +@final +class MiniMokaConfig(ServiceConfig): + """Configuration for the `mini-moka` service.""" + + def __init__( + self, + /, + max_capacity: int | None = None, + root: str | PathLike[str] | None = None, + time_to_idle: str | None = None, + time_to_live: str | None = None, + ) -> None: ... + @property + def max_capacity(self, /) -> int | None: + """ + Sets the max capacity of the cache. + Refer to + [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity). + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """Root path of this backend.""" + @property + def time_to_idle(self, /) -> str | None: + """ + Sets the time to idle of the cache. + Refer to + [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle). + """ + @property + def time_to_live(self, /) -> str | None: + """ + Sets the time to live of the cache. + Refer to + [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live). + """ + +@final +class MokaConfig(ServiceConfig): + """Configuration for the `moka` service.""" + + def __init__( + self, + /, + max_capacity: int | None = None, + name: str | None = None, + root: str | PathLike[str] | None = None, + time_to_idle: str | None = None, + time_to_live: str | None = None, + ) -> None: ... + @property + def max_capacity(self, /) -> int | None: + """ + Sets the max capacity of the cache. + Refer to + [`moka::future::CacheBuilder::max_capacity`](https://docs.rs/moka/latest/moka/future/struct.CacheBuilder.html#method.max_capacity). + """ + @property + def name(self, /) -> str | None: + """Name for this cache instance.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root path of this backend.""" + @property + def time_to_idle(self, /) -> str | None: + """ + Sets the time to idle of the cache. + Refer to + [`moka::future::CacheBuilder::time_to_idle`](https://docs.rs/moka/latest/moka/future/struct.CacheBuilder.html#method.time_to_idle). + """ + @property + def time_to_live(self, /) -> str | None: + """ + Sets the time to live of the cache. + Refer to + [`moka::future::CacheBuilder::time_to_live`](https://docs.rs/moka/latest/moka/future/struct.CacheBuilder.html#method.time_to_live). + """ + +@final +class MongodbConfig(ServiceConfig): + """Configuration for the `mongodb` service.""" + + def __init__( + self, + /, + collection: str | None = None, + connection_string: str | None = None, + database: str | None = None, + key_field: str | None = None, + root: str | PathLike[str] | None = None, + value_field: str | None = None, + ) -> None: ... + @property + def collection(self, /) -> str | None: + """Collection of this backend.""" + @property + def connection_string(self, /) -> str | None: + """Connection string of this backend.""" + @property + def database(self, /) -> str | None: + """Database of this backend.""" + @property + def key_field(self, /) -> str | None: + """Key field of this backend.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root of this backend.""" + @property + def value_field(self, /) -> str | None: + """Value field of this backend.""" + +@final +class MysqlConfig(ServiceConfig): + """Configuration for the `mysql` service.""" + + def __init__( + self, + /, + connection_string: str | None = None, + key_field: str | None = None, + root: str | PathLike[str] | None = None, + table: str | None = None, + value_field: str | None = None, + ) -> None: ... + @property + def connection_string(self, /) -> str | None: + """ + This connection string is used to connect to the mysql service. + There are url based formats. + The format of connect string resembles the url format of the mysql client. + The format is: + `[scheme://][user[:[password]]@]host[:port][/schema][?attribute1=value1&attribute2=value2...` + - `mysql://user@localhost` - `mysql://user:password@localhost` - + `mysql://user:password@localhost:3306` - + `mysql://user:password@localhost:3306/db` For more information, please refer + to . + """ + @property + def key_field(self, /) -> str | None: + """The key field name for mysql.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """The root for mysql.""" + @property + def table(self, /) -> str | None: + """The table name for mysql.""" + @property + def value_field(self, /) -> str | None: + """The value field name for mysql.""" + +@final +class ObsConfig(ServiceConfig): + """Configuration for the `obs` service.""" + + def __init__( + self, + /, + access_key_id: str | None = None, + bucket: str | None = None, + enable_versioning: bool | None = None, + endpoint: str | None = None, + root: str | PathLike[str] | None = None, + secret_access_key: str | None = None, + ) -> None: ... + @property + def access_key_id(self, /) -> str | None: + """Access key id for obs.""" + @property + def bucket(self, /) -> str | None: + """Bucket for obs.""" + @property + def enable_versioning(self, /) -> bool | None: + """ + Deprecated: OBS versioning capability is not controlled by service config. + [Deprecated since 0.57.0] OBS versioning capability is not controlled by + this option and this option is no longer needed. + """ + @property + def endpoint(self, /) -> str | None: + """Endpoint for obs.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root for obs.""" + @property + def secret_access_key(self, /) -> str | None: + """Secret access key for obs.""" + +@final +class OnedriveConfig(ServiceConfig): + """Configuration for the `onedrive` service.""" + + def __init__( + self, + /, + access_token: str | None = None, + client_id: str | None = None, + client_secret: str | None = None, + enable_versioning: bool | None = None, + refresh_token: str | None = None, + root: str | PathLike[str] | None = None, + ) -> None: ... + @property + def access_token(self, /) -> str | None: + """Microsoft Graph API (also OneDrive API) access token.""" + @property + def client_id(self, /) -> str | None: + """ + Microsoft Graph API Application (client) ID that is in the Azure's app + registration portal. + """ + @property + def client_secret(self, /) -> str | None: + """ + Microsoft Graph API Application client secret that is in the Azure's app + registration portal. + """ + @property + def enable_versioning(self, /) -> bool | None: + """ + Deprecated: OneDrive versioning capability is enabled by default. + [Deprecated since 0.57.0] OneDrive versioning capability is enabled by + default and this option is no longer needed. + """ + @property + def refresh_token(self, /) -> str | None: + """Microsoft Graph API (also OneDrive API) refresh token.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """The root path for the OneDrive service for the file access.""" + +@final +class OssConfig(ServiceConfig): + """Configuration for the `oss` service.""" + + def __init__( + self, + /, + bucket: str, + access_key_id: str | None = None, + access_key_secret: str | None = None, + addressing_style: str | None = None, + allow_anonymous: bool | None = None, + batch_max_operations: int | None = None, + delete_max_size: int | None = None, + enable_versioning: bool | None = None, + endpoint: str | None = None, + external_id: str | None = None, + oidc_provider_arn: str | None = None, + oidc_token_file: str | None = None, + presign_addressing_style: str | None = None, + presign_endpoint: str | None = None, + role_arn: str | None = None, + role_session_name: str | None = None, + root: str | PathLike[str] | None = None, + security_token: str | None = None, + server_side_encryption: str | None = None, + server_side_encryption_key_id: str | None = None, + skip_signature: bool | None = None, + sts_endpoint: str | None = None, + ) -> None: ... + @property + def access_key_id(self, /) -> str | None: + """ + Access key id for oss. + - this field if it's `is_some` - env value: `ALIBABA_CLOUD_ACCESS_KEY_ID`. + """ + @property + def access_key_secret(self, /) -> str | None: + """ + Access key secret for oss. + - this field if it's `is_some` - env value: + `ALIBABA_CLOUD_ACCESS_KEY_SECRET`. + """ + @property + def addressing_style(self, /) -> str | None: + """Addressing style for oss.""" + @property + def allow_anonymous(self, /) -> bool | None: + """ + Allow anonymous for oss. + [Deprecated since 0.57.0] Please use `skip_signature` instead of + `allow_anonymous`. + """ + @property + def batch_max_operations(self, /) -> int | None: + """ + Deprecated: OSS delete batch capability is enabled by default. + [Deprecated since 0.57.0] OSS delete batch capability is enabled by default. + Use CapabilityOverrideLayer to override delete_max_size for specific + endpoints. + """ + @property + def bucket(self, /) -> str: + """Bucket for oss.""" + @property + def delete_max_size(self, /) -> int | None: + """ + Deprecated: OSS delete batch capability is enabled by default. + [Deprecated since 0.57.0] OSS delete batch capability is enabled by default. + Use CapabilityOverrideLayer to override delete_max_size for specific + endpoints. + """ + @property + def enable_versioning(self, /) -> bool | None: + """ + Deprecated: OSS versioning capability is enabled by default. + [Deprecated since 0.57.0] OSS versioning capability is enabled by default + and this option is no longer needed. + """ + @property + def endpoint(self, /) -> str | None: + """Endpoint for oss.""" + @property + def external_id(self, /) -> str | None: + """external_id for this backend.""" + @property + def oidc_provider_arn(self, /) -> str | None: + """ + `oidc_provider_arn` will be loaded from - this field if it's `is_some` - env + value: `ALIBABA_CLOUD_OIDC_PROVIDER_ARN`. + """ + @property + def oidc_token_file(self, /) -> str | None: + """ + `oidc_token_file` will be loaded from - this field if it's `is_some` - env + value: `ALIBABA_CLOUD_OIDC_TOKEN_FILE`. + """ + @property + def presign_addressing_style(self, /) -> str | None: + """Pre sign addressing style for oss.""" + @property + def presign_endpoint(self, /) -> str | None: + """Presign endpoint for oss.""" + @property + def role_arn(self, /) -> str | None: + """ + If `role_arn` is set, we will use already known config as source credential + to assume role with `role_arn`. + - this field if it's `is_some` - env value: `ALIBABA_CLOUD_ROLE_ARN`. + """ + @property + def role_session_name(self, /) -> str | None: + """role_session_name for this backend.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root for oss.""" + @property + def security_token(self, /) -> str | None: + """ + `security_token` will be loaded from - this field if it's `is_some` - env + value: `ALIBABA_CLOUD_SECURITY_TOKEN`. + """ + @property + def server_side_encryption(self, /) -> str | None: + """Server side encryption for oss.""" + @property + def server_side_encryption_key_id(self, /) -> str | None: + """Server side encryption key id for oss.""" + @property + def skip_signature(self, /) -> bool | None: + """Skip signature will skip loading credentials and signing requests.""" + @property + def sts_endpoint(self, /) -> str | None: + """ + `sts_endpoint` will be loaded from - this field if it's `is_some` - env + value: `ALIBABA_CLOUD_STS_ENDPOINT`. + """ + +@final +class PersyConfig(ServiceConfig): + """Configuration for the `persy` service.""" + + def __init__( + self, + /, + datafile: str | None = None, + index: str | None = None, + segment: str | None = None, + ) -> None: ... + @property + def datafile(self, /) -> str | None: + """ + That path to the persy data file. + The directory in the path must already exist. + """ + @property + def index(self, /) -> str | None: + """That name of the persy index.""" + @property + def segment(self, /) -> str | None: + """That name of the persy segment.""" + +@final +class PostgresqlConfig(ServiceConfig): + """Configuration for the `postgresql` service.""" + + def __init__( + self, + /, + connection_string: str | None = None, + key_field: str | None = None, + root: str | PathLike[str] | None = None, + table: str | None = None, + value_field: str | None = None, + ) -> None: ... + @property + def connection_string(self, /) -> str | None: + """ + The URL should be with a scheme of either `postgres://` or `postgresql://`. + - `postgresql://user@localhost` - + `postgresql://user:password@%2Fvar%2Flib%2Fpostgresql/mydb?connect_timeout=10`. + - + `postgresql://user@host1:1234,host2,host3:5678?target_session_attrs=read-write` + - `postgresql:///mydb?user=user&host=/var/lib/postgresql` For more + information, please visit + . + """ + @property + def key_field(self, /) -> str | None: + """The key field of postgresql.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + All operations will happen under this root. + Default to `/` if not set. + """ + @property + def table(self, /) -> str | None: + """The table of postgresql.""" + @property + def value_field(self, /) -> str | None: + """The value field of postgresql.""" + +@final +class RedbConfig(ServiceConfig): + """Configuration for the `redb` service.""" + + def __init__( + self, + /, + datadir: str | None = None, + root: str | PathLike[str] | None = None, + table: str | None = None, + ) -> None: ... + @property + def datadir(self, /) -> str | None: + """Path to the redb data directory.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """The root for redb.""" + @property + def table(self, /) -> str | None: + """The table name for redb.""" + +@final +class RedisConfig(ServiceConfig): + """Configuration for the `redis` service.""" + + def __init__( + self, + /, + db: int | None, + cluster_endpoints: str | None = None, + connection_pool_max_size: int | None = None, + default_ttl: str | None = None, + endpoint: str | None = None, + password: str | None = None, + root: str | PathLike[str] | None = None, + username: str | None = None, + ) -> None: ... + @property + def cluster_endpoints(self, /) -> str | None: + """ + Network address of the Redis cluster service. + Can be "tcp://127.0.0.1:6379,tcp://127.0.0.1:6380,tcp://127.0.0.1:6381", + e.g. + default is None. + """ + @property + def connection_pool_max_size(self, /) -> int | None: + """ + The maximum number of connections allowed. + default is 10. + """ + @property + def db(self, /) -> int: + """The number of DBs redis can take is unlimited default is db 0.""" + @property + def default_ttl(self, /) -> str | None: + """ + The default ttl for put operations. + Accepts a humantime duration string (e.g. + "5s"). + """ + @property + def endpoint(self, /) -> str | None: + """ + Network address of the Redis service. + Can be "tcp://127.0.0.1:6379", e.g. + default is "tcp://127.0.0.1:6379". + """ + @property + def password(self, /) -> str | None: + """The password for authentication default is None.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """ + The working directory of the Redis service. + Can be "/path/to/dir" default is "/". + """ + @property + def username(self, /) -> str | None: + """ + The username to connect redis service. + default is None. + """ + +@final +class S3Config(ServiceConfig): + """Configuration for the `s3` service.""" + + def __init__( + self, + /, + bucket: str, + access_key_id: str | None = None, + allow_anonymous: bool | None = None, + assume_role_duration_seconds: int | None = None, + assume_role_session_tags: dict[str, str] | None = None, + batch_max_operations: int | None = None, + checksum_algorithm: str | None = None, + default_acl: str | None = None, + default_storage_class: str | None = None, + delete_max_size: int | None = None, + disable_config_load: bool | None = None, + disable_ec2_metadata: bool | None = None, + disable_list_objects_v2: bool | None = None, + disable_stat_with_override: bool | None = None, + disable_write_with_if_match: bool | None = None, + enable_request_payer: bool | None = None, + enable_versioning: bool | None = None, + enable_virtual_host_style: bool | None = None, + enable_write_with_append: bool | None = None, + endpoint: str | None = None, + external_id: str | None = None, + region: str | None = None, + role_arn: str | None = None, + role_session_name: str | None = None, + root: str | PathLike[str] | None = None, + secret_access_key: str | None = None, + server_side_encryption: str | None = None, + server_side_encryption_aws_kms_key_id: str | None = None, + server_side_encryption_customer_algorithm: str | None = None, + server_side_encryption_customer_key: str | None = None, + server_side_encryption_customer_key_md5: str | None = None, + session_token: str | None = None, + skip_signature: bool | None = None, + ) -> None: ... + @property + def access_key_id(self, /) -> str | None: + """ + access_key_id of this backend. + - If access_key_id is set, we will take user's input first. + - If not, we will try to load it from environment. + """ + @property + def allow_anonymous(self, /) -> bool | None: + """ + Allow anonymous will allow opendal to send request without signing when + credential is not loaded. + [Deprecated since 0.57.0] Please use `skip_signature` instead of + `allow_anonymous`. + """ + @property + def assume_role_duration_seconds(self, /) -> int | None: + """assume_role_duration_seconds for this backend.""" + @property + def assume_role_session_tags(self, /) -> dict[str, str] | None: + """assume_role_session_tags for this backend.""" + @property + def batch_max_operations(self, /) -> int | None: + """ + Deprecated: S3 delete batch capability is enabled by default. + [Deprecated since 0.57.0] S3 delete batch capability is enabled by default. + Use CapabilityOverrideLayer to override delete_max_size for specific + endpoints. + """ + @property + def bucket(self, /) -> str: + """ + Bucket name of this backend. + required. + """ + @property + def checksum_algorithm(self, /) -> str | None: + """ + Checksum Algorithm to use when sending checksums in HTTP headers. + This is necessary when writing to AWS S3 Buckets with Object Lock enabled + for example. + Available options: - "crc32c" - "md5". + """ + @property + def default_acl(self, /) -> str | None: + """ + Default ACL for new objects. + Note that some s3 services like minio do not support this option. + """ + @property + def default_storage_class(self, /) -> str | None: + """ + Default storage_class for this backend. + Available values: - `DEEP_ARCHIVE` - `GLACIER` - `GLACIER_IR` - + `INTELLIGENT_TIERING` - `ONEZONE_IA` - `EXPRESS_ONEZONE` - `OUTPOSTS` - + `REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3 compatible services + don't support all of them. + """ + @property + def delete_max_size(self, /) -> int | None: + """ + Deprecated: S3 delete batch capability is enabled by default. + [Deprecated since 0.57.0] S3 delete batch capability is enabled by default. + Use CapabilityOverrideLayer to override delete_max_size for specific + endpoints. + """ + @property + def disable_config_load(self, /) -> bool | None: + """ + Disable config load so that opendal will not load config from environment. + For examples: - envs like `AWS_ACCESS_KEY_ID` - files like `~/.aws/config`. + """ + @property + def disable_ec2_metadata(self, /) -> bool | None: + """ + Disable load credential from ec2 metadata. + This option is used to disable the default behavior of opendal to load + credential from ec2 metadata, a.k.a., IMDSv2. + """ + @property + def disable_list_objects_v2(self, /) -> bool | None: + """ + OpenDAL uses List Objects V2 by default to list objects. + However, some legacy services do not yet support V2. + This option allows users to switch back to the older List Objects V1. + """ + @property + def disable_stat_with_override(self, /) -> bool | None: + """ + Deprecated: S3 stat override capabilities are enabled by default. + [Deprecated since 0.57.0] S3 stat override capabilities are enabled by + default. + Use CapabilityOverrideLayer to override them for specific endpoints. + """ + @property + def disable_write_with_if_match(self, /) -> bool | None: + """ + Deprecated: S3 write with If-Match capability is enabled by default. + [Deprecated since 0.57.0] S3 write with If-Match capability is enabled by + default and this option is no longer needed. + """ + @property + def enable_request_payer(self, /) -> bool | None: + """ + Indicates whether the client agrees to pay for the requests made to the S3 + bucket. + """ + @property + def enable_versioning(self, /) -> bool | None: + """ + Deprecated: S3 versioning capability is enabled by default. + [Deprecated since 0.57.0] S3 versioning capability is enabled by default and + this option is no longer needed. + """ + @property + def enable_virtual_host_style(self, /) -> bool | None: + """ + Enable virtual host style so that opendal will send API requests in virtual + host style instead of path style. + - By default, opendal will send API to + `https://s3.us-east-1.amazonaws.com/bucket_name` - Enabled, opendal will + send API to `https://bucket_name.s3.us-east-1.amazonaws.com`. + """ + @property + def enable_write_with_append(self, /) -> bool | None: + """ + Deprecated: S3 append capability is enabled by default. + [Deprecated since 0.57.0] S3 append capability is enabled by default and + this option is no longer needed. + """ + @property + def endpoint(self, /) -> str | None: + """ + Endpoint of this backend. + Endpoint must be full uri, e.g. + - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com` + - Cloudflare R2: `https://.r2.cloudflarestorage.com` - Aliyun + OSS: `https://{region}.aliyuncs.com` - Tencent COS: + `https://cos.{region}.myqcloud.com` - Minio: `http://127.0.0.1:9000` If user + inputs endpoint without scheme like "s3.amazonaws.com", we will prepend + "https://" before it. + - If endpoint is set, we will take user's input first. + - If not, we will try to load it from environment. + - If still not set, default to `https://s3.amazonaws.com`. + """ + @property + def external_id(self, /) -> str | None: + """external_id for this backend.""" + @property + def region(self, /) -> str | None: + """ + Region represent the signing region of this endpoint. + This is required if you are using the default AWS S3 endpoint. + If using a custom endpoint, - If region is set, we will take user's input + first. + - If not, we will try to load it from environment. + """ + @property + def role_arn(self, /) -> str | None: + """ + role_arn for this backend. + If `role_arn` is set, we will use already known config as source credential + to assume role with `role_arn`. + """ + @property + def role_session_name(self, /) -> str | None: + """role_session_name for this backend.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + All operations will happen under this root. + default to `/` if not set. + """ + @property + def secret_access_key(self, /) -> str | None: + """ + secret_access_key of this backend. + - If secret_access_key is set, we will take user's input first. + - If not, we will try to load it from environment. + """ + @property + def server_side_encryption(self, /) -> str | None: + """ + server_side_encryption for this backend. + Available values: `AES256`, `aws:kms`. + """ + @property + def server_side_encryption_aws_kms_key_id(self, /) -> str | None: + """ + server_side_encryption_aws_kms_key_id for this backend - If + `server_side_encryption` set to `aws:kms`, and + `server_side_encryption_aws_kms_key_id` is not set, S3 will use aws managed + kms key to encrypt data. + - If `server_side_encryption` set to `aws:kms`, and + `server_side_encryption_aws_kms_key_id` is a valid kms key id, S3 will use + the provided kms key to encrypt data. + - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an + error will be returned. + - If `server_side_encryption` is not `aws:kms`, setting + `server_side_encryption_aws_kms_key_id` is a noop. + """ + @property + def server_side_encryption_customer_algorithm(self, /) -> str | None: + """ + server_side_encryption_customer_algorithm for this backend. + Available values: `AES256`. + """ + @property + def server_side_encryption_customer_key(self, /) -> str | None: + """ + server_side_encryption_customer_key for this backend. + Value: BASE64-encoded key that matches algorithm specified in + `server_side_encryption_customer_algorithm`. + """ + @property + def server_side_encryption_customer_key_md5(self, /) -> str | None: + """ + Set server_side_encryption_customer_key_md5 for this backend. + Value: MD5 digest of key specified in `server_side_encryption_customer_key`. + """ + @property + def session_token(self, /) -> str | None: + """ + session_token (aka, security token) of this backend. + This token will expire after sometime, it's recommended to set session_token + by hand. + """ + @property + def skip_signature(self, /) -> bool | None: + """Skip signature will skip loading credentials and signing requests.""" + @final class Scheme: AliyunDrive: Final[Scheme] @@ -26,7 +1685,6 @@ class Scheme: Azfile: Final[Scheme] B2: Final[Scheme] Cacache: Final[Scheme] - CloudflareKv: Final[Scheme] Cos: Final[Scheme] Dashmap: Final[Scheme] Dropbox: Final[Scheme] @@ -37,7 +1695,6 @@ class Scheme: Ghac: Final[Scheme] Goosefs: Final[Scheme] Gridfs: Final[Scheme] - HdfsNative: Final[Scheme] Hf: Final[Scheme] Http: Final[Scheme] Ipfs: Final[Scheme] @@ -58,14 +1715,12 @@ class Scheme: Redis: Final[Scheme] S3: Final[Scheme] Seafile: Final[Scheme] - Sftp: Final[Scheme] Sled: Final[Scheme] Sqlite: Final[Scheme] Swift: Final[Scheme] Tos: Final[Scheme] Upyun: Final[Scheme] VercelArtifacts: Final[Scheme] - VercelBlob: Final[Scheme] Webdav: Final[Scheme] Webhdfs: Final[Scheme] YandexDisk: Final[Scheme] @@ -77,3 +1732,440 @@ class Scheme: def name(self, /) -> str: ... @property def value(self, /) -> str: ... + +@final +class SeafileConfig(ServiceConfig): + """Configuration for the `seafile` service.""" + + def __init__( + self, + /, + repo_name: str, + endpoint: str | None = None, + password: str | None = None, + root: str | PathLike[str] | None = None, + username: str | None = None, + ) -> None: ... + @property + def endpoint(self, /) -> str | None: + """Endpoint address of this backend.""" + @property + def password(self, /) -> str | None: + """Password of this backend.""" + @property + def repo_name(self, /) -> str: + """ + repo_name of this backend. + required. + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + All operations will happen under this root. + """ + @property + def username(self, /) -> str | None: + """Username of this backend.""" + +class ServiceConfig: + """Base class for all service configs.""" + + @property + def scheme(self, /) -> str: + """The service scheme this config targets, e.g. ``"s3"``.""" + +@final +class SledConfig(ServiceConfig): + """Configuration for the `sled` service.""" + + def __init__( + self, + /, + datadir: str | None = None, + root: str | PathLike[str] | None = None, + tree: str | None = None, + ) -> None: ... + @property + def datadir(self, /) -> str | None: + """That path to the sled data directory.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """The root for sled.""" + @property + def tree(self, /) -> str | None: + """The tree for sled.""" + +@final +class SqliteConfig(ServiceConfig): + """Configuration for the `sqlite` service.""" + + def __init__( + self, + /, + connection_string: str | None = None, + key_field: str | None = None, + root: str | PathLike[str] | None = None, + table: str | None = None, + value_field: str | None = None, + ) -> None: ... + @property + def connection_string(self, /) -> str | None: + """ + Set the connection_string of the sqlite service. + This connection string is used to connect to the sqlite service. + The format of connect string resembles the url format of the sqlite client: + - `sqlite::memory:` - `sqlite:data.db` - `sqlite://data.db` For more + information, please visit + . + """ + @property + def key_field(self, /) -> str | None: + """ + Set the key field name of the sqlite service to read/write. + Default to `key` if not specified. + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Set the working directory, all operations will be performed under it. + default: "/". + """ + @property + def table(self, /) -> str | None: + """Set the table name of the sqlite service to read/write.""" + @property + def value_field(self, /) -> str | None: + """ + Set the value field name of the sqlite service to read/write. + Default to `value` if not specified. + """ + +@final +class SwiftConfig(ServiceConfig): + """Configuration for the `swift` service.""" + + def __init__( + self, + /, + container: str | None = None, + endpoint: str | None = None, + root: str | PathLike[str] | None = None, + temp_url_hash_algorithm: str | None = None, + temp_url_key: str | None = None, + token: str | None = None, + ) -> None: ... + @property + def container(self, /) -> str | None: + """The container for Swift.""" + @property + def endpoint(self, /) -> str | None: + """The endpoint for Swift.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """The root for Swift.""" + @property + def temp_url_hash_algorithm(self, /) -> str | None: + """ + The hash algorithm for TempURL signing. + Supported values: `sha1`, `sha256`, `sha512`. + Defaults to `sha256`. + The cluster must have the chosen algorithm in its `tempurl.allowed_digests` + (check `GET /info`). + """ + @property + def temp_url_key(self, /) -> str | None: + """ + The TempURL key for generating presigned URLs. + This corresponds to the `X-Account-Meta-Temp-URL-Key` or + `X-Container-Meta-Temp-URL-Key` header value configured on the Swift account + or container. + """ + @property + def token(self, /) -> str | None: + """The token for Swift.""" + +@final +class TosConfig(ServiceConfig): + """Configuration for the `tos` service.""" + + def __init__( + self, + /, + bucket: str, + access_key_id: str | None = None, + disable_config_load: bool | None = None, + endpoint: str | None = None, + region: str | None = None, + root: str | PathLike[str] | None = None, + secret_access_key: str | None = None, + security_token: str | None = None, + skip_signature: bool | None = None, + ) -> None: ... + @property + def access_key_id(self, /) -> str | None: + """ + access_key_id of this backend. + - If access_key_id is set, we will take user's input first. + - If not, we will try to load it from environment. + """ + @property + def bucket(self, /) -> str: + """ + Bucket name of this backend. + required. + """ + @property + def disable_config_load(self, /) -> bool | None: + """ + Disable config load so that opendal will not load config from environment. + For examples: - envs like `TOS_ACCESS_KEY_ID`. + """ + @property + def endpoint(self, /) -> str | None: + """ + Endpoint of this backend. + Endpoint must be full uri, e.g. + - TOS: `https://tos-cn-beijing.volces.com` - TOS with region: + `https://tos-{region}.volces.com` If user inputs endpoint without scheme + like "tos-cn-beijing.volces.com", we will prepend "https://" before it. + """ + @property + def region(self, /) -> str | None: + """ + Region represent the signing region of this endpoint. + Required if endpoint is not provided. + - If region is set, we will take user's input first. + - If not, we will try to load it from environment. + - If still not set, default to `cn-beijing`. + """ + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + All operations will happen under this root. + default to `/` if not set. + """ + @property + def secret_access_key(self, /) -> str | None: + """ + secret_access_key of this backend. + - If secret_access_key is set, we will take user's input first. + - If not, we will try to load it from environment. + """ + @property + def security_token(self, /) -> str | None: + """ + security_token of this backend. + This token will expire after sometime, it's recommended to set + security_token by hand. + """ + @property + def skip_signature(self, /) -> bool | None: + """Skip signature will skip loading credentials and signing requests.""" + +@final +class UpyunConfig(ServiceConfig): + """Configuration for the `upyun` service.""" + + def __init__( + self, + /, + bucket: str, + operator: str | None = None, + password: str | None = None, + root: str | PathLike[str] | None = None, + ) -> None: ... + @property + def bucket(self, /) -> str: + """Bucket address of this backend.""" + @property + def operator(self, /) -> str | None: + """Username of this backend.""" + @property + def password(self, /) -> str | None: + """Password of this backend.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + All operations will happen under this root. + """ + +@final +class VercelArtifactsConfig(ServiceConfig): + """Configuration for the `vercel-artifacts` service.""" + + def __init__( + self, + /, + access_token: str | None = None, + endpoint: str | None = None, + team_id: str | None = None, + team_slug: str | None = None, + ) -> None: ... + @property + def access_token(self, /) -> str | None: + """The access token for Vercel.""" + @property + def endpoint(self, /) -> str | None: + """ + The endpoint for the Vercel artifacts API. + Defaults to `https://api.vercel.com`. + """ + @property + def team_id(self, /) -> str | None: + """ + The Vercel team ID. + When set, the `teamId` query parameter is appended to all API requests. + """ + @property + def team_slug(self, /) -> str | None: + """ + The Vercel team slug. + When set, the `slug` query parameter is appended to all API requests. + """ + +@final +class WebdavConfig(ServiceConfig): + """Configuration for the `webdav` service.""" + + def __init__( + self, + /, + disable_copy: bool | None = None, + disable_create_dir: bool | None = None, + enable_conditional_read: bool | None = None, + enable_user_metadata: bool | None = None, + endpoint: str | None = None, + password: str | None = None, + root: str | PathLike[str] | None = None, + token: str | None = None, + user_metadata_prefix: str | None = None, + user_metadata_uri: str | None = None, + username: str | None = None, + ) -> None: ... + @property + def disable_copy(self, /) -> bool | None: + """ + Deprecated: WebDAV copy capability is enabled by default. + [Deprecated since 0.57.0] WebDAV copy capability is enabled by default and + this option is no longer needed. + """ + @property + def disable_create_dir(self, /) -> bool | None: + """ + Disable automatic parent directory creation before write operations. + By default, OpenDAL creates parent directories using MKCOL before writing + files. + This requires PROPFIND support to check directory existence. + Some WebDAV-compatible servers (e.g., bazel-remote) don't support PROPFIND + or don't require explicit directory creation. + Enable this option to skip the MKCOL calls and write files directly. + Default: false. + """ + @property + def enable_conditional_read(self, /) -> bool | None: + """ + Enable conditional read support. + When enabled (the default), OpenDAL forwards the RFC 7232 headers + `If-Match`, `If-None-Match`, `If-Modified-Since` and `If-Unmodified-Since` + to the server when callers provide them. + Some WebDAV-compatible servers (e.g., nginx-dav) don't return ETags in + PROPFIND or don't honor these headers on GET. + Setting this to `false` drops the four `read_with_if_*` capabilities, so + calls like `reader_with(path).if_match(...)` return `ErrorKind::Unsupported` + locally instead of being silently ignored by the server. + Default: true. + """ + @property + def enable_user_metadata(self, /) -> bool | None: + """ + Deprecated: WebDAV user metadata capability is enabled by default. + [Deprecated since 0.57.0] WebDAV user metadata capability is enabled by + default. + Use CapabilityOverrideLayer to override write_with_user_metadata for + endpoints without PROPPATCH support. + """ + @property + def endpoint(self, /) -> str | None: + """Endpoint of this backend.""" + @property + def password(self, /) -> str | None: + """Password of this backend.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root of this backend.""" + @property + def token(self, /) -> str | None: + """Token of this backend.""" + @property + def user_metadata_prefix(self, /) -> str | None: + """ + The XML namespace prefix for user metadata properties. + This prefix is used in PROPPATCH/PROPFIND XML requests. + Different servers may require different prefixes. + Default: "opendal". + """ + @property + def user_metadata_uri(self, /) -> str | None: + """ + The XML namespace URI for user metadata properties. + This URI uniquely identifies the namespace for custom properties. + Different servers may require different namespace URIs. + For example, Nextcloud might work better with its own namespace. + Default: `https://opendal.apache.org/ns`. + """ + @property + def username(self, /) -> str | None: + """Username of this backend.""" + +@final +class WebhdfsConfig(ServiceConfig): + """Configuration for the `webhdfs` service.""" + + def __init__( + self, + /, + atomic_write_dir: str | PathLike[str] | None = None, + delegation: str | None = None, + disable_list_batch: bool | None = None, + endpoint: str | None = None, + root: str | PathLike[str] | None = None, + user_name: str | None = None, + ) -> None: ... + @property + def atomic_write_dir(self, /) -> str | PathLike[str] | None: + """atomic_write_dir of this backend.""" + @property + def delegation(self, /) -> str | None: + """Delegation token for webhdfs.""" + @property + def disable_list_batch(self, /) -> bool | None: + """Disable batch listing.""" + @property + def endpoint(self, /) -> str | None: + """Endpoint for webhdfs.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """Root for webhdfs.""" + @property + def user_name(self, /) -> str | None: + """Name of the user for webhdfs.""" + +@final +class YandexDiskConfig(ServiceConfig): + """Configuration for the `yandex-disk` service.""" + + def __init__( + self, /, access_token: str, root: str | PathLike[str] | None = None + ) -> None: ... + @property + def access_token(self, /) -> str: + """Yandex disk oauth access_token.""" + @property + def root(self, /) -> str | PathLike[str] | None: + """ + Root of this backend. + All operations will happen under this root. + """ diff --git a/bindings/python/ruff.toml b/bindings/python/ruff.toml index 7d494d922df6..90ec6a4d794a 100644 --- a/bindings/python/ruff.toml +++ b/bindings/python/ruff.toml @@ -77,6 +77,8 @@ known-first-party = ["opendal"] [lint.per-file-ignores] "*.pyi" = [ "PYI021", # keep docstrings in generated stubs + "D205", # generated config docstrings copy upstream comments verbatim + "D404", # some docstrings start with `This` "ANN401", # Any in signatures "ANN003", # **kwargs untyped "TID252", # injected relative sibling imports from generated stubs diff --git a/bindings/python/scripts/postprocess_stubs.py b/bindings/python/scripts/postprocess_stubs.py index 80100e69a642..96a34ebfbb1f 100644 --- a/bindings/python/scripts/postprocess_stubs.py +++ b/bindings/python/scripts/postprocess_stubs.py @@ -31,6 +31,7 @@ from __future__ import annotations +import re from pathlib import Path PKG = Path(__file__).resolve().parent.parent / "python" / "opendal" @@ -57,6 +58,9 @@ "from .types import Entry, PresignedRequest\n" ), "file": "import collections.abc\nimport types\nimport typing_extensions\n", + # Generated service configs use `PathBuf` params, which pyo3 renders as + # `str | PathLike[str]`; inject the import the annotation needs. + "services": "from os import PathLike\n", } # Mirrors the `create_exception!` types in `src/errors.rs`; keep in sync. @@ -84,10 +88,38 @@ def fix_incomplete_exceptions() -> None: path.write_text(EXCEPTIONS_STUB) +def rewrite_config_init(text: str) -> str: + """Rewrite generated config ``__new__`` methods as ``__init__``. + + PyO3 emits pyclass constructors as ``__new__(cls, ...) -> XConfig``. Rename + them to ``__init__(self, ...) -> None`` so mkdocstrings' ``merge_init_into_class`` + renders the typed constructor signature on the class. Typing is unchanged -- + ``XConfig(...)`` still checks against the same parameters. + """ + text = re.sub( + r"def __new__\(\n(\s*)cls,", + r"def __init__(\n\1self,", + text, + ) + text = re.sub( + r"def __new__\(cls,", + r"def __init__(self,", + text, + ) + text = re.sub( + r"\) -> [A-Za-z0-9]+Config: \.\.\.", + r") -> None: ...", + text, + ) + return text + + def relocate() -> None: """Move the stubs to the public ``opendal/.pyi`` paths.""" for name in SUBMODULES: text = (GEN / f"{name}.pyi").read_text() + if name == "services": + text = rewrite_config_init(text) (PKG / f"{name}.pyi").write_text(IMPORTS.get(name, "") + text) # Stub for the ``opendal/_opendal.*.so`` extension itself. diff --git a/bindings/python/src/config_types.rs b/bindings/python/src/config_types.rs new file mode 100644 index 000000000000..31c1645bc654 --- /dev/null +++ b/bindings/python/src/config_types.rs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::path::PathBuf; + +use pyo3::prelude::*; +use pyo3::types::PyString; +use pyo3::{Borrowed, IntoPyObject}; + +/// A filesystem path stored as a `String`. +/// +/// Accepts `str`/`os.PathLike` and renders the same type on output, so a config +/// field's constructor parameter and getter share one type. +#[derive(Clone, Debug)] +pub struct PyPath(pub String); + +impl PyPath { + pub fn into_string(self) -> String { + self.0 + } +} + +impl From for PyPath { + fn from(value: String) -> Self { + PyPath(value) + } +} + +impl<'a, 'py> FromPyObject<'a, 'py> for PyPath { + type Error = PyErr; + + const INPUT_TYPE: pyo3::inspect::PyStaticExpr = ::INPUT_TYPE; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + let path = PathBuf::extract(obj)?; + Ok(PyPath(path.to_string_lossy().into_owned())) + } +} + +impl<'py> IntoPyObject<'py> for PyPath { + type Target = PyString; + type Output = Bound<'py, PyString>; + type Error = PyErr; + + // Override the output type to match the input type (`str | os.PathLike[str]`). + const OUTPUT_TYPE: pyo3::inspect::PyStaticExpr = ::INPUT_TYPE; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(PyString::new(py, &self.0)) + } +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index b923c76b019b..9e3652b45a3d 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -38,6 +38,8 @@ mod errors; pub use errors::*; mod options; pub use options::*; +mod config_types; +pub use config_types::*; mod services; pub use services::*; @@ -75,11 +77,9 @@ mod _opendal { use crate::Capability; } - #[pymodule] - mod services { - #[pymodule_export] - use crate::Scheme; - } + // The `services` submodule is defined in the generated `services.rs`. + #[pymodule_export] + use crate::services::services_submodule; #[pymodule] mod layers { diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs index 9782e0385dbd..422b786494f6 100644 --- a/bindings/python/src/operator.rs +++ b/bindings/python/src/operator.rs @@ -67,6 +67,31 @@ fn normalize_scheme(raw: &str) -> String { raw.trim().to_ascii_lowercase().replace('_', "-") } +/// Collect string-valued `**kwargs` into the option map; a non-string value +/// raises a clear error pointing to `from_config`. +fn kwargs_to_map(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult> { + let Some(kwargs) = kwargs else { + return Ok(HashMap::new()); + }; + let mut map = HashMap::with_capacity(kwargs.len()); + for (key, value) in kwargs.iter() { + let key: String = key.extract()?; + let value: String = value.extract().map_err(|_| { + ConfigInvalid::new_err(format!( + "option '{key}' must be a string; got {}. Pass options as strings, \ + or use Operator.from_config(Config(...)) for typed values.", + value + .get_type() + .name() + .map(|n| n.to_string()) + .unwrap_or_else(|_| "?".to_string()), + )) + })?; + map.insert(key, value); + } + Ok(map) +} + /// Rebuild a blocking [`Operator`] while unpickling. /// /// Routes through `from_uri`, not the scheme-based `__new__`, whose scheme @@ -78,6 +103,7 @@ pub fn _reconstruct_operator(scheme: &str, map: HashMap) -> PyRe core: build_blocking_operator_from_uri(scheme, map.clone())?, __scheme: scheme.to_string(), __map: map, + __picklable: true, }) } @@ -93,6 +119,7 @@ pub fn _reconstruct_async_operator( core: build_operator_from_uri(scheme, map.clone())?, __scheme: scheme.to_string(), __map: map, + __picklable: true, }) } @@ -108,6 +135,8 @@ pub struct Operator { core: ocore::blocking::Operator, __scheme: String, __map: HashMap, + /// False when built from a config with a map-valued field (not picklable). + __picklable: bool, } #[pymethods] impl Operator { @@ -126,7 +155,7 @@ impl Operator { /// The new operator. #[new] #[pyo3(signature = (scheme: "str | Scheme", *, **kwargs))] - pub fn new(scheme: Bound, kwargs: Option>) -> PyResult { + pub fn new(scheme: Bound, kwargs: Option>) -> PyResult { let scheme = if let Ok(scheme_str) = scheme.extract::<&str>() { scheme_str.to_string() } else if let Ok(py_scheme) = scheme.extract::() { @@ -137,12 +166,13 @@ impl Operator { )); }; let scheme = normalize_scheme(&scheme); - let map = kwargs.unwrap_or_default(); + let map = kwargs_to_map(kwargs.as_ref())?; Ok(Operator { core: build_blocking_operator(&scheme, map.clone())?, __scheme: scheme, __map: map, + __picklable: true, }) } @@ -181,14 +211,56 @@ impl Operator { pub fn from_uri( _cls: &Bound, uri: &str, - kwargs: Option>, + kwargs: Option>, ) -> PyResult { - let map = kwargs.unwrap_or_default(); + let map = kwargs_to_map(kwargs.as_ref())?; Ok(Operator { core: build_blocking_operator_from_uri(uri, map.clone())?, __scheme: uri.to_string(), __map: map, + __picklable: true, + }) + } + + /// Create a new blocking `Operator` from a typed service config. + /// + /// The config binds its own scheme, so there is no scheme argument and no + /// way to pair a config with the wrong service. + /// + /// Parameters + /// ---------- + /// config : ServiceConfig + /// A service config such as ``opendal.services.S3Config``. + /// + /// Returns + /// ------- + /// Operator + /// The new operator. + /// + /// Examples + /// -------- + /// ```python + /// import opendal + /// from opendal.services import S3Config + /// + /// op = opendal.Operator.from_config(S3Config(bucket="my-bucket")) + /// ``` + #[classmethod] + #[pyo3(signature = (config))] + pub fn from_config( + _cls: &Bound, + config: PyRef, + ) -> PyResult { + let op = config.0.build()?; + let runtime = pyo3_async_runtimes::tokio::get_runtime(); + let _guard = runtime.enter(); + let core = ocore::blocking::Operator::new(op).map_err(format_pyerr)?; + Ok(Operator { + core, + __scheme: config.0.scheme().to_string(), + __map: config.0.to_map(), + __picklable: config.0.is_picklable(), }) } @@ -213,6 +285,7 @@ impl Operator { core: op, __scheme: self.__scheme.clone(), __map: self.__map.clone(), + __picklable: self.__picklable, }) } @@ -764,6 +837,7 @@ impl Operator { core: self.core.clone().into(), __scheme: self.__scheme.clone(), __map: self.__map.clone(), + __picklable: self.__picklable, }) } @@ -781,6 +855,11 @@ impl Operator { } } fn __reduce__(&self, py: Python) -> PyResult> { + if !self.__picklable { + return Err(Unsupported::new_err( + "cannot pickle an operator built from a config with a map-valued field", + )); + } let reconstructor = py .import("opendal._opendal")? .getattr("_reconstruct_operator")?; @@ -801,6 +880,8 @@ pub struct AsyncOperator { core: ocore::Operator, __scheme: String, __map: HashMap, + /// False when built from a config with a map-valued field (not picklable). + __picklable: bool, } #[pymethods] impl AsyncOperator { @@ -819,7 +900,7 @@ impl AsyncOperator { /// The new async operator. #[new] #[pyo3(signature = (scheme: "str | Scheme", * ,**kwargs))] - pub fn new(scheme: Bound, kwargs: Option>) -> PyResult { + pub fn new(scheme: Bound, kwargs: Option>) -> PyResult { let scheme = if let Ok(scheme_str) = scheme.extract::<&str>() { scheme_str.to_string() } else if let Ok(py_scheme) = scheme.extract::() { @@ -831,12 +912,13 @@ impl AsyncOperator { }; let scheme = normalize_scheme(&scheme); - let map = kwargs.unwrap_or_default(); + let map = kwargs_to_map(kwargs.as_ref())?; Ok(AsyncOperator { core: build_operator(&scheme, map.clone())?, __scheme: scheme, __map: map, + __picklable: true, }) } @@ -875,14 +957,52 @@ impl AsyncOperator { pub fn from_uri( _cls: &Bound, uri: &str, - kwargs: Option>, + kwargs: Option>, ) -> PyResult { - let map = kwargs.unwrap_or_default(); + let map = kwargs_to_map(kwargs.as_ref())?; Ok(AsyncOperator { core: build_operator_from_uri(uri, map.clone())?, __scheme: uri.to_string(), __map: map, + __picklable: true, + }) + } + + /// Create a new `AsyncOperator` from a typed service config. + /// + /// The config binds its own scheme, so there is no scheme argument and no + /// way to pair a config with the wrong service. + /// + /// Parameters + /// ---------- + /// config : ServiceConfig + /// A service config such as ``opendal.services.S3Config``. + /// + /// Returns + /// ------- + /// AsyncOperator + /// The new async operator. + /// + /// Examples + /// -------- + /// ```python + /// import opendal + /// from opendal.services import S3Config + /// + /// op = opendal.AsyncOperator.from_config(S3Config(bucket="my-bucket")) + /// ``` + #[classmethod] + #[pyo3(signature = (config))] + pub fn from_config( + _cls: &Bound, + config: PyRef, + ) -> PyResult { + Ok(AsyncOperator { + core: config.0.build()?, + __scheme: config.0.scheme().to_string(), + __map: config.0.to_map(), + __picklable: config.0.is_picklable(), }) } @@ -903,6 +1023,7 @@ impl AsyncOperator { core: op, __scheme: self.__scheme.clone(), __map: self.__map.clone(), + __picklable: self.__picklable, }) } @@ -1826,6 +1947,7 @@ impl AsyncOperator { core: op, __scheme: self.__scheme.clone(), __map: self.__map.clone(), + __picklable: self.__picklable, }) } @@ -1847,6 +1969,11 @@ impl AsyncOperator { } } fn __reduce__(&self, py: Python) -> PyResult> { + if !self.__picklable { + return Err(Unsupported::new_err( + "cannot pickle an operator built from a config with a map-valued field", + )); + } let reconstructor = py .import("opendal._opendal")? .getattr("_reconstruct_async_operator")?; diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs index 227bf217f04b..dbc2bc89bec5 100644 --- a/bindings/python/src/services.rs +++ b/bindings/python/src/services.rs @@ -16,6 +16,7 @@ // under the License. use crate::*; +use std::collections::HashMap; #[pyclass( eq, @@ -43,8 +44,6 @@ pub enum Scheme { B2, #[cfg(feature = "services-cacache")] Cacache, - #[cfg(feature = "services-cloudflare-kv")] - CloudflareKv, #[cfg(feature = "services-cos")] Cos, #[cfg(feature = "services-dashmap")] @@ -65,8 +64,6 @@ pub enum Scheme { Goosefs, #[cfg(feature = "services-gridfs")] Gridfs, - #[cfg(feature = "services-hdfs-native")] - HdfsNative, #[cfg(feature = "services-hf")] Hf, #[cfg(feature = "services-http")] @@ -107,8 +104,6 @@ pub enum Scheme { S3, #[cfg(feature = "services-seafile")] Seafile, - #[cfg(feature = "services-sftp")] - Sftp, #[cfg(feature = "services-sled")] Sled, #[cfg(feature = "services-sqlite")] @@ -121,8 +116,6 @@ pub enum Scheme { Upyun, #[cfg(feature = "services-vercel-artifacts")] VercelArtifacts, - #[cfg(feature = "services-vercel-blob")] - VercelBlob, #[cfg(feature = "services-webdav")] Webdav, #[cfg(feature = "services-webhdfs")] @@ -185,8 +178,6 @@ impl_enum_to_str!( B2 => "b2", #[cfg(feature = "services-cacache")] Cacache => "cacache", - #[cfg(feature = "services-cloudflare-kv")] - CloudflareKv => "cloudflare-kv", #[cfg(feature = "services-cos")] Cos => "cos", #[cfg(feature = "services-dashmap")] @@ -207,8 +198,6 @@ impl_enum_to_str!( Goosefs => "goosefs", #[cfg(feature = "services-gridfs")] Gridfs => "gridfs", - #[cfg(feature = "services-hdfs-native")] - HdfsNative => "hdfs-native", #[cfg(feature = "services-hf")] Hf => "hf", #[cfg(feature = "services-http")] @@ -249,8 +238,6 @@ impl_enum_to_str!( S3 => "s3", #[cfg(feature = "services-seafile")] Seafile => "seafile", - #[cfg(feature = "services-sftp")] - Sftp => "sftp", #[cfg(feature = "services-sled")] Sled => "sled", #[cfg(feature = "services-sqlite")] @@ -263,8 +250,6 @@ impl_enum_to_str!( Upyun => "upyun", #[cfg(feature = "services-vercel-artifacts")] VercelArtifacts => "vercel-artifacts", - #[cfg(feature = "services-vercel-blob")] - VercelBlob => "vercel-blob", #[cfg(feature = "services-webdav")] Webdav => "webdav", #[cfg(feature = "services-webhdfs")] @@ -273,3 +258,7293 @@ impl_enum_to_str!( YandexDisk => "yandex-disk", } ); + +pub trait ConfigBuilder: Send + Sync { + fn build(&self) -> PyResult; + fn scheme(&self) -> &'static str; + fn to_map(&self) -> HashMap; + fn is_picklable(&self) -> bool; +} + +/// Base class for all service configs. +#[pyclass(module = "opendal.services", subclass)] +pub struct ServiceConfig(pub Box); + +#[pymethods] +impl ServiceConfig { + /// The service scheme this config targets, e.g. ``"s3"``. + #[getter] + pub fn scheme(&self) -> &'static str { + self.0.scheme() + } +} + +#[cfg(feature = "services-aliyun-drive")] +/// Configuration for the `aliyun-drive` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct AliyunDriveConfig(ocore::services::AliyunDriveConfig); + +#[cfg(feature = "services-aliyun-drive")] +#[allow(deprecated)] +impl ConfigBuilder for AliyunDriveConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "aliyun-drive" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("drive_type".to_string(), self.0.drive_type.clone()); + if let Some(v) = &self.0.access_token { + map.insert("access_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_id { + map.insert("client_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_secret { + map.insert("client_secret".to_string(), v.clone()); + } + if let Some(v) = &self.0.refresh_token { + map.insert("refresh_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // drive_type: always picklable + // access_token: always picklable + // client_id: always picklable + // client_secret: always picklable + // refresh_token: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-aliyun-drive")] +#[allow(deprecated)] +#[pymethods] +impl AliyunDriveConfig { + #[new] + #[pyo3(signature = ( + drive_type, + access_token = None, + client_id = None, + client_secret = None, + refresh_token = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + drive_type: String, + access_token: Option, + client_id: Option, + client_secret: Option, + refresh_token: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("drive_type".to_string(), drive_type); + if let Some(v) = access_token { + opts.insert("access_token".to_string(), v); + } + if let Some(v) = client_id { + opts.insert("client_id".to_string(), v); + } + if let Some(v) = client_secret { + opts.insert("client_secret".to_string(), v); + } + if let Some(v) = refresh_token { + opts.insert("refresh_token".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // drive_type: not a map field + // access_token: not a map field + // client_id: not a map field + // client_secret: not a map field + // refresh_token: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// The drive_type of this backend. + /// All operations will happen under this type of drive. + /// Available values are `default`, `backup` and `resource`. + /// Fallback to default if not set or no other drives can be found. + #[getter] + fn drive_type(&self) -> String { + self.0.drive_type.clone() + } + /// The access_token of this backend. + /// Solution for client-only purpose. + /// #4733 Required if no client_id, client_secret and refresh_token are + /// provided. + #[getter] + fn access_token(&self) -> Option { + self.0.access_token.clone() + } + /// The client_id of this backend. + /// Required if no access_token is provided. + #[getter] + fn client_id(&self) -> Option { + self.0.client_id.clone() + } + /// The client_secret of this backend. + /// Required if no access_token is provided. + #[getter] + fn client_secret(&self) -> Option { + self.0.client_secret.clone() + } + /// The refresh_token of this backend. + /// Required if no access_token is provided. + #[getter] + fn refresh_token(&self) -> Option { + self.0.refresh_token.clone() + } + /// The Root of this backend. + /// All operations will happen under this root. + /// Default to `/` if not set. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-alluxio")] +/// Configuration for the `alluxio` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct AlluxioConfig(ocore::services::AlluxioConfig); + +#[cfg(feature = "services-alluxio")] +#[allow(deprecated)] +impl ConfigBuilder for AlluxioConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "alluxio" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // endpoint: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-alluxio")] +#[allow(deprecated)] +#[pymethods] +impl AlluxioConfig { + #[new] + #[pyo3(signature = ( + endpoint = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + endpoint: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // endpoint: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// endpoint of this backend. + /// Endpoint must be full uri, mostly like `http://127.0.0.1:39999`. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// root of this backend. + /// All operations will happen under this root. + /// default to `/` if not set. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-azblob")] +/// Configuration for the `azblob` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct AzblobConfig(ocore::services::AzblobConfig); + +#[cfg(feature = "services-azblob")] +#[allow(deprecated)] +impl ConfigBuilder for AzblobConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "azblob" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("container".to_string(), self.0.container.clone()); + if let Some(v) = &self.0.account_key { + map.insert("account_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.account_name { + map.insert("account_name".to_string(), v.clone()); + } + if let Some(v) = &self.0.batch_max_operations { + map.insert("batch_max_operations".to_string(), v.to_string()); + } + if let Some(v) = &self.0.encryption_algorithm { + map.insert("encryption_algorithm".to_string(), v.clone()); + } + if let Some(v) = &self.0.encryption_key { + map.insert("encryption_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.encryption_key_sha256 { + map.insert("encryption_key_sha256".to_string(), v.clone()); + } + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.sas_token { + map.insert("sas_token".to_string(), v.clone()); + } + map.insert( + "skip_signature".to_string(), + if self.0.skip_signature { + "true" + } else { + "false" + } + .to_string(), + ); + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // container: always picklable + // account_key: always picklable + // account_name: always picklable + // batch_max_operations: always picklable + // encryption_algorithm: always picklable + // encryption_key: always picklable + // encryption_key_sha256: always picklable + // endpoint: always picklable + // root: always picklable + // sas_token: always picklable + // skip_signature: always picklable + ok + } +} + +#[cfg(feature = "services-azblob")] +#[allow(deprecated)] +#[pymethods] +impl AzblobConfig { + #[new] + #[pyo3(signature = ( + container, + account_key = None, + account_name = None, + batch_max_operations = None, + encryption_algorithm = None, + encryption_key = None, + encryption_key_sha256 = None, + endpoint = None, + root = None, + sas_token = None, + skip_signature = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + container: String, + account_key: Option, + account_name: Option, + batch_max_operations: Option, + encryption_algorithm: Option, + encryption_key: Option, + encryption_key_sha256: Option, + endpoint: Option, + root: Option, + sas_token: Option, + skip_signature: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("container".to_string(), container); + if let Some(v) = account_key { + opts.insert("account_key".to_string(), v); + } + if let Some(v) = account_name { + opts.insert("account_name".to_string(), v); + } + if let Some(v) = batch_max_operations { + opts.insert("batch_max_operations".to_string(), v.to_string()); + } + if let Some(v) = encryption_algorithm { + opts.insert("encryption_algorithm".to_string(), v); + } + if let Some(v) = encryption_key { + opts.insert("encryption_key".to_string(), v); + } + if let Some(v) = encryption_key_sha256 { + opts.insert("encryption_key_sha256".to_string(), v); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = sas_token { + opts.insert("sas_token".to_string(), v); + } + if let Some(v) = skip_signature { + opts.insert( + "skip_signature".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // container: not a map field + // account_key: not a map field + // account_name: not a map field + // batch_max_operations: not a map field + // encryption_algorithm: not a map field + // encryption_key: not a map field + // encryption_key_sha256: not a map field + // endpoint: not a map field + // root: not a map field + // sas_token: not a map field + // skip_signature: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// The container name of Azblob service backend. + #[getter] + fn container(&self) -> String { + self.0.container.clone() + } + /// The account key of Azblob service backend. + #[getter] + fn account_key(&self) -> Option { + self.0.account_key.clone() + } + /// The account name of Azblob service backend. + #[getter] + fn account_name(&self) -> Option { + self.0.account_name.clone() + } + /// Deprecated: Azblob delete batch capability is enabled by default with Azure + /// Blob's 256-operation batch limit. + /// [Deprecated since 0.57.0] Azblob delete batch capability is enabled by + /// default with Azure Blob's 256-operation batch limit. + /// Use CapabilityOverrideLayer to override delete_max_size for specific + /// endpoints. + #[getter] + fn batch_max_operations(&self) -> Option { + self.0.batch_max_operations + } + /// The encryption algorithm of Azblob service backend. + #[getter] + fn encryption_algorithm(&self) -> Option { + self.0.encryption_algorithm.clone() + } + /// The encryption key of Azblob service backend. + #[getter] + fn encryption_key(&self) -> Option { + self.0.encryption_key.clone() + } + /// The encryption key sha256 of Azblob service backend. + #[getter] + fn encryption_key_sha256(&self) -> Option { + self.0.encryption_key_sha256.clone() + } + /// The endpoint of Azblob service backend. + /// Endpoint must be full uri, e.g. + /// - Azblob: `https://accountname.blob.core.windows.net` - Azurite: + /// `http://127.0.0.1:10000/devstoreaccount1` + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// The root of Azblob service backend. + /// All operations will happen under this root. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// The sas token of Azblob service backend. + #[getter] + fn sas_token(&self) -> Option { + self.0.sas_token.clone() + } + /// Skip signature will skip loading credentials and signing requests. + #[getter] + fn skip_signature(&self) -> Option { + Some(self.0.skip_signature) + } +} + +#[cfg(feature = "services-azdls")] +/// Configuration for the `azdls` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct AzdlsConfig(ocore::services::AzdlsConfig); + +#[cfg(feature = "services-azdls")] +#[allow(deprecated)] +impl ConfigBuilder for AzdlsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "azdls" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("filesystem".to_string(), self.0.filesystem.clone()); + if let Some(v) = &self.0.account_key { + map.insert("account_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.account_name { + map.insert("account_name".to_string(), v.clone()); + } + if let Some(v) = &self.0.authority_host { + map.insert("authority_host".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_id { + map.insert("client_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_secret { + map.insert("client_secret".to_string(), v.clone()); + } + map.insert( + "enable_hns".to_string(), + if self.0.enable_hns { "true" } else { "false" }.to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.sas_token { + map.insert("sas_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.tenant_id { + map.insert("tenant_id".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // filesystem: always picklable + // account_key: always picklable + // account_name: always picklable + // authority_host: always picklable + // client_id: always picklable + // client_secret: always picklable + // enable_hns: always picklable + // endpoint: always picklable + // root: always picklable + // sas_token: always picklable + // tenant_id: always picklable + ok + } +} + +#[cfg(feature = "services-azdls")] +#[allow(deprecated)] +#[pymethods] +impl AzdlsConfig { + #[new] + #[pyo3(signature = ( + filesystem, + account_key = None, + account_name = None, + authority_host = None, + client_id = None, + client_secret = None, + enable_hns = None, + endpoint = None, + root = None, + sas_token = None, + tenant_id = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + filesystem: String, + account_key: Option, + account_name: Option, + authority_host: Option, + client_id: Option, + client_secret: Option, + enable_hns: Option, + endpoint: Option, + root: Option, + sas_token: Option, + tenant_id: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("filesystem".to_string(), filesystem); + if let Some(v) = account_key { + opts.insert("account_key".to_string(), v); + } + if let Some(v) = account_name { + opts.insert("account_name".to_string(), v); + } + if let Some(v) = authority_host { + opts.insert("authority_host".to_string(), v); + } + if let Some(v) = client_id { + opts.insert("client_id".to_string(), v); + } + if let Some(v) = client_secret { + opts.insert("client_secret".to_string(), v); + } + if let Some(v) = enable_hns { + opts.insert( + "enable_hns".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = sas_token { + opts.insert("sas_token".to_string(), v); + } + if let Some(v) = tenant_id { + opts.insert("tenant_id".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // filesystem: not a map field + // account_key: not a map field + // account_name: not a map field + // authority_host: not a map field + // client_id: not a map field + // client_secret: not a map field + // enable_hns: not a map field + // endpoint: not a map field + // root: not a map field + // sas_token: not a map field + // tenant_id: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Filesystem name of this backend. + #[getter] + fn filesystem(&self) -> String { + self.0.filesystem.clone() + } + /// Account key of this backend. + /// - required for shared_key authentication + #[getter] + fn account_key(&self) -> Option { + self.0.account_key.clone() + } + /// Account name of this backend. + #[getter] + fn account_name(&self) -> Option { + self.0.account_name.clone() + } + /// authority_host The authority host of the service principal. + /// - required for client_credentials authentication - default value: + /// `https://login.microsoftonline.com` + #[getter] + fn authority_host(&self) -> Option { + self.0.authority_host.clone() + } + /// client_id The client id of the service principal. + /// - required for client_credentials authentication + #[getter] + fn client_id(&self) -> Option { + self.0.client_id.clone() + } + /// client_secret The client secret of the service principal. + /// - required for client_credentials authentication + #[getter] + fn client_secret(&self) -> Option { + self.0.client_secret.clone() + } + /// Whether hierarchical namespace (HNS) is enabled for the storage account. + /// When enabled, recursive deletion can use pagination to avoid timeouts on + /// large directories. + /// - default value: `false` + #[getter] + fn enable_hns(&self) -> Option { + Some(self.0.enable_hns) + } + /// Endpoint of this backend. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// Root of this backend. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// sas_token The shared access signature token. + /// - required for sas authentication + #[getter] + fn sas_token(&self) -> Option { + self.0.sas_token.clone() + } + /// tenant_id The tenant id of the service principal. + /// - required for client_credentials authentication + #[getter] + fn tenant_id(&self) -> Option { + self.0.tenant_id.clone() + } +} + +#[cfg(feature = "services-azfile")] +/// Configuration for the `azfile` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct AzfileConfig(ocore::services::AzfileConfig); + +#[cfg(feature = "services-azfile")] +#[allow(deprecated)] +impl ConfigBuilder for AzfileConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "azfile" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("share_name".to_string(), self.0.share_name.clone()); + if let Some(v) = &self.0.account_key { + map.insert("account_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.account_name { + map.insert("account_name".to_string(), v.clone()); + } + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.sas_token { + map.insert("sas_token".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // share_name: always picklable + // account_key: always picklable + // account_name: always picklable + // endpoint: always picklable + // root: always picklable + // sas_token: always picklable + ok + } +} + +#[cfg(feature = "services-azfile")] +#[allow(deprecated)] +#[pymethods] +impl AzfileConfig { + #[new] + #[pyo3(signature = ( + share_name, + account_key = None, + account_name = None, + endpoint = None, + root = None, + sas_token = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + share_name: String, + account_key: Option, + account_name: Option, + endpoint: Option, + root: Option, + sas_token: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("share_name".to_string(), share_name); + if let Some(v) = account_key { + opts.insert("account_key".to_string(), v); + } + if let Some(v) = account_name { + opts.insert("account_name".to_string(), v); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = sas_token { + opts.insert("sas_token".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // share_name: not a map field + // account_key: not a map field + // account_name: not a map field + // endpoint: not a map field + // root: not a map field + // sas_token: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// The share name for azfile. + #[getter] + fn share_name(&self) -> String { + self.0.share_name.clone() + } + /// The account key for azfile. + #[getter] + fn account_key(&self) -> Option { + self.0.account_key.clone() + } + /// The account name for azfile. + #[getter] + fn account_name(&self) -> Option { + self.0.account_name.clone() + } + /// The endpoint for azfile. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// The root path for azfile. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// The sas token for azfile. + #[getter] + fn sas_token(&self) -> Option { + self.0.sas_token.clone() + } +} + +#[cfg(feature = "services-b2")] +/// Configuration for the `b2` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct B2Config(ocore::services::B2Config); + +#[cfg(feature = "services-b2")] +#[allow(deprecated)] +impl ConfigBuilder for B2Config { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "b2" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("bucket".to_string(), self.0.bucket.clone()); + map.insert("bucket_id".to_string(), self.0.bucket_id.clone()); + if let Some(v) = &self.0.application_key { + map.insert("application_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.application_key_id { + map.insert("application_key_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // bucket: always picklable + // bucket_id: always picklable + // application_key: always picklable + // application_key_id: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-b2")] +#[allow(deprecated)] +#[pymethods] +impl B2Config { + #[new] + #[pyo3(signature = ( + bucket, + bucket_id, + application_key = None, + application_key_id = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + bucket: String, + bucket_id: String, + application_key: Option, + application_key_id: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("bucket".to_string(), bucket); + opts.insert("bucket_id".to_string(), bucket_id); + if let Some(v) = application_key { + opts.insert("application_key".to_string(), v); + } + if let Some(v) = application_key_id { + opts.insert("application_key_id".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // bucket: not a map field + // bucket_id: not a map field + // application_key: not a map field + // application_key_id: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// bucket of this backend. + /// required. + #[getter] + fn bucket(&self) -> String { + self.0.bucket.clone() + } + /// bucket id of this backend. + /// required. + #[getter] + fn bucket_id(&self) -> String { + self.0.bucket_id.clone() + } + /// applicationKey of this backend. + /// - If application_key is set, we will take user's input first. + /// - If not, we will try to load it from environment. + #[getter] + fn application_key(&self) -> Option { + self.0.application_key.clone() + } + /// keyID of this backend. + /// - If application_key_id is set, we will take user's input first. + /// - If not, we will try to load it from environment. + #[getter] + fn application_key_id(&self) -> Option { + self.0.application_key_id.clone() + } + /// root of this backend. + /// All operations will happen under this root. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-cacache")] +/// Configuration for the `cacache` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct CacacheConfig(ocore::services::CacacheConfig); + +#[cfg(feature = "services-cacache")] +#[allow(deprecated)] +impl ConfigBuilder for CacacheConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "cacache" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.datadir { + map.insert("datadir".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // datadir: always picklable + ok + } +} + +#[cfg(feature = "services-cacache")] +#[allow(deprecated)] +#[pymethods] +impl CacacheConfig { + #[new] + #[pyo3(signature = ( + datadir = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new(datadir: Option) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = datadir { + opts.insert("datadir".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // datadir: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// That path to the cacache data directory. + #[getter] + fn datadir(&self) -> Option { + self.0.datadir.clone() + } +} + +#[cfg(feature = "services-cos")] +/// Configuration for the `cos` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct CosConfig(ocore::services::CosConfig); + +#[cfg(feature = "services-cos")] +#[allow(deprecated)] +impl ConfigBuilder for CosConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "cos" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.bucket { + map.insert("bucket".to_string(), v.clone()); + } + map.insert( + "disable_config_load".to_string(), + if self.0.disable_config_load { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "enable_versioning".to_string(), + if self.0.enable_versioning { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.secret_id { + map.insert("secret_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.secret_key { + map.insert("secret_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.security_token { + map.insert("security_token".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // bucket: always picklable + // disable_config_load: always picklable + // enable_versioning: always picklable + // endpoint: always picklable + // root: always picklable + // secret_id: always picklable + // secret_key: always picklable + // security_token: always picklable + ok + } +} + +#[cfg(feature = "services-cos")] +#[allow(deprecated)] +#[pymethods] +impl CosConfig { + #[new] + #[pyo3(signature = ( + bucket = None, + disable_config_load = None, + enable_versioning = None, + endpoint = None, + root = None, + secret_id = None, + secret_key = None, + security_token = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + bucket: Option, + disable_config_load: Option, + enable_versioning: Option, + endpoint: Option, + root: Option, + secret_id: Option, + secret_key: Option, + security_token: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = bucket { + opts.insert("bucket".to_string(), v); + } + if let Some(v) = disable_config_load { + opts.insert( + "disable_config_load".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = enable_versioning { + opts.insert( + "enable_versioning".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = secret_id { + opts.insert("secret_id".to_string(), v); + } + if let Some(v) = secret_key { + opts.insert("secret_key".to_string(), v); + } + if let Some(v) = security_token { + opts.insert("security_token".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // bucket: not a map field + // disable_config_load: not a map field + // enable_versioning: not a map field + // endpoint: not a map field + // root: not a map field + // secret_id: not a map field + // secret_key: not a map field + // security_token: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Bucket of this backend. + #[getter] + fn bucket(&self) -> Option { + self.0.bucket.clone() + } + /// Disable config load so that opendal will not load config from + #[getter] + fn disable_config_load(&self) -> Option { + Some(self.0.disable_config_load) + } + /// Deprecated: COS versioning capability is enabled by default. + /// [Deprecated since 0.57.0] COS versioning capability is enabled by default + /// and this option is no longer needed. + #[getter] + fn enable_versioning(&self) -> Option { + Some(self.0.enable_versioning) + } + /// Endpoint of this backend. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// Root of this backend. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Secret ID of this backend. + #[getter] + fn secret_id(&self) -> Option { + self.0.secret_id.clone() + } + /// Secret key of this backend. + #[getter] + fn secret_key(&self) -> Option { + self.0.secret_key.clone() + } + /// Security token (a.k.a. + /// session token) of this backend. + /// This is used for temporary credentials issued by Tencent Cloud STS (e.g. + /// `GetFederationToken` / `AssumeRole`). + /// When `security_token` is provided, it will be used together with `secret_id` + /// and `secret_key` to sign requests, and the `x-cos-security-token` header + /// will be attached automatically by the signer. + /// If this field is not set, OpenDAL will also fall back to reading the token + /// from environment variables `TENCENTCLOUD_TOKEN`, + /// `TENCENTCLOUD_SECURITY_TOKEN` or `QCLOUD_SECRET_TOKEN` (unless + /// `disable_config_load` is enabled). + #[getter] + fn security_token(&self) -> Option { + self.0.security_token.clone() + } +} + +#[cfg(feature = "services-dashmap")] +/// Configuration for the `dashmap` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct DashmapConfig(ocore::services::DashmapConfig); + +#[cfg(feature = "services-dashmap")] +#[allow(deprecated)] +impl ConfigBuilder for DashmapConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "dashmap" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // root: always picklable + ok + } +} + +#[cfg(feature = "services-dashmap")] +#[allow(deprecated)] +#[pymethods] +impl DashmapConfig { + #[new] + #[pyo3(signature = ( + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new(root: Option) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// root path of this backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-dropbox")] +/// Configuration for the `dropbox` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct DropboxConfig(ocore::services::DropboxConfig); + +#[cfg(feature = "services-dropbox")] +#[allow(deprecated)] +impl ConfigBuilder for DropboxConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "dropbox" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.access_token { + map.insert("access_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_id { + map.insert("client_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_secret { + map.insert("client_secret".to_string(), v.clone()); + } + if let Some(v) = &self.0.refresh_token { + map.insert("refresh_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // access_token: always picklable + // client_id: always picklable + // client_secret: always picklable + // refresh_token: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-dropbox")] +#[allow(deprecated)] +#[pymethods] +impl DropboxConfig { + #[new] + #[pyo3(signature = ( + access_token = None, + client_id = None, + client_secret = None, + refresh_token = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + access_token: Option, + client_id: Option, + client_secret: Option, + refresh_token: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = access_token { + opts.insert("access_token".to_string(), v); + } + if let Some(v) = client_id { + opts.insert("client_id".to_string(), v); + } + if let Some(v) = client_secret { + opts.insert("client_secret".to_string(), v); + } + if let Some(v) = refresh_token { + opts.insert("refresh_token".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // access_token: not a map field + // client_id: not a map field + // client_secret: not a map field + // refresh_token: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// access token for dropbox. + #[getter] + fn access_token(&self) -> Option { + self.0.access_token.clone() + } + /// client_id for dropbox. + #[getter] + fn client_id(&self) -> Option { + self.0.client_id.clone() + } + /// client_secret for dropbox. + #[getter] + fn client_secret(&self) -> Option { + self.0.client_secret.clone() + } + /// refresh_token for dropbox. + #[getter] + fn refresh_token(&self) -> Option { + self.0.refresh_token.clone() + } + /// root path for dropbox. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-fs")] +/// Configuration for the `fs` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct FsConfig(ocore::services::FsConfig); + +#[cfg(feature = "services-fs")] +#[allow(deprecated)] +impl ConfigBuilder for FsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "fs" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.atomic_write_dir { + map.insert("atomic_write_dir".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // atomic_write_dir: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-fs")] +#[allow(deprecated)] +#[pymethods] +impl FsConfig { + #[new] + #[pyo3(signature = ( + atomic_write_dir = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + atomic_write_dir: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = atomic_write_dir { + opts.insert("atomic_write_dir".to_string(), v.into_string()); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // atomic_write_dir: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// tmp dir for atomic write + #[getter] + fn atomic_write_dir(&self) -> Option { + self.0.atomic_write_dir.clone().map(crate::PyPath::from) + } + /// root dir for backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-ftp")] +/// Configuration for the `ftp` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct FtpConfig(ocore::services::FtpConfig); + +#[cfg(feature = "services-ftp")] +#[allow(deprecated)] +impl ConfigBuilder for FtpConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "ftp" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.password { + map.insert("password".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.user { + map.insert("user".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // endpoint: always picklable + // password: always picklable + // root: always picklable + // user: always picklable + ok + } +} + +#[cfg(feature = "services-ftp")] +#[allow(deprecated)] +#[pymethods] +impl FtpConfig { + #[new] + #[pyo3(signature = ( + endpoint = None, + password = None, + root = None, + user = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + endpoint: Option, + password: Option, + root: Option, + user: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = password { + opts.insert("password".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = user { + opts.insert("user".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // endpoint: not a map field + // password: not a map field + // root: not a map field + // user: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// endpoint of this backend + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// password of this backend + #[getter] + fn password(&self) -> Option { + self.0.password.clone() + } + /// root of this backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// user of this backend + #[getter] + fn user(&self) -> Option { + self.0.user.clone() + } +} + +#[cfg(feature = "services-gcs")] +/// Configuration for the `gcs` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct GcsConfig(ocore::services::GcsConfig); + +#[cfg(feature = "services-gcs")] +#[allow(deprecated)] +impl ConfigBuilder for GcsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "gcs" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("bucket".to_string(), self.0.bucket.clone()); + map.insert( + "allow_anonymous".to_string(), + if self.0.allow_anonymous { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.credential { + map.insert("credential".to_string(), v.clone()); + } + if let Some(v) = &self.0.credential_path { + map.insert("credential_path".to_string(), v.clone()); + } + if let Some(v) = &self.0.default_storage_class { + map.insert("default_storage_class".to_string(), v.clone()); + } + map.insert( + "disable_config_load".to_string(), + if self.0.disable_config_load { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "disable_vm_metadata".to_string(), + if self.0.disable_vm_metadata { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.predefined_acl { + map.insert("predefined_acl".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.scope { + map.insert("scope".to_string(), v.clone()); + } + if let Some(v) = &self.0.service_account { + map.insert("service_account".to_string(), v.clone()); + } + map.insert( + "skip_signature".to_string(), + if self.0.skip_signature { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.token { + map.insert("token".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // bucket: always picklable + // allow_anonymous: always picklable + // credential: always picklable + // credential_path: always picklable + // default_storage_class: always picklable + // disable_config_load: always picklable + // disable_vm_metadata: always picklable + // endpoint: always picklable + // predefined_acl: always picklable + // root: always picklable + // scope: always picklable + // service_account: always picklable + // skip_signature: always picklable + // token: always picklable + ok + } +} + +#[cfg(feature = "services-gcs")] +#[allow(deprecated)] +#[pymethods] +impl GcsConfig { + #[new] + #[pyo3(signature = ( + bucket, + allow_anonymous = None, + credential = None, + credential_path = None, + default_storage_class = None, + disable_config_load = None, + disable_vm_metadata = None, + endpoint = None, + predefined_acl = None, + root = None, + scope = None, + service_account = None, + skip_signature = None, + token = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + bucket: String, + allow_anonymous: Option, + credential: Option, + credential_path: Option, + default_storage_class: Option, + disable_config_load: Option, + disable_vm_metadata: Option, + endpoint: Option, + predefined_acl: Option, + root: Option, + scope: Option, + service_account: Option, + skip_signature: Option, + token: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("bucket".to_string(), bucket); + if let Some(v) = allow_anonymous { + opts.insert( + "allow_anonymous".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = credential { + opts.insert("credential".to_string(), v); + } + if let Some(v) = credential_path { + opts.insert("credential_path".to_string(), v); + } + if let Some(v) = default_storage_class { + opts.insert("default_storage_class".to_string(), v); + } + if let Some(v) = disable_config_load { + opts.insert( + "disable_config_load".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = disable_vm_metadata { + opts.insert( + "disable_vm_metadata".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = predefined_acl { + opts.insert("predefined_acl".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = scope { + opts.insert("scope".to_string(), v); + } + if let Some(v) = service_account { + opts.insert("service_account".to_string(), v); + } + if let Some(v) = skip_signature { + opts.insert( + "skip_signature".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = token { + opts.insert("token".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // bucket: not a map field + // allow_anonymous: not a map field + // credential: not a map field + // credential_path: not a map field + // default_storage_class: not a map field + // disable_config_load: not a map field + // disable_vm_metadata: not a map field + // endpoint: not a map field + // predefined_acl: not a map field + // root: not a map field + // scope: not a map field + // service_account: not a map field + // skip_signature: not a map field + // token: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// bucket name + #[getter] + fn bucket(&self) -> String { + self.0.bucket.clone() + } + /// Allow opendal to send requests without signing when credentials are not + /// loaded. + /// [Deprecated since 0.57.0] Please use `skip_signature` instead of + /// `allow_anonymous` + #[getter] + fn allow_anonymous(&self) -> Option { + Some(self.0.allow_anonymous) + } + /// Credentials string for GCS service OAuth2 authentication. + #[getter] + fn credential(&self) -> Option { + self.0.credential.clone() + } + /// Local path to credentials file for GCS service OAuth2 authentication. + #[getter] + fn credential_path(&self) -> Option { + self.0.credential_path.clone() + } + /// The default storage class used by gcs. + #[getter] + fn default_storage_class(&self) -> Option { + self.0.default_storage_class.clone() + } + /// Disable loading configuration from the environment. + #[getter] + fn disable_config_load(&self) -> Option { + Some(self.0.disable_config_load) + } + /// Disable attempting to load credentials from the GCE metadata server when + /// running within Google Cloud. + #[getter] + fn disable_vm_metadata(&self) -> Option { + Some(self.0.disable_vm_metadata) + } + /// endpoint URI of GCS service, default is `https://storage.googleapis.com` + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// The predefined acl for GCS. + #[getter] + fn predefined_acl(&self) -> Option { + self.0.predefined_acl.clone() + } + /// root URI, all operations happens under `root` + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Scope for gcs. + #[getter] + fn scope(&self) -> Option { + self.0.scope.clone() + } + /// Service Account for gcs. + #[getter] + fn service_account(&self) -> Option { + self.0.service_account.clone() + } + /// Skip signature will skip loading credentials and signing requests. + #[getter] + fn skip_signature(&self) -> Option { + Some(self.0.skip_signature) + } + /// A Google Cloud OAuth2 token. + /// Takes precedence over `credential` and `credential_path`. + #[getter] + fn token(&self) -> Option { + self.0.token.clone() + } +} + +#[cfg(feature = "services-gdrive")] +/// Configuration for the `gdrive` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct GdriveConfig(ocore::services::GdriveConfig); + +#[cfg(feature = "services-gdrive")] +#[allow(deprecated)] +impl ConfigBuilder for GdriveConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "gdrive" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.access_token { + map.insert("access_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_id { + map.insert("client_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_secret { + map.insert("client_secret".to_string(), v.clone()); + } + if let Some(v) = &self.0.refresh_token { + map.insert("refresh_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // access_token: always picklable + // client_id: always picklable + // client_secret: always picklable + // refresh_token: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-gdrive")] +#[allow(deprecated)] +#[pymethods] +impl GdriveConfig { + #[new] + #[pyo3(signature = ( + access_token = None, + client_id = None, + client_secret = None, + refresh_token = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + access_token: Option, + client_id: Option, + client_secret: Option, + refresh_token: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = access_token { + opts.insert("access_token".to_string(), v); + } + if let Some(v) = client_id { + opts.insert("client_id".to_string(), v); + } + if let Some(v) = client_secret { + opts.insert("client_secret".to_string(), v); + } + if let Some(v) = refresh_token { + opts.insert("refresh_token".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // access_token: not a map field + // client_id: not a map field + // client_secret: not a map field + // refresh_token: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Access token for gdrive. + #[getter] + fn access_token(&self) -> Option { + self.0.access_token.clone() + } + /// Client id for gdrive. + #[getter] + fn client_id(&self) -> Option { + self.0.client_id.clone() + } + /// Client secret for gdrive. + #[getter] + fn client_secret(&self) -> Option { + self.0.client_secret.clone() + } + /// Refresh token for gdrive. + #[getter] + fn refresh_token(&self) -> Option { + self.0.refresh_token.clone() + } + /// The root for gdrive + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-ghac")] +/// Configuration for the `ghac` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct GhacConfig(ocore::services::GhacConfig); + +#[cfg(feature = "services-ghac")] +#[allow(deprecated)] +impl ConfigBuilder for GhacConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "ghac" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.runtime_token { + map.insert("runtime_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.version { + map.insert("version".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // endpoint: always picklable + // root: always picklable + // runtime_token: always picklable + // version: always picklable + ok + } +} + +#[cfg(feature = "services-ghac")] +#[allow(deprecated)] +#[pymethods] +impl GhacConfig { + #[new] + #[pyo3(signature = ( + endpoint = None, + root = None, + runtime_token = None, + version = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + endpoint: Option, + root: Option, + runtime_token: Option, + version: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = runtime_token { + opts.insert("runtime_token".to_string(), v); + } + if let Some(v) = version { + opts.insert("version".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // endpoint: not a map field + // root: not a map field + // runtime_token: not a map field + // version: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// The endpoint for ghac service. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// The root path for ghac. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// The runtime token for ghac service. + #[getter] + fn runtime_token(&self) -> Option { + self.0.runtime_token.clone() + } + /// The version that used by cache. + #[getter] + fn version(&self) -> Option { + self.0.version.clone() + } +} + +#[cfg(feature = "services-goosefs")] +/// Configuration for the `goosefs` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct GoosefsConfig(ocore::services::GoosefsConfig); + +#[cfg(feature = "services-goosefs")] +#[allow(deprecated)] +impl ConfigBuilder for GoosefsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "goosefs" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.auth_type { + map.insert("auth_type".to_string(), v.clone()); + } + if let Some(v) = &self.0.auth_username { + map.insert("auth_username".to_string(), v.clone()); + } + if let Some(v) = &self.0.block_size { + map.insert("block_size".to_string(), v.to_string()); + } + if let Some(v) = &self.0.chunk_size { + map.insert("chunk_size".to_string(), v.to_string()); + } + if let Some(v) = &self.0.master_addr { + map.insert("master_addr".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.write_type { + map.insert("write_type".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // auth_type: always picklable + // auth_username: always picklable + // block_size: always picklable + // chunk_size: always picklable + // master_addr: always picklable + // root: always picklable + // write_type: always picklable + ok + } +} + +#[cfg(feature = "services-goosefs")] +#[allow(deprecated)] +#[pymethods] +impl GoosefsConfig { + #[new] + #[pyo3(signature = ( + auth_type = None, + auth_username = None, + block_size = None, + chunk_size = None, + master_addr = None, + root = None, + write_type = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + auth_type: Option, + auth_username: Option, + block_size: Option, + chunk_size: Option, + master_addr: Option, + root: Option, + write_type: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = auth_type { + opts.insert("auth_type".to_string(), v); + } + if let Some(v) = auth_username { + opts.insert("auth_username".to_string(), v); + } + if let Some(v) = block_size { + opts.insert("block_size".to_string(), v.to_string()); + } + if let Some(v) = chunk_size { + opts.insert("chunk_size".to_string(), v.to_string()); + } + if let Some(v) = master_addr { + opts.insert("master_addr".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = write_type { + opts.insert("write_type".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // auth_type: not a map field + // auth_username: not a map field + // block_size: not a map field + // chunk_size: not a map field + // master_addr: not a map field + // root: not a map field + // write_type: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Authentication type. + /// Supported values: `"nosasl"`, `"simple"`. + /// Default: `"simple"` โ€” PLAIN SASL with usernam + /// e. `"nosasl"` โ€” skip authentication entirely. + #[getter] + fn auth_type(&self) -> Option { + self.0.auth_type.clone() + } + /// Authentication username. + /// Used in SIMPLE mode as the login identity. + /// Default: current OS user (`$USER` / `$USERNAME`). + #[getter] + fn auth_username(&self) -> Option { + self.0.auth_username.clone() + } + /// Block size in bytes for new files (default: 64 MiB). + #[getter] + fn block_size(&self) -> Option { + self.0.block_size + } + /// Chunk size in bytes for streaming RPCs (default: 1 MiB). + #[getter] + fn chunk_size(&self) -> Option { + self.0.chunk_size + } + /// Master address(es) in `host:port` format. + /// For single master: `"10.0.0.1:9200"` For HA (comma-separated): + /// `"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"` When multiple addresses are + /// provided, the client uses `PollingMasterInquireClient` to discover the + /// Primary Master automatically. + /// Resolution precedence at `build()` time (highest โ†’ lowest), following + /// `goosefs-sdk` `docs/CLIENT_CONFIGURATION.md` ยง1: + /// 1. This field (when set on the builder / OpenDAL config map) + /// 2. `GOOSEFS_MASTER_ADDR` environment variable + /// 3. `goosefs.master.rpc.addresses` / `goosefs.master.hostname` in + /// `goosefs-site.properties` `build()` fails with `ConfigInvalid` only when + /// **none** of the above supplies a master address. + #[getter] + fn master_addr(&self) -> Option { + self.0.master_addr.clone() + } + /// Root path of this backend. + /// All operations will happen under this root. + /// Default to `/` if not set. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Default write type for new files. + /// Supported values: `"must_cache"`, `"cache_through"`, `"through"`, + /// `"async_through"`. + /// Default: `"must_cache"`. + #[getter] + fn write_type(&self) -> Option { + self.0.write_type.clone() + } +} + +#[cfg(feature = "services-gridfs")] +/// Configuration for the `gridfs` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct GridfsConfig(ocore::services::GridfsConfig); + +#[cfg(feature = "services-gridfs")] +#[allow(deprecated)] +impl ConfigBuilder for GridfsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "gridfs" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.bucket { + map.insert("bucket".to_string(), v.clone()); + } + if let Some(v) = &self.0.chunk_size { + map.insert("chunk_size".to_string(), v.to_string()); + } + if let Some(v) = &self.0.connection_string { + map.insert("connection_string".to_string(), v.clone()); + } + if let Some(v) = &self.0.database { + map.insert("database".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // bucket: always picklable + // chunk_size: always picklable + // connection_string: always picklable + // database: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-gridfs")] +#[allow(deprecated)] +#[pymethods] +impl GridfsConfig { + #[new] + #[pyo3(signature = ( + bucket = None, + chunk_size = None, + connection_string = None, + database = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + bucket: Option, + chunk_size: Option, + connection_string: Option, + database: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = bucket { + opts.insert("bucket".to_string(), v); + } + if let Some(v) = chunk_size { + opts.insert("chunk_size".to_string(), v.to_string()); + } + if let Some(v) = connection_string { + opts.insert("connection_string".to_string(), v); + } + if let Some(v) = database { + opts.insert("database".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // bucket: not a map field + // chunk_size: not a map field + // connection_string: not a map field + // database: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// The bucket name of the MongoDB GridFs service to read/write. + #[getter] + fn bucket(&self) -> Option { + self.0.bucket.clone() + } + /// The chunk size of the MongoDB GridFs service used to break the user file + /// into chunks. + #[getter] + fn chunk_size(&self) -> Option { + self.0.chunk_size + } + /// The connection string of the MongoDB service. + #[getter] + fn connection_string(&self) -> Option { + self.0.connection_string.clone() + } + /// The database name of the MongoDB GridFs service to read/write. + #[getter] + fn database(&self) -> Option { + self.0.database.clone() + } + /// The working directory, all operations will be performed under it. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-hf")] +/// Configuration for the `hf` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct HfConfig(ocore::services::HfConfig); + +#[cfg(feature = "services-hf")] +#[allow(deprecated)] +impl ConfigBuilder for HfConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "hf" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + // download_mode: opaque field omitted from flat option map + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.repo_id { + map.insert("repo_id".to_string(), v.clone()); + } + // repo_type: opaque field omitted from flat option map + if let Some(v) = &self.0.revision { + map.insert("revision".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.token { + map.insert("token".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // download_mode: always picklable + // endpoint: always picklable + // repo_id: always picklable + // repo_type: always picklable + // revision: always picklable + // root: always picklable + // token: always picklable + ok + } +} + +#[cfg(feature = "services-hf")] +#[allow(deprecated)] +#[pymethods] +impl HfConfig { + #[new] + #[pyo3(signature = ( + download_mode = None, + endpoint = None, + repo_id = None, + repo_type = None, + revision = None, + root = None, + token = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + download_mode: Option, + endpoint: Option, + repo_id: Option, + repo_type: Option, + revision: Option, + root: Option, + token: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = download_mode { + opts.insert("download_mode".to_string(), v); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = repo_id { + opts.insert("repo_id".to_string(), v); + } + if let Some(v) = repo_type { + opts.insert("repo_type".to_string(), v); + } + if let Some(v) = revision { + opts.insert("revision".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = token { + opts.insert("token".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // download_mode: not a map field + // endpoint: not a map field + // repo_id: not a map field + // repo_type: not a map field + // revision: not a map field + // root: not a map field + // token: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + // download_mode: opaque field, no getter + /// Endpoint of the Hugging Face Hub. + /// Default is "https://huggingface.co". + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// Repo id of this backend. + /// This is required. + #[getter] + fn repo_id(&self) -> Option { + self.0.repo_id.clone() + } + // repo_type: opaque field, no getter + /// Revision of this backend. + /// Default is main. + #[getter] + fn revision(&self) -> Option { + self.0.revision.clone() + } + /// Root of this backend. + /// Can be "/path/to/dir". + /// Default is "/". + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Token of this backend. + /// This is optional. + #[getter] + fn token(&self) -> Option { + self.0.token.clone() + } +} + +#[cfg(feature = "services-http")] +/// Configuration for the `http` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct HttpConfig(ocore::services::HttpConfig); + +#[cfg(feature = "services-http")] +#[allow(deprecated)] +impl ConfigBuilder for HttpConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "http" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.password { + map.insert("password".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.token { + map.insert("token".to_string(), v.clone()); + } + if let Some(v) = &self.0.username { + map.insert("username".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // endpoint: always picklable + // password: always picklable + // root: always picklable + // token: always picklable + // username: always picklable + ok + } +} + +#[cfg(feature = "services-http")] +#[allow(deprecated)] +#[pymethods] +impl HttpConfig { + #[new] + #[pyo3(signature = ( + endpoint = None, + password = None, + root = None, + token = None, + username = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + endpoint: Option, + password: Option, + root: Option, + token: Option, + username: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = password { + opts.insert("password".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = token { + opts.insert("token".to_string(), v); + } + if let Some(v) = username { + opts.insert("username".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // endpoint: not a map field + // password: not a map field + // root: not a map field + // token: not a map field + // username: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// endpoint of this backend + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// password of this backend + #[getter] + fn password(&self) -> Option { + self.0.password.clone() + } + /// root of this backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// token of this backend + #[getter] + fn token(&self) -> Option { + self.0.token.clone() + } + /// username of this backend + #[getter] + fn username(&self) -> Option { + self.0.username.clone() + } +} + +#[cfg(feature = "services-ipfs")] +/// Configuration for the `ipfs` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct IpfsConfig(ocore::services::IpfsConfig); + +#[cfg(feature = "services-ipfs")] +#[allow(deprecated)] +impl ConfigBuilder for IpfsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "ipfs" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // endpoint: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-ipfs")] +#[allow(deprecated)] +#[pymethods] +impl IpfsConfig { + #[new] + #[pyo3(signature = ( + endpoint = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + endpoint: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // endpoint: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// IPFS gateway endpoint. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// IPFS root. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-ipmfs")] +/// Configuration for the `ipmfs` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct IpmfsConfig(ocore::services::IpmfsConfig); + +#[cfg(feature = "services-ipmfs")] +#[allow(deprecated)] +impl ConfigBuilder for IpmfsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "ipmfs" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // endpoint: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-ipmfs")] +#[allow(deprecated)] +#[pymethods] +impl IpmfsConfig { + #[new] + #[pyo3(signature = ( + endpoint = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + endpoint: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // endpoint: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Endpoint for ipfs. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// Root for ipfs. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-koofr")] +/// Configuration for the `koofr` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct KoofrConfig(ocore::services::KoofrConfig); + +#[cfg(feature = "services-koofr")] +#[allow(deprecated)] +impl ConfigBuilder for KoofrConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "koofr" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("email".to_string(), self.0.email.clone()); + map.insert("endpoint".to_string(), self.0.endpoint.clone()); + if let Some(v) = &self.0.password { + map.insert("password".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // email: always picklable + // endpoint: always picklable + // password: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-koofr")] +#[allow(deprecated)] +#[pymethods] +impl KoofrConfig { + #[new] + #[pyo3(signature = ( + email, + endpoint, + password = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + email: String, + endpoint: String, + password: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("email".to_string(), email); + opts.insert("endpoint".to_string(), endpoint); + if let Some(v) = password { + opts.insert("password".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // email: not a map field + // endpoint: not a map field + // password: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Koofr email. + #[getter] + fn email(&self) -> String { + self.0.email.clone() + } + /// Koofr endpoint. + #[getter] + fn endpoint(&self) -> String { + self.0.endpoint.clone() + } + /// password of this backend. + /// (Must be the application password) + #[getter] + fn password(&self) -> Option { + self.0.password.clone() + } + /// root of this backend. + /// All operations will happen under this root. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-memcached")] +/// Configuration for the `memcached` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct MemcachedConfig(ocore::services::MemcachedConfig); + +#[cfg(feature = "services-memcached")] +#[allow(deprecated)] +impl ConfigBuilder for MemcachedConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "memcached" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.connection_pool_max_size { + map.insert("connection_pool_max_size".to_string(), v.to_string()); + } + if let Some(d) = &self.0.default_ttl { + map.insert("default_ttl".to_string(), format!("{}s", d.as_secs())); + } + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.password { + map.insert("password".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.username { + map.insert("username".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // connection_pool_max_size: always picklable + // default_ttl: always picklable + // endpoint: always picklable + // password: always picklable + // root: always picklable + // username: always picklable + ok + } +} + +#[cfg(feature = "services-memcached")] +#[allow(deprecated)] +#[pymethods] +impl MemcachedConfig { + #[new] + #[pyo3(signature = ( + connection_pool_max_size = None, + default_ttl = None, + endpoint = None, + password = None, + root = None, + username = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + connection_pool_max_size: Option, + default_ttl: Option, + endpoint: Option, + password: Option, + root: Option, + username: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = connection_pool_max_size { + opts.insert("connection_pool_max_size".to_string(), v.to_string()); + } + if let Some(v) = default_ttl { + opts.insert("default_ttl".to_string(), v); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = password { + opts.insert("password".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = username { + opts.insert("username".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // connection_pool_max_size: not a map field + // default_ttl: not a map field + // endpoint: not a map field + // password: not a map field + // root: not a map field + // username: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// The maximum number of connections allowed. + /// default is 10 + #[getter] + fn connection_pool_max_size(&self) -> Option { + self.0.connection_pool_max_size + } + /// The default ttl for put operations. + /// Accepts a humantime duration string (e.g. + /// "5s"). + #[getter] + fn default_ttl(&self) -> Option { + self.0.default_ttl.map(|d| format!("{}s", d.as_secs())) + } + /// network address of the memcached service. + /// For example: "tcp://localhost:11211" + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// Memcached password, optional. + #[getter] + fn password(&self) -> Option { + self.0.password.clone() + } + /// the working directory of the service. + /// Can be "/path/to/dir" default is "/" + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Memcached username, optional. + #[getter] + fn username(&self) -> Option { + self.0.username.clone() + } +} + +#[cfg(feature = "services-memory")] +/// Configuration for the `memory` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct MemoryConfig(ocore::services::MemoryConfig); + +#[cfg(feature = "services-memory")] +#[allow(deprecated)] +impl ConfigBuilder for MemoryConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "memory" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // root: always picklable + ok + } +} + +#[cfg(feature = "services-memory")] +#[allow(deprecated)] +#[pymethods] +impl MemoryConfig { + #[new] + #[pyo3(signature = ( + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new(root: Option) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// root of the backend. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-mini-moka")] +/// Configuration for the `mini-moka` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct MiniMokaConfig(ocore::services::MiniMokaConfig); + +#[cfg(feature = "services-mini-moka")] +#[allow(deprecated)] +impl ConfigBuilder for MiniMokaConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "mini-moka" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.max_capacity { + map.insert("max_capacity".to_string(), v.to_string()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.time_to_idle { + map.insert("time_to_idle".to_string(), v.clone()); + } + if let Some(v) = &self.0.time_to_live { + map.insert("time_to_live".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // max_capacity: always picklable + // root: always picklable + // time_to_idle: always picklable + // time_to_live: always picklable + ok + } +} + +#[cfg(feature = "services-mini-moka")] +#[allow(deprecated)] +#[pymethods] +impl MiniMokaConfig { + #[new] + #[pyo3(signature = ( + max_capacity = None, + root = None, + time_to_idle = None, + time_to_live = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + max_capacity: Option, + root: Option, + time_to_idle: Option, + time_to_live: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = max_capacity { + opts.insert("max_capacity".to_string(), v.to_string()); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = time_to_idle { + opts.insert("time_to_idle".to_string(), v); + } + if let Some(v) = time_to_live { + opts.insert("time_to_live".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // max_capacity: not a map field + // root: not a map field + // time_to_idle: not a map field + // time_to_live: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Sets the max capacity of the cache. + /// Refer to + /// [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity) + #[getter] + fn max_capacity(&self) -> Option { + self.0.max_capacity + } + /// root path of this backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Sets the time to idle of the cache. + /// Refer to + /// [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle) + #[getter] + fn time_to_idle(&self) -> Option { + self.0.time_to_idle.clone() + } + /// Sets the time to live of the cache. + /// Refer to + /// [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live) + #[getter] + fn time_to_live(&self) -> Option { + self.0.time_to_live.clone() + } +} + +#[cfg(feature = "services-moka")] +/// Configuration for the `moka` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct MokaConfig(ocore::services::MokaConfig); + +#[cfg(feature = "services-moka")] +#[allow(deprecated)] +impl ConfigBuilder for MokaConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "moka" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.max_capacity { + map.insert("max_capacity".to_string(), v.to_string()); + } + if let Some(v) = &self.0.name { + map.insert("name".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.time_to_idle { + map.insert("time_to_idle".to_string(), v.clone()); + } + if let Some(v) = &self.0.time_to_live { + map.insert("time_to_live".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // max_capacity: always picklable + // name: always picklable + // root: always picklable + // time_to_idle: always picklable + // time_to_live: always picklable + ok + } +} + +#[cfg(feature = "services-moka")] +#[allow(deprecated)] +#[pymethods] +impl MokaConfig { + #[new] + #[pyo3(signature = ( + max_capacity = None, + name = None, + root = None, + time_to_idle = None, + time_to_live = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + max_capacity: Option, + name: Option, + root: Option, + time_to_idle: Option, + time_to_live: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = max_capacity { + opts.insert("max_capacity".to_string(), v.to_string()); + } + if let Some(v) = name { + opts.insert("name".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = time_to_idle { + opts.insert("time_to_idle".to_string(), v); + } + if let Some(v) = time_to_live { + opts.insert("time_to_live".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // max_capacity: not a map field + // name: not a map field + // root: not a map field + // time_to_idle: not a map field + // time_to_live: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Sets the max capacity of the cache. + /// Refer to + /// [`moka::future::CacheBuilder::max_capacity`](https://docs.rs/moka/latest/moka/future/struct.CacheBuilder.html#method.max_capacity) + #[getter] + fn max_capacity(&self) -> Option { + self.0.max_capacity + } + /// Name for this cache instance. + #[getter] + fn name(&self) -> Option { + self.0.name.clone() + } + /// root path of this backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Sets the time to idle of the cache. + /// Refer to + /// [`moka::future::CacheBuilder::time_to_idle`](https://docs.rs/moka/latest/moka/future/struct.CacheBuilder.html#method.time_to_idle) + #[getter] + fn time_to_idle(&self) -> Option { + self.0.time_to_idle.clone() + } + /// Sets the time to live of the cache. + /// Refer to + /// [`moka::future::CacheBuilder::time_to_live`](https://docs.rs/moka/latest/moka/future/struct.CacheBuilder.html#method.time_to_live) + #[getter] + fn time_to_live(&self) -> Option { + self.0.time_to_live.clone() + } +} + +#[cfg(feature = "services-mongodb")] +/// Configuration for the `mongodb` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct MongodbConfig(ocore::services::MongodbConfig); + +#[cfg(feature = "services-mongodb")] +#[allow(deprecated)] +impl ConfigBuilder for MongodbConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "mongodb" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.collection { + map.insert("collection".to_string(), v.clone()); + } + if let Some(v) = &self.0.connection_string { + map.insert("connection_string".to_string(), v.clone()); + } + if let Some(v) = &self.0.database { + map.insert("database".to_string(), v.clone()); + } + if let Some(v) = &self.0.key_field { + map.insert("key_field".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.value_field { + map.insert("value_field".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // collection: always picklable + // connection_string: always picklable + // database: always picklable + // key_field: always picklable + // root: always picklable + // value_field: always picklable + ok + } +} + +#[cfg(feature = "services-mongodb")] +#[allow(deprecated)] +#[pymethods] +impl MongodbConfig { + #[new] + #[pyo3(signature = ( + collection = None, + connection_string = None, + database = None, + key_field = None, + root = None, + value_field = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + collection: Option, + connection_string: Option, + database: Option, + key_field: Option, + root: Option, + value_field: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = collection { + opts.insert("collection".to_string(), v); + } + if let Some(v) = connection_string { + opts.insert("connection_string".to_string(), v); + } + if let Some(v) = database { + opts.insert("database".to_string(), v); + } + if let Some(v) = key_field { + opts.insert("key_field".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = value_field { + opts.insert("value_field".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // collection: not a map field + // connection_string: not a map field + // database: not a map field + // key_field: not a map field + // root: not a map field + // value_field: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// collection of this backend + #[getter] + fn collection(&self) -> Option { + self.0.collection.clone() + } + /// connection string of this backend + #[getter] + fn connection_string(&self) -> Option { + self.0.connection_string.clone() + } + /// database of this backend + #[getter] + fn database(&self) -> Option { + self.0.database.clone() + } + /// key field of this backend + #[getter] + fn key_field(&self) -> Option { + self.0.key_field.clone() + } + /// root of this backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// value field of this backend + #[getter] + fn value_field(&self) -> Option { + self.0.value_field.clone() + } +} + +#[cfg(feature = "services-mysql")] +/// Configuration for the `mysql` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct MysqlConfig(ocore::services::MysqlConfig); + +#[cfg(feature = "services-mysql")] +#[allow(deprecated)] +impl ConfigBuilder for MysqlConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "mysql" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.connection_string { + map.insert("connection_string".to_string(), v.clone()); + } + if let Some(v) = &self.0.key_field { + map.insert("key_field".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.table { + map.insert("table".to_string(), v.clone()); + } + if let Some(v) = &self.0.value_field { + map.insert("value_field".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // connection_string: always picklable + // key_field: always picklable + // root: always picklable + // table: always picklable + // value_field: always picklable + ok + } +} + +#[cfg(feature = "services-mysql")] +#[allow(deprecated)] +#[pymethods] +impl MysqlConfig { + #[new] + #[pyo3(signature = ( + connection_string = None, + key_field = None, + root = None, + table = None, + value_field = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + connection_string: Option, + key_field: Option, + root: Option, + table: Option, + value_field: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = connection_string { + opts.insert("connection_string".to_string(), v); + } + if let Some(v) = key_field { + opts.insert("key_field".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = table { + opts.insert("table".to_string(), v); + } + if let Some(v) = value_field { + opts.insert("value_field".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // connection_string: not a map field + // key_field: not a map field + // root: not a map field + // table: not a map field + // value_field: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// This connection string is used to connect to the mysql service. + /// There are url based formats. + /// The format of connect string resembles the url format of the mysql client. + /// The format is: + /// `[scheme://][user[:[password]]@]host[:port][/schema][?attribute1=value1&attribute2=value2...` + /// - `mysql://user@localhost` - `mysql://user:password@localhost` - + /// `mysql://user:password@localhost:3306` - + /// `mysql://user:password@localhost:3306/db` For more information, please refer + /// to . + #[getter] + fn connection_string(&self) -> Option { + self.0.connection_string.clone() + } + /// The key field name for mysql. + #[getter] + fn key_field(&self) -> Option { + self.0.key_field.clone() + } + /// The root for mysql. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// The table name for mysql. + #[getter] + fn table(&self) -> Option { + self.0.table.clone() + } + /// The value field name for mysql. + #[getter] + fn value_field(&self) -> Option { + self.0.value_field.clone() + } +} + +#[cfg(feature = "services-obs")] +/// Configuration for the `obs` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct ObsConfig(ocore::services::ObsConfig); + +#[cfg(feature = "services-obs")] +#[allow(deprecated)] +impl ConfigBuilder for ObsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "obs" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.access_key_id { + map.insert("access_key_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.bucket { + map.insert("bucket".to_string(), v.clone()); + } + map.insert( + "enable_versioning".to_string(), + if self.0.enable_versioning { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.secret_access_key { + map.insert("secret_access_key".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // access_key_id: always picklable + // bucket: always picklable + // enable_versioning: always picklable + // endpoint: always picklable + // root: always picklable + // secret_access_key: always picklable + ok + } +} + +#[cfg(feature = "services-obs")] +#[allow(deprecated)] +#[pymethods] +impl ObsConfig { + #[new] + #[pyo3(signature = ( + access_key_id = None, + bucket = None, + enable_versioning = None, + endpoint = None, + root = None, + secret_access_key = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + access_key_id: Option, + bucket: Option, + enable_versioning: Option, + endpoint: Option, + root: Option, + secret_access_key: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = access_key_id { + opts.insert("access_key_id".to_string(), v); + } + if let Some(v) = bucket { + opts.insert("bucket".to_string(), v); + } + if let Some(v) = enable_versioning { + opts.insert( + "enable_versioning".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = secret_access_key { + opts.insert("secret_access_key".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // access_key_id: not a map field + // bucket: not a map field + // enable_versioning: not a map field + // endpoint: not a map field + // root: not a map field + // secret_access_key: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Access key id for obs. + #[getter] + fn access_key_id(&self) -> Option { + self.0.access_key_id.clone() + } + /// Bucket for obs. + #[getter] + fn bucket(&self) -> Option { + self.0.bucket.clone() + } + /// Deprecated: OBS versioning capability is not controlled by service config. + /// [Deprecated since 0.57.0] OBS versioning capability is not controlled by + /// this option and this option is no longer needed. + #[getter] + fn enable_versioning(&self) -> Option { + Some(self.0.enable_versioning) + } + /// Endpoint for obs. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// Root for obs. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Secret access key for obs. + #[getter] + fn secret_access_key(&self) -> Option { + self.0.secret_access_key.clone() + } +} + +#[cfg(feature = "services-onedrive")] +/// Configuration for the `onedrive` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct OnedriveConfig(ocore::services::OnedriveConfig); + +#[cfg(feature = "services-onedrive")] +#[allow(deprecated)] +impl ConfigBuilder for OnedriveConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "onedrive" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.access_token { + map.insert("access_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_id { + map.insert("client_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.client_secret { + map.insert("client_secret".to_string(), v.clone()); + } + map.insert( + "enable_versioning".to_string(), + if self.0.enable_versioning { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.refresh_token { + map.insert("refresh_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // access_token: always picklable + // client_id: always picklable + // client_secret: always picklable + // enable_versioning: always picklable + // refresh_token: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-onedrive")] +#[allow(deprecated)] +#[pymethods] +impl OnedriveConfig { + #[new] + #[pyo3(signature = ( + access_token = None, + client_id = None, + client_secret = None, + enable_versioning = None, + refresh_token = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + access_token: Option, + client_id: Option, + client_secret: Option, + enable_versioning: Option, + refresh_token: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = access_token { + opts.insert("access_token".to_string(), v); + } + if let Some(v) = client_id { + opts.insert("client_id".to_string(), v); + } + if let Some(v) = client_secret { + opts.insert("client_secret".to_string(), v); + } + if let Some(v) = enable_versioning { + opts.insert( + "enable_versioning".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = refresh_token { + opts.insert("refresh_token".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // access_token: not a map field + // client_id: not a map field + // client_secret: not a map field + // enable_versioning: not a map field + // refresh_token: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Microsoft Graph API (also OneDrive API) access token + #[getter] + fn access_token(&self) -> Option { + self.0.access_token.clone() + } + /// Microsoft Graph API Application (client) ID that is in the Azure's app + /// registration portal + #[getter] + fn client_id(&self) -> Option { + self.0.client_id.clone() + } + /// Microsoft Graph API Application client secret that is in the Azure's app + /// registration portal + #[getter] + fn client_secret(&self) -> Option { + self.0.client_secret.clone() + } + /// Deprecated: OneDrive versioning capability is enabled by default. + /// [Deprecated since 0.57.0] OneDrive versioning capability is enabled by + /// default and this option is no longer needed. + #[getter] + fn enable_versioning(&self) -> Option { + Some(self.0.enable_versioning) + } + /// Microsoft Graph API (also OneDrive API) refresh token + #[getter] + fn refresh_token(&self) -> Option { + self.0.refresh_token.clone() + } + /// The root path for the OneDrive service for the file access + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-oss")] +/// Configuration for the `oss` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct OssConfig(ocore::services::OssConfig); + +#[cfg(feature = "services-oss")] +#[allow(deprecated)] +impl ConfigBuilder for OssConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "oss" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("bucket".to_string(), self.0.bucket.clone()); + if let Some(v) = &self.0.access_key_id { + map.insert("access_key_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.access_key_secret { + map.insert("access_key_secret".to_string(), v.clone()); + } + if let Some(v) = &self.0.addressing_style { + map.insert("addressing_style".to_string(), v.clone()); + } + map.insert( + "allow_anonymous".to_string(), + if self.0.allow_anonymous { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.batch_max_operations { + map.insert("batch_max_operations".to_string(), v.to_string()); + } + if let Some(v) = &self.0.delete_max_size { + map.insert("delete_max_size".to_string(), v.to_string()); + } + map.insert( + "enable_versioning".to_string(), + if self.0.enable_versioning { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.external_id { + map.insert("external_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.oidc_provider_arn { + map.insert("oidc_provider_arn".to_string(), v.clone()); + } + if let Some(v) = &self.0.oidc_token_file { + map.insert("oidc_token_file".to_string(), v.clone()); + } + if let Some(v) = &self.0.presign_addressing_style { + map.insert("presign_addressing_style".to_string(), v.clone()); + } + if let Some(v) = &self.0.presign_endpoint { + map.insert("presign_endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.role_arn { + map.insert("role_arn".to_string(), v.clone()); + } + if let Some(v) = &self.0.role_session_name { + map.insert("role_session_name".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.security_token { + map.insert("security_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.server_side_encryption { + map.insert("server_side_encryption".to_string(), v.clone()); + } + if let Some(v) = &self.0.server_side_encryption_key_id { + map.insert("server_side_encryption_key_id".to_string(), v.clone()); + } + map.insert( + "skip_signature".to_string(), + if self.0.skip_signature { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.sts_endpoint { + map.insert("sts_endpoint".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // bucket: always picklable + // access_key_id: always picklable + // access_key_secret: always picklable + // addressing_style: always picklable + // allow_anonymous: always picklable + // batch_max_operations: always picklable + // delete_max_size: always picklable + // enable_versioning: always picklable + // endpoint: always picklable + // external_id: always picklable + // oidc_provider_arn: always picklable + // oidc_token_file: always picklable + // presign_addressing_style: always picklable + // presign_endpoint: always picklable + // role_arn: always picklable + // role_session_name: always picklable + // root: always picklable + // security_token: always picklable + // server_side_encryption: always picklable + // server_side_encryption_key_id: always picklable + // skip_signature: always picklable + // sts_endpoint: always picklable + ok + } +} + +#[cfg(feature = "services-oss")] +#[allow(deprecated)] +#[pymethods] +impl OssConfig { + #[new] + #[pyo3(signature = ( + bucket, + access_key_id = None, + access_key_secret = None, + addressing_style = None, + allow_anonymous = None, + batch_max_operations = None, + delete_max_size = None, + enable_versioning = None, + endpoint = None, + external_id = None, + oidc_provider_arn = None, + oidc_token_file = None, + presign_addressing_style = None, + presign_endpoint = None, + role_arn = None, + role_session_name = None, + root = None, + security_token = None, + server_side_encryption = None, + server_side_encryption_key_id = None, + skip_signature = None, + sts_endpoint = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + bucket: String, + access_key_id: Option, + access_key_secret: Option, + addressing_style: Option, + allow_anonymous: Option, + batch_max_operations: Option, + delete_max_size: Option, + enable_versioning: Option, + endpoint: Option, + external_id: Option, + oidc_provider_arn: Option, + oidc_token_file: Option, + presign_addressing_style: Option, + presign_endpoint: Option, + role_arn: Option, + role_session_name: Option, + root: Option, + security_token: Option, + server_side_encryption: Option, + server_side_encryption_key_id: Option, + skip_signature: Option, + sts_endpoint: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("bucket".to_string(), bucket); + if let Some(v) = access_key_id { + opts.insert("access_key_id".to_string(), v); + } + if let Some(v) = access_key_secret { + opts.insert("access_key_secret".to_string(), v); + } + if let Some(v) = addressing_style { + opts.insert("addressing_style".to_string(), v); + } + if let Some(v) = allow_anonymous { + opts.insert( + "allow_anonymous".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = batch_max_operations { + opts.insert("batch_max_operations".to_string(), v.to_string()); + } + if let Some(v) = delete_max_size { + opts.insert("delete_max_size".to_string(), v.to_string()); + } + if let Some(v) = enable_versioning { + opts.insert( + "enable_versioning".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = external_id { + opts.insert("external_id".to_string(), v); + } + if let Some(v) = oidc_provider_arn { + opts.insert("oidc_provider_arn".to_string(), v); + } + if let Some(v) = oidc_token_file { + opts.insert("oidc_token_file".to_string(), v); + } + if let Some(v) = presign_addressing_style { + opts.insert("presign_addressing_style".to_string(), v); + } + if let Some(v) = presign_endpoint { + opts.insert("presign_endpoint".to_string(), v); + } + if let Some(v) = role_arn { + opts.insert("role_arn".to_string(), v); + } + if let Some(v) = role_session_name { + opts.insert("role_session_name".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = security_token { + opts.insert("security_token".to_string(), v); + } + if let Some(v) = server_side_encryption { + opts.insert("server_side_encryption".to_string(), v); + } + if let Some(v) = server_side_encryption_key_id { + opts.insert("server_side_encryption_key_id".to_string(), v); + } + if let Some(v) = skip_signature { + opts.insert( + "skip_signature".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = sts_endpoint { + opts.insert("sts_endpoint".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // bucket: not a map field + // access_key_id: not a map field + // access_key_secret: not a map field + // addressing_style: not a map field + // allow_anonymous: not a map field + // batch_max_operations: not a map field + // delete_max_size: not a map field + // enable_versioning: not a map field + // endpoint: not a map field + // external_id: not a map field + // oidc_provider_arn: not a map field + // oidc_token_file: not a map field + // presign_addressing_style: not a map field + // presign_endpoint: not a map field + // role_arn: not a map field + // role_session_name: not a map field + // root: not a map field + // security_token: not a map field + // server_side_encryption: not a map field + // server_side_encryption_key_id: not a map field + // skip_signature: not a map field + // sts_endpoint: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Bucket for oss. + #[getter] + fn bucket(&self) -> String { + self.0.bucket.clone() + } + /// Access key id for oss. + /// - this field if it's `is_some` - env value: `ALIBABA_CLOUD_ACCESS_KEY_ID` + #[getter] + fn access_key_id(&self) -> Option { + self.0.access_key_id.clone() + } + /// Access key secret for oss. + /// - this field if it's `is_some` - env value: + /// `ALIBABA_CLOUD_ACCESS_KEY_SECRET` + #[getter] + fn access_key_secret(&self) -> Option { + self.0.access_key_secret.clone() + } + /// Addressing style for oss. + #[getter] + fn addressing_style(&self) -> Option { + self.0.addressing_style.clone() + } + /// Allow anonymous for oss. + /// [Deprecated since 0.57.0] Please use `skip_signature` instead of + /// `allow_anonymous` + #[getter] + fn allow_anonymous(&self) -> Option { + Some(self.0.allow_anonymous) + } + /// Deprecated: OSS delete batch capability is enabled by default. + /// [Deprecated since 0.57.0] OSS delete batch capability is enabled by default. + /// Use CapabilityOverrideLayer to override delete_max_size for specific + /// endpoints. + #[getter] + fn batch_max_operations(&self) -> Option { + self.0.batch_max_operations + } + /// Deprecated: OSS delete batch capability is enabled by default. + /// [Deprecated since 0.57.0] OSS delete batch capability is enabled by default. + /// Use CapabilityOverrideLayer to override delete_max_size for specific + /// endpoints. + #[getter] + fn delete_max_size(&self) -> Option { + self.0.delete_max_size + } + /// Deprecated: OSS versioning capability is enabled by default. + /// [Deprecated since 0.57.0] OSS versioning capability is enabled by default + /// and this option is no longer needed. + #[getter] + fn enable_versioning(&self) -> Option { + Some(self.0.enable_versioning) + } + /// Endpoint for oss. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// external_id for this backend. + #[getter] + fn external_id(&self) -> Option { + self.0.external_id.clone() + } + /// `oidc_provider_arn` will be loaded from - this field if it's `is_some` - env + /// value: `ALIBABA_CLOUD_OIDC_PROVIDER_ARN` + #[getter] + fn oidc_provider_arn(&self) -> Option { + self.0.oidc_provider_arn.clone() + } + /// `oidc_token_file` will be loaded from - this field if it's `is_some` - env + /// value: `ALIBABA_CLOUD_OIDC_TOKEN_FILE` + #[getter] + fn oidc_token_file(&self) -> Option { + self.0.oidc_token_file.clone() + } + /// Pre sign addressing style for oss. + #[getter] + fn presign_addressing_style(&self) -> Option { + self.0.presign_addressing_style.clone() + } + /// Presign endpoint for oss. + #[getter] + fn presign_endpoint(&self) -> Option { + self.0.presign_endpoint.clone() + } + /// If `role_arn` is set, we will use already known config as source credential + /// to assume role with `role_arn`. + /// - this field if it's `is_some` - env value: `ALIBABA_CLOUD_ROLE_ARN` + #[getter] + fn role_arn(&self) -> Option { + self.0.role_arn.clone() + } + /// role_session_name for this backend. + #[getter] + fn role_session_name(&self) -> Option { + self.0.role_session_name.clone() + } + /// Root for oss. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// `security_token` will be loaded from - this field if it's `is_some` - env + /// value: `ALIBABA_CLOUD_SECURITY_TOKEN` + #[getter] + fn security_token(&self) -> Option { + self.0.security_token.clone() + } + /// Server side encryption for oss. + #[getter] + fn server_side_encryption(&self) -> Option { + self.0.server_side_encryption.clone() + } + /// Server side encryption key id for oss. + #[getter] + fn server_side_encryption_key_id(&self) -> Option { + self.0.server_side_encryption_key_id.clone() + } + /// Skip signature will skip loading credentials and signing requests. + #[getter] + fn skip_signature(&self) -> Option { + Some(self.0.skip_signature) + } + /// `sts_endpoint` will be loaded from - this field if it's `is_some` - env + /// value: `ALIBABA_CLOUD_STS_ENDPOINT` + #[getter] + fn sts_endpoint(&self) -> Option { + self.0.sts_endpoint.clone() + } +} + +#[cfg(feature = "services-persy")] +/// Configuration for the `persy` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct PersyConfig(ocore::services::PersyConfig); + +#[cfg(feature = "services-persy")] +#[allow(deprecated)] +impl ConfigBuilder for PersyConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "persy" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.datafile { + map.insert("datafile".to_string(), v.clone()); + } + if let Some(v) = &self.0.index { + map.insert("index".to_string(), v.clone()); + } + if let Some(v) = &self.0.segment { + map.insert("segment".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // datafile: always picklable + // index: always picklable + // segment: always picklable + ok + } +} + +#[cfg(feature = "services-persy")] +#[allow(deprecated)] +#[pymethods] +impl PersyConfig { + #[new] + #[pyo3(signature = ( + datafile = None, + index = None, + segment = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + datafile: Option, + index: Option, + segment: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = datafile { + opts.insert("datafile".to_string(), v); + } + if let Some(v) = index { + opts.insert("index".to_string(), v); + } + if let Some(v) = segment { + opts.insert("segment".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // datafile: not a map field + // index: not a map field + // segment: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// That path to the persy data file. + /// The directory in the path must already exist. + #[getter] + fn datafile(&self) -> Option { + self.0.datafile.clone() + } + /// That name of the persy index. + #[getter] + fn index(&self) -> Option { + self.0.index.clone() + } + /// That name of the persy segment. + #[getter] + fn segment(&self) -> Option { + self.0.segment.clone() + } +} + +#[cfg(feature = "services-postgresql")] +/// Configuration for the `postgresql` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct PostgresqlConfig(ocore::services::PostgresqlConfig); + +#[cfg(feature = "services-postgresql")] +#[allow(deprecated)] +impl ConfigBuilder for PostgresqlConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "postgresql" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.connection_string { + map.insert("connection_string".to_string(), v.clone()); + } + if let Some(v) = &self.0.key_field { + map.insert("key_field".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.table { + map.insert("table".to_string(), v.clone()); + } + if let Some(v) = &self.0.value_field { + map.insert("value_field".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // connection_string: always picklable + // key_field: always picklable + // root: always picklable + // table: always picklable + // value_field: always picklable + ok + } +} + +#[cfg(feature = "services-postgresql")] +#[allow(deprecated)] +#[pymethods] +impl PostgresqlConfig { + #[new] + #[pyo3(signature = ( + connection_string = None, + key_field = None, + root = None, + table = None, + value_field = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + connection_string: Option, + key_field: Option, + root: Option, + table: Option, + value_field: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = connection_string { + opts.insert("connection_string".to_string(), v); + } + if let Some(v) = key_field { + opts.insert("key_field".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = table { + opts.insert("table".to_string(), v); + } + if let Some(v) = value_field { + opts.insert("value_field".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // connection_string: not a map field + // key_field: not a map field + // root: not a map field + // table: not a map field + // value_field: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// The URL should be with a scheme of either `postgres://` or `postgresql://`. + /// - `postgresql://user@localhost` - + /// `postgresql://user:password@%2Fvar%2Flib%2Fpostgresql/mydb?connect_timeout=10` + /// - + /// `postgresql://user@host1:1234,host2,host3:5678?target_session_attrs=read-write` + /// - `postgresql:///mydb?user=user&host=/var/lib/postgresql` For more + /// information, please visit + /// . + #[getter] + fn connection_string(&self) -> Option { + self.0.connection_string.clone() + } + /// the key field of postgresql + #[getter] + fn key_field(&self) -> Option { + self.0.key_field.clone() + } + /// Root of this backend. + /// All operations will happen under this root. + /// Default to `/` if not set. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// the table of postgresql + #[getter] + fn table(&self) -> Option { + self.0.table.clone() + } + /// the value field of postgresql + #[getter] + fn value_field(&self) -> Option { + self.0.value_field.clone() + } +} + +#[cfg(feature = "services-redb")] +/// Configuration for the `redb` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct RedbConfig(ocore::services::RedbConfig); + +#[cfg(feature = "services-redb")] +#[allow(deprecated)] +impl ConfigBuilder for RedbConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "redb" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.datadir { + map.insert("datadir".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.table { + map.insert("table".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // datadir: always picklable + // root: always picklable + // table: always picklable + ok + } +} + +#[cfg(feature = "services-redb")] +#[allow(deprecated)] +#[pymethods] +impl RedbConfig { + #[new] + #[pyo3(signature = ( + datadir = None, + root = None, + table = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + datadir: Option, + root: Option, + table: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = datadir { + opts.insert("datadir".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = table { + opts.insert("table".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // datadir: not a map field + // root: not a map field + // table: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// path to the redb data directory. + #[getter] + fn datadir(&self) -> Option { + self.0.datadir.clone() + } + /// The root for redb. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// The table name for redb. + #[getter] + fn table(&self) -> Option { + self.0.table.clone() + } +} + +#[cfg(feature = "services-redis")] +/// Configuration for the `redis` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct RedisConfig(ocore::services::RedisConfig); + +#[cfg(feature = "services-redis")] +#[allow(deprecated)] +impl ConfigBuilder for RedisConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "redis" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("db".to_string(), self.0.db.to_string()); + if let Some(v) = &self.0.cluster_endpoints { + map.insert("cluster_endpoints".to_string(), v.clone()); + } + if let Some(v) = &self.0.connection_pool_max_size { + map.insert("connection_pool_max_size".to_string(), v.to_string()); + } + if let Some(d) = &self.0.default_ttl { + map.insert("default_ttl".to_string(), format!("{}s", d.as_secs())); + } + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.password { + map.insert("password".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.username { + map.insert("username".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // db: always picklable + // cluster_endpoints: always picklable + // connection_pool_max_size: always picklable + // default_ttl: always picklable + // endpoint: always picklable + // password: always picklable + // root: always picklable + // username: always picklable + ok + } +} + +#[cfg(feature = "services-redis")] +#[allow(deprecated)] +#[pymethods] +impl RedisConfig { + #[new] + #[pyo3(signature = ( + db, + cluster_endpoints = None, + connection_pool_max_size = None, + default_ttl = None, + endpoint = None, + password = None, + root = None, + username = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + db: Option, + cluster_endpoints: Option, + connection_pool_max_size: Option, + default_ttl: Option, + endpoint: Option, + password: Option, + root: Option, + username: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = db { + opts.insert("db".to_string(), v.to_string()); + } + if let Some(v) = cluster_endpoints { + opts.insert("cluster_endpoints".to_string(), v); + } + if let Some(v) = connection_pool_max_size { + opts.insert("connection_pool_max_size".to_string(), v.to_string()); + } + if let Some(v) = default_ttl { + opts.insert("default_ttl".to_string(), v); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = password { + opts.insert("password".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = username { + opts.insert("username".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // db: not a map field + // cluster_endpoints: not a map field + // connection_pool_max_size: not a map field + // default_ttl: not a map field + // endpoint: not a map field + // password: not a map field + // root: not a map field + // username: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// the number of DBs redis can take is unlimited default is db 0 + #[getter] + fn db(&self) -> i64 { + self.0.db + } + /// network address of the Redis cluster service. + /// Can be "tcp://127.0.0.1:6379,tcp://127.0.0.1:6380,tcp://127.0.0.1:6381", + /// e.g. + /// default is None + #[getter] + fn cluster_endpoints(&self) -> Option { + self.0.cluster_endpoints.clone() + } + /// The maximum number of connections allowed. + /// default is 10 + #[getter] + fn connection_pool_max_size(&self) -> Option { + self.0.connection_pool_max_size + } + /// The default ttl for put operations. + /// Accepts a humantime duration string (e.g. + /// "5s"). + #[getter] + fn default_ttl(&self) -> Option { + self.0.default_ttl.map(|d| format!("{}s", d.as_secs())) + } + /// network address of the Redis service. + /// Can be "tcp://127.0.0.1:6379", e.g. + /// default is "tcp://127.0.0.1:6379" + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// the password for authentication default is None + #[getter] + fn password(&self) -> Option { + self.0.password.clone() + } + /// the working directory of the Redis service. + /// Can be "/path/to/dir" default is "/" + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// the username to connect redis service. + /// default is None + #[getter] + fn username(&self) -> Option { + self.0.username.clone() + } +} + +#[cfg(feature = "services-s3")] +/// Configuration for the `s3` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct S3Config(ocore::services::S3Config); + +#[cfg(feature = "services-s3")] +#[allow(deprecated)] +impl ConfigBuilder for S3Config { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "s3" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("bucket".to_string(), self.0.bucket.clone()); + if let Some(v) = &self.0.access_key_id { + map.insert("access_key_id".to_string(), v.clone()); + } + map.insert( + "allow_anonymous".to_string(), + if self.0.allow_anonymous { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.assume_role_duration_seconds { + map.insert("assume_role_duration_seconds".to_string(), v.to_string()); + } + // assume_role_session_tags: map field omitted from flat option map + if let Some(v) = &self.0.batch_max_operations { + map.insert("batch_max_operations".to_string(), v.to_string()); + } + if let Some(v) = &self.0.checksum_algorithm { + map.insert("checksum_algorithm".to_string(), v.clone()); + } + if let Some(v) = &self.0.default_acl { + map.insert("default_acl".to_string(), v.clone()); + } + if let Some(v) = &self.0.default_storage_class { + map.insert("default_storage_class".to_string(), v.clone()); + } + if let Some(v) = &self.0.delete_max_size { + map.insert("delete_max_size".to_string(), v.to_string()); + } + map.insert( + "disable_config_load".to_string(), + if self.0.disable_config_load { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "disable_ec2_metadata".to_string(), + if self.0.disable_ec2_metadata { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "disable_list_objects_v2".to_string(), + if self.0.disable_list_objects_v2 { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "disable_stat_with_override".to_string(), + if self.0.disable_stat_with_override { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "disable_write_with_if_match".to_string(), + if self.0.disable_write_with_if_match { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "enable_request_payer".to_string(), + if self.0.enable_request_payer { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "enable_versioning".to_string(), + if self.0.enable_versioning { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "enable_virtual_host_style".to_string(), + if self.0.enable_virtual_host_style { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "enable_write_with_append".to_string(), + if self.0.enable_write_with_append { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.external_id { + map.insert("external_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.region { + map.insert("region".to_string(), v.clone()); + } + if let Some(v) = &self.0.role_arn { + map.insert("role_arn".to_string(), v.clone()); + } + if let Some(v) = &self.0.role_session_name { + map.insert("role_session_name".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.secret_access_key { + map.insert("secret_access_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.server_side_encryption { + map.insert("server_side_encryption".to_string(), v.clone()); + } + if let Some(v) = &self.0.server_side_encryption_aws_kms_key_id { + map.insert( + "server_side_encryption_aws_kms_key_id".to_string(), + v.clone(), + ); + } + if let Some(v) = &self.0.server_side_encryption_customer_algorithm { + map.insert( + "server_side_encryption_customer_algorithm".to_string(), + v.clone(), + ); + } + if let Some(v) = &self.0.server_side_encryption_customer_key { + map.insert("server_side_encryption_customer_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.server_side_encryption_customer_key_md5 { + map.insert( + "server_side_encryption_customer_key_md5".to_string(), + v.clone(), + ); + } + if let Some(v) = &self.0.session_token { + map.insert("session_token".to_string(), v.clone()); + } + map.insert( + "skip_signature".to_string(), + if self.0.skip_signature { + "true" + } else { + "false" + } + .to_string(), + ); + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // bucket: always picklable + // access_key_id: always picklable + // allow_anonymous: always picklable + // assume_role_duration_seconds: always picklable + if self.0.assume_role_session_tags.is_some() { + ok = false; + } + // batch_max_operations: always picklable + // checksum_algorithm: always picklable + // default_acl: always picklable + // default_storage_class: always picklable + // delete_max_size: always picklable + // disable_config_load: always picklable + // disable_ec2_metadata: always picklable + // disable_list_objects_v2: always picklable + // disable_stat_with_override: always picklable + // disable_write_with_if_match: always picklable + // enable_request_payer: always picklable + // enable_versioning: always picklable + // enable_virtual_host_style: always picklable + // enable_write_with_append: always picklable + // endpoint: always picklable + // external_id: always picklable + // region: always picklable + // role_arn: always picklable + // role_session_name: always picklable + // root: always picklable + // secret_access_key: always picklable + // server_side_encryption: always picklable + // server_side_encryption_aws_kms_key_id: always picklable + // server_side_encryption_customer_algorithm: always picklable + // server_side_encryption_customer_key: always picklable + // server_side_encryption_customer_key_md5: always picklable + // session_token: always picklable + // skip_signature: always picklable + ok + } +} + +#[cfg(feature = "services-s3")] +#[allow(deprecated)] +#[pymethods] +impl S3Config { + #[new] + #[pyo3(signature = ( + bucket, + access_key_id = None, + allow_anonymous = None, + assume_role_duration_seconds = None, + assume_role_session_tags = None, + batch_max_operations = None, + checksum_algorithm = None, + default_acl = None, + default_storage_class = None, + delete_max_size = None, + disable_config_load = None, + disable_ec2_metadata = None, + disable_list_objects_v2 = None, + disable_stat_with_override = None, + disable_write_with_if_match = None, + enable_request_payer = None, + enable_versioning = None, + enable_virtual_host_style = None, + enable_write_with_append = None, + endpoint = None, + external_id = None, + region = None, + role_arn = None, + role_session_name = None, + root = None, + secret_access_key = None, + server_side_encryption = None, + server_side_encryption_aws_kms_key_id = None, + server_side_encryption_customer_algorithm = None, + server_side_encryption_customer_key = None, + server_side_encryption_customer_key_md5 = None, + session_token = None, + skip_signature = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + bucket: String, + access_key_id: Option, + allow_anonymous: Option, + assume_role_duration_seconds: Option, + assume_role_session_tags: Option>, + batch_max_operations: Option, + checksum_algorithm: Option, + default_acl: Option, + default_storage_class: Option, + delete_max_size: Option, + disable_config_load: Option, + disable_ec2_metadata: Option, + disable_list_objects_v2: Option, + disable_stat_with_override: Option, + disable_write_with_if_match: Option, + enable_request_payer: Option, + enable_versioning: Option, + enable_virtual_host_style: Option, + enable_write_with_append: Option, + endpoint: Option, + external_id: Option, + region: Option, + role_arn: Option, + role_session_name: Option, + root: Option, + secret_access_key: Option, + server_side_encryption: Option, + server_side_encryption_aws_kms_key_id: Option, + server_side_encryption_customer_algorithm: Option, + server_side_encryption_customer_key: Option, + server_side_encryption_customer_key_md5: Option, + session_token: Option, + skip_signature: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("bucket".to_string(), bucket); + if let Some(v) = access_key_id { + opts.insert("access_key_id".to_string(), v); + } + if let Some(v) = allow_anonymous { + opts.insert( + "allow_anonymous".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = assume_role_duration_seconds { + opts.insert("assume_role_duration_seconds".to_string(), v.to_string()); + } + // assume_role_session_tags: map field set after from_iter + if let Some(v) = batch_max_operations { + opts.insert("batch_max_operations".to_string(), v.to_string()); + } + if let Some(v) = checksum_algorithm { + opts.insert("checksum_algorithm".to_string(), v); + } + if let Some(v) = default_acl { + opts.insert("default_acl".to_string(), v); + } + if let Some(v) = default_storage_class { + opts.insert("default_storage_class".to_string(), v); + } + if let Some(v) = delete_max_size { + opts.insert("delete_max_size".to_string(), v.to_string()); + } + if let Some(v) = disable_config_load { + opts.insert( + "disable_config_load".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = disable_ec2_metadata { + opts.insert( + "disable_ec2_metadata".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = disable_list_objects_v2 { + opts.insert( + "disable_list_objects_v2".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = disable_stat_with_override { + opts.insert( + "disable_stat_with_override".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = disable_write_with_if_match { + opts.insert( + "disable_write_with_if_match".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = enable_request_payer { + opts.insert( + "enable_request_payer".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = enable_versioning { + opts.insert( + "enable_versioning".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = enable_virtual_host_style { + opts.insert( + "enable_virtual_host_style".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = enable_write_with_append { + opts.insert( + "enable_write_with_append".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = external_id { + opts.insert("external_id".to_string(), v); + } + if let Some(v) = region { + opts.insert("region".to_string(), v); + } + if let Some(v) = role_arn { + opts.insert("role_arn".to_string(), v); + } + if let Some(v) = role_session_name { + opts.insert("role_session_name".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = secret_access_key { + opts.insert("secret_access_key".to_string(), v); + } + if let Some(v) = server_side_encryption { + opts.insert("server_side_encryption".to_string(), v); + } + if let Some(v) = server_side_encryption_aws_kms_key_id { + opts.insert("server_side_encryption_aws_kms_key_id".to_string(), v); + } + if let Some(v) = server_side_encryption_customer_algorithm { + opts.insert("server_side_encryption_customer_algorithm".to_string(), v); + } + if let Some(v) = server_side_encryption_customer_key { + opts.insert("server_side_encryption_customer_key".to_string(), v); + } + if let Some(v) = server_side_encryption_customer_key_md5 { + opts.insert("server_side_encryption_customer_key_md5".to_string(), v); + } + if let Some(v) = session_token { + opts.insert("session_token".to_string(), v); + } + if let Some(v) = skip_signature { + opts.insert( + "skip_signature".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // bucket: not a map field + // access_key_id: not a map field + // allow_anonymous: not a map field + // assume_role_duration_seconds: not a map field + if let Some(v) = assume_role_session_tags { + cfg.assume_role_session_tags = Some(v); + } + // batch_max_operations: not a map field + // checksum_algorithm: not a map field + // default_acl: not a map field + // default_storage_class: not a map field + // delete_max_size: not a map field + // disable_config_load: not a map field + // disable_ec2_metadata: not a map field + // disable_list_objects_v2: not a map field + // disable_stat_with_override: not a map field + // disable_write_with_if_match: not a map field + // enable_request_payer: not a map field + // enable_versioning: not a map field + // enable_virtual_host_style: not a map field + // enable_write_with_append: not a map field + // endpoint: not a map field + // external_id: not a map field + // region: not a map field + // role_arn: not a map field + // role_session_name: not a map field + // root: not a map field + // secret_access_key: not a map field + // server_side_encryption: not a map field + // server_side_encryption_aws_kms_key_id: not a map field + // server_side_encryption_customer_algorithm: not a map field + // server_side_encryption_customer_key: not a map field + // server_side_encryption_customer_key_md5: not a map field + // session_token: not a map field + // skip_signature: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// bucket name of this backend. + /// required. + #[getter] + fn bucket(&self) -> String { + self.0.bucket.clone() + } + /// access_key_id of this backend. + /// - If access_key_id is set, we will take user's input first. + /// - If not, we will try to load it from environment. + #[getter] + fn access_key_id(&self) -> Option { + self.0.access_key_id.clone() + } + /// Allow anonymous will allow opendal to send request without signing when + /// credential is not loaded. + /// [Deprecated since 0.57.0] Please use `skip_signature` instead of + /// `allow_anonymous` + #[getter] + fn allow_anonymous(&self) -> Option { + Some(self.0.allow_anonymous) + } + /// assume_role_duration_seconds for this backend. + #[getter] + fn assume_role_duration_seconds(&self) -> Option { + self.0.assume_role_duration_seconds + } + /// assume_role_session_tags for this backend. + #[getter] + fn assume_role_session_tags(&self) -> Option> { + self.0.assume_role_session_tags.clone() + } + /// Deprecated: S3 delete batch capability is enabled by default. + /// [Deprecated since 0.57.0] S3 delete batch capability is enabled by default. + /// Use CapabilityOverrideLayer to override delete_max_size for specific + /// endpoints. + #[getter] + fn batch_max_operations(&self) -> Option { + self.0.batch_max_operations + } + /// Checksum Algorithm to use when sending checksums in HTTP headers. + /// This is necessary when writing to AWS S3 Buckets with Object Lock enabled + /// for example. + /// Available options: - "crc32c" - "md5" + #[getter] + fn checksum_algorithm(&self) -> Option { + self.0.checksum_algorithm.clone() + } + /// Default ACL for new objects. + /// Note that some s3 services like minio do not support this option. + #[getter] + fn default_acl(&self) -> Option { + self.0.default_acl.clone() + } + /// default storage_class for this backend. + /// Available values: - `DEEP_ARCHIVE` - `GLACIER` - `GLACIER_IR` - + /// `INTELLIGENT_TIERING` - `ONEZONE_IA` - `EXPRESS_ONEZONE` - `OUTPOSTS` - + /// `REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3 compatible services + /// don't support all of them + #[getter] + fn default_storage_class(&self) -> Option { + self.0.default_storage_class.clone() + } + /// Deprecated: S3 delete batch capability is enabled by default. + /// [Deprecated since 0.57.0] S3 delete batch capability is enabled by default. + /// Use CapabilityOverrideLayer to override delete_max_size for specific + /// endpoints. + #[getter] + fn delete_max_size(&self) -> Option { + self.0.delete_max_size + } + /// Disable config load so that opendal will not load config from environment. + /// For examples: - envs like `AWS_ACCESS_KEY_ID` - files like `~/.aws/config` + #[getter] + fn disable_config_load(&self) -> Option { + Some(self.0.disable_config_load) + } + /// Disable load credential from ec2 metadata. + /// This option is used to disable the default behavior of opendal to load + /// credential from ec2 metadata, a.k.a., IMDSv2 + #[getter] + fn disable_ec2_metadata(&self) -> Option { + Some(self.0.disable_ec2_metadata) + } + /// OpenDAL uses List Objects V2 by default to list objects. + /// However, some legacy services do not yet support V2. + /// This option allows users to switch back to the older List Objects V1. + #[getter] + fn disable_list_objects_v2(&self) -> Option { + Some(self.0.disable_list_objects_v2) + } + /// Deprecated: S3 stat override capabilities are enabled by default. + /// [Deprecated since 0.57.0] S3 stat override capabilities are enabled by + /// default. + /// Use CapabilityOverrideLayer to override them for specific endpoints. + #[getter] + fn disable_stat_with_override(&self) -> Option { + Some(self.0.disable_stat_with_override) + } + /// Deprecated: S3 write with If-Match capability is enabled by default. + /// [Deprecated since 0.57.0] S3 write with If-Match capability is enabled by + /// default and this option is no longer needed. + #[getter] + fn disable_write_with_if_match(&self) -> Option { + Some(self.0.disable_write_with_if_match) + } + /// Indicates whether the client agrees to pay for the requests made to the S3 + /// bucket. + #[getter] + fn enable_request_payer(&self) -> Option { + Some(self.0.enable_request_payer) + } + /// Deprecated: S3 versioning capability is enabled by default. + /// [Deprecated since 0.57.0] S3 versioning capability is enabled by default and + /// this option is no longer needed. + #[getter] + fn enable_versioning(&self) -> Option { + Some(self.0.enable_versioning) + } + /// Enable virtual host style so that opendal will send API requests in virtual + /// host style instead of path style. + /// - By default, opendal will send API to + /// `https://s3.us-east-1.amazonaws.com/bucket_name` - Enabled, opendal will + /// send API to `https://bucket_name.s3.us-east-1.amazonaws.com` + #[getter] + fn enable_virtual_host_style(&self) -> Option { + Some(self.0.enable_virtual_host_style) + } + /// Deprecated: S3 append capability is enabled by default. + /// [Deprecated since 0.57.0] S3 append capability is enabled by default and + /// this option is no longer needed. + #[getter] + fn enable_write_with_append(&self) -> Option { + Some(self.0.enable_write_with_append) + } + /// endpoint of this backend. + /// Endpoint must be full uri, e.g. + /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com` + /// - Cloudflare R2: `https://.r2.cloudflarestorage.com` - Aliyun + /// OSS: `https://{region}.aliyuncs.com` - Tencent COS: + /// `https://cos.{region}.myqcloud.com` - Minio: `http://127.0.0.1:9000` If user + /// inputs endpoint without scheme like "s3.amazonaws.com", we will prepend + /// "https://" before it. + /// - If endpoint is set, we will take user's input first. + /// - If not, we will try to load it from environment. + /// - If still not set, default to `https://s3.amazonaws.com`. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// external_id for this backend. + #[getter] + fn external_id(&self) -> Option { + self.0.external_id.clone() + } + /// Region represent the signing region of this endpoint. + /// This is required if you are using the default AWS S3 endpoint. + /// If using a custom endpoint, - If region is set, we will take user's input + /// first. + /// - If not, we will try to load it from environment. + #[getter] + fn region(&self) -> Option { + self.0.region.clone() + } + /// role_arn for this backend. + /// If `role_arn` is set, we will use already known config as source credential + /// to assume role with `role_arn`. + #[getter] + fn role_arn(&self) -> Option { + self.0.role_arn.clone() + } + /// role_session_name for this backend. + #[getter] + fn role_session_name(&self) -> Option { + self.0.role_session_name.clone() + } + /// root of this backend. + /// All operations will happen under this root. + /// default to `/` if not set. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// secret_access_key of this backend. + /// - If secret_access_key is set, we will take user's input first. + /// - If not, we will try to load it from environment. + #[getter] + fn secret_access_key(&self) -> Option { + self.0.secret_access_key.clone() + } + /// server_side_encryption for this backend. + /// Available values: `AES256`, `aws:kms`. + #[getter] + fn server_side_encryption(&self) -> Option { + self.0.server_side_encryption.clone() + } + /// server_side_encryption_aws_kms_key_id for this backend - If + /// `server_side_encryption` set to `aws:kms`, and + /// `server_side_encryption_aws_kms_key_id` is not set, S3 will use aws managed + /// kms key to encrypt data. + /// - If `server_side_encryption` set to `aws:kms`, and + /// `server_side_encryption_aws_kms_key_id` is a valid kms key id, S3 will use + /// the provided kms key to encrypt data. + /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an + /// error will be returned. + /// - If `server_side_encryption` is not `aws:kms`, setting + /// `server_side_encryption_aws_kms_key_id` is a noop. + #[getter] + fn server_side_encryption_aws_kms_key_id(&self) -> Option { + self.0.server_side_encryption_aws_kms_key_id.clone() + } + /// server_side_encryption_customer_algorithm for this backend. + /// Available values: `AES256`. + #[getter] + fn server_side_encryption_customer_algorithm(&self) -> Option { + self.0.server_side_encryption_customer_algorithm.clone() + } + /// server_side_encryption_customer_key for this backend. + /// Value: BASE64-encoded key that matches algorithm specified in + /// `server_side_encryption_customer_algorithm`. + #[getter] + fn server_side_encryption_customer_key(&self) -> Option { + self.0.server_side_encryption_customer_key.clone() + } + /// Set server_side_encryption_customer_key_md5 for this backend. + /// Value: MD5 digest of key specified in `server_side_encryption_customer_key`. + #[getter] + fn server_side_encryption_customer_key_md5(&self) -> Option { + self.0.server_side_encryption_customer_key_md5.clone() + } + /// session_token (aka, security token) of this backend. + /// This token will expire after sometime, it's recommended to set session_token + /// by hand. + #[getter] + fn session_token(&self) -> Option { + self.0.session_token.clone() + } + /// Skip signature will skip loading credentials and signing requests. + #[getter] + fn skip_signature(&self) -> Option { + Some(self.0.skip_signature) + } +} + +#[cfg(feature = "services-seafile")] +/// Configuration for the `seafile` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct SeafileConfig(ocore::services::SeafileConfig); + +#[cfg(feature = "services-seafile")] +#[allow(deprecated)] +impl ConfigBuilder for SeafileConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "seafile" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("repo_name".to_string(), self.0.repo_name.clone()); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.password { + map.insert("password".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.username { + map.insert("username".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // repo_name: always picklable + // endpoint: always picklable + // password: always picklable + // root: always picklable + // username: always picklable + ok + } +} + +#[cfg(feature = "services-seafile")] +#[allow(deprecated)] +#[pymethods] +impl SeafileConfig { + #[new] + #[pyo3(signature = ( + repo_name, + endpoint = None, + password = None, + root = None, + username = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + repo_name: String, + endpoint: Option, + password: Option, + root: Option, + username: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("repo_name".to_string(), repo_name); + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = password { + opts.insert("password".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = username { + opts.insert("username".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // repo_name: not a map field + // endpoint: not a map field + // password: not a map field + // root: not a map field + // username: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// repo_name of this backend. + /// required. + #[getter] + fn repo_name(&self) -> String { + self.0.repo_name.clone() + } + /// endpoint address of this backend. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// password of this backend. + #[getter] + fn password(&self) -> Option { + self.0.password.clone() + } + /// root of this backend. + /// All operations will happen under this root. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// username of this backend. + #[getter] + fn username(&self) -> Option { + self.0.username.clone() + } +} + +#[cfg(feature = "services-sled")] +/// Configuration for the `sled` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct SledConfig(ocore::services::SledConfig); + +#[cfg(feature = "services-sled")] +#[allow(deprecated)] +impl ConfigBuilder for SledConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "sled" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.datadir { + map.insert("datadir".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.tree { + map.insert("tree".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // datadir: always picklable + // root: always picklable + // tree: always picklable + ok + } +} + +#[cfg(feature = "services-sled")] +#[allow(deprecated)] +#[pymethods] +impl SledConfig { + #[new] + #[pyo3(signature = ( + datadir = None, + root = None, + tree = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + datadir: Option, + root: Option, + tree: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = datadir { + opts.insert("datadir".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = tree { + opts.insert("tree".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // datadir: not a map field + // root: not a map field + // tree: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// That path to the sled data directory. + #[getter] + fn datadir(&self) -> Option { + self.0.datadir.clone() + } + /// The root for sled. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// The tree for sled. + #[getter] + fn tree(&self) -> Option { + self.0.tree.clone() + } +} + +#[cfg(feature = "services-sqlite")] +/// Configuration for the `sqlite` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct SqliteConfig(ocore::services::SqliteConfig); + +#[cfg(feature = "services-sqlite")] +#[allow(deprecated)] +impl ConfigBuilder for SqliteConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "sqlite" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.connection_string { + map.insert("connection_string".to_string(), v.clone()); + } + if let Some(v) = &self.0.key_field { + map.insert("key_field".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.table { + map.insert("table".to_string(), v.clone()); + } + if let Some(v) = &self.0.value_field { + map.insert("value_field".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // connection_string: always picklable + // key_field: always picklable + // root: always picklable + // table: always picklable + // value_field: always picklable + ok + } +} + +#[cfg(feature = "services-sqlite")] +#[allow(deprecated)] +#[pymethods] +impl SqliteConfig { + #[new] + #[pyo3(signature = ( + connection_string = None, + key_field = None, + root = None, + table = None, + value_field = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + connection_string: Option, + key_field: Option, + root: Option, + table: Option, + value_field: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = connection_string { + opts.insert("connection_string".to_string(), v); + } + if let Some(v) = key_field { + opts.insert("key_field".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = table { + opts.insert("table".to_string(), v); + } + if let Some(v) = value_field { + opts.insert("value_field".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // connection_string: not a map field + // key_field: not a map field + // root: not a map field + // table: not a map field + // value_field: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Set the connection_string of the sqlite service. + /// This connection string is used to connect to the sqlite service. + /// The format of connect string resembles the url format of the sqlite client: + /// - `sqlite::memory:` - `sqlite:data.db` - `sqlite://data.db` For more + /// information, please visit + /// . + #[getter] + fn connection_string(&self) -> Option { + self.0.connection_string.clone() + } + /// Set the key field name of the sqlite service to read/write. + /// Default to `key` if not specified. + #[getter] + fn key_field(&self) -> Option { + self.0.key_field.clone() + } + /// set the working directory, all operations will be performed under it. + /// default: "/" + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Set the table name of the sqlite service to read/write. + #[getter] + fn table(&self) -> Option { + self.0.table.clone() + } + /// Set the value field name of the sqlite service to read/write. + /// Default to `value` if not specified. + #[getter] + fn value_field(&self) -> Option { + self.0.value_field.clone() + } +} + +#[cfg(feature = "services-swift")] +/// Configuration for the `swift` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct SwiftConfig(ocore::services::SwiftConfig); + +#[cfg(feature = "services-swift")] +#[allow(deprecated)] +impl ConfigBuilder for SwiftConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "swift" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.container { + map.insert("container".to_string(), v.clone()); + } + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.temp_url_hash_algorithm { + map.insert("temp_url_hash_algorithm".to_string(), v.clone()); + } + if let Some(v) = &self.0.temp_url_key { + map.insert("temp_url_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.token { + map.insert("token".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // container: always picklable + // endpoint: always picklable + // root: always picklable + // temp_url_hash_algorithm: always picklable + // temp_url_key: always picklable + // token: always picklable + ok + } +} + +#[cfg(feature = "services-swift")] +#[allow(deprecated)] +#[pymethods] +impl SwiftConfig { + #[new] + #[pyo3(signature = ( + container = None, + endpoint = None, + root = None, + temp_url_hash_algorithm = None, + temp_url_key = None, + token = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + container: Option, + endpoint: Option, + root: Option, + temp_url_hash_algorithm: Option, + temp_url_key: Option, + token: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = container { + opts.insert("container".to_string(), v); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = temp_url_hash_algorithm { + opts.insert("temp_url_hash_algorithm".to_string(), v); + } + if let Some(v) = temp_url_key { + opts.insert("temp_url_key".to_string(), v); + } + if let Some(v) = token { + opts.insert("token".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // container: not a map field + // endpoint: not a map field + // root: not a map field + // temp_url_hash_algorithm: not a map field + // temp_url_key: not a map field + // token: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// The container for Swift. + #[getter] + fn container(&self) -> Option { + self.0.container.clone() + } + /// The endpoint for Swift. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// The root for Swift. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// The hash algorithm for TempURL signing. + /// Supported values: `sha1`, `sha256`, `sha512`. + /// Defaults to `sha256`. + /// The cluster must have the chosen algorithm in its `tempurl.allowed_digests` + /// (check `GET /info`). + #[getter] + fn temp_url_hash_algorithm(&self) -> Option { + self.0.temp_url_hash_algorithm.clone() + } + /// The TempURL key for generating presigned URLs. + /// This corresponds to the `X-Account-Meta-Temp-URL-Key` or + /// `X-Container-Meta-Temp-URL-Key` header value configured on the Swift account + /// or container. + #[getter] + fn temp_url_key(&self) -> Option { + self.0.temp_url_key.clone() + } + /// The token for Swift. + #[getter] + fn token(&self) -> Option { + self.0.token.clone() + } +} + +#[cfg(feature = "services-tos")] +/// Configuration for the `tos` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct TosConfig(ocore::services::TosConfig); + +#[cfg(feature = "services-tos")] +#[allow(deprecated)] +impl ConfigBuilder for TosConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "tos" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("bucket".to_string(), self.0.bucket.clone()); + if let Some(v) = &self.0.access_key_id { + map.insert("access_key_id".to_string(), v.clone()); + } + map.insert( + "disable_config_load".to_string(), + if self.0.disable_config_load { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.region { + map.insert("region".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.secret_access_key { + map.insert("secret_access_key".to_string(), v.clone()); + } + if let Some(v) = &self.0.security_token { + map.insert("security_token".to_string(), v.clone()); + } + map.insert( + "skip_signature".to_string(), + if self.0.skip_signature { + "true" + } else { + "false" + } + .to_string(), + ); + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // bucket: always picklable + // access_key_id: always picklable + // disable_config_load: always picklable + // endpoint: always picklable + // region: always picklable + // root: always picklable + // secret_access_key: always picklable + // security_token: always picklable + // skip_signature: always picklable + ok + } +} + +#[cfg(feature = "services-tos")] +#[allow(deprecated)] +#[pymethods] +impl TosConfig { + #[new] + #[pyo3(signature = ( + bucket, + access_key_id = None, + disable_config_load = None, + endpoint = None, + region = None, + root = None, + secret_access_key = None, + security_token = None, + skip_signature = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + bucket: String, + access_key_id: Option, + disable_config_load: Option, + endpoint: Option, + region: Option, + root: Option, + secret_access_key: Option, + security_token: Option, + skip_signature: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("bucket".to_string(), bucket); + if let Some(v) = access_key_id { + opts.insert("access_key_id".to_string(), v); + } + if let Some(v) = disable_config_load { + opts.insert( + "disable_config_load".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = region { + opts.insert("region".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = secret_access_key { + opts.insert("secret_access_key".to_string(), v); + } + if let Some(v) = security_token { + opts.insert("security_token".to_string(), v); + } + if let Some(v) = skip_signature { + opts.insert( + "skip_signature".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // bucket: not a map field + // access_key_id: not a map field + // disable_config_load: not a map field + // endpoint: not a map field + // region: not a map field + // root: not a map field + // secret_access_key: not a map field + // security_token: not a map field + // skip_signature: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// bucket name of this backend. + /// required. + #[getter] + fn bucket(&self) -> String { + self.0.bucket.clone() + } + /// access_key_id of this backend. + /// - If access_key_id is set, we will take user's input first. + /// - If not, we will try to load it from environment. + #[getter] + fn access_key_id(&self) -> Option { + self.0.access_key_id.clone() + } + /// Disable config load so that opendal will not load config from environment. + /// For examples: - envs like `TOS_ACCESS_KEY_ID` + #[getter] + fn disable_config_load(&self) -> Option { + Some(self.0.disable_config_load) + } + /// endpoint of this backend. + /// Endpoint must be full uri, e.g. + /// - TOS: `https://tos-cn-beijing.volces.com` - TOS with region: + /// `https://tos-{region}.volces.com` If user inputs endpoint without scheme + /// like "tos-cn-beijing.volces.com", we will prepend "https://" before it. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// Region represent the signing region of this endpoint. + /// Required if endpoint is not provided. + /// - If region is set, we will take user's input first. + /// - If not, we will try to load it from environment. + /// - If still not set, default to `cn-beijing`. + #[getter] + fn region(&self) -> Option { + self.0.region.clone() + } + /// root of this backend. + /// All operations will happen under this root. + /// default to `/` if not set. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// secret_access_key of this backend. + /// - If secret_access_key is set, we will take user's input first. + /// - If not, we will try to load it from environment. + #[getter] + fn secret_access_key(&self) -> Option { + self.0.secret_access_key.clone() + } + /// security_token of this backend. + /// This token will expire after sometime, it's recommended to set + /// security_token by hand. + #[getter] + fn security_token(&self) -> Option { + self.0.security_token.clone() + } + /// Skip signature will skip loading credentials and signing requests. + #[getter] + fn skip_signature(&self) -> Option { + Some(self.0.skip_signature) + } +} + +#[cfg(feature = "services-upyun")] +/// Configuration for the `upyun` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct UpyunConfig(ocore::services::UpyunConfig); + +#[cfg(feature = "services-upyun")] +#[allow(deprecated)] +impl ConfigBuilder for UpyunConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "upyun" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("bucket".to_string(), self.0.bucket.clone()); + if let Some(v) = &self.0.operator { + map.insert("operator".to_string(), v.clone()); + } + if let Some(v) = &self.0.password { + map.insert("password".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // bucket: always picklable + // operator: always picklable + // password: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-upyun")] +#[allow(deprecated)] +#[pymethods] +impl UpyunConfig { + #[new] + #[pyo3(signature = ( + bucket, + operator = None, + password = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + bucket: String, + operator: Option, + password: Option, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("bucket".to_string(), bucket); + if let Some(v) = operator { + opts.insert("operator".to_string(), v); + } + if let Some(v) = password { + opts.insert("password".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // bucket: not a map field + // operator: not a map field + // password: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// bucket address of this backend. + #[getter] + fn bucket(&self) -> String { + self.0.bucket.clone() + } + /// username of this backend. + #[getter] + fn operator(&self) -> Option { + self.0.operator.clone() + } + /// password of this backend. + #[getter] + fn password(&self) -> Option { + self.0.password.clone() + } + /// root of this backend. + /// All operations will happen under this root. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +#[cfg(feature = "services-vercel-artifacts")] +/// Configuration for the `vercel-artifacts` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct VercelArtifactsConfig(ocore::services::VercelArtifactsConfig); + +#[cfg(feature = "services-vercel-artifacts")] +#[allow(deprecated)] +impl ConfigBuilder for VercelArtifactsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "vercel-artifacts" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.access_token { + map.insert("access_token".to_string(), v.clone()); + } + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.team_id { + map.insert("team_id".to_string(), v.clone()); + } + if let Some(v) = &self.0.team_slug { + map.insert("team_slug".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // access_token: always picklable + // endpoint: always picklable + // team_id: always picklable + // team_slug: always picklable + ok + } +} + +#[cfg(feature = "services-vercel-artifacts")] +#[allow(deprecated)] +#[pymethods] +impl VercelArtifactsConfig { + #[new] + #[pyo3(signature = ( + access_token = None, + endpoint = None, + team_id = None, + team_slug = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + access_token: Option, + endpoint: Option, + team_id: Option, + team_slug: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = access_token { + opts.insert("access_token".to_string(), v); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = team_id { + opts.insert("team_id".to_string(), v); + } + if let Some(v) = team_slug { + opts.insert("team_slug".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = + ::from_iter(opts) + .map_err(format_pyerr)?; + // access_token: not a map field + // endpoint: not a map field + // team_id: not a map field + // team_slug: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// The access token for Vercel. + #[getter] + fn access_token(&self) -> Option { + self.0.access_token.clone() + } + /// The endpoint for the Vercel artifacts API. + /// Defaults to `https://api.vercel.com`. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// The Vercel team ID. + /// When set, the `teamId` query parameter is appended to all API requests. + #[getter] + fn team_id(&self) -> Option { + self.0.team_id.clone() + } + /// The Vercel team slug. + /// When set, the `slug` query parameter is appended to all API requests. + #[getter] + fn team_slug(&self) -> Option { + self.0.team_slug.clone() + } +} + +#[cfg(feature = "services-webdav")] +/// Configuration for the `webdav` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct WebdavConfig(ocore::services::WebdavConfig); + +#[cfg(feature = "services-webdav")] +#[allow(deprecated)] +impl ConfigBuilder for WebdavConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "webdav" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert( + "disable_copy".to_string(), + if self.0.disable_copy { "true" } else { "false" }.to_string(), + ); + map.insert( + "disable_create_dir".to_string(), + if self.0.disable_create_dir { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "enable_conditional_read".to_string(), + if self.0.enable_conditional_read { + "true" + } else { + "false" + } + .to_string(), + ); + map.insert( + "enable_user_metadata".to_string(), + if self.0.enable_user_metadata { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.password { + map.insert("password".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.token { + map.insert("token".to_string(), v.clone()); + } + if let Some(v) = &self.0.user_metadata_prefix { + map.insert("user_metadata_prefix".to_string(), v.clone()); + } + if let Some(v) = &self.0.user_metadata_uri { + map.insert("user_metadata_uri".to_string(), v.clone()); + } + if let Some(v) = &self.0.username { + map.insert("username".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // disable_copy: always picklable + // disable_create_dir: always picklable + // enable_conditional_read: always picklable + // enable_user_metadata: always picklable + // endpoint: always picklable + // password: always picklable + // root: always picklable + // token: always picklable + // user_metadata_prefix: always picklable + // user_metadata_uri: always picklable + // username: always picklable + ok + } +} + +#[cfg(feature = "services-webdav")] +#[allow(deprecated)] +#[pymethods] +impl WebdavConfig { + #[new] + #[pyo3(signature = ( + disable_copy = None, + disable_create_dir = None, + enable_conditional_read = None, + enable_user_metadata = None, + endpoint = None, + password = None, + root = None, + token = None, + user_metadata_prefix = None, + user_metadata_uri = None, + username = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + disable_copy: Option, + disable_create_dir: Option, + enable_conditional_read: Option, + enable_user_metadata: Option, + endpoint: Option, + password: Option, + root: Option, + token: Option, + user_metadata_prefix: Option, + user_metadata_uri: Option, + username: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = disable_copy { + opts.insert( + "disable_copy".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = disable_create_dir { + opts.insert( + "disable_create_dir".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = enable_conditional_read { + opts.insert( + "enable_conditional_read".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = enable_user_metadata { + opts.insert( + "enable_user_metadata".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = password { + opts.insert("password".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = token { + opts.insert("token".to_string(), v); + } + if let Some(v) = user_metadata_prefix { + opts.insert("user_metadata_prefix".to_string(), v); + } + if let Some(v) = user_metadata_uri { + opts.insert("user_metadata_uri".to_string(), v); + } + if let Some(v) = username { + opts.insert("username".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // disable_copy: not a map field + // disable_create_dir: not a map field + // enable_conditional_read: not a map field + // enable_user_metadata: not a map field + // endpoint: not a map field + // password: not a map field + // root: not a map field + // token: not a map field + // user_metadata_prefix: not a map field + // user_metadata_uri: not a map field + // username: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Deprecated: WebDAV copy capability is enabled by default. + /// [Deprecated since 0.57.0] WebDAV copy capability is enabled by default and + /// this option is no longer needed. + #[getter] + fn disable_copy(&self) -> Option { + Some(self.0.disable_copy) + } + /// Disable automatic parent directory creation before write operations. + /// By default, OpenDAL creates parent directories using MKCOL before writing + /// files. + /// This requires PROPFIND support to check directory existence. + /// Some WebDAV-compatible servers (e.g., bazel-remote) don't support PROPFIND + /// or don't require explicit directory creation. + /// Enable this option to skip the MKCOL calls and write files directly. + /// Default: false + #[getter] + fn disable_create_dir(&self) -> Option { + Some(self.0.disable_create_dir) + } + /// Enable conditional read support. + /// When enabled (the default), OpenDAL forwards the RFC 7232 headers + /// `If-Match`, `If-None-Match`, `If-Modified-Since` and `If-Unmodified-Since` + /// to the server when callers provide them. + /// Some WebDAV-compatible servers (e.g., nginx-dav) don't return ETags in + /// PROPFIND or don't honor these headers on GET. + /// Setting this to `false` drops the four `read_with_if_*` capabilities, so + /// calls like `reader_with(path).if_match(...)` return `ErrorKind::Unsupported` + /// locally instead of being silently ignored by the server. + /// Default: true + #[getter] + fn enable_conditional_read(&self) -> Option { + Some(self.0.enable_conditional_read) + } + /// Deprecated: WebDAV user metadata capability is enabled by default. + /// [Deprecated since 0.57.0] WebDAV user metadata capability is enabled by + /// default. + /// Use CapabilityOverrideLayer to override write_with_user_metadata for + /// endpoints without PROPPATCH support. + #[getter] + fn enable_user_metadata(&self) -> Option { + Some(self.0.enable_user_metadata) + } + /// endpoint of this backend + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// password of this backend + #[getter] + fn password(&self) -> Option { + self.0.password.clone() + } + /// root of this backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// token of this backend + #[getter] + fn token(&self) -> Option { + self.0.token.clone() + } + /// The XML namespace prefix for user metadata properties. + /// This prefix is used in PROPPATCH/PROPFIND XML requests. + /// Different servers may require different prefixes. + /// Default: "opendal" + #[getter] + fn user_metadata_prefix(&self) -> Option { + self.0.user_metadata_prefix.clone() + } + /// The XML namespace URI for user metadata properties. + /// This URI uniquely identifies the namespace for custom properties. + /// Different servers may require different namespace URIs. + /// For example, Nextcloud might work better with its own namespace. + /// Default: `https://opendal.apache.org/ns` + #[getter] + fn user_metadata_uri(&self) -> Option { + self.0.user_metadata_uri.clone() + } + /// username of this backend + #[getter] + fn username(&self) -> Option { + self.0.username.clone() + } +} + +#[cfg(feature = "services-webhdfs")] +/// Configuration for the `webhdfs` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct WebhdfsConfig(ocore::services::WebhdfsConfig); + +#[cfg(feature = "services-webhdfs")] +#[allow(deprecated)] +impl ConfigBuilder for WebhdfsConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "webhdfs" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + if let Some(v) = &self.0.atomic_write_dir { + map.insert("atomic_write_dir".to_string(), v.clone()); + } + if let Some(v) = &self.0.delegation { + map.insert("delegation".to_string(), v.clone()); + } + map.insert( + "disable_list_batch".to_string(), + if self.0.disable_list_batch { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.user_name { + map.insert("user_name".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // atomic_write_dir: always picklable + // delegation: always picklable + // disable_list_batch: always picklable + // endpoint: always picklable + // root: always picklable + // user_name: always picklable + ok + } +} + +#[cfg(feature = "services-webhdfs")] +#[allow(deprecated)] +#[pymethods] +impl WebhdfsConfig { + #[new] + #[pyo3(signature = ( + atomic_write_dir = None, + delegation = None, + disable_list_batch = None, + endpoint = None, + root = None, + user_name = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + atomic_write_dir: Option, + delegation: Option, + disable_list_batch: Option, + endpoint: Option, + root: Option, + user_name: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = atomic_write_dir { + opts.insert("atomic_write_dir".to_string(), v.into_string()); + } + if let Some(v) = delegation { + opts.insert("delegation".to_string(), v); + } + if let Some(v) = disable_list_batch { + opts.insert( + "disable_list_batch".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = user_name { + opts.insert("user_name".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // atomic_write_dir: not a map field + // delegation: not a map field + // disable_list_batch: not a map field + // endpoint: not a map field + // root: not a map field + // user_name: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// atomic_write_dir of this backend + #[getter] + fn atomic_write_dir(&self) -> Option { + self.0.atomic_write_dir.clone().map(crate::PyPath::from) + } + /// Delegation token for webhdfs. + #[getter] + fn delegation(&self) -> Option { + self.0.delegation.clone() + } + /// Disable batch listing + #[getter] + fn disable_list_batch(&self) -> Option { + Some(self.0.disable_list_batch) + } + /// Endpoint for webhdfs. + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// Root for webhdfs. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// Name of the user for webhdfs. + #[getter] + fn user_name(&self) -> Option { + self.0.user_name.clone() + } +} + +#[cfg(feature = "services-yandex-disk")] +/// Configuration for the `yandex-disk` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct YandexDiskConfig(ocore::services::YandexDiskConfig); + +#[cfg(feature = "services-yandex-disk")] +#[allow(deprecated)] +impl ConfigBuilder for YandexDiskConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "yandex-disk" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert("access_token".to_string(), self.0.access_token.clone()); + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + // access_token: always picklable + // root: always picklable + ok + } +} + +#[cfg(feature = "services-yandex-disk")] +#[allow(deprecated)] +#[pymethods] +impl YandexDiskConfig { + #[new] + #[pyo3(signature = ( + access_token, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + access_token: String, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + opts.insert("access_token".to_string(), access_token); + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + // access_token: not a map field + // root: not a map field + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// yandex disk oauth access_token. + #[getter] + fn access_token(&self) -> String { + self.0.access_token.clone() + } + /// root of this backend. + /// All operations will happen under this root. + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + +// Defined here (not in `lib.rs`) so pyo3 introspection sees the config +// `#[pymodule_export]`s as literal items. +#[pyo3::pymodule(submodule, name = "services")] +pub mod services_submodule { + #[cfg(feature = "services-aliyun-drive")] + #[pymodule_export] + use crate::AliyunDriveConfig; + #[cfg(feature = "services-alluxio")] + #[pymodule_export] + use crate::AlluxioConfig; + #[cfg(feature = "services-azblob")] + #[pymodule_export] + use crate::AzblobConfig; + #[cfg(feature = "services-azdls")] + #[pymodule_export] + use crate::AzdlsConfig; + #[cfg(feature = "services-azfile")] + #[pymodule_export] + use crate::AzfileConfig; + #[cfg(feature = "services-b2")] + #[pymodule_export] + use crate::B2Config; + #[cfg(feature = "services-cacache")] + #[pymodule_export] + use crate::CacacheConfig; + #[cfg(feature = "services-cos")] + #[pymodule_export] + use crate::CosConfig; + #[cfg(feature = "services-dashmap")] + #[pymodule_export] + use crate::DashmapConfig; + #[cfg(feature = "services-dropbox")] + #[pymodule_export] + use crate::DropboxConfig; + #[cfg(feature = "services-fs")] + #[pymodule_export] + use crate::FsConfig; + #[cfg(feature = "services-ftp")] + #[pymodule_export] + use crate::FtpConfig; + #[cfg(feature = "services-gcs")] + #[pymodule_export] + use crate::GcsConfig; + #[cfg(feature = "services-gdrive")] + #[pymodule_export] + use crate::GdriveConfig; + #[cfg(feature = "services-ghac")] + #[pymodule_export] + use crate::GhacConfig; + #[cfg(feature = "services-goosefs")] + #[pymodule_export] + use crate::GoosefsConfig; + #[cfg(feature = "services-gridfs")] + #[pymodule_export] + use crate::GridfsConfig; + #[cfg(feature = "services-hf")] + #[pymodule_export] + use crate::HfConfig; + #[cfg(feature = "services-http")] + #[pymodule_export] + use crate::HttpConfig; + #[cfg(feature = "services-ipfs")] + #[pymodule_export] + use crate::IpfsConfig; + #[cfg(feature = "services-ipmfs")] + #[pymodule_export] + use crate::IpmfsConfig; + #[cfg(feature = "services-koofr")] + #[pymodule_export] + use crate::KoofrConfig; + #[cfg(feature = "services-memcached")] + #[pymodule_export] + use crate::MemcachedConfig; + #[cfg(feature = "services-memory")] + #[pymodule_export] + use crate::MemoryConfig; + #[cfg(feature = "services-mini-moka")] + #[pymodule_export] + use crate::MiniMokaConfig; + #[cfg(feature = "services-moka")] + #[pymodule_export] + use crate::MokaConfig; + #[cfg(feature = "services-mongodb")] + #[pymodule_export] + use crate::MongodbConfig; + #[cfg(feature = "services-mysql")] + #[pymodule_export] + use crate::MysqlConfig; + #[cfg(feature = "services-obs")] + #[pymodule_export] + use crate::ObsConfig; + #[cfg(feature = "services-onedrive")] + #[pymodule_export] + use crate::OnedriveConfig; + #[cfg(feature = "services-oss")] + #[pymodule_export] + use crate::OssConfig; + #[cfg(feature = "services-persy")] + #[pymodule_export] + use crate::PersyConfig; + #[cfg(feature = "services-postgresql")] + #[pymodule_export] + use crate::PostgresqlConfig; + #[cfg(feature = "services-redb")] + #[pymodule_export] + use crate::RedbConfig; + #[cfg(feature = "services-redis")] + #[pymodule_export] + use crate::RedisConfig; + #[cfg(feature = "services-s3")] + #[pymodule_export] + use crate::S3Config; + #[pymodule_export] + use crate::Scheme; + #[cfg(feature = "services-seafile")] + #[pymodule_export] + use crate::SeafileConfig; + #[pymodule_export] + use crate::ServiceConfig; + #[cfg(feature = "services-sled")] + #[pymodule_export] + use crate::SledConfig; + #[cfg(feature = "services-sqlite")] + #[pymodule_export] + use crate::SqliteConfig; + #[cfg(feature = "services-swift")] + #[pymodule_export] + use crate::SwiftConfig; + #[cfg(feature = "services-tos")] + #[pymodule_export] + use crate::TosConfig; + #[cfg(feature = "services-upyun")] + #[pymodule_export] + use crate::UpyunConfig; + #[cfg(feature = "services-vercel-artifacts")] + #[pymodule_export] + use crate::VercelArtifactsConfig; + #[cfg(feature = "services-webdav")] + #[pymodule_export] + use crate::WebdavConfig; + #[cfg(feature = "services-webhdfs")] + #[pymodule_export] + use crate::WebhdfsConfig; + #[cfg(feature = "services-yandex-disk")] + #[pymodule_export] + use crate::YandexDiskConfig; +} diff --git a/bindings/python/tests/test_from_config.py b/bindings/python/tests/test_from_config.py new file mode 100644 index 000000000000..61ff8e41330f --- /dev/null +++ b/bindings/python/tests/test_from_config.py @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for typed service configs and cross-path operator equivalence. + +Uses the ``memory`` service so the suite runs without ``OPENDAL_TEST``; the +kwargs value-type error tests use ``s3`` but fail before any backend call. +""" + +import pickle +from pathlib import PurePosixPath +from uuid import uuid4 + +import pytest + +import opendal +from opendal.services import MemoryConfig, S3Config, ServiceConfig + + +@pytest.fixture(autouse=True) +def check_capability() -> None: + # Override conftest's autouse fixture, which requires OPENDAL_TEST. + return None + + +def test_config_subclasses_service_config(): + assert issubclass(MemoryConfig, ServiceConfig) + assert issubclass(S3Config, ServiceConfig) + + +def test_scheme_is_bound_and_not_a_field(): + assert S3Config(bucket="b").scheme == "s3" + assert MemoryConfig().scheme == "memory" + with pytest.raises(TypeError): + S3Config(bucket="b", scheme="hdfs") + + +def test_required_field_enforced(): + with pytest.raises(TypeError): + S3Config() + + +def test_unknown_field_rejected(): + with pytest.raises(TypeError): + S3Config(bucket="b", not_a_real_field=1) + + +def test_getters_read_back_values(): + cfg = S3Config(bucket="b", region="us-east-1", assume_role_duration_seconds=900) + assert cfg.bucket == "b" + assert cfg.region == "us-east-1" + assert cfg.assume_role_duration_seconds == 900 + + +def test_path_field_accepts_pathlike(): + cfg = S3Config(bucket="b", root=PurePosixPath("/data/x")) + assert cfg.root == "/data/x" + + +def test_from_config_builds_blocking_operator(): + op = opendal.Operator.from_config(MemoryConfig()) + assert isinstance(op, opendal.Operator) + key = f"k-{uuid4()}" + op.write(key, b"hello") + assert op.read(key) == b"hello" + + +@pytest.mark.asyncio +async def test_from_config_builds_async_operator(): + op = opendal.AsyncOperator.from_config(MemoryConfig()) + assert isinstance(op, opendal.AsyncOperator) + key = f"k-{uuid4()}" + await op.write(key, b"world") + assert (await op.read(key)) == b"world" + + +def test_from_config_matches_constructor_capability(): + from_cfg = opendal.Operator.from_config(MemoryConfig()) + from_kwargs = opendal.Operator("memory") + assert from_cfg.capability().write == from_kwargs.capability().write + assert from_cfg.capability().read == from_kwargs.capability().read + + +def test_from_config_rejects_non_config(): + with pytest.raises(TypeError): + opendal.Operator.from_config("memory") + + +def test_from_config_supports_map_valued_field(): + cfg = S3Config(bucket="b", assume_role_session_tags={"team": "eng"}) + assert cfg.assume_role_session_tags == {"team": "eng"} + op = opendal.Operator.from_config(cfg) + assert isinstance(op, opendal.Operator) + + +def test_from_config_pickle_roundtrip(): + op = opendal.Operator.from_config(MemoryConfig()) + restored = pickle.loads(pickle.dumps(op)) + assert isinstance(restored, opendal.Operator) + key = f"k-{uuid4()}" + restored.write(key, b"data") + assert restored.read(key) == b"data" + + +def test_pickle_raises_when_map_field_set(): + cfg = S3Config(bucket="b", assume_role_session_tags={"team": "eng"}) + op = opendal.Operator.from_config(cfg) + with pytest.raises(opendal.exceptions.Unsupported): + pickle.dumps(op) + + +def _blocking_paths(root: str) -> dict[str, opendal.Operator]: + return { + "kwargs": opendal.Operator("memory", root=root), + "from_uri": opendal.Operator.from_uri("memory://", root=root), + "from_config": opendal.Operator.from_config(MemoryConfig(root=root)), + } + + +def _async_paths(root: str) -> dict[str, opendal.AsyncOperator]: + return { + "kwargs": opendal.AsyncOperator("memory", root=root), + "from_uri": opendal.AsyncOperator.from_uri("memory://", root=root), + "from_config": opendal.AsyncOperator.from_config(MemoryConfig(root=root)), + } + + +def _info(op: opendal.Operator) -> tuple: + cap = op.capability() + return ( + op.__class__.__name__, + cap.read, + cap.write, + cap.delete, + cap.list, + cap.copy, + cap.rename, + ) + + +def test_paths_produce_same_scheme_and_root(): + for op in _blocking_paths("/equiv/").values(): + assert 'Operator("memory"' in repr(op) + assert 'root="/equiv/"' in repr(op) + + +def test_paths_produce_same_capability(): + infos = {name: _info(op) for name, op in _blocking_paths("/cap/").items()} + assert infos["kwargs"] == infos["from_uri"] == infos["from_config"] + + +def test_paths_read_write_roundtrip_identically(): + for name, op in _blocking_paths("/rw/").items(): + key = f"k-{uuid4()}" + op.write(key, b"payload") + assert op.read(key) == b"payload", name + assert op.stat(key).content_length == len(b"payload"), name + + +@pytest.mark.asyncio +async def test_async_paths_read_write_roundtrip_identically(): + for name, op in _async_paths("/arw/").items(): + key = f"k-{uuid4()}" + await op.write(key, b"payload") + assert (await op.read(key)) == b"payload", name + + +def test_paths_pickle_consistently(): + for name, op in _blocking_paths("/pk/").items(): + restored = pickle.loads(pickle.dumps(op)) + assert isinstance(restored, opendal.Operator), name + assert 'root="/pk/"' in repr(restored), name + + +def test_kwargs_and_config_agree_on_string_options(): + from_kwargs = opendal.Operator("memory", root="/agree/") + from_config = opendal.Operator.from_config(MemoryConfig(root="/agree/")) + assert _info(from_kwargs) == _info(from_config) + + +@pytest.mark.parametrize( + "build", + [ + lambda: opendal.Operator("s3", bucket="b", enable_versioning=True), + lambda: opendal.AsyncOperator("s3", bucket="b", enable_versioning=True), + lambda: opendal.Operator.from_uri("s3://b", enable_versioning=True), + lambda: opendal.AsyncOperator.from_uri("s3://b", enable_versioning=True), + ], +) +def test_non_string_kwarg_raises_clear_error(build): + with pytest.raises(opendal.exceptions.ConfigInvalid, match="must be a string"): + build() + + +def test_string_kwarg_accepted(): + op = opendal.Operator("s3", bucket="b", enable_versioning="true") + assert isinstance(op, opendal.Operator) diff --git a/dev/src/generate/python.j2 b/dev/src/generate/python.j2 index 7ff45f5231ab..57a240b484bb 100644 --- a/dev/src/generate/python.j2 +++ b/dev/src/generate/python.j2 @@ -16,6 +16,7 @@ // under the License. use crate::*; +use std::collections::HashMap; #[pyclass( eq, @@ -81,3 +82,108 @@ impl_enum_to_str!( {%- endfor %} } ); + +pub trait ConfigBuilder: Send + Sync { + fn build(&self) -> PyResult; + fn scheme(&self) -> &'static str; + fn to_map(&self) -> HashMap; + fn is_picklable(&self) -> bool; +} + +/// Base class for all service configs. +#[pyclass(module = "opendal.services", subclass)] +pub struct ServiceConfig(pub Box); + +#[pymethods] +impl ServiceConfig { + /// The service scheme this config targets, e.g. ``"s3"``. + #[getter] + pub fn scheme(&self) -> &'static str { + self.0.scheme() + } +} +{% for name in srvs %} +#[cfg(feature = "{{ service_to_feature(name) }}")] +{{ config_class_doc(name) }} +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct {{ service_to_pascal(name) }}Config(ocore::services::{{ service_to_pascal(name) }}Config); + +#[cfg(feature = "{{ service_to_feature(name) }}")] +#[allow(deprecated)] +impl ConfigBuilder for {{ service_to_pascal(name) }}Config { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "{{ snake_to_kebab_case(name) }}" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); +{%- for field in srvs[name].config %} + {{ config_to_map(field) }} +{%- endfor %} + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; +{%- for field in srvs[name].config %} + {{ config_picklable(field) }} +{%- endfor %} + ok + } +} + +#[cfg(feature = "{{ service_to_feature(name) }}")] +#[allow(deprecated)] +#[pymethods] +impl {{ service_to_pascal(name) }}Config { + #[new] + #[pyo3(signature = ( +{%- for field in srvs[name].config %} + {{ config_new_param(field) }}, +{%- endfor %} + ))] + #[allow(clippy::too_many_arguments)] + fn new( +{%- for field in srvs[name].config %} + {{ field.name }}: {{ config_param_type(field) }}, +{%- endfor %} + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); +{%- for field in srvs[name].config %} + {{ config_to_opts(field) }} +{%- endfor %} + #[allow(unused_mut)] + let mut cfg = + ::from_iter(opts) + .map_err(format_pyerr)?; +{%- for field in srvs[name].config %} + {{ config_assign_map(field) }} +{%- endfor %} + let this = Self(cfg); + Ok(pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))).add_subclass(this)) + } +{% for field in srvs[name].config %} + {{ config_getter(field) }} +{%- endfor %} +} +{% endfor %} +// Defined here (not in `lib.rs`) so pyo3 introspection sees the config +// `#[pymodule_export]`s as literal items. +#[pyo3::pymodule(submodule, name = "services")] +pub mod services_submodule { + #[pymodule_export] + use crate::Scheme; + #[pymodule_export] + use crate::ServiceConfig; +{%- for name in srvs %} + #[cfg(feature = "{{ service_to_feature(name) }}")] + #[pymodule_export] + use crate::{{ service_to_pascal(name) }}Config; +{%- endfor %} +} + diff --git a/dev/src/generate/python.rs b/dev/src/generate/python.rs index 2cab43e09e0f..6a4d055a2861 100644 --- a/dev/src/generate/python.rs +++ b/dev/src/generate/python.rs @@ -15,25 +15,69 @@ // specific language governing permissions and limitations // under the License. -use crate::generate::parser::{ConfigType, Services, sorted_services}; +use crate::generate::parser::{Config, ConfigType, Service, Services, sorted_services}; use anyhow::Result; use minijinja::value::ViaDeserialize; use minijinja::{Environment, context}; use std::fs; use std::path::PathBuf; +/// Path-like fields, widened to accept `os.PathLike` in addition to `str`. +fn is_path_like_field(name: &str) -> bool { + matches!(name, "root" | "atomic_write_dir") +} + +/// Whether a field is keyword-optional in Python (optional core fields and bools). +fn field_is_optional(field: &Config) -> bool { + field.optional || field.value == ConfigType::Bool +} + +/// Order fields so required (no-default) params precede optional ones, as pyo3 +/// signatures require. Order within each group stays alphabetical. +fn ordered_for_signature(services: Services) -> Services { + services + .into_iter() + .map(|(k, srv)| { + let mut config = srv.config; + config.sort_by(|a, b| { + field_is_optional(a) + .cmp(&field_is_optional(b)) + .then_with(|| a.name.cmp(&b.name)) + }); + (k, Service { config }) + }) + .collect() +} + fn enabled_service(srv: &str) -> bool { - match srv { - // not enabled in bindings/python/Cargo.toml - "etcd" | "foundationdb" | "hdfs" | "rocksdb" | "tikv" | "github" | "cloudflare_kv" - | "monoiofs" | "dbfs" | "surrealdb" | "d1" | "opfs" | "compfs" | "lakefs" | "pcloud" - | "vercel_blob" | "foyer" => false, - _ => true, - } + // Services keyed by their directory name (kebab-case). Excludes services not + // built by `bindings/python/Cargo.toml`'s `services-all` feature. + !matches!( + srv, + "etcd" + | "foundationdb" + | "hdfs" + | "hdfs-native" + | "rocksdb" + | "tikv" + | "github" + | "cloudflare-kv" + | "monoiofs" + | "dbfs" + | "surrealdb" + | "d1" + | "opfs" + | "compfs" + | "lakefs" + | "pcloud" + | "vercel-blob" + | "sftp" + | "foyer" + ) } pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> { - let srvs = sorted_services(services, enabled_service); + let srvs = ordered_for_signature(sorted_services(services, enabled_service)); let mut env = Environment::new(); env.add_template("python", include_str!("python.j2"))?; env.add_function("snake_to_kebab_case", snake_to_kebab_case); @@ -42,6 +86,15 @@ pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> { env.add_function("make_python_type", make_python_type); env.add_function("make_pydoc_param_header", make_pydoc_param_header); env.add_function("make_pydoc_param", make_pydoc_param); + env.add_function("config_param_type", config_param_type); + env.add_function("config_new_param", config_new_param); + env.add_function("config_to_opts", config_to_opts); + env.add_function("config_assign_map", config_assign_map); + env.add_function("config_getter", config_getter); + env.add_function("config_to_map", config_to_map); + env.add_function("config_picklable", config_picklable); + env.add_function("config_class_doc", config_class_doc); + env.add_function("config_field_doc", config_field_doc); let tmpl = env.get_template("python")?; let output = workspace_dir.join("bindings/python/src/services.rs"); @@ -50,6 +103,7 @@ pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> { rendered.push('\n'); } fs::write(output, rendered)?; + Ok(()) } @@ -244,3 +298,281 @@ fn make_python_type(ty: ViaDeserialize) -> Result FieldKind { + match field.value { + ConfigType::Bool => FieldKind::Bool, + ConfigType::Usize | ConfigType::U64 | ConfigType::I64 | ConfigType::U32 | ConfigType::U16 => { + FieldKind::Num + } + ConfigType::Vec => FieldKind::Vec, + ConfigType::HashMap => FieldKind::Map, + ConfigType::Duration => FieldKind::Duration, + ConfigType::String => { + if is_opaque_string_fallback(&field.name) { + FieldKind::Other + } else if is_path_like_field(&field.name) { + FieldKind::Path + } else if field.optional { + FieldKind::Str + } else { + FieldKind::StrRequired + } + } + } +} + +/// Config fields whose core type is a string-parsed enum, not a `String`. +fn is_opaque_string_fallback(field: &str) -> bool { + matches!(field, "repo_type" | "download_mode") +} + +fn int_type(ty: ConfigType) -> &'static str { + match ty { + ConfigType::Usize => "usize", + ConfigType::U64 => "u64", + ConfigType::I64 => "i64", + ConfigType::U32 => "u32", + ConfigType::U16 => "u16", + _ => "usize", + } +} + +/// The Rust `#[new]` parameter type, chosen so introspection renders the exact +/// Python type. +fn config_param_type(field: ViaDeserialize) -> Result { + Ok(match field_kind(&field) { + FieldKind::Str | FieldKind::Duration | FieldKind::Other => "Option".to_string(), + FieldKind::StrRequired => "String".to_string(), + FieldKind::Path => "Option".to_string(), + FieldKind::Bool => "Option".to_string(), + FieldKind::Num => format!("Option<{}>", int_type(field.value)), + FieldKind::Vec => "Option>".to_string(), + FieldKind::Map => "Option>".to_string(), + }) +} + +/// The `#[new]` signature parameter, defaulting optional fields to `None`. +fn config_new_param(field: ViaDeserialize) -> Result { + if field_is_optional(&field) { + Ok(format!("{} = None", field.name)) + } else { + Ok(field.name.to_string()) + } +} + +/// Insert a `#[new]` parameter into the option map handed to `from_iter`. +fn config_to_opts(field: ViaDeserialize) -> Result { + let name = &field.name; + let key = &field.name; + Ok(match field_kind(&field) { + FieldKind::StrRequired => format!("opts.insert(\"{key}\".to_string(), {name});"), + FieldKind::Str | FieldKind::Duration | FieldKind::Other => { + format!("if let Some(v) = {name} {{ opts.insert(\"{key}\".to_string(), v); }}") + } + FieldKind::Path => format!( + "if let Some(v) = {name} {{ opts.insert(\"{key}\".to_string(), v.into_string()); }}" + ), + FieldKind::Bool => format!( + "if let Some(v) = {name} {{ opts.insert(\"{key}\".to_string(), if v {{ \"true\" }} else {{ \"false\" }}.to_string()); }}" + ), + FieldKind::Num => format!( + "if let Some(v) = {name} {{ opts.insert(\"{key}\".to_string(), v.to_string()); }}" + ), + FieldKind::Vec => format!( + "if let Some(v) = {name} {{ opts.insert(\"{key}\".to_string(), v.join(\",\")); }}" + ), + // Map fields are set on the config directly after `from_iter`. + FieldKind::Map => format!("// {name}: map field set after from_iter"), + }) +} + +/// Set a map-valued field on the core config after `from_iter` (map values +/// cannot pass through the flat option map). No-op for other kinds. +fn config_assign_map(field: ViaDeserialize) -> Result { + let name = &field.name; + Ok(match field_kind(&field) { + FieldKind::Map => format!("if let Some(v) = {name} {{ cfg.{name} = Some(v); }}"), + _ => format!("// {name}: not a map field"), + }) +} + +/// A getter (with doc comment) for a config field. Opaque fields have none. +fn config_getter(field: ViaDeserialize) -> Result { + let name = &field.name; + let kind = field_kind(&field); + + if kind == FieldKind::Other { + return Ok(format!("// {name}: opaque field, no getter")); + } + + let body = match kind { + FieldKind::StrRequired => { + format!("#[getter]\n fn {name}(&self) -> String {{ self.0.{name}.clone() }}") + } + FieldKind::Str => { + format!("#[getter]\n fn {name}(&self) -> Option {{ self.0.{name}.clone() }}") + } + // `PyPath`/`Option` keep the getter type equal to the constructor. + FieldKind::Path => format!( + "#[getter]\n fn {name}(&self) -> Option {{ self.0.{name}.clone().map(crate::PyPath::from) }}" + ), + FieldKind::Bool => { + format!("#[getter]\n fn {name}(&self) -> Option {{ Some(self.0.{name}) }}") + } + FieldKind::Num => { + let ty = int_type(field.value); + if field.optional { + format!("#[getter]\n fn {name}(&self) -> Option<{ty}> {{ self.0.{name} }}") + } else { + format!("#[getter]\n fn {name}(&self) -> {ty} {{ self.0.{name} }}") + } + } + FieldKind::Vec => { + format!( + "#[getter]\n fn {name}(&self) -> Option> {{ self.0.{name}.clone() }}" + ) + } + FieldKind::Map => format!( + "#[getter]\n fn {name}(&self) -> Option> {{ self.0.{name}.clone() }}" + ), + FieldKind::Duration => format!( + "#[getter]\n fn {name}(&self) -> Option {{ self.0.{name}.map(|d| format!(\"{{}}s\", d.as_secs())) }}" + ), + FieldKind::Other => unreachable!(), + }; + + let doc = config_field_doc(field)?; + if doc.is_empty() { + Ok(body) + } else { + Ok(format!("{doc}\n {body}")) + } +} + +/// Insert a field into the flat option map used for `repr`/pickle. +fn config_to_map(field: ViaDeserialize) -> Result { + let name = &field.name; + let key = &field.name; + Ok(match field_kind(&field) { + FieldKind::StrRequired => { + format!("map.insert(\"{key}\".to_string(), self.0.{name}.clone());") + } + FieldKind::Str | FieldKind::Path => format!( + "if let Some(v) = &self.0.{name} {{ map.insert(\"{key}\".to_string(), v.clone()); }}" + ), + FieldKind::Bool => format!( + "map.insert(\"{key}\".to_string(), if self.0.{name} {{ \"true\" }} else {{ \"false\" }}.to_string());" + ), + FieldKind::Num => { + if field.optional { + format!( + "if let Some(v) = &self.0.{name} {{ map.insert(\"{key}\".to_string(), v.to_string()); }}" + ) + } else { + format!("map.insert(\"{key}\".to_string(), self.0.{name}.to_string());") + } + } + FieldKind::Vec => format!( + "if let Some(v) = &self.0.{name} {{ map.insert(\"{key}\".to_string(), v.join(\",\")); }}" + ), + FieldKind::Duration => format!( + "if let Some(d) = &self.0.{name} {{ map.insert(\"{key}\".to_string(), format!(\"{{}}s\", d.as_secs())); }}" + ), + // Map/opaque fields are omitted from the flat option map. + FieldKind::Map => format!("// {name}: map field omitted from flat option map"), + FieldKind::Other => format!("// {name}: opaque field omitted from flat option map"), + }) +} + +/// Flip `ok` to false when a map-valued field is set (breaks flat-map pickling). +fn config_picklable(field: ViaDeserialize) -> Result { + let name = &field.name; + Ok(match field_kind(&field) { + FieldKind::Map => format!("if self.0.{name}.is_some() {{ ok = false; }}"), + _ => format!("// {name}: always picklable"), + }) +} + +/// Render text as Rust `///` doc-comment lines (lifted into `.pyi` docstrings). +fn to_doc_lines(text: &str, indent: usize) -> String { + let pad = " ".repeat(indent); + let normalized: String = text.split_whitespace().collect::>().join(" "); + if normalized.is_empty() { + return String::new(); + } + format_text(&normalized, 0, 76) + .lines() + .map(|line| { + let line = line.trim_end(); + if line.is_empty() { + format!("{pad}///") + } else { + format!("{pad}/// {line}") + } + }) + .collect::>() + .join("\n") +} + +/// The class-level doc comment for a service config. +fn config_class_doc(service: &str) -> Result { + Ok(to_doc_lines( + &format!( + "Configuration for the `{}` service.", + snake_to_kebab_case(service) + ), + 0, + )) +} + +/// The field doc comment, including any deprecation note. +fn config_field_doc(field: ViaDeserialize) -> Result { + let mut text = field.comments.trim().to_string(); + + match field_kind(&field) { + FieldKind::Duration => { + if !text.is_empty() { + text.push(' '); + } + text.push_str("Accepts a humantime duration string (e.g. \"5s\")."); + } + FieldKind::Vec => { + if !text.is_empty() { + text.push(' '); + } + text.push_str("A list of strings, serialized as a \",\" separated value."); + } + _ => {} + } + + if let Some(deprecated) = &field.deprecated { + if !text.is_empty() { + text.push(' '); + } + text.push_str(&format!( + "[Deprecated since {}] {}", + deprecated.since, + deprecated.note.trim() + )); + } + + Ok(to_doc_lines(&text, 4)) +} + + From 189640915827f04601209b2d1872fcecc66f8ca1 Mon Sep 17 00:00:00 2001 From: chitralverma Date: Wed, 8 Jul 2026 15:07:35 +0530 Subject: [PATCH 2/3] chore(bindings/python): drop no-op placeholder comments from generated services.rs --- bindings/python/src/services.rs | 574 -------------------------------- dev/src/generate/python.j2 | 25 +- dev/src/generate/python.rs | 11 +- 3 files changed, 25 insertions(+), 585 deletions(-) diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs index dbc2bc89bec5..21616140ff68 100644 --- a/bindings/python/src/services.rs +++ b/bindings/python/src/services.rs @@ -318,12 +318,6 @@ impl ConfigBuilder for AliyunDriveConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // drive_type: always picklable - // access_token: always picklable - // client_id: always picklable - // client_secret: always picklable - // refresh_token: always picklable - // root: always picklable ok } } @@ -371,12 +365,6 @@ impl AliyunDriveConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // drive_type: not a map field - // access_token: not a map field - // client_id: not a map field - // client_secret: not a map field - // refresh_token: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -456,8 +444,6 @@ impl ConfigBuilder for AlluxioConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // endpoint: always picklable - // root: always picklable ok } } @@ -487,8 +473,6 @@ impl AlluxioConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // endpoint: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -571,17 +555,6 @@ impl ConfigBuilder for AzblobConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // container: always picklable - // account_key: always picklable - // account_name: always picklable - // batch_max_operations: always picklable - // encryption_algorithm: always picklable - // encryption_key: always picklable - // encryption_key_sha256: always picklable - // endpoint: always picklable - // root: always picklable - // sas_token: always picklable - // skip_signature: always picklable ok } } @@ -657,17 +630,6 @@ impl AzblobConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // container: not a map field - // account_key: not a map field - // account_name: not a map field - // batch_max_operations: not a map field - // encryption_algorithm: not a map field - // encryption_key: not a map field - // encryption_key_sha256: not a map field - // endpoint: not a map field - // root: not a map field - // sas_token: not a map field - // skip_signature: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -796,17 +758,6 @@ impl ConfigBuilder for AzdlsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // filesystem: always picklable - // account_key: always picklable - // account_name: always picklable - // authority_host: always picklable - // client_id: always picklable - // client_secret: always picklable - // enable_hns: always picklable - // endpoint: always picklable - // root: always picklable - // sas_token: always picklable - // tenant_id: always picklable ok } } @@ -882,17 +833,6 @@ impl AzdlsConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // filesystem: not a map field - // account_key: not a map field - // account_name: not a map field - // authority_host: not a map field - // client_id: not a map field - // client_secret: not a map field - // enable_hns: not a map field - // endpoint: not a map field - // root: not a map field - // sas_token: not a map field - // tenant_id: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -1006,12 +946,6 @@ impl ConfigBuilder for AzfileConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // share_name: always picklable - // account_key: always picklable - // account_name: always picklable - // endpoint: always picklable - // root: always picklable - // sas_token: always picklable ok } } @@ -1059,12 +993,6 @@ impl AzfileConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // share_name: not a map field - // account_key: not a map field - // account_name: not a map field - // endpoint: not a map field - // root: not a map field - // sas_token: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -1138,11 +1066,6 @@ impl ConfigBuilder for B2Config { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // bucket: always picklable - // bucket_id: always picklable - // application_key: always picklable - // application_key_id: always picklable - // root: always picklable ok } } @@ -1183,11 +1106,6 @@ impl B2Config { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // bucket: not a map field - // bucket_id: not a map field - // application_key: not a map field - // application_key_id: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -1255,7 +1173,6 @@ impl ConfigBuilder for CacacheConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // datadir: always picklable ok } } @@ -1278,7 +1195,6 @@ impl CacacheConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // datadir: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -1352,14 +1268,6 @@ impl ConfigBuilder for CosConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // bucket: always picklable - // disable_config_load: always picklable - // enable_versioning: always picklable - // endpoint: always picklable - // root: always picklable - // secret_id: always picklable - // secret_key: always picklable - // security_token: always picklable ok } } @@ -1425,14 +1333,6 @@ impl CosConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // bucket: not a map field - // disable_config_load: not a map field - // enable_versioning: not a map field - // endpoint: not a map field - // root: not a map field - // secret_id: not a map field - // secret_key: not a map field - // security_token: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -1520,7 +1420,6 @@ impl ConfigBuilder for DashmapConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // root: always picklable ok } } @@ -1543,7 +1442,6 @@ impl DashmapConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -1596,11 +1494,6 @@ impl ConfigBuilder for DropboxConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // access_token: always picklable - // client_id: always picklable - // client_secret: always picklable - // refresh_token: always picklable - // root: always picklable ok } } @@ -1645,11 +1538,6 @@ impl DropboxConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // access_token: not a map field - // client_id: not a map field - // client_secret: not a map field - // refresh_token: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -1713,8 +1601,6 @@ impl ConfigBuilder for FsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // atomic_write_dir: always picklable - // root: always picklable ok } } @@ -1744,8 +1630,6 @@ impl FsConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // atomic_write_dir: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -1800,10 +1684,6 @@ impl ConfigBuilder for FtpConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // endpoint: always picklable - // password: always picklable - // root: always picklable - // user: always picklable ok } } @@ -1843,10 +1723,6 @@ impl FtpConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // endpoint: not a map field - // password: not a map field - // root: not a map field - // user: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -1963,20 +1839,6 @@ impl ConfigBuilder for GcsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // bucket: always picklable - // allow_anonymous: always picklable - // credential: always picklable - // credential_path: always picklable - // default_storage_class: always picklable - // disable_config_load: always picklable - // disable_vm_metadata: always picklable - // endpoint: always picklable - // predefined_acl: always picklable - // root: always picklable - // scope: always picklable - // service_account: always picklable - // skip_signature: always picklable - // token: always picklable ok } } @@ -2076,20 +1938,6 @@ impl GcsConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // bucket: not a map field - // allow_anonymous: not a map field - // credential: not a map field - // credential_path: not a map field - // default_storage_class: not a map field - // disable_config_load: not a map field - // disable_vm_metadata: not a map field - // endpoint: not a map field - // predefined_acl: not a map field - // root: not a map field - // scope: not a map field - // service_account: not a map field - // skip_signature: not a map field - // token: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -2212,11 +2060,6 @@ impl ConfigBuilder for GdriveConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // access_token: always picklable - // client_id: always picklable - // client_secret: always picklable - // refresh_token: always picklable - // root: always picklable ok } } @@ -2261,11 +2104,6 @@ impl GdriveConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // access_token: not a map field - // client_id: not a map field - // client_secret: not a map field - // refresh_token: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -2335,10 +2173,6 @@ impl ConfigBuilder for GhacConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // endpoint: always picklable - // root: always picklable - // runtime_token: always picklable - // version: always picklable ok } } @@ -2378,10 +2212,6 @@ impl GhacConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // endpoint: not a map field - // root: not a map field - // runtime_token: not a map field - // version: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -2455,13 +2285,6 @@ impl ConfigBuilder for GoosefsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // auth_type: always picklable - // auth_username: always picklable - // block_size: always picklable - // chunk_size: always picklable - // master_addr: always picklable - // root: always picklable - // write_type: always picklable ok } } @@ -2516,13 +2339,6 @@ impl GoosefsConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // auth_type: not a map field - // auth_username: not a map field - // block_size: not a map field - // chunk_size: not a map field - // master_addr: not a map field - // root: not a map field - // write_type: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -2626,11 +2442,6 @@ impl ConfigBuilder for GridfsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // bucket: always picklable - // chunk_size: always picklable - // connection_string: always picklable - // database: always picklable - // root: always picklable ok } } @@ -2675,11 +2486,6 @@ impl GridfsConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // bucket: not a map field - // chunk_size: not a map field - // connection_string: not a map field - // database: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -2733,14 +2539,12 @@ impl ConfigBuilder for HfConfig { fn to_map(&self) -> HashMap { #[allow(unused_mut)] let mut map = HashMap::new(); - // download_mode: opaque field omitted from flat option map if let Some(v) = &self.0.endpoint { map.insert("endpoint".to_string(), v.clone()); } if let Some(v) = &self.0.repo_id { map.insert("repo_id".to_string(), v.clone()); } - // repo_type: opaque field omitted from flat option map if let Some(v) = &self.0.revision { map.insert("revision".to_string(), v.clone()); } @@ -2755,13 +2559,6 @@ impl ConfigBuilder for HfConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // download_mode: always picklable - // endpoint: always picklable - // repo_id: always picklable - // repo_type: always picklable - // revision: always picklable - // root: always picklable - // token: always picklable ok } } @@ -2816,13 +2613,6 @@ impl HfConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // download_mode: not a map field - // endpoint: not a map field - // repo_id: not a map field - // repo_type: not a map field - // revision: not a map field - // root: not a map field - // token: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -2830,7 +2620,6 @@ impl HfConfig { ) } - // download_mode: opaque field, no getter /// Endpoint of the Hugging Face Hub. /// Default is "https://huggingface.co". #[getter] @@ -2843,7 +2632,6 @@ impl HfConfig { fn repo_id(&self) -> Option { self.0.repo_id.clone() } - // repo_type: opaque field, no getter /// Revision of this backend. /// Default is main. #[getter] @@ -2903,11 +2691,6 @@ impl ConfigBuilder for HttpConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // endpoint: always picklable - // password: always picklable - // root: always picklable - // token: always picklable - // username: always picklable ok } } @@ -2952,11 +2735,6 @@ impl HttpConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // endpoint: not a map field - // password: not a map field - // root: not a map field - // token: not a map field - // username: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -3020,8 +2798,6 @@ impl ConfigBuilder for IpfsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // endpoint: always picklable - // root: always picklable ok } } @@ -3051,8 +2827,6 @@ impl IpfsConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // endpoint: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -3101,8 +2875,6 @@ impl ConfigBuilder for IpmfsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // endpoint: always picklable - // root: always picklable ok } } @@ -3132,8 +2904,6 @@ impl IpmfsConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // endpoint: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -3184,10 +2954,6 @@ impl ConfigBuilder for KoofrConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // email: always picklable - // endpoint: always picklable - // password: always picklable - // root: always picklable ok } } @@ -3223,10 +2989,6 @@ impl KoofrConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // email: not a map field - // endpoint: not a map field - // password: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -3299,12 +3061,6 @@ impl ConfigBuilder for MemcachedConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // connection_pool_max_size: always picklable - // default_ttl: always picklable - // endpoint: always picklable - // password: always picklable - // root: always picklable - // username: always picklable ok } } @@ -3354,12 +3110,6 @@ impl MemcachedConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // connection_pool_max_size: not a map field - // default_ttl: not a map field - // endpoint: not a map field - // password: not a map field - // root: not a map field - // username: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -3430,7 +3180,6 @@ impl ConfigBuilder for MemoryConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // root: always picklable ok } } @@ -3453,7 +3202,6 @@ impl MemoryConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -3503,10 +3251,6 @@ impl ConfigBuilder for MiniMokaConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // max_capacity: always picklable - // root: always picklable - // time_to_idle: always picklable - // time_to_live: always picklable ok } } @@ -3546,10 +3290,6 @@ impl MiniMokaConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // max_capacity: not a map field - // root: not a map field - // time_to_idle: not a map field - // time_to_live: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -3623,11 +3363,6 @@ impl ConfigBuilder for MokaConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // max_capacity: always picklable - // name: always picklable - // root: always picklable - // time_to_idle: always picklable - // time_to_live: always picklable ok } } @@ -3672,11 +3407,6 @@ impl MokaConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // max_capacity: not a map field - // name: not a map field - // root: not a map field - // time_to_idle: not a map field - // time_to_live: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -3758,12 +3488,6 @@ impl ConfigBuilder for MongodbConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // collection: always picklable - // connection_string: always picklable - // database: always picklable - // key_field: always picklable - // root: always picklable - // value_field: always picklable ok } } @@ -3813,12 +3537,6 @@ impl MongodbConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // collection: not a map field - // connection_string: not a map field - // database: not a map field - // key_field: not a map field - // root: not a map field - // value_field: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -3896,11 +3614,6 @@ impl ConfigBuilder for MysqlConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // connection_string: always picklable - // key_field: always picklable - // root: always picklable - // table: always picklable - // value_field: always picklable ok } } @@ -3945,11 +3658,6 @@ impl MysqlConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // connection_string: not a map field - // key_field: not a map field - // root: not a map field - // table: not a map field - // value_field: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -4039,12 +3747,6 @@ impl ConfigBuilder for ObsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // access_key_id: always picklable - // bucket: always picklable - // enable_versioning: always picklable - // endpoint: always picklable - // root: always picklable - // secret_access_key: always picklable ok } } @@ -4097,12 +3799,6 @@ impl ObsConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // access_key_id: not a map field - // bucket: not a map field - // enable_versioning: not a map field - // endpoint: not a map field - // root: not a map field - // secret_access_key: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -4191,12 +3887,6 @@ impl ConfigBuilder for OnedriveConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // access_token: always picklable - // client_id: always picklable - // client_secret: always picklable - // enable_versioning: always picklable - // refresh_token: always picklable - // root: always picklable ok } } @@ -4249,12 +3939,6 @@ impl OnedriveConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // access_token: not a map field - // client_id: not a map field - // client_secret: not a map field - // enable_versioning: not a map field - // refresh_token: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -4403,28 +4087,6 @@ impl ConfigBuilder for OssConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // bucket: always picklable - // access_key_id: always picklable - // access_key_secret: always picklable - // addressing_style: always picklable - // allow_anonymous: always picklable - // batch_max_operations: always picklable - // delete_max_size: always picklable - // enable_versioning: always picklable - // endpoint: always picklable - // external_id: always picklable - // oidc_provider_arn: always picklable - // oidc_token_file: always picklable - // presign_addressing_style: always picklable - // presign_endpoint: always picklable - // role_arn: always picklable - // role_session_name: always picklable - // root: always picklable - // security_token: always picklable - // server_side_encryption: always picklable - // server_side_encryption_key_id: always picklable - // skip_signature: always picklable - // sts_endpoint: always picklable ok } } @@ -4561,28 +4223,6 @@ impl OssConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // bucket: not a map field - // access_key_id: not a map field - // access_key_secret: not a map field - // addressing_style: not a map field - // allow_anonymous: not a map field - // batch_max_operations: not a map field - // delete_max_size: not a map field - // enable_versioning: not a map field - // endpoint: not a map field - // external_id: not a map field - // oidc_provider_arn: not a map field - // oidc_token_file: not a map field - // presign_addressing_style: not a map field - // presign_endpoint: not a map field - // role_arn: not a map field - // role_session_name: not a map field - // root: not a map field - // security_token: not a map field - // server_side_encryption: not a map field - // server_side_encryption_key_id: not a map field - // skip_signature: not a map field - // sts_endpoint: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -4753,9 +4393,6 @@ impl ConfigBuilder for PersyConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // datafile: always picklable - // index: always picklable - // segment: always picklable ok } } @@ -4790,9 +4427,6 @@ impl PersyConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // datafile: not a map field - // index: not a map field - // segment: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -4856,11 +4490,6 @@ impl ConfigBuilder for PostgresqlConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // connection_string: always picklable - // key_field: always picklable - // root: always picklable - // table: always picklable - // value_field: always picklable ok } } @@ -4905,11 +4534,6 @@ impl PostgresqlConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // connection_string: not a map field - // key_field: not a map field - // root: not a map field - // table: not a map field - // value_field: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -4985,9 +4609,6 @@ impl ConfigBuilder for RedbConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // datadir: always picklable - // root: always picklable - // table: always picklable ok } } @@ -5022,9 +4643,6 @@ impl RedbConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // datadir: not a map field - // root: not a map field - // table: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -5094,14 +4712,6 @@ impl ConfigBuilder for RedisConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // db: always picklable - // cluster_endpoints: always picklable - // connection_pool_max_size: always picklable - // default_ttl: always picklable - // endpoint: always picklable - // password: always picklable - // root: always picklable - // username: always picklable ok } } @@ -5161,14 +4771,6 @@ impl RedisConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // db: not a map field - // cluster_endpoints: not a map field - // connection_pool_max_size: not a map field - // default_ttl: not a map field - // endpoint: not a map field - // password: not a map field - // root: not a map field - // username: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -5262,7 +4864,6 @@ impl ConfigBuilder for S3Config { if let Some(v) = &self.0.assume_role_duration_seconds { map.insert("assume_role_duration_seconds".to_string(), v.to_string()); } - // assume_role_session_tags: map field omitted from flat option map if let Some(v) = &self.0.batch_max_operations { map.insert("batch_max_operations".to_string(), v.to_string()); } @@ -5421,41 +5022,9 @@ impl ConfigBuilder for S3Config { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // bucket: always picklable - // access_key_id: always picklable - // allow_anonymous: always picklable - // assume_role_duration_seconds: always picklable if self.0.assume_role_session_tags.is_some() { ok = false; } - // batch_max_operations: always picklable - // checksum_algorithm: always picklable - // default_acl: always picklable - // default_storage_class: always picklable - // delete_max_size: always picklable - // disable_config_load: always picklable - // disable_ec2_metadata: always picklable - // disable_list_objects_v2: always picklable - // disable_stat_with_override: always picklable - // disable_write_with_if_match: always picklable - // enable_request_payer: always picklable - // enable_versioning: always picklable - // enable_virtual_host_style: always picklable - // enable_write_with_append: always picklable - // endpoint: always picklable - // external_id: always picklable - // region: always picklable - // role_arn: always picklable - // role_session_name: always picklable - // root: always picklable - // secret_access_key: always picklable - // server_side_encryption: always picklable - // server_side_encryption_aws_kms_key_id: always picklable - // server_side_encryption_customer_algorithm: always picklable - // server_side_encryption_customer_key: always picklable - // server_side_encryption_customer_key_md5: always picklable - // session_token: always picklable - // skip_signature: always picklable ok } } @@ -5551,7 +5120,6 @@ impl S3Config { if let Some(v) = assume_role_duration_seconds { opts.insert("assume_role_duration_seconds".to_string(), v.to_string()); } - // assume_role_session_tags: map field set after from_iter if let Some(v) = batch_max_operations { opts.insert("batch_max_operations".to_string(), v.to_string()); } @@ -5669,41 +5237,9 @@ impl S3Config { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // bucket: not a map field - // access_key_id: not a map field - // allow_anonymous: not a map field - // assume_role_duration_seconds: not a map field if let Some(v) = assume_role_session_tags { cfg.assume_role_session_tags = Some(v); } - // batch_max_operations: not a map field - // checksum_algorithm: not a map field - // default_acl: not a map field - // default_storage_class: not a map field - // delete_max_size: not a map field - // disable_config_load: not a map field - // disable_ec2_metadata: not a map field - // disable_list_objects_v2: not a map field - // disable_stat_with_override: not a map field - // disable_write_with_if_match: not a map field - // enable_request_payer: not a map field - // enable_versioning: not a map field - // enable_virtual_host_style: not a map field - // enable_write_with_append: not a map field - // endpoint: not a map field - // external_id: not a map field - // region: not a map field - // role_arn: not a map field - // role_session_name: not a map field - // root: not a map field - // secret_access_key: not a map field - // server_side_encryption: not a map field - // server_side_encryption_aws_kms_key_id: not a map field - // server_side_encryption_customer_algorithm: not a map field - // server_side_encryption_customer_key: not a map field - // server_side_encryption_customer_key_md5: not a map field - // session_token: not a map field - // skip_signature: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -5990,11 +5526,6 @@ impl ConfigBuilder for SeafileConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // repo_name: always picklable - // endpoint: always picklable - // password: always picklable - // root: always picklable - // username: always picklable ok } } @@ -6037,11 +5568,6 @@ impl SeafileConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // repo_name: not a map field - // endpoint: not a map field - // password: not a map field - // root: not a map field - // username: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -6110,9 +5636,6 @@ impl ConfigBuilder for SledConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // datadir: always picklable - // root: always picklable - // tree: always picklable ok } } @@ -6147,9 +5670,6 @@ impl SledConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // datadir: not a map field - // root: not a map field - // tree: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -6212,11 +5732,6 @@ impl ConfigBuilder for SqliteConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // connection_string: always picklable - // key_field: always picklable - // root: always picklable - // table: always picklable - // value_field: always picklable ok } } @@ -6261,11 +5776,6 @@ impl SqliteConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // connection_string: not a map field - // key_field: not a map field - // root: not a map field - // table: not a map field - // value_field: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -6349,12 +5859,6 @@ impl ConfigBuilder for SwiftConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // container: always picklable - // endpoint: always picklable - // root: always picklable - // temp_url_hash_algorithm: always picklable - // temp_url_key: always picklable - // token: always picklable ok } } @@ -6404,12 +5908,6 @@ impl SwiftConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // container: not a map field - // endpoint: not a map field - // root: not a map field - // temp_url_hash_algorithm: not a map field - // temp_url_key: not a map field - // token: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -6516,15 +6014,6 @@ impl ConfigBuilder for TosConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // bucket: always picklable - // access_key_id: always picklable - // disable_config_load: always picklable - // endpoint: always picklable - // region: always picklable - // root: always picklable - // secret_access_key: always picklable - // security_token: always picklable - // skip_signature: always picklable ok } } @@ -6593,15 +6082,6 @@ impl TosConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // bucket: not a map field - // access_key_id: not a map field - // disable_config_load: not a map field - // endpoint: not a map field - // region: not a map field - // root: not a map field - // secret_access_key: not a map field - // security_token: not a map field - // skip_signature: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -6707,10 +6187,6 @@ impl ConfigBuilder for UpyunConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // bucket: always picklable - // operator: always picklable - // password: always picklable - // root: always picklable ok } } @@ -6748,10 +6224,6 @@ impl UpyunConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // bucket: not a map field - // operator: not a map field - // password: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -6817,10 +6289,6 @@ impl ConfigBuilder for VercelArtifactsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // access_token: always picklable - // endpoint: always picklable - // team_id: always picklable - // team_slug: always picklable ok } } @@ -6861,10 +6329,6 @@ impl VercelArtifactsConfig { let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // access_token: not a map field - // endpoint: not a map field - // team_id: not a map field - // team_slug: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -6972,17 +6436,6 @@ impl ConfigBuilder for WebdavConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // disable_copy: always picklable - // disable_create_dir: always picklable - // enable_conditional_read: always picklable - // enable_user_metadata: always picklable - // endpoint: always picklable - // password: always picklable - // root: always picklable - // token: always picklable - // user_metadata_prefix: always picklable - // user_metadata_uri: always picklable - // username: always picklable ok } } @@ -7069,17 +6522,6 @@ impl WebdavConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // disable_copy: not a map field - // disable_create_dir: not a map field - // enable_conditional_read: not a map field - // enable_user_metadata: not a map field - // endpoint: not a map field - // password: not a map field - // root: not a map field - // token: not a map field - // user_metadata_prefix: not a map field - // user_metadata_uri: not a map field - // username: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -7220,12 +6662,6 @@ impl ConfigBuilder for WebhdfsConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // atomic_write_dir: always picklable - // delegation: always picklable - // disable_list_batch: always picklable - // endpoint: always picklable - // root: always picklable - // user_name: always picklable ok } } @@ -7278,12 +6714,6 @@ impl WebhdfsConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // atomic_write_dir: not a map field - // delegation: not a map field - // disable_list_batch: not a map field - // endpoint: not a map field - // root: not a map field - // user_name: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) @@ -7350,8 +6780,6 @@ impl ConfigBuilder for YandexDiskConfig { fn is_picklable(&self) -> bool { #[allow(unused_mut)] let mut ok = true; - // access_token: always picklable - // root: always picklable ok } } @@ -7379,8 +6807,6 @@ impl YandexDiskConfig { #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; - // access_token: not a map field - // root: not a map field let this = Self(cfg); Ok( pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) diff --git a/dev/src/generate/python.j2 b/dev/src/generate/python.j2 index 57a240b484bb..b415fb204d64 100644 --- a/dev/src/generate/python.j2 +++ b/dev/src/generate/python.j2 @@ -122,7 +122,10 @@ impl ConfigBuilder for {{ service_to_pascal(name) }}Config { #[allow(unused_mut)] let mut map = HashMap::new(); {%- for field in srvs[name].config %} - {{ config_to_map(field) }} +{%- set line = config_to_map(field) %} +{%- if line %} + {{ line }} +{%- endif %} {%- endfor %} map } @@ -130,7 +133,10 @@ impl ConfigBuilder for {{ service_to_pascal(name) }}Config { #[allow(unused_mut)] let mut ok = true; {%- for field in srvs[name].config %} - {{ config_picklable(field) }} +{%- set line = config_picklable(field) %} +{%- if line %} + {{ line }} +{%- endif %} {%- endfor %} ok } @@ -155,20 +161,29 @@ impl {{ service_to_pascal(name) }}Config { #[allow(unused_mut)] let mut opts: HashMap = HashMap::new(); {%- for field in srvs[name].config %} - {{ config_to_opts(field) }} +{%- set line = config_to_opts(field) %} +{%- if line %} + {{ line }} +{%- endif %} {%- endfor %} #[allow(unused_mut)] let mut cfg = ::from_iter(opts) .map_err(format_pyerr)?; {%- for field in srvs[name].config %} - {{ config_assign_map(field) }} +{%- set line = config_assign_map(field) %} +{%- if line %} + {{ line }} +{%- endif %} {%- endfor %} let this = Self(cfg); Ok(pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))).add_subclass(this)) } {% for field in srvs[name].config %} - {{ config_getter(field) }} +{%- set getter = config_getter(field) %} +{%- if getter %} + {{ getter }} +{%- endif %} {%- endfor %} } {% endfor %} diff --git a/dev/src/generate/python.rs b/dev/src/generate/python.rs index 6a4d055a2861..00fcd9673efd 100644 --- a/dev/src/generate/python.rs +++ b/dev/src/generate/python.rs @@ -398,7 +398,7 @@ fn config_to_opts(field: ViaDeserialize) -> Result format!("// {name}: map field set after from_iter"), + FieldKind::Map => String::new(), }) } @@ -408,7 +408,7 @@ fn config_assign_map(field: ViaDeserialize) -> Result format!("if let Some(v) = {name} {{ cfg.{name} = Some(v); }}"), - _ => format!("// {name}: not a map field"), + _ => String::new(), }) } @@ -418,7 +418,7 @@ fn config_getter(field: ViaDeserialize) -> Result) -> Result format!("// {name}: map field omitted from flat option map"), - FieldKind::Other => format!("// {name}: opaque field omitted from flat option map"), + FieldKind::Map | FieldKind::Other => String::new(), }) } @@ -505,7 +504,7 @@ fn config_picklable(field: ViaDeserialize) -> Result format!("if self.0.{name}.is_some() {{ ok = false; }}"), - _ => format!("// {name}: always picklable"), + _ => String::new(), }) } From 77b22165e86162970f87da70b7eb84a65312112b Mon Sep 17 00:00:00 2001 From: chitralverma Date: Wed, 8 Jul 2026 16:50:13 +0530 Subject: [PATCH 3/3] fix(bindings/python): keep Sftp and HdfsNative in the Scheme enum --- bindings/python/python/opendal/services.pyi | 2 + bindings/python/src/services.rs | 263 ++++++++++++++++++++ dev/src/generate/python.rs | 8 +- 3 files changed, 269 insertions(+), 4 deletions(-) diff --git a/bindings/python/python/opendal/services.pyi b/bindings/python/python/opendal/services.pyi index 432055c0e117..68b0b791da17 100644 --- a/bindings/python/python/opendal/services.pyi +++ b/bindings/python/python/opendal/services.pyi @@ -1695,6 +1695,7 @@ class Scheme: Ghac: Final[Scheme] Goosefs: Final[Scheme] Gridfs: Final[Scheme] + HdfsNative: Final[Scheme] Hf: Final[Scheme] Http: Final[Scheme] Ipfs: Final[Scheme] @@ -1715,6 +1716,7 @@ class Scheme: Redis: Final[Scheme] S3: Final[Scheme] Seafile: Final[Scheme] + Sftp: Final[Scheme] Sled: Final[Scheme] Sqlite: Final[Scheme] Swift: Final[Scheme] diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs index 21616140ff68..c6685c6464fc 100644 --- a/bindings/python/src/services.rs +++ b/bindings/python/src/services.rs @@ -64,6 +64,8 @@ pub enum Scheme { Goosefs, #[cfg(feature = "services-gridfs")] Gridfs, + #[cfg(feature = "services-hdfs-native")] + HdfsNative, #[cfg(feature = "services-hf")] Hf, #[cfg(feature = "services-http")] @@ -104,6 +106,8 @@ pub enum Scheme { S3, #[cfg(feature = "services-seafile")] Seafile, + #[cfg(feature = "services-sftp")] + Sftp, #[cfg(feature = "services-sled")] Sled, #[cfg(feature = "services-sqlite")] @@ -198,6 +202,8 @@ impl_enum_to_str!( Goosefs => "goosefs", #[cfg(feature = "services-gridfs")] Gridfs => "gridfs", + #[cfg(feature = "services-hdfs-native")] + HdfsNative => "hdfs-native", #[cfg(feature = "services-hf")] Hf => "hf", #[cfg(feature = "services-http")] @@ -238,6 +244,8 @@ impl_enum_to_str!( S3 => "s3", #[cfg(feature = "services-seafile")] Seafile => "seafile", + #[cfg(feature = "services-sftp")] + Sftp => "sftp", #[cfg(feature = "services-sled")] Sled => "sled", #[cfg(feature = "services-sqlite")] @@ -2521,6 +2529,120 @@ impl GridfsConfig { } } +#[cfg(feature = "services-hdfs-native")] +/// Configuration for the `hdfs-native` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct HdfsNativeConfig(ocore::services::HdfsNativeConfig); + +#[cfg(feature = "services-hdfs-native")] +#[allow(deprecated)] +impl ConfigBuilder for HdfsNativeConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "hdfs-native" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert( + "enable_append".to_string(), + if self.0.enable_append { + "true" + } else { + "false" + } + .to_string(), + ); + if let Some(v) = &self.0.name_node { + map.insert("name_node".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + if self.0.options.is_some() { + ok = false; + } + ok + } +} + +#[cfg(feature = "services-hdfs-native")] +#[allow(deprecated)] +#[pymethods] +impl HdfsNativeConfig { + #[new] + #[pyo3(signature = ( + enable_append = None, + name_node = None, + options = None, + root = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + enable_append: Option, + name_node: Option, + options: Option>, + root: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = enable_append { + opts.insert( + "enable_append".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = name_node { + opts.insert("name_node".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + if let Some(v) = options { + cfg.options = Some(v); + } + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Deprecated: HDFS Native append capability is enabled by default. + /// [Deprecated since 0.57.0] HDFS Native append capability is enabled by + /// default and this option is no longer needed. + #[getter] + fn enable_append(&self) -> Option { + Some(self.0.enable_append) + } + /// name_node of this backend + #[getter] + fn name_node(&self) -> Option { + self.0.name_node.clone() + } + /// other options for hdfs client + #[getter] + fn options(&self) -> Option> { + self.0.options.clone() + } + /// work dir of this backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } +} + #[cfg(feature = "services-hf")] /// Configuration for the `hf` service. #[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] @@ -5604,6 +5726,141 @@ impl SeafileConfig { } } +#[cfg(feature = "services-sftp")] +/// Configuration for the `sftp` service. +#[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct SftpConfig(ocore::services::SftpConfig); + +#[cfg(feature = "services-sftp")] +#[allow(deprecated)] +impl ConfigBuilder for SftpConfig { + fn build(&self) -> PyResult { + ocore::Operator::from_config(self.0.clone()).map_err(format_pyerr) + } + fn scheme(&self) -> &'static str { + "sftp" + } + fn to_map(&self) -> HashMap { + #[allow(unused_mut)] + let mut map = HashMap::new(); + map.insert( + "enable_copy".to_string(), + if self.0.enable_copy { "true" } else { "false" }.to_string(), + ); + if let Some(v) = &self.0.endpoint { + map.insert("endpoint".to_string(), v.clone()); + } + if let Some(v) = &self.0.key { + map.insert("key".to_string(), v.clone()); + } + if let Some(v) = &self.0.known_hosts_strategy { + map.insert("known_hosts_strategy".to_string(), v.clone()); + } + if let Some(v) = &self.0.root { + map.insert("root".to_string(), v.clone()); + } + if let Some(v) = &self.0.user { + map.insert("user".to_string(), v.clone()); + } + map + } + fn is_picklable(&self) -> bool { + #[allow(unused_mut)] + let mut ok = true; + ok + } +} + +#[cfg(feature = "services-sftp")] +#[allow(deprecated)] +#[pymethods] +impl SftpConfig { + #[new] + #[pyo3(signature = ( + enable_copy = None, + endpoint = None, + key = None, + known_hosts_strategy = None, + root = None, + user = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + enable_copy: Option, + endpoint: Option, + key: Option, + known_hosts_strategy: Option, + root: Option, + user: Option, + ) -> PyResult> { + #[allow(unused_mut)] + let mut opts: HashMap = HashMap::new(); + if let Some(v) = enable_copy { + opts.insert( + "enable_copy".to_string(), + if v { "true" } else { "false" }.to_string(), + ); + } + if let Some(v) = endpoint { + opts.insert("endpoint".to_string(), v); + } + if let Some(v) = key { + opts.insert("key".to_string(), v); + } + if let Some(v) = known_hosts_strategy { + opts.insert("known_hosts_strategy".to_string(), v); + } + if let Some(v) = root { + opts.insert("root".to_string(), v.into_string()); + } + if let Some(v) = user { + opts.insert("user".to_string(), v); + } + #[allow(unused_mut)] + let mut cfg = ::from_iter(opts) + .map_err(format_pyerr)?; + let this = Self(cfg); + Ok( + pyo3::PyClassInitializer::from(ServiceConfig(Box::new(this.clone()))) + .add_subclass(this), + ) + } + + /// Deprecated: SFTP copy capability is enabled by default. + /// [Deprecated since 0.57.0] SFTP copy capability is enabled by default and + /// this option is no longer needed. + #[getter] + fn enable_copy(&self) -> Option { + Some(self.0.enable_copy) + } + /// endpoint of this backend + #[getter] + fn endpoint(&self) -> Option { + self.0.endpoint.clone() + } + /// key of this backend + #[getter] + fn key(&self) -> Option { + self.0.key.clone() + } + /// known_hosts_strategy of this backend + #[getter] + fn known_hosts_strategy(&self) -> Option { + self.0.known_hosts_strategy.clone() + } + /// root of this backend + #[getter] + fn root(&self) -> Option { + self.0.root.clone().map(crate::PyPath::from) + } + /// user of this backend + #[getter] + fn user(&self) -> Option { + self.0.user.clone() + } +} + #[cfg(feature = "services-sled")] /// Configuration for the `sled` service. #[pyclass(module = "opendal.services", extends = ServiceConfig, frozen, skip_from_py_object)] @@ -6882,6 +7139,9 @@ pub mod services_submodule { #[cfg(feature = "services-gridfs")] #[pymodule_export] use crate::GridfsConfig; + #[cfg(feature = "services-hdfs-native")] + #[pymodule_export] + use crate::HdfsNativeConfig; #[cfg(feature = "services-hf")] #[pymodule_export] use crate::HfConfig; @@ -6946,6 +7206,9 @@ pub mod services_submodule { use crate::SeafileConfig; #[pymodule_export] use crate::ServiceConfig; + #[cfg(feature = "services-sftp")] + #[pymodule_export] + use crate::SftpConfig; #[cfg(feature = "services-sled")] #[pymodule_export] use crate::SledConfig; diff --git a/dev/src/generate/python.rs b/dev/src/generate/python.rs index 00fcd9673efd..948141679848 100644 --- a/dev/src/generate/python.rs +++ b/dev/src/generate/python.rs @@ -50,14 +50,15 @@ fn ordered_for_signature(services: Services) -> Services { } fn enabled_service(srv: &str) -> bool { - // Services keyed by their directory name (kebab-case). Excludes services not - // built by `bindings/python/Cargo.toml`'s `services-all` feature. + // Services (keyed by kebab-case directory name) that are not exposed to the + // Python binding. Everything else is emitted `#[cfg(feature = + // "services-")]`-gated, so services absent from `services-all` (e.g. + // sftp, hdfs-native) still appear in the `Scheme` enum but compile out. !matches!( srv, "etcd" | "foundationdb" | "hdfs" - | "hdfs-native" | "rocksdb" | "tikv" | "github" @@ -71,7 +72,6 @@ fn enabled_service(srv: &str) -> bool { | "lakefs" | "pcloud" | "vercel-blob" - | "sftp" | "foyer" ) }