diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index bf57397945e..84c38d73e34 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -47,7 +47,7 @@ default = [ # NOTE: this is the feature we used to build pypi wheels. # When enable or disable some features, -# also need to update dev/src/generate/binding_python.rs `enabled_service` to match it. +# Also, you need to update the `enabled_service` function in dev/src/generate/python.rs to match it. services-all = [ "default", "services-aliyun-drive", diff --git a/bindings/python/python/opendal/__base.pyi b/bindings/python/python/opendal/__base.pyi index c9e794b8198..6096af429bf 100644 --- a/bindings/python/python/opendal/__base.pyi +++ b/bindings/python/python/opendal/__base.pyi @@ -1,9 +1,3 @@ -""" -this file is generated by opendal/dev/generate/binding_python.rs, and opendal.__base doesn't exists. - -DO NOT EDIT IT Manually -""" - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -21,6 +15,12 @@ DO NOT EDIT IT Manually # specific language governing permissions and limitations # under the License. +""" +> DO NOT EDIT IT MANUALLY < + +This file is generated by opendal/dev/generate/python.rs. `opendal.__base` doesn't exists. +""" + from typing import overload, Literal, TypeAlias # `true`/`false`` in any case, for example, `true`/`True`/`TRUE` `false`/`False`/`FALSE` @@ -33,6 +33,7 @@ _int: TypeAlias = str # for more details _duration: TypeAlias = str + # A "," separated string, for example `"127.0.0.1:1,127.0.0.1:2"` _strings: TypeAlias = str @@ -42,6 +43,7 @@ class _Base: The services list here is support by opendal pypi wheel. """ + @overload def __init__( self, @@ -55,6 +57,7 @@ class _Base: client_secret: str = ..., refresh_token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -64,6 +67,7 @@ class _Base: root: str = ..., endpoint: str = ..., ) -> None: ... + @overload def __init__( self, @@ -76,6 +80,7 @@ class _Base: public_key: str = ..., parent_resource_id: str = ..., ) -> None: ... + @overload def __init__( self, @@ -93,6 +98,7 @@ class _Base: sas_token: str = ..., batch_max_operations: _int = ..., ) -> None: ... + @overload def __init__( self, @@ -105,6 +111,7 @@ class _Base: account_name: str = ..., account_key: str = ..., ) -> None: ... + @overload def __init__( self, @@ -118,6 +125,7 @@ class _Base: account_key: str = ..., sas_token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -130,6 +138,7 @@ class _Base: application_key_id: str = ..., application_key: str = ..., ) -> None: ... + @overload def __init__( self, @@ -138,6 +147,7 @@ class _Base: *, datadir: str = ..., ) -> None: ... + @overload def __init__( self, @@ -148,6 +158,7 @@ class _Base: root: str = ..., api_key: str = ..., ) -> None: ... + @overload def __init__( self, @@ -159,6 +170,7 @@ class _Base: namespace_id: str = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -167,6 +179,7 @@ class _Base: *, root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -180,6 +193,7 @@ class _Base: bucket: str = ..., disable_config_load: _bool = ..., ) -> None: ... + @overload def __init__( self, @@ -194,6 +208,7 @@ class _Base: key_field: str = ..., value_field: str = ..., ) -> None: ... + @overload def __init__( self, @@ -202,6 +217,7 @@ class _Base: *, root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -212,6 +228,7 @@ class _Base: endpoint: str = ..., token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -224,6 +241,7 @@ class _Base: client_id: str = ..., client_secret: str = ..., ) -> None: ... + @overload def __init__( self, @@ -233,6 +251,7 @@ class _Base: root: str = ..., atomic_write_dir: str = ..., ) -> None: ... + @overload def __init__( self, @@ -253,6 +272,7 @@ class _Base: disable_config_load: _bool = ..., token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -265,6 +285,7 @@ class _Base: client_id: str = ..., client_secret: str = ..., ) -> None: ... + @overload def __init__( self, @@ -276,6 +297,7 @@ class _Base: endpoint: str = ..., runtime_token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -287,6 +309,7 @@ class _Base: root: str = ..., token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -299,6 +322,7 @@ class _Base: chunk_size: _int = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -309,6 +333,7 @@ class _Base: url: str = ..., enable_append: _bool = ..., ) -> None: ... + @overload def __init__( self, @@ -321,6 +346,7 @@ class _Base: token: str = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -333,6 +359,7 @@ class _Base: root: str = ..., token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -346,6 +373,7 @@ class _Base: ds_web_auth_token: str = ..., is_china_mainland: _bool = ..., ) -> None: ... + @overload def __init__( self, @@ -355,6 +383,7 @@ class _Base: endpoint: str = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -364,6 +393,7 @@ class _Base: root: str = ..., endpoint: str = ..., ) -> None: ... + @overload def __init__( self, @@ -375,6 +405,7 @@ class _Base: root: str = ..., password: str = ..., ) -> None: ... + @overload def __init__( self, @@ -388,6 +419,7 @@ class _Base: repository: str = ..., branch: str = ..., ) -> None: ... + @overload def __init__( self, @@ -401,6 +433,7 @@ class _Base: value_field: str = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -413,6 +446,7 @@ class _Base: password: str = ..., default_ttl: _duration = ..., ) -> None: ... + @overload def __init__( self, @@ -421,6 +455,7 @@ class _Base: *, root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -432,6 +467,7 @@ class _Base: time_to_idle: _duration = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -445,6 +481,7 @@ class _Base: num_segments: _int = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -458,6 +495,7 @@ class _Base: key_field: str = ..., value_field: str = ..., ) -> None: ... + @overload def __init__( self, @@ -466,6 +504,7 @@ class _Base: *, root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -478,6 +517,7 @@ class _Base: value_field: str = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -494,6 +534,7 @@ class _Base: value_field: str = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -506,6 +547,7 @@ class _Base: secret_access_key: str = ..., bucket: str = ..., ) -> None: ... + @overload def __init__( self, @@ -515,6 +557,7 @@ class _Base: access_token: str = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -539,6 +582,7 @@ class _Base: oidc_token_file: str = ..., sts_endpoint: str = ..., ) -> None: ... + @overload def __init__( self, @@ -550,6 +594,7 @@ class _Base: username: str = ..., password: str = ..., ) -> None: ... + @overload def __init__( self, @@ -560,6 +605,7 @@ class _Base: segment: str = ..., index: str = ..., ) -> None: ... + @overload def __init__( self, @@ -572,6 +618,7 @@ class _Base: key_field: str = ..., value_field: str = ..., ) -> None: ... + @overload def __init__( self, @@ -582,6 +629,7 @@ class _Base: root: str = ..., table: str = ..., ) -> None: ... + @overload def __init__( self, @@ -596,6 +644,7 @@ class _Base: root: str = ..., default_ttl: _duration = ..., ) -> None: ... + @overload def __init__( self, @@ -630,6 +679,7 @@ class _Base: checksum_algorithm: str = ..., disable_write_with_if_match: _bool = ..., ) -> None: ... + @overload def __init__( self, @@ -642,6 +692,7 @@ class _Base: username: str = ..., password: str = ..., ) -> None: ... + @overload def __init__( self, @@ -655,6 +706,7 @@ class _Base: known_hosts_strategy: str = ..., enable_copy: _bool = ..., ) -> None: ... + @overload def __init__( self, @@ -665,6 +717,7 @@ class _Base: root: str = ..., tree: str = ..., ) -> None: ... + @overload def __init__( self, @@ -677,6 +730,7 @@ class _Base: value_field: str = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -688,6 +742,7 @@ class _Base: endpoint: str = ..., key: str = ..., ) -> None: ... + @overload def __init__( self, @@ -704,6 +759,7 @@ class _Base: value_field: str = ..., root: str = ..., ) -> None: ... + @overload def __init__( self, @@ -715,6 +771,7 @@ class _Base: root: str = ..., token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -726,6 +783,7 @@ class _Base: operator: str = ..., password: str = ..., ) -> None: ... + @overload def __init__( self, @@ -734,6 +792,7 @@ class _Base: *, access_token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -743,6 +802,7 @@ class _Base: root: str = ..., token: str = ..., ) -> None: ... + @overload def __init__( self, @@ -756,6 +816,7 @@ class _Base: root: str = ..., disable_copy: _bool = ..., ) -> None: ... + @overload def __init__( self, @@ -768,6 +829,7 @@ class _Base: disable_list_batch: _bool = ..., atomic_write_dir: str = ..., ) -> None: ... + @overload def __init__( self, @@ -777,5 +839,7 @@ class _Base: access_token: str, root: str = ..., ) -> None: ... + + @overload - def __init__(self, scheme: str, /, **kwargs: str) -> None: ... + def __init__(self, scheme: str, /, **kwargs: str) -> None: ... \ No newline at end of file diff --git a/dev/Cargo.lock b/dev/Cargo.lock index f0479e85386..9557672df84 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -66,15 +66,6 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" -[[package]] -name = "basic-toml" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "823388e228f614e9558c6804262db37960ec8821856535f5c3f59913140558f8" -dependencies = [ - "serde", -] - [[package]] name = "clap" version = "4.5.23" @@ -156,15 +147,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "humansize" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" -dependencies = [ - "libm", -] - [[package]] name = "humantime" version = "2.1.0" @@ -177,18 +159,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itoa" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" - -[[package]] -name = "libm" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" - [[package]] name = "log" version = "0.4.22" @@ -202,35 +172,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "nom" -version = "7.1.3" +name = "minijinja" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +checksum = "2c37e1b517d1dcd0e51dc36c4567b9d5a29262b3ec8da6cb5d35e27a8fb529b5" dependencies = [ - "memchr", - "minimal-lexical", + "serde", ] [[package]] @@ -241,17 +188,12 @@ dependencies = [ "clap", "env_logger", "log", + "minijinja", "pretty_assertions", - "rinja", + "serde", "syn", ] -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - [[package]] name = "pretty_assertions" version = "1.4.1" @@ -309,53 +251,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" -[[package]] -name = "rinja" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc4940d00595430b3d7d5a01f6222b5e5b51395d1120bdb28d854bb8abb17a5" -dependencies = [ - "humansize", - "itoa", - "percent-encoding", - "rinja_derive", -] - -[[package]] -name = "rinja_derive" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d9ed0146aef6e2825f1b1515f074510549efba38d71f4554eec32eb36ba18b" -dependencies = [ - "basic-toml", - "memchr", - "mime", - "mime_guess", - "proc-macro2", - "quote", - "rinja_parser", - "rustc-hash", - "serde", - "syn", -] - -[[package]] -name = "rinja_parser" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f9a866e2e00a7a1fb27e46e9e324a6f7c0e7edc4543cae1d38f4e4a100c610" -dependencies = [ - "memchr", - "nom", - "serde", -] - -[[package]] -name = "rustc-hash" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" - [[package]] name = "serde" version = "1.0.216" @@ -393,12 +288,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "unicase" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" - [[package]] name = "unicode-ident" version = "1.0.14" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 9d0e98acd22..2acaeecae7c 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -28,11 +28,12 @@ rust-version = "1.75" version = "0.0.1" [dependencies] -anyhow = "1.0.95" +anyhow = { version = "1.0.95" } clap = { version = "4.5.23", features = ["derive"] } -env_logger = "0.11.6" -log = "0.4.22" -rinja = "0.3.5" +env_logger = { version = "0.11.5" } +log = { version = "0.4.22" } +minijinja = { version = "2.5.0" } +serde = { version = "1.0.216", features = ["derive"] } syn = { version = "2.0.91", features = ["visit", "full", "extra-traits"] } [dev-dependencies] diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs deleted file mode 100644 index 68bd9b4577d..00000000000 --- a/dev/src/generate/binding_python.rs +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::generate::parser::Services; -use anyhow::Result; -use rinja::Template; -use std::fs; -use std::path::PathBuf; - -use super::parser::{ConfigType, Service}; - -// Using the template in this path, relative -// to the `templates` dir in the crate root -#[derive(Template)] -#[template(path = "python.py.jinja2", escape = "none")] -struct PythonTemplate { - services: Vec<(String, Service)>, -} - -/// TODO: add a common utils to parse enabled features from cargo.toml -fn enabled_service(srv: &str) -> bool { - match srv { - // not enabled in bindings/python/Cargo.toml - "etcd" | "foundationdb" | "ftp" | "hdfs" | "rocksdb" | "tikv" => false, - _ => true, - } -} - -pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { - let mut v = Vec::from_iter( - services - .clone() - .into_iter() - .filter(|x| enabled_service(x.0.as_str())), - ); - - // move required options at beginning. - for srv in &mut v { - let mut v = Vec::from_iter(srv.1.config.clone().into_iter().enumerate()); - - v.sort_by_key(|a| (a.1.optional, a.0)); - - srv.1.config = v.iter().map(|f| f.1.clone()).collect(); - } - - let tmpl = PythonTemplate { services: v }; - - let t = tmpl.render().expect("should render template"); - - let output_file: String = project_root - .join("bindings/python/python/opendal/__base.pyi") - .to_str() - .expect("should build output file path") - .into(); - - fs::write(output_file, t).expect("failed to write result to file"); - - Ok(()) -} - -impl ConfigType { - pub fn python_type(&self) -> String { - match self { - ConfigType::Bool => "_bool".into(), - ConfigType::Duration => "_duration".into(), - ConfigType::I64 - | ConfigType::Usize - | ConfigType::U64 - | ConfigType::U32 - | ConfigType::U16 => "_int".into(), - ConfigType::Vec => "_strings".into(), - ConfigType::String => "str".into(), - } - } -} diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs index 71ec4442999..780f19a7406 100644 --- a/dev/src/generate/mod.rs +++ b/dev/src/generate/mod.rs @@ -16,20 +16,18 @@ // under the License. mod parser; +mod python; -mod binding_python; - +use crate::workspace_dir; use anyhow::Result; -use std::path::PathBuf; pub fn run(language: &str) -> Result<()> { - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let services_path = manifest_dir.join("../core/src/services").canonicalize()?; - let project_root = manifest_dir.join("..").canonicalize()?; + let workspace_dir = workspace_dir(); + let services_path = workspace_dir.join("core/src/services").canonicalize()?; let services = parser::parse(&services_path.to_string_lossy())?; match language { - "python" | "py" => binding_python::generate(project_root, &services), - _ => Err(anyhow::anyhow!("Unsupported language: {}", language)), + "python" | "py" => python::generate(workspace_dir, services), + _ => anyhow::bail!("unsupported language: {}", language), } } diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index 45cfe7cb455..086d7be5a37 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -15,40 +15,29 @@ // specific language governing permissions and limitations // under the License. -use anyhow::Result; use anyhow::{anyhow, Context}; +use anyhow::{bail, Result}; use log::debug; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::fs; use std::fs::read_dir; use std::str::FromStr; -use std::{fs, vec}; use syn::{ Expr, ExprLit, Field, GenericArgument, Item, Lit, LitStr, Meta, PathArguments, Type, TypePath, }; -#[derive(Debug, Clone)] -pub struct Services(HashMap); - -impl IntoIterator for Services { - type Item = (String, Service); - type IntoIter = vec::IntoIter<(String, Service)>; - - fn into_iter(self) -> Self::IntoIter { - let mut v = Vec::from_iter(self.0); - v.sort(); - v.into_iter() - } -} +pub type Services = HashMap; /// Service represents a service supported by opendal core, like `s3` and `fs` -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] pub struct Service { /// All configurations for this service. pub config: Vec, } /// Config represents a configuration item for a service. -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] pub struct Config { /// The name of this config, for example, `access_key_id` and `secret_access_key` pub name: String, @@ -64,7 +53,7 @@ pub struct Config { pub comments: String, } -#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] pub enum ConfigType { /// Mapping to rust's `bool` Bool, @@ -94,7 +83,7 @@ impl FromStr for ConfigType { type Err = anyhow::Error; fn from_str(s: &str) -> Result { - let ct = match s { + Ok(match s { "bool" => ConfigType::Bool, "String" => ConfigType::String, "Duration" => ConfigType::Duration, @@ -106,10 +95,9 @@ impl FromStr for ConfigType { "u16" => ConfigType::U16, "Vec" => ConfigType::Vec, - v => return Err(anyhow!("unsupported config type {v:?}")), - }; - Ok(ct) + v => bail!("unsupported config type {v}"), + }) } } @@ -135,11 +123,11 @@ impl FromStr for ConfigType { /// ``` /// /// - since = "0.52.0" -#[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd)] +#[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] pub struct AttrDeprecated { - /// The since of this deprecated field. + /// The `since` of this deprecated field. pub since: String, - /// The note for this deprecated field. + /// The `note` for this deprecated field. pub note: String, } @@ -163,7 +151,7 @@ pub fn parse(path: &str) -> Result { map.insert(parser.service, service); } - Ok(Services(map)) + Ok(map) } /// ServiceParser is used to parse a service config file. diff --git a/dev/templates/python.py.jinja2 b/dev/src/generate/python.j2 similarity index 71% rename from dev/templates/python.py.jinja2 rename to dev/src/generate/python.j2 index 7c8b2732efc..772407d5297 100644 --- a/dev/templates/python.py.jinja2 +++ b/dev/src/generate/python.j2 @@ -1,9 +1,3 @@ -""" -this file is generated by opendal/dev/generate/binding_python.rs, and opendal.__base doesn't exists. - -DO NOT EDIT IT Manually -""" - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -21,6 +15,12 @@ DO NOT EDIT IT Manually # specific language governing permissions and limitations # under the License. +""" +> DO NOT EDIT IT MANUALLY < + +This file is generated by opendal/dev/generate/python.rs. `opendal.__base` doesn't exists. +""" + from typing import overload, Literal, TypeAlias # `true`/`false`` in any case, for example, `true`/`True`/`TRUE` `false`/`False`/`FALSE` @@ -43,20 +43,21 @@ class _Base: The services list here is support by opendal pypi wheel. """ -{% for srv in services %} +{% for srv in srvs %} @overload - def __init__(self, - scheme: Literal["{{srv.0}}"], + def __init__( + self, + scheme: Literal["{{srv}}"], /, *, - {% for field in srv.1.config.clone().into_iter() %} - {% if field.deprecated.is_some() %} - # deprecated: {{ field.deprecated.unwrap().note }} - {% endif %} - {{field.name}}: {{field.value.python_type()}} {% if field.optional %} = ... {% endif %}, - {% endfor %} - ) -> None: ... + {%- for field in srvs[srv].config -%} + {% if field.deprecated %} + # deprecated: {{field.deprecated["note"]}} + {%- endif %} + {{field.name}}: {{make_pytype(field.value)}}{% if field.optional %} = ...{% endif %}, + {%- endfor %} + ) -> None: ... {% endfor %} @overload - def __init__(self, scheme:str, /, **kwargs: str) -> None: ... + def __init__(self, scheme: str, /, **kwargs: str) -> None: ... diff --git a/dev/src/generate/python.rs b/dev/src/generate/python.rs new file mode 100644 index 00000000000..268a6e89d50 --- /dev/null +++ b/dev/src/generate/python.rs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::generate::parser::{ConfigType, Service, Services}; +use anyhow::Result; +use minijinja::value::ViaDeserialize; +use minijinja::{context, Environment}; +use std::fs; +use std::path::PathBuf; + +/// TODO: add a common utils to parse enabled features from cargo.toml +fn enabled_service(srv: &str) -> bool { + match srv { + // not enabled in bindings/python/Cargo.toml + "etcd" | "foundationdb" | "ftp" | "hdfs" | "rocksdb" | "tikv" => false, + _ => true, + } +} + +pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> { + let mut srvs = Services::new(); + for (k, srv) in services.into_iter() { + if !enabled_service(k.as_str()) { + continue; + } + + let mut sorted = srv.config.into_iter().enumerate().collect::>(); + sorted.sort_by_key(|(i, v)| (v.optional, *i)); + let config = sorted.into_iter().map(|(_, v)| v).collect(); + srvs.insert(k, Service { config }); + } + + let mut env = Environment::new(); + env.add_template("python", include_str!("python.j2"))?; + env.add_function("make_pytype", make_pytype); + let tmpl = env.get_template("python")?; + + let output = workspace_dir.join("bindings/python/python/opendal/__base.pyi"); + fs::write(output, tmpl.render(context! { srvs => srvs })?)?; + + Ok(()) +} + +fn make_pytype(ty: ViaDeserialize) -> Result { + Ok(match ty.0 { + ConfigType::Bool => "_bool", + ConfigType::Duration => "_duration", + ConfigType::I64 + | ConfigType::Usize + | ConfigType::U64 + | ConfigType::U32 + | ConfigType::U16 => "_int", + ConfigType::Vec => "_strings", + ConfigType::String => "str", + } + .to_string()) +} diff --git a/dev/src/main.rs b/dev/src/main.rs index a56430abf8a..1858e89b1f8 100644 --- a/dev/src/main.rs +++ b/dev/src/main.rs @@ -19,10 +19,21 @@ mod generate; use anyhow::Result; use clap::{Parser, Subcommand}; +use std::path::PathBuf; + +fn manifest_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .canonicalize() + .unwrap() +} + +fn workspace_dir() -> PathBuf { + manifest_dir().join("..").canonicalize().unwrap() +} #[derive(Parser)] #[command(version, about, long_about = None)] -struct Cli { +struct Cmd { #[command(subcommand)] command: Commands, } @@ -39,13 +50,7 @@ enum Commands { fn main() -> Result<()> { env_logger::init(); - let cli = Cli::parse(); - - match cli.command { - Commands::Generate { language } => { - generate::run(&language)?; - } + match Cmd::parse().command { + Commands::Generate { language } => generate::run(&language), } - - Ok(()) }