diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfc649e..cac8008 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,3 +52,4 @@ jobs: AWS_ENDPOINT: http://localhost:9000 AWS_ACCESS_KEY_ID: minioadmin AWS_SECRET_ACCESS_KEY: minioadmin + AWS_REGION: us-east-1 diff --git a/Cargo.lock b/Cargo.lock index 55a86b1..08b4ecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "ahash" version = "0.7.8" @@ -136,6 +142,48 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" + +[[package]] +name = "apache-avro" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" +dependencies = [ + "bigdecimal", + "digest", + "libflate", + "log", + "num-bigint", + "quad-rand", + "rand", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "strum", + "strum_macros", + "thiserror 1.0.69", + "typed-builder", + "uuid", +] + +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "arrow" version = "52.2.0" @@ -278,7 +326,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap", + "indexmap 2.6.0", "lexical-core", "num", "serde", @@ -362,7 +410,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -386,6 +434,18 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "backon" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" +dependencies = [ + "fastrand", + "futures-core", + "pin-project", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -413,6 +473,26 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bigdecimal" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f850665a0385e070b64c38d2354e6c104c8479c59868d1e48a0c13ee2c7a1c1" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + [[package]] name = "bitflags" version = "1.3.2" @@ -425,6 +505,18 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -434,6 +526,29 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borsh" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2506947f73ad44e344215ccd6403ac2ae18cd8e046e581a441bf8d199f257f03" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2593a3b8b938bd68373196c9832f516be11fa487ef4ae745eb282e6a56a7244" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "brotli" version = "6.0.0" @@ -461,11 +576,33 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytemuck" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" [[package]] name = "byteorder" @@ -570,7 +707,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -597,13 +734,19 @@ dependencies = [ "nom", "pathdiff", "ron", - "rust-ini", + "rust-ini 0.18.0", "serde", "serde_json", "toml", "yaml-rust", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-random" version = "0.1.18" @@ -634,21 +777,49 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + [[package]] name = "cpufeatures" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -701,6 +872,47 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.89", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.89", +] + +[[package]] +name = "dary_heap" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" + [[package]] name = "dashmap" version = "6.1.0" @@ -733,7 +945,7 @@ dependencies = [ "delta_kernel_derive", "either", "fix-hidden-lifetime-bug", - "indexmap", + "indexmap 2.6.0", "itertools", "lazy_static", "parquet", @@ -758,7 +970,7 @@ checksum = "ec5c4fb5b59b1bd55ed8ebcf941f27a327d600c19a4a4103546846c358be93ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -798,7 +1010,7 @@ dependencies = [ "fix-hidden-lifetime-bug", "futures", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.6.0", "itertools", "lazy_static", "libc", @@ -827,6 +1039,47 @@ dependencies = [ "z85", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.89", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.89", +] + [[package]] name = "digest" version = "0.10.7" @@ -834,6 +1087,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] @@ -846,7 +1100,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -855,6 +1109,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -938,6 +1201,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "flagset" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ea1ec5f8307826a5b71094dd91fc04d4ae75d5709b20ad351c7fb4815c86ec" + [[package]] name = "flatbuffers" version = "24.3.25" @@ -988,6 +1257,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.31" @@ -1044,7 +1319,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1108,9 +1383,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "h2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", @@ -1118,7 +1393,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -1157,9 +1432,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heck" @@ -1179,6 +1454,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hmac" version = "0.12.1" @@ -1188,6 +1469,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" version = "1.1.0" @@ -1236,9 +1526,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", @@ -1270,6 +1560,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -1314,6 +1605,49 @@ dependencies = [ "cc", ] +[[package]] +name = "iceberg" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "651dfca7c429918e164607a549287cfdd1e7814d2e4cb577d0d6dc57fe19b785" +dependencies = [ + "anyhow", + "apache-avro", + "array-init", + "arrow-arith", + "arrow-array", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", + "async-trait", + "bimap", + "bitvec", + "bytes", + "chrono", + "derive_builder", + "fnv", + "futures", + "itertools", + "murmur3", + "once_cell", + "opendal", + "ordered-float 4.5.0", + "parquet", + "reqwest", + "rust_decimal", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_repr", + "serde_with", + "tokio", + "typed-builder", + "url", + "uuid", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -1429,9 +1763,15 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -1453,6 +1793,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.6.0" @@ -1460,7 +1811,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.1", + "hashbrown 0.15.2", + "serde", ] [[package]] @@ -1492,9 +1844,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2" [[package]] name = "jobserver" @@ -1538,13 +1890,17 @@ dependencies = [ "config", "deltalake", "env_logger", + "fastrand", "futures", + "iceberg", "log", "native-tls", "object_store", "parquet", "postgres", "postgres-native-tls", + "regex", + "serde_json", "tempfile", "thiserror 1.0.69", "tokio", @@ -1629,6 +1985,30 @@ version = "0.2.164" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +[[package]] +name = "libflate" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" +dependencies = [ + "adler32", + "core2", + "crc32fast", + "dary_heap", + "libflate_lz77", +] + +[[package]] +name = "libflate_lz77" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" +dependencies = [ + "core2", + "hashbrown 0.14.5", + "rle-decode-fast", +] + [[package]] name = "libm" version = "0.2.11" @@ -1649,9 +2029,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "lock_api" @@ -1733,6 +2113,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + [[package]] name = "native-tls" version = "0.2.12" @@ -1745,7 +2131,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -1782,6 +2168,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", + "serde", ] [[package]] @@ -1793,6 +2180,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -1870,7 +2263,7 @@ dependencies = [ "md-5", "parking_lot", "percent-encoding", - "quick-xml", + "quick-xml 0.36.2", "rand", "reqwest", "ring", @@ -1890,7 +2283,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] -name = "openssl" +name = "opendal" +version = "0.49.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b04d09b9822c2f75a1d2fc513a2c1279c70e91e7407936fffdf6a6976ec530a" +dependencies = [ + "anyhow", + "async-trait", + "backon", + "base64 0.22.1", + "bytes", + "chrono", + "crc32c", + "flagset", + "futures", + "getrandom", + "http", + "log", + "md-5", + "once_cell", + "percent-encoding", + "quick-xml 0.36.2", + "reqsign", + "reqwest", + "serde", + "serde_json", + "tokio", + "uuid", +] + +[[package]] +name = "openssl" version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" @@ -1912,7 +2335,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1942,16 +2365,35 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" dependencies = [ - "dlv-list", + "dlv-list 0.3.0", "hashbrown 0.12.3", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list 0.5.2", + "hashbrown 0.14.5", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -2069,7 +2511,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2121,6 +2563,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "pin-project-lite" version = "0.2.15" @@ -2190,6 +2652,12 @@ dependencies = [ "postgres-protocol", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -2199,15 +2667,60 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quick-xml" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e446ed58cef1bbfe847bc2fda0e2e4ea9f0e57b90c507d4781292590d72a4e" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick-xml" version = "0.36.2" @@ -2279,6 +2792,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.5" @@ -2341,12 +2860,55 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "reqsign" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb0075a66c8bfbf4cc8b70dca166e722e1f55a3ea9250ecbb85f4d92a5f64149" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "chrono", + "form_urlencoded", + "getrandom", + "hex", + "hmac", + "home", + "http", + "log", + "percent-encoding", + "quick-xml 0.35.0", + "rand", + "reqwest", + "rust-ini 0.21.1", + "serde", + "serde_json", + "sha1", + "sha2", +] + [[package]] name = "reqwest" version = "0.12.9" @@ -2389,6 +2951,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots", "windows-registry", ] @@ -2407,6 +2970,41 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "roaring" version = "0.10.6" @@ -2435,7 +3033,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" dependencies = [ "cfg-if", - "ordered-multimap", + "ordered-multimap 0.4.3", +] + +[[package]] +name = "rust-ini" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" +dependencies = [ + "cfg-if", + "ordered-multimap 0.7.3", + "trim-in-place", +] + +[[package]] +name = "rust_decimal" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand", + "rkyv", + "serde", + "serde_json", ] [[package]] @@ -2474,9 +3099,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.17" +version = "0.23.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" +checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" dependencies = [ "once_cell", "ring", @@ -2488,15 +3113,14 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.0.1", ] [[package]] @@ -2564,6 +3188,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "2.11.1" @@ -2571,7 +3201,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.6.0", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +dependencies = [ + "bitflags 2.6.0", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -2608,6 +3251,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.215" @@ -2616,7 +3268,7 @@ checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2631,6 +3283,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2643,6 +3306,47 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.6.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.89", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.8" @@ -2669,6 +3373,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "0.3.11" @@ -2791,7 +3501,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2813,9 +3523,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.87" +version = "2.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" dependencies = [ "proc-macro2", "quote", @@ -2824,9 +3534,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -2839,9 +3549,15 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.14.0" @@ -2881,7 +3597,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2892,7 +3608,7 @@ checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2903,7 +3619,38 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "ordered-float", + "ordered-float 2.10.1", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", ] [[package]] @@ -2966,7 +3713,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -3037,6 +3784,23 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap 2.6.0", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.3" @@ -3063,7 +3827,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -3075,6 +3839,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "trim-in-place" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" + [[package]] name = "try-lock" version = "0.2.5" @@ -3091,6 +3861,26 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typed-builder" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "typenum" version = "1.17.0" @@ -3111,9 +3901,9 @@ checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-normalization" @@ -3138,9 +3928,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -3201,7 +3991,7 @@ checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -3257,7 +4047,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "wasm-bindgen-shared", ] @@ -3291,7 +4081,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3335,6 +4125,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "whoami" version = "1.5.2" @@ -3476,6 +4275,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + [[package]] name = "write16" version = "1.0.0" @@ -3488,6 +4296,15 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "yaml-rust" version = "0.4.5" @@ -3499,9 +4316,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", @@ -3511,13 +4328,13 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "synstructure", ] @@ -3545,27 +4362,27 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "synstructure", ] @@ -3594,7 +4411,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2617910..1fd6b90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,13 +14,16 @@ clap = { version = "4.5", features = [ "derive" ] } config = "0.13.3" deltalake = { version = "0.21" } env_logger = "0.11.1" +fastrand = "2.2.0" futures = "0.3" +iceberg = "0.3.0" log = "0.4" native-tls = "0.2.11" object_store = { version = "0.10", features = ["aws"] } parquet = { version = "52" } postgres = { version = "0.19.7", git = "https://github.com/splitgraph/rust-postgres", rev = "88c2c7714a4558aed6a63e2e2b140a8359568858" } postgres-native-tls = { version = "0.5.0", git = "https://github.com/splitgraph/rust-postgres", rev = "88c2c7714a4558aed6a63e2e2b140a8359568858" } +serde_json = "1" tempfile = "3" thiserror = "1" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "signal", "process"] } @@ -28,6 +31,9 @@ tokio-postgres = { version = "0.7.10", git = "https://github.com/splitgraph/rust url = "2.5.0" uuid = "1.2.1" +[dev-dependencies] +regex = "1.11" + [profile.release] codegen-units = 1 lto = true diff --git a/docker-compose.yml b/docker-compose.yml index 1d568e6..0e270e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,8 +27,8 @@ services: entrypoint: > /bin/sh -c " until (/usr/bin/mc config host add test-minio http://minio:9000 minioadmin minioadmin) do echo '...waiting...' && sleep 1; done; - /usr/bin/mc rm -r --force test-minio/sdl-test-bucket; - /usr/bin/mc mb test-minio/sdl-test-bucket; + /usr/bin/mc rm -r --force test-minio/lhl-test-bucket; + /usr/bin/mc mb test-minio/lhl-test-bucket; /usr/bin/mc admin user add test-minio test-user test-pass; /usr/bin/mc admin policy attach test-minio readwrite --user test-user; exit 0; " diff --git a/src/delta.rs b/src/delta_destination.rs similarity index 100% rename from src/delta.rs rename to src/delta_destination.rs diff --git a/src/error.rs b/src/error.rs index dfa5fc0..6a1465a 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,6 +10,8 @@ pub enum DataLoadingError { ArrowError(#[from] arrow::error::ArrowError), #[error("Delta Table error")] DeltaTableError(#[from] deltalake::DeltaTableError), + #[error("Iceberg error")] + IcebergError(#[from] iceberg::Error), #[error("I/O error")] IoError(#[from] std::io::Error), #[error("Object store error")] diff --git a/src/iceberg_destination.rs b/src/iceberg_destination.rs new file mode 100644 index 0000000..bf28a52 --- /dev/null +++ b/src/iceberg_destination.rs @@ -0,0 +1,224 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use arrow::array::RecordBatch; +use arrow_schema::{Field, Schema, SchemaRef}; +use futures::{pin_mut, StreamExt, TryStream}; +use iceberg::io::FileIO; +use iceberg::spec::{ + DataContentType, DataFileFormat, FormatVersion, Manifest, ManifestContentType, ManifestEntry, + ManifestFile, ManifestListWriter, ManifestMetadata, ManifestStatus, ManifestWriter, Operation, + PartitionSpec, Snapshot, Struct, Summary, TableMetadata, TableMetadataBuilder, +}; +use iceberg::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, +}; +use iceberg::writer::file_writer::{FileWriter, FileWriterBuilder, ParquetWriterBuilder}; +use iceberg::TableCreation; +use log::info; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use parquet::file::properties::WriterProperties; +use url::Url; +use uuid::Uuid; + +use crate::error::DataLoadingError; + +fn create_file_io(target_url: String) -> Result { + let mut file_io_props: Vec<(String, String)> = vec![]; + if let Ok(aws_endpoint) = std::env::var("AWS_ENDPOINT") { + file_io_props.push(("s3.endpoint".to_string(), aws_endpoint)); + } + + let file_io = FileIO::from_path(target_url.clone())? + .with_props(file_io_props) + .build()?; + Ok(file_io) +} + +// Create the v0 metadata object. This one will contain no snapshot +fn create_metadata_v0( + iceberg_schema: &iceberg::spec::Schema, + target_url: String, +) -> Result { + let table_creation = TableCreation::builder() + .name("dummy_name".to_string()) // Required by TableCreationBuilder. Doesn't affect output + .schema(iceberg_schema.clone()) + .location(target_url.to_string()) + .build(); + + let table_metadata = TableMetadataBuilder::from_table_creation(table_creation)?.build()?; + Ok(table_metadata) +} + +// Clone an arrow schema, assigning sequential field IDs starting from 1 +fn assign_field_ids(arrow_schema: Arc) -> Schema { + let mut field_id_counter = 1; + let new_fields: Vec = arrow_schema + .fields + .iter() + .map(|field_ref| { + let mut field: Field = (**field_ref).clone(); + let mut metadata = field_ref.metadata().clone(); + metadata.insert( + PARQUET_FIELD_ID_META_KEY.to_owned(), + field_id_counter.to_string(), + ); + field_id_counter += 1; + field.set_metadata(metadata); + field + }) + .collect(); + Schema::new_with_metadata(new_fields, arrow_schema.metadata.clone()) +} + +// Create the v1 metadata object by adding a snapshot to the v0 metadata object +fn create_metadata_v1( + metadata_v0: &TableMetadata, + snapshot: Snapshot, +) -> Result { + let snapshot_id = snapshot.snapshot_id(); + // Copy metadata v0, modifying current snapshot ID + let mut metadata_v0_json = serde_json::to_value(metadata_v0).unwrap(); + if let Some(obj) = metadata_v0_json.as_object_mut() { + obj.insert( + "current-snapshot-id".to_string(), + serde_json::Value::from(snapshot_id), + ); + } + let mut metadata_v1: TableMetadata = serde_json::from_value(metadata_v0_json).unwrap(); + metadata_v1.append_snapshot(snapshot); + Ok(metadata_v1) +} + +const DEFAULT_SCHEMA_ID: i32 = 0; + +pub async fn record_batches_to_iceberg( + record_batch_stream: impl TryStream>, + arrow_schema: SchemaRef, + target_url: Url, +) -> Result<(), DataLoadingError> { + pin_mut!(record_batch_stream); + + let file_io = create_file_io(target_url.to_string())?; + let arrow_schema_with_ids = assign_field_ids(arrow_schema.clone()); + let iceberg_schema = iceberg::arrow::arrow_schema_to_schema(&arrow_schema_with_ids)?; + let metadata_v0 = create_metadata_v0(&iceberg_schema, target_url.to_string())?; + let metadata_v0_location = format!("{}/metadata/v0.metadata.json", target_url); + + file_io + .new_output(&metadata_v0_location)? + .write(serde_json::to_vec(&metadata_v0).unwrap().into()) + .await?; + info!("Wrote v0 metadata: {:?}", metadata_v0_location); + + let file_writer_builder = ParquetWriterBuilder::new( + WriterProperties::builder().build(), + Arc::new(iceberg_schema.clone()), + file_io.clone(), + DefaultLocationGenerator::new(metadata_v0.clone()).unwrap(), + DefaultFileNameGenerator::new( + "part".to_string(), + Some(Uuid::new_v4().to_string()), + DataFileFormat::Parquet, + ), + ); + let mut file_writer = file_writer_builder.build().await.unwrap(); + + while let Some(maybe_batch) = record_batch_stream.next().await { + let batch = maybe_batch?; + file_writer.write(&batch).await?; + } + let data_files: Vec<_> = file_writer + .close() + .await? + .iter_mut() + .map(|data_file_builder| { + let data_file = data_file_builder + .content(DataContentType::Data) + .partition(Struct::empty()) + .build() + .unwrap(); + info!("Wrote data file: {:?}", data_file.file_path()); + data_file + }) + .collect(); + + let snapshot_id = fastrand::i64(..); + let sequence_number = 1; + + let manifest_file_path = format!("{}/metadata/manifest-{}.avro", target_url, Uuid::new_v4()); + let manifest_file_output = file_io.new_output(manifest_file_path)?; + let manifest_writer: ManifestWriter = + ManifestWriter::new(manifest_file_output, snapshot_id, vec![]); + let manifest_metadata = ManifestMetadata::builder() + .schema_id(DEFAULT_SCHEMA_ID) + .schema(iceberg_schema.clone()) + .partition_spec( + PartitionSpec::builder(&iceberg_schema) + .with_spec_id(0) + .build()?, + ) + .content(ManifestContentType::Data) + .format_version(FormatVersion::V2) + .build(); + let manifest = Manifest::new( + manifest_metadata, + data_files + .iter() + .map(|data_file| { + ManifestEntry::builder() + .status(ManifestStatus::Added) + .snapshot_id(snapshot_id) + .data_file(data_file.clone()) + .build() + }) + .collect(), + ); + let manifest_file: ManifestFile = manifest_writer.write(manifest).await?; + info!("Wrote manifest file: {:?}", manifest_file.manifest_path); + + let manifest_list_path = format!( + "{}/metadata/manifest-list-{}.avro", + target_url, + Uuid::new_v4() + ); + let manifest_file_output = file_io.new_output(manifest_list_path.clone())?; + let mut manifest_list_writer: ManifestListWriter = ManifestListWriter::v2( + manifest_file_output, + snapshot_id, + -1, // parent_snapshot_id is optional. Ideally ManifestListWriter wouldn't write it at all + sequence_number, + ); + manifest_list_writer.add_manifests(vec![manifest_file].into_iter())?; + manifest_list_writer.close().await?; + info!("Wrote manifest list: {:?}", manifest_list_path); + + let snapshot = Snapshot::builder() + .with_snapshot_id(snapshot_id) + .with_schema_id(DEFAULT_SCHEMA_ID) + .with_manifest_list(manifest_list_path.clone()) + .with_sequence_number(sequence_number) + .with_timestamp_ms( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64, + ) + .with_summary(Summary { + operation: Operation::Append, + other: HashMap::new(), + }) + .build(); + + let metadata_v1 = create_metadata_v1(&metadata_v0, snapshot)?; + let metadata_v1_location = format!("{}/metadata/v1.metadata.json", target_url,); + + file_io + .new_output(&metadata_v1_location)? + .write(serde_json::to_vec(&metadata_v1).unwrap().into()) + .await?; + info!("Wrote v1 metadata: {:?}", metadata_v1_location); + + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index f825103..f0f78fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,15 @@ use clap::{Parser, Subcommand}; -use delta::record_batches_to_delta; +use delta_destination::record_batches_to_delta; use error::DataLoadingError; use futures::TryStreamExt; +use iceberg_destination::record_batches_to_iceberg; use log::info; use parquet::arrow::async_reader::ParquetRecordBatchStreamBuilder; use url::Url; -pub mod delta; +pub mod delta_destination; pub mod error; +pub mod iceberg_destination; pub mod pg_arrow_source; mod pg_datetime; mod pg_numeric; @@ -47,6 +49,26 @@ enum Commands { )] batch_size: usize, }, + #[command(arg_required_else_help = true)] + ParquetToIceberg { + source_file: String, + target_url: Url, + }, + #[command(arg_required_else_help = true)] + PgToIceberg { + connection_string: Url, + target_url: Url, + #[clap(long, short, action, help("SQL text to extract the data"))] + query: String, + #[clap( + long, + short, + action, + default_value_t = 10000, + help("Number of rows to process per batch") + )] + batch_size: usize, + }, } pub async fn do_main(args: Cli) -> Result<(), DataLoadingError> { @@ -86,8 +108,39 @@ pub async fn do_main(args: Cli) -> Result<(), DataLoadingError> { info!("Rowset schema: {}", arrow_schema); record_batches_to_delta(record_batch_stream, arrow_schema, target_url, overwrite).await } + Commands::ParquetToIceberg { + source_file, + target_url, + } => { + let file = tokio::fs::File::open(source_file).await?; + let record_batch_reader = ParquetRecordBatchStreamBuilder::new(file) + .await? + .build() + .unwrap(); + let schema = record_batch_reader.schema().clone(); + info!("File schema: {}", schema); + record_batches_to_iceberg( + record_batch_reader.map_err(DataLoadingError::ParquetError), + schema, + target_url, + ) + .await + } + Commands::PgToIceberg { + connection_string, + target_url, + query, + batch_size, + } => { + let mut source = PgArrowSource::new(connection_string.as_ref(), &query, batch_size) + .await + .map_err(DataLoadingError::PostgresError)?; + let arrow_schema = source.get_arrow_schema(); + let record_batch_stream = source.get_record_batch_stream(); + info!("Rowset schema: {}", arrow_schema); + record_batches_to_iceberg(record_batch_stream, arrow_schema, target_url).await + } } - // TODO // - custom parquet writer settings (e.g. bloom filters) // - bypass the disk, write files to ram one-by-one and emit them out diff --git a/tests/basic_integration.rs b/tests/basic_integration.rs index 49f3501..11bf390 100644 --- a/tests/basic_integration.rs +++ b/tests/basic_integration.rs @@ -5,15 +5,16 @@ use arrow::array::{ use arrow::datatypes::DataType; use clap::Parser; use futures::{StreamExt, TryStreamExt}; -use lakehouse_loader::delta::object_store_keys_from_env; +use lakehouse_loader::delta_destination::object_store_keys_from_env; use lakehouse_loader::pg_arrow_source::PgArrowSource; use lakehouse_loader::{do_main, Cli}; use object_store::path::Path; +use regex::Regex; use url::Url; #[tokio::test] async fn test_pg_to_delta_e2e() { - let target_url = "s3://sdl-test-bucket/abc"; + let target_url = "s3://lhl-test-bucket/delta"; // WHEN valid arguments are passed to the command let parsed_args = Cli::parse_from(vec![ "lakehouse-loader", @@ -28,33 +29,73 @@ async fn test_pg_to_delta_e2e() { let config = object_store_keys_from_env("s3"); - // Handle some deltalake weirdness let (store, path) = object_store::parse_url_opts(&Url::parse(target_url).unwrap(), config).unwrap(); - let paths = store + let mut paths = store .list(Some(&path)) .map_ok(|m| m.location) .boxed() .try_collect::>() .await .unwrap(); + paths.sort(); assert_eq!(paths.len(), 3); // THEN delta log files are written assert_eq!( paths[0].to_string(), - "abc/_delta_log/00000000000000000000.json" + "delta/_delta_log/00000000000000000000.json" ); assert_eq!( paths[1].to_string(), - "abc/_delta_log/00000000000000000001.json" + "delta/_delta_log/00000000000000000001.json" ); // THEN a delta content file is written - assert!(paths[2].to_string().starts_with("abc/part-00000-")); + assert!(paths[2].to_string().starts_with("delta/part-00000-")); assert!(paths[2].to_string().ends_with("-c000.snappy.parquet")); } +#[tokio::test] +async fn test_pg_to_iceberg() { + let target_url = "s3://lhl-test-bucket/iceberg"; + // WHEN valid arguments are passed to the command + let parsed_args = Cli::parse_from(vec![ + "lakehouse-loader", + "pg-to-iceberg", + "postgres://test-user:test-password@localhost:5432/test-db", + "-q", + // We have to cherry-pick fields as not all types are supported by iceberg + "select cint4, cint8, ctext, cbool from t1 order by id", + target_url, + ]); + // THEN the command runs successfully + do_main(parsed_args).await.unwrap(); + + let config = object_store_keys_from_env("s3"); + + let (store, path) = + object_store::parse_url_opts(&Url::parse(target_url).unwrap(), config).unwrap(); + + let mut paths = store + .list(Some(&path)) + .map_ok(|m| m.location) + .boxed() + .try_collect::>() + .await + .unwrap(); + + paths.sort(); + + // THEN iceberg data and metadata files are written + assert_eq!(paths.len(), 5); + assert!(Regex::new(r"^iceberg/data/part-00000-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}.parquet$").unwrap().is_match(paths[0].as_ref())); + assert!(Regex::new(r"^iceberg/metadata/manifest-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}.avro$").unwrap().is_match(paths[1].as_ref())); + assert!(Regex::new(r"^iceberg/metadata/manifest-list-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}.avro$").unwrap().is_match(paths[2].as_ref())); + assert_eq!(&paths[3].to_string(), "iceberg/metadata/v0.metadata.json"); + assert_eq!(&paths[4].to_string(), "iceberg/metadata/v1.metadata.json"); +} + #[tokio::test] async fn test_pg_arrow_source() { // WHEN 25001 rows are split into batches of 10000