diff --git a/Cargo.lock b/Cargo.lock index 08b4ecd..cfdfc3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,7 +168,7 @@ dependencies = [ "strum", "strum_macros", "thiserror 1.0.69", - "typed-builder", + "typed-builder 0.19.1", "uuid", ] @@ -186,9 +186,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85" +checksum = "c91839b07e474b3995035fd8ac33ee54f9c9ccbbb1ea33d9909c71bffdf1259d" dependencies = [ "arrow-arith", "arrow-array", @@ -207,9 +207,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c" +checksum = "855c57c4efd26722b044dcd3e348252560e3e0333087fb9f6479dc0bf744054f" dependencies = [ "arrow-array", "arrow-buffer", @@ -222,9 +222,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c" +checksum = "bd03279cea46569acf9295f6224fbc370c5df184b4d2ecfe97ccb131d5615a7f" dependencies = [ "ahash 0.8.11", "arrow-buffer", @@ -233,15 +233,15 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "num", ] [[package]] name = "arrow-buffer" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1" +checksum = "9e4a9b9b1d6d7117f6138e13bc4dd5daa7f94e671b70e8c9c4dc37b4f5ecfc16" dependencies = [ "bytes", "half", @@ -250,9 +250,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e" +checksum = "bc70e39916e60c5b7af7a8e2719e3ae589326039e1e863675a008bee5ffe90fd" dependencies = [ "arrow-array", "arrow-buffer", @@ -270,9 +270,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2" +checksum = "789b2af43c1049b03a8d088ff6b2257cdcea1756cd76b174b1f2600356771b97" dependencies = [ "arrow-array", "arrow-buffer", @@ -289,9 +289,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5" +checksum = "e4e75edf21ffd53744a9b8e3ed11101f610e7ceb1a29860432824f1834a1f623" dependencies = [ "arrow-buffer", "arrow-schema", @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f" +checksum = "d186a909dece9160bf8312f5124d797884f608ef5435a36d9d608e0b2a9bcbf8" dependencies = [ "arrow-array", "arrow-buffer", @@ -315,9 +315,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445" +checksum = "b66ff2fedc1222942d0bd2fd391cb14a85baa3857be95c9373179bd616753b85" dependencies = [ "arrow-array", "arrow-buffer", @@ -335,9 +335,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc" +checksum = "ece7b5bc1180e6d82d1a60e1688c199829e8842e38497563c3ab6ea813e527fd" dependencies = [ "arrow-array", "arrow-buffer", @@ -350,9 +350,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c" +checksum = "745c114c8f0e8ce211c83389270de6fbe96a9088a7b32c2a041258a443fe83ff" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -364,18 +364,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8" +checksum = "b95513080e728e4cec37f1ff5af4f12c9688d47795d17cda80b6ec2cf74d4678" dependencies = [ "serde", ] [[package]] name = "arrow-select" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3" +checksum = "8e415279094ea70323c032c6e739c48ad8d80e78a09bef7117b8718ad5bf3722" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -387,9 +387,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6" +checksum = "11d956cae7002eb8d83a27dbd34daaea1cf5b75852f0b84deb4d93a276e92bbf" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,6 +402,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.83" @@ -436,13 +447,12 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backon" -version = "0.4.4" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" +checksum = "ba5289ec98f68f28dd809fd601059e6aa908bb8f6108620930828283d4ee23d7" dependencies = [ "fastrand", - "futures-core", - "pin-project", + "gloo-timers", "tokio", ] @@ -551,9 +561,9 @@ dependencies = [ [[package]] name = "brotli" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -656,9 +666,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" dependencies = [ "chrono", "chrono-tz-build", @@ -667,12 +677,11 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" dependencies = [ "parse-zoneinfo", - "phf", "phf_codegen", ] @@ -722,6 +731,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "config" version = "0.13.4" @@ -829,6 +847,24 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -929,12 +965,13 @@ dependencies = [ [[package]] name = "delta_kernel" -version = "0.3.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa08a82239f51e6d3d249c38f0f5bf7c8a78b28587e1b466893c9eac84d252d8" +checksum = "2efadff110c892910f5b1b09a764845dd136f5dabdd802e64f2968586fbe19e6" dependencies = [ "arrow-arith", "arrow-array", + "arrow-buffer", "arrow-cast", "arrow-json", "arrow-ord", @@ -943,11 +980,9 @@ dependencies = [ "bytes", "chrono", "delta_kernel_derive", - "either", "fix-hidden-lifetime-bug", "indexmap 2.6.0", "itertools", - "lazy_static", "parquet", "roaring", "rustc_version", @@ -964,9 +999,9 @@ dependencies = [ [[package]] name = "delta_kernel_derive" -version = "0.3.1" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec5c4fb5b59b1bd55ed8ebcf941f27a327d600c19a4a4103546846c358be93ff" +checksum = "49fe89c064f509a4a00c00c895af4b966e4b592e7ddf6a476eae856a9b883623" dependencies = [ "proc-macro2", "quote", @@ -975,18 +1010,18 @@ dependencies = [ [[package]] name = "deltalake" -version = "0.21.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b12986bb181170e5bd15d6c49ed73488c444d7e9d5a08afcf37e7e69504f93ae" +checksum = "950788ee777d7fa22043fd329854809cbbf418d25dc04456509e56e885973a7a" dependencies = [ "deltalake-core", ] [[package]] name = "deltalake-core" -version = "0.21.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a95bfec2cdf24ccc7b9d7dda2d92f5d91ed892e79d0928f61fc5e7e4e27be5e" +checksum = "9f254e14de9b1ff46419253c83ddfc65bc1bb254825ba0d28dd8ce9e38c6782d" dependencies = [ "arrow", "arrow-arith", @@ -1009,7 +1044,7 @@ dependencies = [ "errno", "fix-hidden-lifetime-bug", "futures", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "indexmap 2.6.0", "itertools", "lazy_static", @@ -1030,7 +1065,7 @@ dependencies = [ "serde", "serde_json", "sqlparser", - "thiserror 1.0.69", + "thiserror 2.0.3", "tokio", "tracing", "url", @@ -1118,12 +1153,6 @@ dependencies = [ "const-random", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "either" version = "1.13.0" @@ -1169,6 +1198,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "event-listener" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1233,6 +1283,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1381,6 +1437,18 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.7" @@ -1435,6 +1503,11 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -1608,14 +1681,14 @@ dependencies = [ [[package]] name = "iceberg" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "651dfca7c429918e164607a549287cfdd1e7814d2e4cb577d0d6dc57fe19b785" +source = "git+https://github.com/splitgraph/iceberg-rust?rev=eeca14d13b23f2a92e57f503c141a860a3407226#eeca14d13b23f2a92e57f503c141a860a3407226" dependencies = [ "anyhow", "apache-avro", "array-init", "arrow-arith", "arrow-array", + "arrow-cast", "arrow-ord", "arrow-schema", "arrow-select", @@ -1629,11 +1702,14 @@ dependencies = [ "fnv", "futures", "itertools", + "moka", "murmur3", "once_cell", "opendal", "ordered-float 4.5.0", "parquet", + "paste", + "rand", "reqwest", "rust_decimal", "serde", @@ -1643,7 +1719,7 @@ dependencies = [ "serde_repr", "serde_with", "tokio", - "typed-builder", + "typed-builder 0.20.0", "url", "uuid", ] @@ -1917,9 +1993,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -1930,9 +2006,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -1941,9 +2017,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "0.8.6" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" dependencies = [ "lexical-util", "static_assertions", @@ -1951,18 +2027,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "0.8.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" dependencies = [ "lexical-util", "lexical-write-integer", @@ -1971,9 +2047,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" dependencies = [ "lexical-util", "static_assertions", @@ -2113,6 +2189,30 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "moka" +version = "0.12.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cf62eb4dd975d2dde76432fb1075c49e3ee2331cf36f1f8fd4b66550d32b6f" +dependencies = [ + "async-lock", + "async-trait", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "event-listener", + "futures-util", + "once_cell", + "parking_lot", + "quanta", + "rustc_version", + "smallvec", + "tagptr", + "thiserror 1.0.69", + "triomphe", + "uuid", +] + [[package]] name = "murmur3" version = "0.5.2" @@ -2248,9 +2348,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.10.2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" +checksum = "6eb4c22c6154a1e759d7099f9ffad7cc5ef8245f9efbab4a41b92623079c82f3" dependencies = [ "async-trait", "base64 0.22.1", @@ -2284,9 +2384,9 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "opendal" -version = "0.49.2" +version = "0.50.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b04d09b9822c2f75a1d2fc513a2c1279c70e91e7407936fffdf6a6976ec530a" +checksum = "cb28bb6c64e116ceaf8dd4e87099d3cfea4a58e85e62b104fef74c91afba0f44" dependencies = [ "anyhow", "async-trait", @@ -2394,6 +2494,12 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.3" @@ -2419,9 +2525,9 @@ dependencies = [ [[package]] name = "parquet" -version = "52.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd" +checksum = "2b449890367085eb65d7d3321540abc3d7babbd179ce31df0016e90719114191" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -2438,7 +2544,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "lz4_flex", "num", "num-bigint", @@ -2563,26 +2669,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pin-project" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.89", -] - [[package]] name = "pin-project-lite" version = "0.2.15" @@ -2711,6 +2797,21 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" +[[package]] +name = "quanta" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quick-xml" version = "0.35.0" @@ -2828,6 +2929,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "raw-cpuid" +version = "11.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -3402,24 +3512,23 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "snafu" -version = "0.7.5" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" dependencies = [ - "doc-comment", "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.7.5" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.89", ] [[package]] @@ -3552,6 +3661,12 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -3845,6 +3960,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" +[[package]] +name = "triomphe" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" + [[package]] name = "try-lock" version = "0.2.5" @@ -3867,7 +3988,16 @@ version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" dependencies = [ - "typed-builder-macro", + "typed-builder-macro 0.19.1", +] + +[[package]] +name = "typed-builder" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e14ed59dc8b7b26cacb2a92bad2e8b1f098806063898ab42a3bd121d7d45e75" +dependencies = [ + "typed-builder-macro 0.20.0", ] [[package]] @@ -3881,6 +4011,17 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "typed-builder-macro" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "560b82d656506509d43abe30e0ba64c56b1953ab3d4fe7ba5902747a7a3cedd5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "typenum" version = "1.17.0" @@ -4145,6 +4286,22 @@ dependencies = [ "web-sys", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.9" @@ -4154,6 +4311,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 1fd6b90..4a7cf19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,22 +5,22 @@ edition = "2021" [dependencies] -arrow = { version = "52" } -arrow-buffer = { version = "52" } -arrow-schema = { version = "52" } +arrow = { version = "53" } +arrow-buffer = { version = "53" } +arrow-schema = { version = "53" } async-trait = "0.1" bytes = "1.4.0" clap = { version = "4.5", features = [ "derive" ] } config = "0.13.3" -deltalake = { version = "0.21" } +deltalake = { version = "0.22" } env_logger = "0.11.1" fastrand = "2.2.0" futures = "0.3" -iceberg = "0.3.0" +iceberg = { git = "https://github.com/splitgraph/iceberg-rust", rev = "eeca14d13b23f2a92e57f503c141a860a3407226" } log = "0.4" native-tls = "0.2.11" -object_store = { version = "0.10", features = ["aws"] } -parquet = { version = "52" } +object_store = { version = "0.11", features = ["aws"] } +parquet = { version = "53" } postgres = { version = "0.19.7", git = "https://github.com/splitgraph/rust-postgres", rev = "88c2c7714a4558aed6a63e2e2b140a8359568858" } postgres-native-tls = { version = "0.5.0", git = "https://github.com/splitgraph/rust-postgres", rev = "88c2c7714a4558aed6a63e2e2b140a8359568858" } serde_json = "1" diff --git a/src/delta_destination.rs b/src/delta_destination.rs index b2d9cfc..17ba47d 100644 --- a/src/delta_destination.rs +++ b/src/delta_destination.rs @@ -207,8 +207,15 @@ pub async fn record_batches_to_object_store( // Create the corresponding Add action; currently we don't support partition columns // which simplifies things. - let add = - create_add(&Default::default(), file_name, size, &metadata, -1, &None).unwrap(); + let add = create_add( + &Default::default(), + file_name, + size, + &metadata, + -1, + &None::>, + ) + .unwrap(); Ok(add) }); diff --git a/src/iceberg_destination.rs b/src/iceberg_destination.rs index bf28a52..fa8c87a 100644 --- a/src/iceberg_destination.rs +++ b/src/iceberg_destination.rs @@ -7,9 +7,10 @@ use arrow_schema::{Field, Schema, SchemaRef}; use futures::{pin_mut, StreamExt, TryStream}; use iceberg::io::FileIO; use iceberg::spec::{ - DataContentType, DataFileFormat, FormatVersion, Manifest, ManifestContentType, ManifestEntry, - ManifestFile, ManifestListWriter, ManifestMetadata, ManifestStatus, ManifestWriter, Operation, - PartitionSpec, Snapshot, Struct, Summary, TableMetadata, TableMetadataBuilder, + BoundPartitionSpec, DataContentType, DataFileFormat, FormatVersion, Manifest, + ManifestContentType, ManifestEntry, ManifestFile, ManifestListWriter, ManifestMetadata, + ManifestStatus, ManifestWriter, Operation, Snapshot, Struct, Summary, TableMetadata, + TableMetadataBuilder, }; use iceberg::writer::file_writer::location_generator::{ DefaultFileNameGenerator, DefaultLocationGenerator, @@ -78,16 +79,18 @@ fn create_metadata_v1( snapshot: Snapshot, ) -> Result { let snapshot_id = snapshot.snapshot_id(); + let mut metadata = metadata_v0.clone(); + metadata.append_snapshot(snapshot); + // Copy metadata v0, modifying current snapshot ID - let mut metadata_v0_json = serde_json::to_value(metadata_v0).unwrap(); - if let Some(obj) = metadata_v0_json.as_object_mut() { + let mut metadata_json = serde_json::to_value(metadata).unwrap(); + if let Some(obj) = metadata_json.as_object_mut() { obj.insert( "current-snapshot-id".to_string(), serde_json::Value::from(snapshot_id), ); } - let mut metadata_v1: TableMetadata = serde_json::from_value(metadata_v0_json).unwrap(); - metadata_v1.append_snapshot(snapshot); + let metadata_v1: TableMetadata = serde_json::from_value(metadata_json).unwrap(); Ok(metadata_v1) } @@ -102,7 +105,9 @@ pub async fn record_batches_to_iceberg( let file_io = create_file_io(target_url.to_string())?; let arrow_schema_with_ids = assign_field_ids(arrow_schema.clone()); - let iceberg_schema = iceberg::arrow::arrow_schema_to_schema(&arrow_schema_with_ids)?; + let iceberg_schema = Arc::new(iceberg::arrow::arrow_schema_to_schema( + &arrow_schema_with_ids, + )?); let metadata_v0 = create_metadata_v0(&iceberg_schema, target_url.to_string())?; let metadata_v0_location = format!("{}/metadata/v0.metadata.json", target_url); @@ -114,7 +119,7 @@ pub async fn record_batches_to_iceberg( let file_writer_builder = ParquetWriterBuilder::new( WriterProperties::builder().build(), - Arc::new(iceberg_schema.clone()), + iceberg_schema.clone(), file_io.clone(), DefaultLocationGenerator::new(metadata_v0.clone()).unwrap(), DefaultFileNameGenerator::new( @@ -155,7 +160,7 @@ pub async fn record_batches_to_iceberg( .schema_id(DEFAULT_SCHEMA_ID) .schema(iceberg_schema.clone()) .partition_spec( - PartitionSpec::builder(&iceberg_schema) + BoundPartitionSpec::builder(iceberg_schema.clone()) .with_spec_id(0) .build()?, )