diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aeeb877..f63d11a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -333,7 +333,7 @@ jobs: # SurrealDB SurrealKV Engine - name: surrealdb-embedded-surrealkv database: surrealdb - enabled: false + enabled: true endpoint: -e surrealkv:~/crud-bench description: SurrealDB embedded with SurrealKV storage skipped: SurrealDB with SurrealKV storage benchmark awaiting fixes diff --git a/Cargo.lock b/Cargo.lock index 2bb6d20..3efb301 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1213,8 +1213,8 @@ dependencies = [ "surrealdb", "surrealdb-rocksdb", "surrealdb-types", - "surrealkv", - "surrealmx 0.15.0", + "surrealkv 0.16.0", + "surrealmx", "sysinfo 0.37.2", "tokio", "tokio-postgres", @@ -2581,7 +2581,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.61.2", ] [[package]] @@ -3109,7 +3109,7 @@ dependencies = [ "guardian", "interval-heap", "log", - "lz4_flex", + "lz4_flex 0.11.5", "path-absolutize", "quick_cache", "rustc-hash", @@ -3148,6 +3148,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +dependencies = [ + "twox-hash", +] + [[package]] name = "mac" version = "0.1.1" @@ -4753,10 +4762,11 @@ checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" [[package]] name = "revision" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b762d27bdb312a92b2a1786e9dec700186f3c3e8a98b56b36c1b078e6bf2804a" +checksum = "1afd502a7bd15659a0b81479de42a249afa2dd3bdcc689871d6da4b3d005018d" dependencies = [ + "bytes", "chrono", "geo 0.31.0", "regex", @@ -4768,9 +4778,9 @@ dependencies = [ [[package]] name = "revision-derive" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e05651c26aaf33d5e642e380a43b1e5112f76668085aab9377130694160d5177" +checksum = "400e8b48caaca9faa40f590f643c5c155d6c8700378e700de1bd8f453e11578d" dependencies = [ "proc-macro2", "quote", @@ -5158,7 +5168,7 @@ dependencies = [ "bytes", "chrono", "itertools 0.14.0", - "lz4_flex", + "lz4_flex 0.11.5", "scylla-macros", "snap", "stable_deref_trait", @@ -5554,19 +5564,20 @@ checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" [[package]] name = "storekey" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79f0a20b9cc597130e6b3ae22f2bc0892095c0def0361b4b86e64a609ef222ef" +checksum = "ac0a0c60667da4e707563b1f1616534926d4a2508d6a7cdb26414a9103f63e1c" dependencies = [ + "bytes", "storekey-derive", "uuid", ] [[package]] name = "storekey-derive" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b17310dd353bab243079e8669927142cc4bde8e6b91bc966c65e347e78563a3" +checksum = "7555a4e3498505187316766c9f4fc23d54ac1ee4455e6c931f87a6990efbb8ba" dependencies = [ "proc-macro2", "quote", @@ -5652,8 +5663,7 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "surrealdb" version = "3.0.0-alpha.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9feee672723cb93404ecc4e546bc87db5546f18c007fd6104a657e3b358751d0" +source = "git+https://github.com/surrealdb/surrealdb?branch=arriqaaq%2Fskvfixes#179a46a244447369ffd2d0497f5b0f59ee923a50" dependencies = [ "arrayvec", "async-channel", @@ -5695,8 +5705,7 @@ dependencies = [ [[package]] name = "surrealdb-core" version = "3.0.0-alpha.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fb7e4aae83485f3056bef658918dda357b426d9efba2077b8cc69aef744850c" +source = "git+https://github.com/surrealdb/surrealdb?branch=arriqaaq%2Fskvfixes#179a46a244447369ffd2d0497f5b0f59ee923a50" dependencies = [ "addr", "affinitypool", @@ -5771,8 +5780,8 @@ dependencies = [ "surrealdb-protocol", "surrealdb-rocksdb", "surrealdb-types", - "surrealkv", - "surrealmx 0.14.4", + "surrealkv 0.15.1", + "surrealmx", "sysinfo 0.33.1", "tempfile", "thiserror 1.0.69", @@ -5841,8 +5850,7 @@ dependencies = [ [[package]] name = "surrealdb-types" version = "3.0.0-alpha.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "735155fe32da2ecd6adcb5f96b8ffb244c6b8ecce65db21537bb854c32690879" +source = "git+https://github.com/surrealdb/surrealdb?branch=arriqaaq%2Fskvfixes#179a46a244447369ffd2d0497f5b0f59ee923a50" dependencies = [ "anyhow", "bytes", @@ -5867,8 +5875,7 @@ dependencies = [ [[package]] name = "surrealdb-types-derive" version = "3.0.0-alpha.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a20facc0d9ca1e9c8e035066968ac575f4b25a6fc3736713496f4cd0763c408" +source = "git+https://github.com/surrealdb/surrealdb?branch=arriqaaq%2Fskvfixes#179a46a244447369ffd2d0497f5b0f59ee923a50" dependencies = [ "proc-macro2", "quote", @@ -5878,8 +5885,7 @@ dependencies = [ [[package]] name = "surrealkv" version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae2e4384e4e61954e97d74798f9b13423988b32f60e8377b0b49efbcd7540ba5" +source = "git+https://github.com/surrealdb/surrealkv?branch=fix-keys-only-bug#99bf2344e5bf538f9d032ca28c1d035187e9df59" dependencies = [ "arc-swap", "async-channel", @@ -5899,6 +5905,7 @@ dependencies = [ "integer-encoding", "interval-heap", "log", + "lz4_flex 0.12.0", "memmap2", "parking_lot", "quick_cache", @@ -5910,24 +5917,38 @@ dependencies = [ ] [[package]] -name = "surrealmx" -version = "0.14.4" +name = "surrealkv" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a285508f077d3ff9069d408a677ea9af290cdc841f65e769e4e6385aca0d6db" +checksum = "1b25dacdb399a9ec6b67e4b9b626a668b7f4e641d14ad6c5babc3241cb7e2faa" dependencies = [ - "ahash 0.8.12", "arc-swap", - "bincode 2.0.1", + "async-channel", + "async-trait", + "bit-vec", + "byteorder", "bytes", - "crossbeam-deque", - "crossbeam-queue", + "chrono", + "crc32fast", + "crossbeam", "crossbeam-skiplist", - "lz4", + "double-ended-peekable", + "fs2", + "futures", + "getrandom 0.3.4", + "guardian", + "integer-encoding", + "interval-heap", + "log", + "lz4_flex 0.12.0", + "memmap2", "parking_lot", - "serde", - "smallvec", - "thiserror 2.0.17", - "tracing", + "quick_cache", + "rand 0.9.2", + "scopeguard", + "sha2", + "snap", + "tokio", ] [[package]] @@ -7068,19 +7089,6 @@ dependencies = [ "windows-strings 0.4.2", ] -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement 0.60.2", - "windows-interface 0.59.3", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - [[package]] name = "windows-future" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 42beac3..e7ab020 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,9 +106,9 @@ scylla = { version = "1.4.1", optional = true } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" serial_test = "3.2.0" -surrealdb = { version = "3.0.0-alpha.16", default-features = false, optional = true } -surrealdb-types = { version = "3.0.0-alpha.16", default-features = false, optional = true } -surrealkv = { version = "0.15.1", optional = true } +surrealdb = { git = "https://github.com/surrealdb/surrealdb", branch = "arriqaaq/skvfixes", default-features = false, optional = true } +surrealdb-types = { git = "https://github.com/surrealdb/surrealdb", branch = "arriqaaq/skvfixes", default-features = false, optional = true } +surrealkv = { version = "0.16.0", optional = true } surrealmx = { version = "0.15.0", optional = true } sysinfo = { version = "0.37.2", features = ["serde"] } tokio = { version = "1.48.0", features = ["macros", "time", "rt-multi-thread"] } diff --git a/src/surrealkv.rs b/src/surrealkv.rs index 04a83ba..1000480 100644 --- a/src/surrealkv.rs +++ b/src/surrealkv.rs @@ -22,17 +22,11 @@ const DATABASE_DIR: &str = "surrealkv"; const BLOCK_SIZE: usize = 64 * 1024; /// Calculate SurrealKV specific memory allocation -fn calculate_surrealkv_memory() -> (u64, u64) { +fn calculate_surrealkv_memory() -> u64 { // Load the system memory let memory = Config::new(); - // Calculate total available cache memory in bytes - let total_cache_bytes = memory.cache_gb * 1024 * 1024 * 1024; - // Allocate 70% for block cache - let block_cache_bytes = (total_cache_bytes * 70) / 100; - // Allocate 30% for value cache - let value_cache_bytes = (total_cache_bytes * 30) / 100; // Return configuration - (block_cache_bytes, value_cache_bytes) + memory.cache_gb * 1024 * 1024 * 1024 } pub(crate) struct SurrealKVClientProvider { @@ -50,15 +44,13 @@ impl BenchmarkEngine for SurrealKVClientProvider { // Cleanup the data directory std::fs::remove_dir_all(DATABASE_DIR).ok(); // Calculate memory allocation - let (block_cache_bytes, value_cache_bytes) = calculate_surrealkv_memory(); + let block_cache_bytes = calculate_surrealkv_memory(); // Configure custom options let builder = TreeBuilder::new(); // Enable max memtable size let builder = builder.with_max_memtable_size(256 * 1024 * 1024); // Enable the block cache capacity let builder = builder.with_block_cache_capacity(block_cache_bytes); - // Enable the block cache capacity - let builder = builder.with_vlog_cache_capacity(value_cache_bytes); // Disable versioned queries let builder = builder.with_versioning(false, 0); // Enable separated keys and values