diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a7fada2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +target/**/* +.vscode/**/* diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..8341606 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,914 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "backtrace" +version = "0.3.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bincode" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bindgen" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cexpr 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "clang-sys 0.28.1 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "which 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bindings" +version = "0.1.0" +dependencies = [ + "bindgen 0.52.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bstr" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byteorder" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cexpr" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clang-sys" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "clap" +version = "2.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "clap-verbosity-flag" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "const-cstr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "crossbeam-channel" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "derive_more" +version = "0.99.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "either" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "env_logger" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "exitfailure" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "failure" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "failure_derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fnv" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "globset" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", + "bstr 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "globwalk" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ignore 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-segmentation 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hermit-abi" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ignore" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "globset 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "same-file 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "is_sorted" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazycell" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libloading" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memchr" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memoffset" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nom" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num_cpus" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nvpair-sys" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro2" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quicli" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "clap-verbosity-flag 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "exitfailure 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "globwalk 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rayon" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon-core 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rayon-core" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-queue 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "remove_dir_all" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc-hash" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "same-file" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "scopeguard" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "structopt" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt-derive 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "structopt-derive" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synstructure" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termcolor" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wincolor 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thread_local" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "walkdir" +version = "2.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "same-file 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "which" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wincolor" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zquash" +version = "0.1.0" +dependencies = [ + "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bindings 0.1.0", + "const-cstr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "dotenv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "is_sorted 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "nvpair-sys 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quicli 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" +"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" +"checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea" +"checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" +"checksum bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" +"checksum bindgen 0.52.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f1c85344eb535a31b62f0af37be84441ba9e7f0f4111eb0530f43d15e513fe57" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum bstr 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8d6c2c5b58ab920a4f5aeaaca34b4488074e8cc7596af94e6f8c6ff247c60245" +"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +"checksum cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)" = "f52a465a666ca3d838ebbf08b241383421412fe7ebb463527bba275526d89f76" +"checksum cexpr 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum clang-sys 0.28.1 (registry+https://github.com/rust-lang/crates.io-index)" = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853" +"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +"checksum clap-verbosity-flag 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bda14f5323b2b747f52908c5b7b8af7790784088bc7c2957a11695e39ad476dc" +"checksum const-cstr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed3d0b5ff30645a68f35ece8cea4556ca14ef8a1651455f789a099a0513532a6" +"checksum crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa" +"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca" +"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" +"checksum crossbeam-queue 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dfd6515864a82d2f877b42813d4553292c6659498c9a2aa31bab5a15243c2700" +"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" +"checksum derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2159be042979966de68315bce7034bb000c775f22e3e834e1c52ff78f041cae8" +"checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" +"checksum dotenv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" +"checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" +"checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +"checksum exitfailure 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2ff5bd832af37f366c6c194d813a11cd90ac484f124f079294f28e357ae40515" +"checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" +"checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" +"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +"checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +"checksum globset 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "925aa2cac82d8834e2b2a4415b6f6879757fb5c0928fc445ae76461a12eed8f2" +"checksum globwalk 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce5d04da8cf35b507b2cbec92bbf2d5085292d07cd87637994fd437fe1617bbb" +"checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +"checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120" +"checksum humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +"checksum ignore 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0ec16832258409d571aaef8273f3c3cc5b060d784e159d1a0f3b0017308f84a7" +"checksum is_sorted 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "357376465c37db3372ef6a00585d336ed3d0f11d4345eef77ebcb05865392b21" +"checksum itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +"checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" +"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" +"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" +"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" +"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +"checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72" +"checksum nvpair-sys 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5fc4cc751960db1094a7a732dd0706927aeae9194bf66c30fa227b70fc62b6c8" +"checksum peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +"checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" +"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum quicli 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9e8539e98d5a5e3cb0398aedac3e9642ead7d3047a459893526710cb5ad86f6c" +"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +"checksum rayon 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "43739f8831493b276363637423d3622d4bd6394ab6f0a9c4a552e208aeb7fddd" +"checksum rayon-core 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8bf17de6f23b05473c437eb958b9c850bfc8af0961fe17b4cc92d5a627b4791" +"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd" +"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" +"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" +"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +"checksum rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7540fc8b0c49f096ee9c961cda096467dce8084bec6bdca2fc83895fd9b28cb8" +"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum same-file 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "585e8ddcedc187886a30fa705c47985c3fa88d06624095856b36ca0b82ff4421" +"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "1217f97ab8e8904b57dd22eb61cde455fa7446a9c1cf43966066da047c1f3702" +"checksum serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c6faef9a2e64b0064f48570289b4bf8823b7581f1d6157c1b52152306651d0" +"checksum shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +"checksum structopt 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "16c2cdbf9cc375f15d1b4141bc48aeef444806655cd0e904207edc8d68d86ed7" +"checksum structopt-derive 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "53010261a84b37689f9ed7d395165029f9cc7abb9f56bbfe86bee2597ed25107" +"checksum syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)" = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +"checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238" +"checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" +"checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e" +"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +"checksum unicode-segmentation 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" +"checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" +"checksum walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9658c94fa8b940eab2250bd5a457f9c48b748420d71293b165c8cdbe2f55f71e" +"checksum which 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5475d47078209a02e60614f7ba5e645ef3ed60f771920ac1906d7c1cc65024c8" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum wincolor 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "96f5016b18804d24db43cebf3c77269e7569b8954a8464501c216cc5e070eaa9" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..2d30118 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["bindings", "zquash"] diff --git a/bindings/Cargo.toml b/bindings/Cargo.toml new file mode 100644 index 0000000..2242973 --- /dev/null +++ b/bindings/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "bindings" +version = "0.1.0" +authors = ["Christian Schwarz "] +edition = "2018" +build = "build.rs" + +[dependencies] +serde = { version = "*", features = ["derive"] } + +[build-dependencies] +bindgen = "0.52" +cc = "~1" \ No newline at end of file diff --git a/bindings/build.rs b/bindings/build.rs new file mode 100644 index 0000000..105ad8a --- /dev/null +++ b/bindings/build.rs @@ -0,0 +1,31 @@ +extern crate bindgen; + +use std::env; +use std::path::{Path, PathBuf}; + +fn main() { + let dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let helpers = Path::new(&dir).join("./helpers"); + + // compile helpers lib + cc::Build::new().file(helpers.join("helpers.c")).flag("-Werror").flag("-Wall").flag("-Wpedantic").compile("helpers"); + + let clang_args = [format!("-I{}", helpers.display())]; + let bindings = bindgen::Builder::default() + .rust_target(bindgen::RustTarget::Nightly) + .header("wrapper.h") + .header(helpers.join("helpers.h").to_str().unwrap()) + .clang_args(&clang_args) + .derive_debug(true) + .whitelist_recursively(true) + .whitelist_function("drr_effective_payload_len") + .whitelist_type("dmu_replay_record") + .generate() + .expect("unable to generate bindings"); + + // Write the bindings to the $OUT_DIR/bindings.rs file. + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("couldn't write bindings!"); +} diff --git a/bindings/helpers/helpers.c b/bindings/helpers/helpers.c new file mode 100644 index 0000000..392ff00 --- /dev/null +++ b/bindings/helpers/helpers.c @@ -0,0 +1,30 @@ +#include "helpers.h" + +ssize_t drr_effective_payload_len(const dmu_replay_record_t *drr) { + switch (drr->drr_type) { + case DRR_OBJECT: + return DRR_OBJECT_PAYLOAD_SIZE(&drr->drr_u.drr_object); + case DRR_WRITE: + return DRR_WRITE_PAYLOAD_SIZE(&drr->drr_u.drr_write); + case DRR_SPILL: + return DRR_SPILL_PAYLOAD_SIZE(&drr->drr_u.drr_spill); + case DRR_BEGIN: + /* fallthrough */ + case DRR_FREEOBJECTS: + /* fallthrough */ + case DRR_FREE: + /* fallthrough */ + case DRR_END: + /* fallthrough */ + case DRR_WRITE_BYREF: + /* fallthrough */ + case DRR_WRITE_EMBEDDED: + /* fallthrough */ + case DRR_OBJECT_RANGE: + /* fallthrough */ + case DRR_NUMTYPES: + /* fallthrough */ + break; + } + return drr->drr_payloadlen; +} diff --git a/bindings/helpers/helpers.h b/bindings/helpers/helpers.h new file mode 100644 index 0000000..9584efa --- /dev/null +++ b/bindings/helpers/helpers.h @@ -0,0 +1,3 @@ +#include "zfs_ioctl.expanded.h" + +ssize_t drr_effective_payload_len(const dmu_replay_record_t *drr); diff --git a/bindings/helpers/zfs_ioctl.expanded.h b/bindings/helpers/zfs_ioctl.expanded.h new file mode 100644 index 0000000..aec09ef --- /dev/null +++ b/bindings/helpers/zfs_ioctl.expanded.h @@ -0,0 +1,369 @@ +/* + * The contents of this file were taken from the ZFS on Linux project: + * + * https://github.com/zfsonlinux/zfs/blob/master/include/sys/zfs_ioctl.h + * + * Some macros were expanded manually. + * + */ + +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2018 by Delphix. All rights reserved. + * Copyright 2016 RackTop Systems. + * Copyright (c) 2017, Intel Corporation. + */ + +#include +#include + +#define B_FALSE 0 +#define B_TRUE 1 +#define MAXNAMELEN 256 + +#define ZIO_DATA_SALT_LEN 8 +#define ZIO_DATA_IV_LEN 12 +#define ZIO_DATA_MAC_LEN 16 + +typedef enum dmu_objset_type { + DMU_OST_NONE, + DMU_OST_META, + DMU_OST_ZFS, + DMU_OST_ZVOL, + DMU_OST_OTHER, /* For testing only! */ + DMU_OST_ANY, /* Be careful! */ + DMU_OST_NUMTYPES +} dmu_objset_type_t; + +typedef struct zio_cksum { + uint64_t zc_word[4]; +} zio_cksum_t; + +typedef struct ddt_key { + zio_cksum_t ddk_cksum; /* 256-bit block checksum */ + /* + * Encoded with logical & physical size, encryption, and compression, + * as follows: + * +-------+-------+-------+-------+-------+-------+-------+-------+ + * | 0 | 0 | 0 |X| comp| PSIZE | LSIZE | + * +-------+-------+-------+-------+-------+-------+-------+-------+ + */ + uint64_t ddk_prop; +} ddt_key_t; + +typedef enum dmu_object_byteswap { + DMU_BSWAP_UINT8, + DMU_BSWAP_UINT16, + DMU_BSWAP_UINT32, + DMU_BSWAP_UINT64, + DMU_BSWAP_ZAP, + DMU_BSWAP_DNODE, + DMU_BSWAP_OBJSET, + DMU_BSWAP_ZNODE, + DMU_BSWAP_OLDACL, + DMU_BSWAP_ACL, + /* + * Allocating a new byteswap type number makes the on-disk format + * incompatible with any other format that uses the same number. + * + * Data can usually be structured to work with one of the + * DMU_BSWAP_UINT* or DMU_BSWAP_ZAP types. + */ + DMU_BSWAP_NUMFUNCS +} dmu_object_byteswap_t; + +#define DMU_OT_NEWTYPE 0x80 +#define DMU_OT_METADATA 0x40 +#define DMU_OT_ENCRYPTED 0x20 +#define DMU_OT_BYTESWAP_MASK 0x1f + +#define DMU_OT(byteswap, metadata, encrypted) \ + (DMU_OT_NEWTYPE | \ + ((metadata) ? DMU_OT_METADATA : 0) | \ + ((encrypted) ? DMU_OT_ENCRYPTED : 0) | \ + ((byteswap) & DMU_OT_BYTESWAP_MASK)) + +typedef enum dmu_object_type { + DMU_OT_NONE, + /* general: */ + DMU_OT_OBJECT_DIRECTORY, /* ZAP */ + DMU_OT_OBJECT_ARRAY, /* UINT64 */ + DMU_OT_PACKED_NVLIST, /* UINT8 (XDR by nvlist_pack/unpack) */ + DMU_OT_PACKED_NVLIST_SIZE, /* UINT64 */ + DMU_OT_BPOBJ, /* UINT64 */ + DMU_OT_BPOBJ_HDR, /* UINT64 */ + /* spa: */ + DMU_OT_SPACE_MAP_HEADER, /* UINT64 */ + DMU_OT_SPACE_MAP, /* UINT64 */ + /* zil: */ + DMU_OT_INTENT_LOG, /* UINT64 */ + /* dmu: */ + DMU_OT_DNODE, /* DNODE */ + DMU_OT_OBJSET, /* OBJSET */ + /* dsl: */ + DMU_OT_DSL_DIR, /* UINT64 */ + DMU_OT_DSL_DIR_CHILD_MAP, /* ZAP */ + DMU_OT_DSL_DS_SNAP_MAP, /* ZAP */ + DMU_OT_DSL_PROPS, /* ZAP */ + DMU_OT_DSL_DATASET, /* UINT64 */ + /* zpl: */ + DMU_OT_ZNODE, /* ZNODE */ + DMU_OT_OLDACL, /* Old ACL */ + DMU_OT_PLAIN_FILE_CONTENTS, /* UINT8 */ + DMU_OT_DIRECTORY_CONTENTS, /* ZAP */ + DMU_OT_MASTER_NODE, /* ZAP */ + DMU_OT_UNLINKED_SET, /* ZAP */ + /* zvol: */ + DMU_OT_ZVOL, /* UINT8 */ + DMU_OT_ZVOL_PROP, /* ZAP */ + /* other; for testing only! */ + DMU_OT_PLAIN_OTHER, /* UINT8 */ + DMU_OT_UINT64_OTHER, /* UINT64 */ + DMU_OT_ZAP_OTHER, /* ZAP */ + /* new object types: */ + DMU_OT_ERROR_LOG, /* ZAP */ + DMU_OT_SPA_HISTORY, /* UINT8 */ + DMU_OT_SPA_HISTORY_OFFSETS, /* spa_his_phys_t */ + DMU_OT_POOL_PROPS, /* ZAP */ + DMU_OT_DSL_PERMS, /* ZAP */ + DMU_OT_ACL, /* ACL */ + DMU_OT_SYSACL, /* SYSACL */ + DMU_OT_FUID, /* FUID table (Packed NVLIST UINT8) */ + DMU_OT_FUID_SIZE, /* FUID table size UINT64 */ + DMU_OT_NEXT_CLONES, /* ZAP */ + DMU_OT_SCAN_QUEUE, /* ZAP */ + DMU_OT_USERGROUP_USED, /* ZAP */ + DMU_OT_USERGROUP_QUOTA, /* ZAP */ + DMU_OT_USERREFS, /* ZAP */ + DMU_OT_DDT_ZAP, /* ZAP */ + DMU_OT_DDT_STATS, /* ZAP */ + DMU_OT_SA, /* System attr */ + DMU_OT_SA_MASTER_NODE, /* ZAP */ + DMU_OT_SA_ATTR_REGISTRATION, /* ZAP */ + DMU_OT_SA_ATTR_LAYOUTS, /* ZAP */ + DMU_OT_SCAN_XLATE, /* ZAP */ + DMU_OT_DEDUP, /* fake dedup BP from ddt_bp_create() */ + DMU_OT_DEADLIST, /* ZAP */ + DMU_OT_DEADLIST_HDR, /* UINT64 */ + DMU_OT_DSL_CLONES, /* ZAP */ + DMU_OT_BPOBJ_SUBOBJ, /* UINT64 */ + /* + * Do not allocate new object types here. Doing so makes the on-disk + * format incompatible with any other format that uses the same object + * type number. + * + * When creating an object which does not have one of the above types + * use the DMU_OTN_* type with the correct byteswap and metadata + * values. + * + * The DMU_OTN_* types do not have entries in the dmu_ot table, + * use the DMU_OT_IS_METADATA() and DMU_OT_BYTESWAP() macros instead + * of indexing into dmu_ot directly (this works for both DMU_OT_* types + * and DMU_OTN_* types). + */ + DMU_OT_NUMTYPES, + + /* + * Names for valid types declared with DMU_OT(). + */ + DMU_OTN_UINT8_DATA = DMU_OT(DMU_BSWAP_UINT8, B_FALSE, B_FALSE), + DMU_OTN_UINT8_METADATA = DMU_OT(DMU_BSWAP_UINT8, B_TRUE, B_FALSE), + DMU_OTN_UINT16_DATA = DMU_OT(DMU_BSWAP_UINT16, B_FALSE, B_FALSE), + DMU_OTN_UINT16_METADATA = DMU_OT(DMU_BSWAP_UINT16, B_TRUE, B_FALSE), + DMU_OTN_UINT32_DATA = DMU_OT(DMU_BSWAP_UINT32, B_FALSE, B_FALSE), + DMU_OTN_UINT32_METADATA = DMU_OT(DMU_BSWAP_UINT32, B_TRUE, B_FALSE), + DMU_OTN_UINT64_DATA = DMU_OT(DMU_BSWAP_UINT64, B_FALSE, B_FALSE), + DMU_OTN_UINT64_METADATA = DMU_OT(DMU_BSWAP_UINT64, B_TRUE, B_FALSE), + DMU_OTN_ZAP_DATA = DMU_OT(DMU_BSWAP_ZAP, B_FALSE, B_FALSE), + DMU_OTN_ZAP_METADATA = DMU_OT(DMU_BSWAP_ZAP, B_TRUE, B_FALSE), + + DMU_OTN_UINT8_ENC_DATA = DMU_OT(DMU_BSWAP_UINT8, B_FALSE, B_TRUE), + DMU_OTN_UINT8_ENC_METADATA = DMU_OT(DMU_BSWAP_UINT8, B_TRUE, B_TRUE), + DMU_OTN_UINT16_ENC_DATA = DMU_OT(DMU_BSWAP_UINT16, B_FALSE, B_TRUE), + DMU_OTN_UINT16_ENC_METADATA = DMU_OT(DMU_BSWAP_UINT16, B_TRUE, B_TRUE), + DMU_OTN_UINT32_ENC_DATA = DMU_OT(DMU_BSWAP_UINT32, B_FALSE, B_TRUE), + DMU_OTN_UINT32_ENC_METADATA = DMU_OT(DMU_BSWAP_UINT32, B_TRUE, B_TRUE), + DMU_OTN_UINT64_ENC_DATA = DMU_OT(DMU_BSWAP_UINT64, B_FALSE, B_TRUE), + DMU_OTN_UINT64_ENC_METADATA = DMU_OT(DMU_BSWAP_UINT64, B_TRUE, B_TRUE), + DMU_OTN_ZAP_ENC_DATA = DMU_OT(DMU_BSWAP_ZAP, B_FALSE, B_TRUE), + DMU_OTN_ZAP_ENC_METADATA = DMU_OT(DMU_BSWAP_ZAP, B_TRUE, B_TRUE), +} dmu_object_type_t; + +#define P2ROUNDUP(x, align) ((((x) - 1) | ((align) - 1)) + 1) +#define DRR_WRITE_COMPRESSED(drrw) ((drrw)->drr_compressiontype != 0) +#define DRR_WRITE_PAYLOAD_SIZE(drrw) \ + (DRR_WRITE_COMPRESSED(drrw) ? (drrw)->drr_compressed_size : \ + (drrw)->drr_logical_size) +#define DRR_SPILL_PAYLOAD_SIZE(drrs) \ + ((drrs)->drr_compressed_size ? \ + (drrs)->drr_compressed_size : (drrs)->drr_length) +#define DRR_OBJECT_PAYLOAD_SIZE(drro) \ + ((drro)->drr_raw_bonuslen != 0 ? \ + (drro)->drr_raw_bonuslen : P2ROUNDUP((drro)->drr_bonuslen, 8)) +typedef struct dmu_replay_record { + enum { + DRR_BEGIN, DRR_OBJECT, DRR_FREEOBJECTS, + DRR_WRITE, DRR_FREE, DRR_END, DRR_WRITE_BYREF, + DRR_SPILL, DRR_WRITE_EMBEDDED, DRR_OBJECT_RANGE, + DRR_NUMTYPES + } drr_type; + uint32_t drr_payloadlen; + union { + struct drr_begin { + uint64_t drr_magic; + uint64_t drr_versioninfo; /* was drr_version */ + uint64_t drr_creation_time; + dmu_objset_type_t drr_type; + uint32_t drr_flags; + uint64_t drr_toguid; + uint64_t drr_fromguid; + char drr_toname[MAXNAMELEN]; + } drr_begin; + struct drr_end { + zio_cksum_t drr_checksum; + uint64_t drr_toguid; + } drr_end; + struct drr_object { + uint64_t drr_object; + dmu_object_type_t drr_type; + dmu_object_type_t drr_bonustype; + uint32_t drr_blksz; + uint32_t drr_bonuslen; + uint8_t drr_checksumtype; + uint8_t drr_compress; + uint8_t drr_dn_slots; + uint8_t drr_flags; + uint32_t drr_raw_bonuslen; + uint64_t drr_toguid; + /* only (possibly) nonzero for raw streams */ + uint8_t drr_indblkshift; + uint8_t drr_nlevels; + uint8_t drr_nblkptr; + uint8_t drr_pad[5]; + uint64_t drr_maxblkid; + /* bonus content follows */ + } drr_object; + struct drr_freeobjects { + uint64_t drr_firstobj; + uint64_t drr_numobjs; + uint64_t drr_toguid; + } drr_freeobjects; + struct drr_write { + uint64_t drr_object; + dmu_object_type_t drr_type; + uint32_t drr_pad; + uint64_t drr_offset; + uint64_t drr_logical_size; + uint64_t drr_toguid; + uint8_t drr_checksumtype; + uint8_t drr_flags; + uint8_t drr_compressiontype; + uint8_t drr_pad2[5]; + /* deduplication key */ + ddt_key_t drr_key; + /* only nonzero if drr_compressiontype is not 0 */ + uint64_t drr_compressed_size; + /* only nonzero for raw streams */ + uint8_t drr_salt[ZIO_DATA_SALT_LEN]; + uint8_t drr_iv[ZIO_DATA_IV_LEN]; + uint8_t drr_mac[ZIO_DATA_MAC_LEN]; + /* content follows */ + } drr_write; + struct drr_free { + uint64_t drr_object; + uint64_t drr_offset; + uint64_t drr_length; + uint64_t drr_toguid; + } drr_free; + struct drr_write_byref { + /* where to put the data */ + uint64_t drr_object; + uint64_t drr_offset; + uint64_t drr_length; + uint64_t drr_toguid; + /* where to find the prior copy of the data */ + uint64_t drr_refguid; + uint64_t drr_refobject; + uint64_t drr_refoffset; + /* properties of the data */ + uint8_t drr_checksumtype; + uint8_t drr_flags; + uint8_t drr_pad2[6]; + ddt_key_t drr_key; /* deduplication key */ + } drr_write_byref; + struct drr_spill { + uint64_t drr_object; + uint64_t drr_length; + uint64_t drr_toguid; + uint8_t drr_flags; + uint8_t drr_compressiontype; + uint8_t drr_pad[6]; + /* only nonzero for raw streams */ + uint64_t drr_compressed_size; + uint8_t drr_salt[ZIO_DATA_SALT_LEN]; + uint8_t drr_iv[ZIO_DATA_IV_LEN]; + uint8_t drr_mac[ZIO_DATA_MAC_LEN]; + dmu_object_type_t drr_type; + /* spill data follows */ + } drr_spill; + struct drr_write_embedded { + uint64_t drr_object; + uint64_t drr_offset; + /* logical length, should equal blocksize */ + uint64_t drr_length; + uint64_t drr_toguid; + uint8_t drr_compression; + uint8_t drr_etype; + uint8_t drr_pad[6]; + uint32_t drr_lsize; /* uncompressed size of payload */ + uint32_t drr_psize; /* compr. (real) size of payload */ + /* (possibly compressed) content follows */ + } drr_write_embedded; + struct drr_object_range { + uint64_t drr_firstobj; + uint64_t drr_numslots; + uint64_t drr_toguid; + uint8_t drr_salt[ZIO_DATA_SALT_LEN]; + uint8_t drr_iv[ZIO_DATA_IV_LEN]; + uint8_t drr_mac[ZIO_DATA_MAC_LEN]; + uint8_t drr_flags; + uint8_t drr_pad[3]; + } drr_object_range; + + /* + * Nore: drr_checksum is overlaid with all record types + * except DRR_BEGIN. Therefore its (non-pad) members + * must not overlap with members from the other structs. + * We accomplish this by putting its members at the very + * end of the struct. + */ + struct drr_checksum { + uint64_t drr_pad[34]; + /* + * fletcher-4 checksum of everything preceding the + * checksum. + */ + zio_cksum_t drr_checksum; + } drr_checksum; + } drr_u; +} dmu_replay_record_t; diff --git a/bindings/src/generated.rs b/bindings/src/generated.rs new file mode 100644 index 0000000..df021ba --- /dev/null +++ b/bindings/src/generated.rs @@ -0,0 +1,7 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +#[allow(clippy::all)] + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/bindings/src/lib.rs b/bindings/src/lib.rs new file mode 100644 index 0000000..61d438c --- /dev/null +++ b/bindings/src/lib.rs @@ -0,0 +1,143 @@ + + +mod generated; +pub use generated::*; + +use std::fmt; + +pub type drr_begin = dmu_replay_record__bindgen_ty_2_drr_begin; +pub type drr_end = dmu_replay_record__bindgen_ty_2_drr_end; + +impl fmt::Debug for drr_begin { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + /* + pub struct drr_begin { + pub drr_magic: u64, + pub drr_versioninfo: u64, + pub drr_creation_time: u64, + pub drr_type: dmu_objset_type_t, + pub drr_flags: u32, + pub drr_toguid: u64, + pub drr_fromguid: u64, + pub drr_toname: [::std::os::raw::c_char; 256usize], + } + */ + let drr_toname = unsafe { + std::slice::from_raw_parts( + self.drr_toname[..].as_ptr() as *const u8, + self.drr_toname.len(), + ) + }; + let term = drr_toname + .iter() + .enumerate() + .filter_map(|(i, c)| if *c == 0 { Some(i) } else { None }) + .next() + .expect("drr_toname should be null terminated"); + use std::ffi::CStr; + let drr_toname = unsafe { CStr::from_bytes_with_nul_unchecked(&drr_toname[0..=term]) }; + out.debug_struct("drr_begin") + .field("drr_magic", &self.drr_magic) + .field("drr_versioninfo", &self.drr_versioninfo) + .field("drr_creation_time", &self.drr_creation_time) + .field("drr_type", &self.drr_type) + .field("drr_flags", &self.drr_flags) + .field("drr_toguid", &self.drr_toguid) + .field("drr_fromguid", &self.drr_fromguid) + .field("drr_toname", &drr_toname) + .finish() + } +} + +use std::borrow::Borrow; + +pub struct drr_debug>(pub R); + +impl> fmt::Debug for drr_debug { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + let drr = self.0.borrow(); + let dbg: &dyn fmt::Debug = unsafe { + if drr.drr_type == dmu_replay_record_DRR_OBJECT { + &drr.drr_u.drr_object + } else if drr.drr_type == dmu_replay_record_DRR_FREEOBJECTS { + &drr.drr_u.drr_freeobjects + } else if drr.drr_type == dmu_replay_record_DRR_WRITE { + &drr.drr_u.drr_write + } else if drr.drr_type == dmu_replay_record_DRR_FREE { + &drr.drr_u.drr_free + } else if drr.drr_type == dmu_replay_record_DRR_BEGIN { + &drr.drr_u.drr_begin + } else if drr.drr_type == dmu_replay_record_DRR_END { + &drr.drr_u.drr_end + } else if drr.drr_type == dmu_replay_record_DRR_SPILL { + &drr.drr_u.drr_spill + } else if drr.drr_type == dmu_replay_record_DRR_OBJECT_RANGE { + &drr.drr_u.drr_object_range + } else { + unreachable!() + } + }; + if !out.alternate() { + write!(out, "{:?}", dbg) + } else { + write!(out, "{:#?}", dbg) + } + } +} + +pub mod byte_serialize_impls { + + use super::dmu_replay_record; + use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer}; + use std::fmt; + + struct dmu_replay_record_visitor; + + impl<'de> Visitor<'de> for dmu_replay_record_visitor { + type Value = dmu_replay_record; + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!( + formatter, + "expecting a byte array of size {}", + std::mem::size_of::() + ) + } + fn visit_bytes(self, v: &[u8]) -> Result + where + E: serde::de::Error, + { + assert!(v.len() == std::mem::size_of::()); + let mut record: dmu_replay_record; + unsafe { + record = std::mem::zeroed(); + let record_aligned_ptr = &mut record as *mut _ as *mut u8; + std::ptr::copy(v.as_ptr(), record_aligned_ptr, v.len()); + } + Ok(record) + } + } + + impl Serialize for dmu_replay_record { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let as_bytes = unsafe { + std::slice::from_raw_parts( + self as *const _ as *const u8, + std::mem::size_of_val(&*self), + ) + }; + serializer.serialize_bytes(as_bytes) + } + } + + impl<'de> Deserialize<'de> for dmu_replay_record { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_bytes(dmu_replay_record_visitor) + } + } +} diff --git a/bindings/wrapper.h b/bindings/wrapper.h new file mode 100644 index 0000000..e69de29 diff --git a/zquash/Cargo.toml b/zquash/Cargo.toml new file mode 100644 index 0000000..a545376 --- /dev/null +++ b/zquash/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "zquash" +version = "0.1.0" +authors = ["Christian Schwarz "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bindings = { path = "../bindings" } + +structopt = "~0.2" +dotenv = "*" +failure = "*" +quicli = "0.4" + +serde = "*" +bincode = "*" +itertools = "0.8" +difference = "*" +log = "*" +derive_more = "*" + +is_sorted = "~0.1" + +nvpair-sys = "0.1" +const-cstr = "0.3" +libc = "*" diff --git a/zquash/src/dmu_stream.rs b/zquash/src/dmu_stream.rs new file mode 100644 index 0000000..b26a625 --- /dev/null +++ b/zquash/src/dmu_stream.rs @@ -0,0 +1,198 @@ +use bindings::*; +use failure::{Error, ResultExt}; +use std::io; +use std::io::{BufReader, Read}; + +use crate::fletcher4::Fletcher4; + +pub trait ReplayRecordExt { + fn bytes(&self) -> &[u8]; + fn bytes_until_checksum(&self) -> &[u8]; + fn checksum(&self) -> &[u64; 4]; + fn checksum_bytes(&self) -> &[u8]; +} + +impl ReplayRecordExt for dmu_replay_record { + #[inline] + fn bytes(&self) -> &[u8] { + unsafe { + std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of_val(&*self)) + } + } + #[inline] + fn bytes_until_checksum(&self) -> &[u8] { + // checksum is at the end of DRR + unsafe { + std::slice::from_raw_parts( + self as *const _ as *const u8, + std::mem::size_of_val(&*self) + - std::mem::size_of_val(&self.drr_u.drr_checksum.drr_checksum), + ) + } + } + #[inline] + fn checksum(&self) -> &[u64; 4] { + assert!(self.drr_type != dmu_replay_record_DRR_BEGIN); + unsafe { &self.drr_u.drr_checksum.drr_checksum.zc_word } + } + #[inline] + fn checksum_bytes(&self) -> &[u8] { + assert!(self.drr_type != dmu_replay_record_DRR_BEGIN); + unsafe { + std::slice::from_raw_parts( + &self.drr_u.drr_checksum.drr_checksum.zc_word as *const _ as *const u8, + std::mem::size_of_val(&self.drr_u.drr_checksum.drr_checksum.zc_word), + ) + } + } +} + + +#[derive(Clone)] +pub struct RecordWithPayload { + pub drr: dmu_replay_record, + pub payload: Vec, +} + +use std::fmt; + +impl fmt::Debug for RecordWithPayload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RecordWithPayload") + .field("drr", &drr_debug(&self.drr)) + .finish() + } +} + +pub struct Record<'r> { + pub header: dmu_replay_record, + pub payload_len: u64, + pub payload_reader: &'r mut io::Read, +} + +fn drain(buf: &mut [u8], r: &mut io::Read) -> Result<(), Error> { + loop { + match r.read(buf) { + Ok(0) => return Ok(()), + Ok(_) => (), + Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(()), + Err(e) => failure::bail!(e), + } + } +} + +pub fn read_with_callback(input: &mut io::Read, mut cb: C) -> Result<(), Error> +where + C: FnMut(Record<'_>) -> Result<(), Error>, +{ + let mut input = BufReader::new(input); + let mut drain_buf = vec![0 as u8; 1 << 15]; + let mut header: dmu_replay_record = unsafe { std::mem::zeroed() }; + loop { + unsafe { + let header_slice = std::slice::from_raw_parts_mut( + &mut header as *mut dmu_replay_record as *mut u8, + std::mem::size_of::(), + ); + input + .read_exact(&mut header_slice[..]) + .context("read header")?; + } + let payload_len = unsafe { drr_effective_payload_len(&header as *const _) } as u64; + let mut payload_reader = input.by_ref().take(payload_len); + let r = Record { + header, + payload_len, + payload_reader: &mut payload_reader, + }; + cb(r)?; + // always drain the reader + drain(&mut drain_buf, &mut payload_reader).context("drain remaining payload")?; + if header.drr_type == dmu_replay_record_DRR_END { + break; + } + } + Ok(()) +} + +struct RecordPayloadWriter<'f, W> { + fletcher: &'f mut Fletcher4, + w: W, + written: usize, +} + +impl Drop for RecordPayloadWriter<'_, A> { + fn drop(&mut self) { + assert!(self.written % 4 == 0); + } +} + +impl io::Write for RecordPayloadWriter<'_, W> +where + W: Write, +{ + fn write(&mut self, buf: &[u8]) -> io::Result { + let written = self.w.write(buf)?; + self.written += written; + self.fletcher.feed_bytes(&buf[0..written]); + Ok((written)) + } + fn flush(&mut self) -> io::Result<()> { + self.w.flush() + } +} + +pub struct StreamWriter { + checksum: Fletcher4, + expect_begin: bool, + seen_end: bool, + out: W, +} + +use std::io::Write; + +impl StreamWriter { + pub fn new(out: W) -> StreamWriter { + StreamWriter { + checksum: Fletcher4::new(), + expect_begin: true, + seen_end: false, + out, + } + } + + pub fn write_record_and_payload_reader( + &mut self, + header: &dmu_replay_record, + payload: &mut P, + ) -> Result<(), Error> { + let mut header: dmu_replay_record = *header; // copy + if header.drr_type == dmu_replay_record_DRR_END { + self.seen_end = true; + unsafe { + header.drr_u.drr_end.drr_checksum.zc_word = self.checksum.checksum(); + } + } + if self.expect_begin { + assert!(!self.seen_end); + assert_eq!(header.drr_type, dmu_replay_record_DRR_BEGIN); + self.expect_begin = false; + self.checksum.feed_bytes(header.bytes()); + } else { + self.checksum.feed_bytes(header.bytes_until_checksum()); + unsafe { + header.drr_u.drr_checksum.drr_checksum.zc_word = self.checksum.checksum(); + } + self.checksum.feed_bytes(header.checksum_bytes()); + } + self.out.write_all(header.bytes()).context("write header")?; + let mut w = RecordPayloadWriter { + fletcher: &mut self.checksum, + w: &mut self.out, + written: 0, + }; + let n = io::copy(payload, &mut w)?; + assert!(n % 4 == 0); + Ok(()) + } +} diff --git a/zquash/src/fletcher4.rs b/zquash/src/fletcher4.rs new file mode 100644 index 0000000..1558473 --- /dev/null +++ b/zquash/src/fletcher4.rs @@ -0,0 +1,67 @@ +use std::io; +use std::io::prelude::*; + +#[derive(Debug)] +pub struct Fletcher4 { + a: u64, + b: u64, + c: u64, + d: u64, +} + +impl Fletcher4 { + pub fn new() -> Fletcher4 { + Fletcher4 { + a: 0, + b: 0, + c: 0, + d: 0, + } + } + + // data.len() % 4 == 0 + #[inline] + pub fn feed_bytes(&mut self, data: &[u8]) { + assert!(data.len() % 4 == 0); + self.feed_iter(data.chunks_exact(4).map(|s: &[u8]| { + debug_assert!(s.len() == 4); + let s: [u8; 4] = [s[0], s[1], s[2], s[3]]; + // let s = std::mem::transmute::<_, [u8; 4]>(s[0..4]); + u32::from_ne_bytes(s) + })) + } + + pub fn feed_iter(&mut self, data: I) + where + I: IntoIterator, + { + data.into_iter().for_each(|f| self.feed(f)) + } + + #[inline] + pub fn feed(&mut self, f: u32) { + /* from zfs_fletcher.c: + * a = a + f + * i i-1 i-1 + * + * b = b + a + * i i-1 i + * + * c = c + b (fletcher-4 only) + * i i-1 i + * + * d = d + c (fletcher-4 only) + * i i-1 i + */ + self.a = self.a.wrapping_add(u64::from(f)); + self.b = self.b.wrapping_add(self.a); + self.c = self.c.wrapping_add(self.b); + self.d = self.d.wrapping_add(self.c); + } + + #[inline] + pub fn checksum(&self) -> [u64; 4] { + [self.a, self.b, self.c, self.d] + } + +} diff --git a/zquash/src/lib.rs b/zquash/src/lib.rs new file mode 100644 index 0000000..ce65c4b --- /dev/null +++ b/zquash/src/lib.rs @@ -0,0 +1,7 @@ +#[macro_use] +extern crate derive_more; + +pub mod dmu_stream; +pub mod fletcher4; +pub mod lsm; +pub mod split_tree; \ No newline at end of file diff --git a/zquash/src/lsm/lsm.rs b/zquash/src/lsm/lsm.rs new file mode 100644 index 0000000..3f1d7a6 --- /dev/null +++ b/zquash/src/lsm/lsm.rs @@ -0,0 +1,284 @@ + + +extern crate bincode; +extern crate serde; + +use serde::de::DeserializeOwned; +use serde::Serialize; + +struct FileIdentifier { + level: usize, + num_in_level: usize, +} + +impl FileIdentifier { + fn to_path(&self, directory: &std::path::Path) -> std::path::PathBuf { + directory.join(format!("{}_{}", self.level, self.num_in_level)) + } +} + +fn read_to_opt(reader: R) -> Option { + let obj = bincode::deserialize_from(reader); + if let Ok(obj) = obj { + return Some(obj); + } + //TODO check if at end of stream some better way + match obj { + Err(err) => match err.as_ref() { + bincode::ErrorKind::Io(err) => match err.kind() { + std::io::ErrorKind::UnexpectedEof => None, + _ => panic!(format!("IO Error: {:?}", err)), + }, + _ => panic!(format!("Deserialize Error")), + }, + _ => unreachable!(), + } +} + +pub struct LSMWriter { + btree: std::collections::BTreeMap, + directory: std::path::PathBuf, + level_counts: Vec, + max_entries_in_ram: usize, + max_files_per_level: usize, +} + +impl LSMWriter { + pub fn new( + directory: std::path::PathBuf, + max_entries_in_ram: usize, + max_files_per_level: usize, + ) -> LSMWriter { + std::fs::create_dir_all(&directory).unwrap(); + LSMWriter { + btree: std::collections::BTreeMap::new(), + directory, + level_counts: Vec::new(), + max_entries_in_ram, + max_files_per_level, + } + } + + pub fn insert(&mut self, key: K, value: V) { + self.btree.insert(key, value); + if self.btree.len() > self.max_entries_in_ram { + self.write_btree_to_file(); + self.btree.clear(); + self.merge_full_levels(); + } + } + + fn new_file_at_level(&mut self, level: usize) -> FileIdentifier { + if level > 20 { + panic!("Suspiciously high level") + } + while self.level_counts.len() <= level { + self.level_counts.push(0); + } + let id = FileIdentifier { + level: level, + num_in_level: self.level_counts[level], + }; + self.level_counts[level] += 1; + id + } + + fn merge_level_up(&mut self, level: usize) { + assert!(self.level_counts[level] > 1); //Merging just one file would work but should not be called + let merge_paths: Vec<_> = (0..self.level_counts[level]) + .map(|num| { + FileIdentifier { + level: level, + num_in_level: num, + } + .to_path(&self.directory) + }) + .collect(); + let out_path = self.new_file_at_level(level + 1).to_path(&self.directory); + self.merge_files(&merge_paths, &out_path); + self.level_counts[level] = 0; + } + + fn merge_full_levels(&mut self) { + let mut level = 0; + while level < self.level_counts.len() { + if self.level_counts[level] >= self.max_files_per_level { + self.merge_level_up(level); + level += 1; + } else { + //Levels can only become full if a new file from a lower level propagates up + break; + } + } + } + + pub fn merge_completely(mut self, out_path: &std::path::Path) { + self.write_btree_to_file(); + + let mut all_paths = Vec::new(); + + for (level, count) in self.level_counts.iter().enumerate() { + (0..*count) + .map(|num| { + FileIdentifier { + level: level, + num_in_level: num, + } + .to_path(&self.directory) + }) + .for_each(|path| all_paths.push(path)); + } + + //Maybe it is not most efficient to merge all files at the same time + //as there is probably a sweet spot somewhere for how many files one should stream from at once + //But easier for now + self.merge_files(&all_paths, &out_path); + std::fs::remove_dir(self.directory).unwrap(); + } + + fn write_btree_to_file(&mut self) { + let path = self.new_file_at_level(0).to_path(&self.directory); + let file = std::fs::File::create(path).unwrap(); + let mut stream = std::io::BufWriter::new(file); + for (key, value) in self.btree.iter() { + bincode::serialize_into(&mut stream, &key).unwrap(); + bincode::serialize_into(&mut stream, &value).unwrap(); + } + } + + fn merge_files(&self, paths: &Vec, out_path: &std::path::Path) { + let mut streams: Vec<_> = paths + .iter() + .map(|path| std::fs::File::open(path).unwrap()) + .map(|file| std::io::BufReader::new(file)) + .collect(); + let mut out_stream = std::io::BufWriter::new(std::fs::File::create(out_path).unwrap()); + + fn cmp_opt(x: &Option, y: &Option) -> std::cmp::Ordering { + use std::cmp::Ordering; + match (x, y) { + (Some(x), Some(y)) => x.cmp(y), + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => Ordering::Equal, + } + } + + //Vec of lowest key of each stream, is None if stream is empty + let mut keys: Vec> = streams + .iter_mut() + .map(|stream| read_to_opt(stream)) + .collect(); + + loop { + let (i, min_k) = keys + .iter() + .enumerate() + .min_by(|(_, k1), (_, k2)| cmp_opt(k1, k2)) + .unwrap(); + if let Some(min_k) = min_k { + bincode::serialize_into(&mut out_stream, &min_k).unwrap(); + let min_v: V = read_to_opt(&mut streams[i]) + .expect("Key was read from file but value is missing"); + bincode::serialize_into(&mut out_stream, &min_v).unwrap(); + keys[i] = read_to_opt(&mut streams[i]); + } else { + break; + } + } + + for p in paths { + std::fs::remove_file(p).unwrap(); + } + } +} + +pub struct LSMReader { + file: std::io::BufReader, + oh: std::marker::PhantomData, + why: std::marker::PhantomData, +} + +impl LSMReader { + pub fn open(path: &std::path::Path) -> LSMReader { + LSMReader { + file: std::io::BufReader::new(std::fs::File::open(path).unwrap()), + oh: std::marker::PhantomData, + why: std::marker::PhantomData, + } + } +} + +impl Iterator for LSMReader { + type Item = (K, V); + + fn next(&mut self) -> Option { + let key: Option = read_to_opt(&mut self.file); + if let Some(key) = key { + let value: V = + read_to_opt(&mut self.file).expect("Key was read from file but value is missing"); + Some((key, value)) + } else { + None + } + } +} + +pub struct SortedLSMWriter { + file: std::io::BufWriter, + last_entry: Option, + phantom: std::marker::PhantomData, +} + +impl SortedLSMWriter { + pub fn new( + path: &std::path::Path + ) -> SortedLSMWriter { + SortedLSMWriter { + file: std::io::BufWriter::new(std::fs::File::create(path).unwrap()), + last_entry: None, + phantom: std::marker::PhantomData, + } + } + + pub fn insert(&mut self, key: K, value: V) { + if let Some(last_key) = &self.last_entry { + assert!(last_key.cmp(&key) == std::cmp::Ordering::Less); + } + bincode::serialize_into(&mut self.file, &key).unwrap(); + bincode::serialize_into(&mut self.file, &value).unwrap(); + self.last_entry = Some(key); + } +} + +#[cfg(test)] +mod tests { + extern crate tempfile; + use super::*; + + impl LSMWriter { + pub fn into_reader(mut self, out_path: &std::path::Path) -> LSMReader { + self.merge_completely(out_path); + LSMReader::open(out_path) + } + } + + #[test] + fn sort_reversed_numbers() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().to_path_buf(); + let dir2 = tempfile::tempdir().unwrap(); + let pathout = dir2.path().join("outfile"); + std::fs::remove_dir_all(&path); + std::fs::remove_file(&pathout); + let mut lsm = LSMWriter::new(path, 2, 3); + for i in (0..100).rev() { + lsm.insert(i, i + 10); + } + let reader = lsm.into_reader(&pathout); + assert!(reader.eq((0..100).map(|x| (x, x + 10)))); + std::fs::remove_file(&pathout); + } +} + + diff --git a/zquash/src/lsm/lsm_srv.rs b/zquash/src/lsm/lsm_srv.rs new file mode 100644 index 0000000..88b9354 --- /dev/null +++ b/zquash/src/lsm/lsm_srv.rs @@ -0,0 +1,998 @@ +use std::io; + +use failure::{self, format_err, Error, ResultExt}; +use std::fs; +use std::io::prelude::*; +use std::iter::Peekable; +use std::path::PathBuf; + +use crate::dmu_stream; +use crate::dmu_stream::RecordWithPayload; + +use super::lsm; +use super::object_merge::ObjectMergeHelper; + +use crate::split_tree::{self, SplitTree}; + +use std::cell::Cell; +use std::cell::RefCell; +use std::rc::Rc; + +pub struct LSMSrvConfig { + pub root_dir: PathBuf, +} + +fn stream_dir(config: &LSMSrvConfig, name: String) -> PathBuf { + config.root_dir.join(name) +} + +fn sorted_stream_path(config: &LSMSrvConfig, name: String) -> PathBuf { + config.root_dir.join(name).join("sorted.bin") +} + +pub fn read_stream( + config: &LSMSrvConfig, + stream: &mut io::Read, + name: String, +) -> Result<(), failure::Error> { + let stream_dir = stream_dir(config, name.clone()); + if stream_dir.exists() { + return Err(format_err!( + "stream dir {:?} exists, db inconsistent or duplicate name", + stream_dir + ))?; + } + fs::create_dir_all(&stream_dir).context("create stream dir")?; + + let writer_tmp = stream_dir.join("writer"); + fs::create_dir(&writer_tmp).context("create writer tmp dir")?; + + let stream_sorted_out = sorted_stream_path(config, name.clone()); + + use super::lsm::LSMWriter; + + let mut writer = LSMWriter::new(writer_tmp.clone(), 1 << 10, 10); // FIXME + + dmu_stream::read_with_callback(stream, |mut record| -> Result<(), Error> { + let dmu_stream::Record { + header: drr, + payload_len, + mut payload_reader, + } = record; + + let lsm_k = LSMKey(drr); + let mut payload = Vec::new(); + payload_reader + .read_to_end(&mut payload) + .context("read payload")?; + let lsm_v = payload; + writer.insert(lsm_k, lsm_v); + + Ok(()) + }) + .unwrap(); + + writer.merge_completely(&stream_sorted_out); // FIXME + + Ok(()) +} + +trait LSMReaderIterTrait: Iterator)> {} + +impl)>> LSMReaderIterTrait for I {} + +type LSMReaderIter = Rc>>>; + +struct ObjectRangeIter { + stream_idx: usize, + stream: LSMReaderIter, + current_objects_within_range_drained: Rc>, +} + +use std::fmt; + +impl fmt::Debug for ObjectRangeIter { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + out.debug_struct("ObjectRangeIter") + .field("stream_idx", &self.stream_idx) + .finish() + } +} + +#[derive(Debug)] +struct ObjectRangeInfo { + object_range_drr: RecordWithPayload, + objid_space: SplitTree, +} + +impl ObjectRangeInfo { + fn firstobj(&self) -> u64 { + assert!(self.object_range_drr.drr.drr_type == dmu_replay_record_DRR_OBJECT_RANGE); + unsafe { + self.object_range_drr + .drr + .drr_u + .drr_object_range + .drr_firstobj + } + } + fn contains_id(&self, obj_id: u64) -> bool { + assert!(self.object_range_drr.drr.drr_type == dmu_replay_record_DRR_OBJECT_RANGE); + unsafe { + let drr_o = &self.object_range_drr.drr.drr_u.drr_object_range; + drr_o.drr_firstobj <= obj_id && obj_id < (drr_o.drr_firstobj + drr_o.drr_numslots) + } + } +} + +struct ObjectsWithinObjectRangeIterator { + stream_idx: usize, + stream: LSMReaderIter, + drained: Rc>, + object_iter_drained: Rc>, +} + +impl fmt::Debug for ObjectsWithinObjectRangeIterator { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + out.debug_struct("ObjectsWithinObjectRangeIterator") + .field("stream_idx", &self.stream_idx) + .finish() + } +} + +struct ObjectIter { + obj_id: u64, + stream: LSMReaderIter, + drained: Rc>, +} + +impl Iterator for ObjectRangeIter { + type Item = (ObjectRangeInfo, ObjectsWithinObjectRangeIterator); + fn next(&mut self) -> Option { + if !self.current_objects_within_range_drained.get() { + panic!("must drain previously returned iterator") + } + + let mut stream = self.stream.borrow_mut(); + let (end_record, _) = stream.peek().unwrap(); + if end_record.drr_type == dmu_replay_record_DRR_END { + return None; + } + if end_record.drr_type == dmu_replay_record_DRR_FREEOBJECTS + && unsafe { + end_record.drr_u.drr_freeobjects.drr_firstobj > 0 + && end_record.drr_u.drr_freeobjects.drr_numobjs == 0 + } + { + // the trailing FREEOBJECTS record + return None; + } + + let (or_record, payload) = stream.next().expect("expecting a record"); + assert_eq!( + or_record.drr_type, + dmu_replay_record_DRR_OBJECT_RANGE, + "{:?}", + drr_debug(or_record) + ); + use itertools::Itertools; + let object_range_drr = RecordWithPayload { + drr: or_record, + payload, + }; + + let object_range_info_records = stream.peeking_take_while(|(drr, _)| { + drr.drr_type == dmu_replay_record_DRR_FREEOBJECTS + || drr.drr_type == dmu_replay_record_DRR_OBJECT + }); + + let mut objid_space = SplitTree::new(); + for (drr, payload) in object_range_info_records { + let (offset, length) = unsafe { + match drr.drr_type { + dmu_replay_record_DRR_FREEOBJECTS => ( + drr.drr_u.drr_freeobjects.drr_firstobj, + drr.drr_u.drr_freeobjects.drr_numobjs, + ), + dmu_replay_record_DRR_OBJECT => (drr.drr_u.drr_object.drr_object, 1), + _ => panic!("unexpected drr_type {:?}", drr_debug(drr)), // FIXME + } + }; + objid_space.insert(offset, length, RecordWithPayload { drr, payload }); + } + + let or_info = ObjectRangeInfo { + object_range_drr, + objid_space, + }; + + drop(stream); + + self.current_objects_within_range_drained = Rc::new(Cell::new(false)); + + let iter = ObjectsWithinObjectRangeIterator { + stream_idx: self.stream_idx, + stream: self.stream.clone(), + drained: self.current_objects_within_range_drained.clone(), + object_iter_drained: Rc::new(Cell::new(true)), + }; + return Some((or_info, iter)); + } +} + +impl Iterator for ObjectsWithinObjectRangeIterator { + type Item = (u64, ObjectIter); + fn next(&mut self) -> Option { + if !self.object_iter_drained.get() { + panic!("must drain previously returned iterator") + } + + if self.drained.get() { + return None; + } + + let mut stream = self.stream.borrow_mut(); + + let (next_record, _) = stream.peek().unwrap(); + if Self::is_follow(next_record) { + self.drained.set(true); + return None; + } + + assert!( + self.should_consume(next_record) + + " unexpected record type {:?}", + drr_debug(next_record) + ); + let obj_id = unsafe { LSMKey(*next_record).lower_obj_id() }.unwrap(); // FIXME + + drop(stream); + + self.object_iter_drained = Rc::new(Cell::new(false)); + + let object_iter = ObjectIter { + obj_id, + stream: self.stream.clone(), + drained: self.object_iter_drained.clone(), + }; + + Some((obj_id, object_iter)) + } +} + +impl ObjectsWithinObjectRangeIterator { + // returns `true` iff drr is a follow-element of an object range stream, + // i.e. the a stream element not consumed by this iterator + fn is_follow(drr: &dmu_replay_record) -> bool { + // next OBJECT_RANGE + if drr.drr_type == dmu_replay_record_DRR_OBJECT_RANGE { + return true; + } + + // Trailing FREEOBJECTS is not preceded by OBJECT_RANGE + if drr.drr_type == dmu_replay_record_DRR_FREEOBJECTS + && unsafe { + drr.drr_u.drr_freeobjects.drr_firstobj > 0 + && drr.drr_u.drr_freeobjects.drr_numobjs == 0 + } + { + return true; + } + + return false; + } + + fn should_consume(&self, drr: &dmu_replay_record) -> bool { + if drr.drr_type == dmu_replay_record_DRR_WRITE || drr.drr_type == dmu_replay_record_DRR_FREE + { + let _obj_id = unsafe { LSMKey(*drr).lower_obj_id() }.unwrap(); + // TODO validate that obj_id is within this iterator's object range + return true; + } + return false; + } + + fn peek_objid(&self) -> Option { + if self.drained.get() { + return None; + } + match self.stream.borrow_mut().peek() { + Some((drr, _)) => { + if Self::is_follow(drr) { + None + } else { + assert!( + self.should_consume(drr) + " unexpected record type {:?}", + drr_debug(drr) + ); // FIXME + Some(unsafe { LSMKey(*drr).lower_obj_id() }.unwrap()) + } + } + None => None, + } + } +} + +impl Iterator for ObjectIter { + type Item = RecordWithPayload; + fn next(&mut self) -> Option { + if self.drained.get() { + return None; + } + + let mut stream = self.stream.borrow_mut(); + + let (next_record, _) = stream.peek().unwrap(); + let same_objid = + unsafe { LSMKey(*next_record).lower_obj_id() }.map_or(false, |id| self.obj_id == id); + if !same_objid { + // TODO need check for invalid record type here? + self.drained.set(true); + return None; + } + + unsafe { + match next_record.drr_type { + dmu_replay_record_DRR_WRITE | dmu_replay_record_DRR_FREE => { + let (drr, payload) = stream.next().unwrap(); // checked above + Some(RecordWithPayload { drr, payload }) + } + dmu_replay_record_DRR_OBJECT_RANGE => None, + _ => panic!("unexpected drr_type {:?}", drr_debug(next_record)), + } + } + } +} + +unsafe fn consume_until_object_range_return_begin( + stream: Rc>>>, +) -> Option { + let s = &mut *stream.borrow_mut(); + let mut begin = None; + loop { + let (r, pay) = s.peek().unwrap(); + if r.drr_type == dmu_replay_record_DRR_OBJECT_RANGE { + break; + } else if r.drr_type == dmu_replay_record_DRR_BEGIN { + assert!(begin.is_none()); // TODO error handling + begin = Some(RecordWithPayload { + drr: *r, + payload: pay.clone(), + }); + s.next(); + } else { + dbg!(drr_debug(r)); + s.next(); + } + } + return begin; +} + +unsafe fn symbolic_dump_consume_lsm_reader( + stream: Rc>>>, +) { + println!("suck up until OBJECT_RANGE begins"); + let begin = consume_until_object_range_return_begin(stream.clone()); + dbg!(begin); + println!("dump ObjectRangeIter and child iterators"); + let mut iter = ObjectRangeIter { + stream_idx: 0, + stream: stream.clone(), + current_objects_within_range_drained: Rc::default(), + }; + for (or, it) in iter { + dbg!(or); + for (o, rec_it) in it { + dbg!(o); + for rec in rec_it { + dbg!(rec); + } + } + } + println!("dump remainder of the stream"); + { + // scope for Drop + let s = &mut *stream.borrow_mut(); + loop { + if let Some((r, _)) = s.next() { + dbg!(drr_debug(r)); + } else { + println!("end of stream"); + break; + } + } + } +} + +pub unsafe fn merge_streams( + config: &LSMSrvConfig, + streams_newest_to_oldest: &[&str], + target: String, +) -> Result<(), failure::Error> { + let mut streams: Vec>> = vec![]; + for (stream, stream_path) in streams_newest_to_oldest.iter().enumerate() { + let x: Box)>> = Box::new(lsm::LSMReader::open( + &sorted_stream_path(config, (*stream_path).to_owned()), + )); + // let x: Box)>> = + let x: Box = + Box::new(x.map(|(LSMKey(drr), payload): (LSMKey, Vec)| (drr, payload))); + let x = x.peekable(); + streams.push(x); + } + let mut streams: Vec>>>> = streams + .into_iter() + .map(|b| Rc::new(RefCell::new(b))) + .collect(); + + let writer_out = sorted_stream_path(config, target); + fs::create_dir_all(writer_out.parent().unwrap())?; + let mut target: lsm::SortedLSMWriter> = lsm::SortedLSMWriter::new(&writer_out); // FIXME + + // merge BEGIN record, drop noop FREEOBJECTS(0, 0) record + use std::collections::VecDeque; + let mut begins_oldest_to_newest = streams + .iter() + .rev() + .map(|s| consume_until_object_range_return_begin(s.clone()).unwrap()) + .collect::>(); + assert!(streams.len() > 0); + let begin = begins_oldest_to_newest.pop_front().unwrap(); + let begin = begins_oldest_to_newest.into_iter().fold(begin, |mut b, r| { + unsafe { + assert_eq!(r.drr.drr_type, dmu_replay_record_DRR_BEGIN); + assert_eq!( + dbg!(&b).drr.drr_u.drr_begin.drr_toguid, + dbg!(&r).drr.drr_u.drr_begin.drr_fromguid + ); // TODO error + let fromguid = b.drr.drr_u.drr_begin.drr_fromguid; + b = r; + b.drr.drr_u.drr_begin.drr_fromguid = fromguid; + b + } + }); + target.insert(LSMKey(begin.drr.clone()), begin.payload); + let drr_toguid = unsafe { begin.drr.drr_u.drr_begin.drr_toguid }; + + let mut streams: Vec<_> = streams + .into_iter() + .enumerate() + .map(|(stream_idx, stream)| { + ObjectRangeIter { + stream_idx, + stream: stream.clone(), + current_objects_within_range_drained: Rc::new(Cell::new(true)), + } + .peekable() + }) + .collect(); + + let mut next_object_range = 0; // we know every stream touches object range [0, 32) + loop { + assert_eq!(next_object_range % 32, 0); + + let this_range_streams = streams + .iter_mut() + .enumerate() + .filter_map(|(i, s)| { + let (ori, or_iter) = s.peek()?; // TODO + if ori.firstobj() == next_object_range { + Some(i) + } else { + None + } + }) + .collect::>(); + + println!("this_range_streams = {:?}", this_range_streams); + + if this_range_streams.len() == 0 { + // check if there is any next range + let min = streams + .iter_mut() + .enumerate() + .filter_map(|(i, s)| Some((i, s.peek()?))) + .min_by_key(|(i, (ori, _))| ori.firstobj()); + if let Some((min_idx, (ori, _))) = min { + next_object_range = ori.firstobj(); + continue; + } else { + break; + } + } else { + // advance next_object_range + next_object_range = streams[this_range_streams[0]].peek().unwrap().0.firstobj(); + } + assert!(is_sorted::IsSorted::is_sorted( + &mut this_range_streams.iter() + )); + assert!(this_range_streams.len() > 0); + + // the highest-level (= first) OBJECT_RANGE record wins + // and with it, all FREEOBJECTS and OBJECT records (encoded in objid_space) + // + // However, we still gotta merge the READ and WRITE records per object + + let this_range_streams = this_range_streams + .into_iter() + // consume from those streams that have the current object range + .map(|stream_idx| streams[stream_idx].next().unwrap()) + .collect::>(); + + // split the (ObjectRangeInfo, ObjectsWithinObjectRange) iterators into two separate iterators + let (oris, mut or_iters) = this_range_streams.into_iter().fold( + (vec![], vec![]), + |(mut oris, mut or_iters), (ori, or_iter)| { + oris.push(ori); + or_iters.push(or_iter); + (oris, or_iters) + }, + ); + + let ori: &ObjectRangeInfo = &oris[0]; // most recent OBJECT_RANGE wins + target.insert(LSMKey(ori.object_range_drr.drr), Vec::::new()); + // insert FREEOBJECTS and OBJECT records + for (obj_id, len, occ_obj) in ori.objid_space.iter() { + match occ_obj { + split_tree::Occ::Unknown => (), + split_tree::Occ::Free => { + let freeobjects = unsafe { + let mut r: dmu_replay_record = std::mem::zeroed(); + r.drr_type = dmu_replay_record_DRR_FREEOBJECTS; + r.drr_payloadlen = 0; + r.drr_u.drr_freeobjects = dmu_replay_record__bindgen_ty_2_drr_freeobjects { + drr_firstobj: obj_id, + drr_numobjs: len, + drr_toguid, + }; + r + }; + // sw.write_record_and_payload_reader(or.unwrap(), &mut io::empty())?; + target.insert(LSMKey(freeobjects), vec![]); + } + split_tree::Occ::Occupied(obj) => { + target.insert(LSMKey(obj.drr), obj.payload.clone()); + } + } + } + let mut next_object_id = ori.firstobj(); + + while ori.contains_id(next_object_id) { + println!("next_object_id = {:?}", next_object_id); + + let mut this_object_streams = or_iters + .iter_mut() + .filter_map(|or_iter| { + if let Some(obj_id) = or_iter.peek_objid() { + if obj_id == next_object_id { + Some(or_iter) + } else { + None + } + } else { + None + } + }) + .collect::>(); + + println!("this_object_streams = {:?}", this_object_streams); + + let this_object_id = next_object_id; + next_object_id += 1; + if this_object_streams.len() == 0 { + continue; + } + + // merge (= consume) iterators for this object + let mut merger: ObjectMergeHelper = + ObjectMergeHelper::new(this_object_streams.len()); + let object_iters = + this_object_streams + .iter_mut() + .enumerate() + .map(|(level, oior_iter)| { + let (obj_id, obj_iter) = oior_iter.next().unwrap(); // consume object() + assert_eq!(obj_id, this_object_id); + obj_iter.map(move |drr| (drr, level)) + }); + let unified = itertools::kmerge_by( + object_iters, + |(drr_a, level_a): &(RecordWithPayload, usize), + (drr_b, level_b): &(RecordWithPayload, usize)| { + let offset_ord = LSMKey(drr_a.drr) + .offset_in_object_if_any() + .cmp(&LSMKey(drr_b.drr).offset_in_object_if_any()); + if offset_ord == Ordering::Equal { + level_a < level_b + } else { + offset_ord == Ordering::Less + } + }, + ); + + #[derive(Clone, Debug, From, Into)] + struct DrrWrapper(RecordWithPayload); + + impl Default for DrrWrapper { + fn default() -> Self { + let drr = unsafe { + let drr: dmu_replay_record = std::mem::zeroed(); + drr + }; + DrrWrapper(RecordWithPayload { + drr, + payload: vec![], + }) + } + } + let mut out = VecDeque::new(); + for (record, level) in unified { + assert_eq!( + Some(this_object_id), + unsafe { LSMKey(record.drr).lower_obj_id() }, + "unexpected object id {:?}", + record + ); + if record.drr.drr_type == dmu_replay_record_DRR_WRITE { + let o = &record.drr.drr_u.drr_write; + merger.insert_write( + &mut out, + level, + o.drr_offset, + o.drr_logical_size, + record.into(), + ); + } else if record.drr.drr_type == dmu_replay_record_DRR_FREE { + assert_eq!(record.payload.len(), 0); + let o = &dbg!(record).drr.drr_u.drr_free; + let len = if o.drr_length == std::u64::MAX { + std::u64::MAX - o.drr_offset // free to end + } else { + o.drr_length + }; + merger.insert_free(&mut out, level, o.drr_offset, len); + } else { + panic!(" unexpected record type {:?}", record); + } + } + // for each level, end it + for level in 0..this_object_streams.len() { + merger.insert_end(&mut out, level) + } + + // write out object_info + for knowledge in out { + use super::object_merge::KnowledgeKind::{End, Free, Occupied}; + match knowledge.kind { + Occupied(DrrWrapper(RecordWithPayload { drr, payload })) => { + target.insert(LSMKey(drr), payload); + } + Free => { + let free = unsafe { + let mut r: dmu_replay_record = std::mem::zeroed(); + r.drr_type = dmu_replay_record_DRR_FREE; + r.drr_payloadlen = 0; + r.drr_u.drr_free = dmu_replay_record__bindgen_ty_2_drr_free { + drr_object: this_object_id, + drr_offset: knowledge.from, + drr_length: knowledge.len, + drr_toguid, + }; + r + }; + target.insert(LSMKey(free), Vec::new()); + } + End => panic!("merger emitted End record, unsupported"), + } + } + } + // After while ori.contains_id(next_object_id) + for stream in or_iters { + println!("draining stream {:?}", stream); + let stream_dbg = format!("{:?}", stream); + for (obj_id, obj_iter) in stream { + println!("draining object {:?}", obj_id); + obj_iter.for_each(|x| { + println!( + "drain leftovers of object range {:?} {:?} {:?} {:?}", + ori, + stream_dbg, + obj_id, + drr_debug(&x.drr) + ) + }); + } + } + } + + // synthesize END record + let end_drr = unsafe { + let mut drr: dmu_replay_record = std::mem::zeroed(); + drr.drr_type = dmu_replay_record_DRR_END; + drr.drr_payloadlen = 0; + drr.drr_u.drr_end.drr_toguid = drr_toguid; + drr + }; + target.insert(LSMKey(end_drr), vec![]); + + Ok(()) +} + +pub unsafe fn write_stream( + config: &LSMSrvConfig, + name: String, + out: &mut std::io::Write, +) -> Result<(), failure::Error> { + let mut out = dmu_stream::StreamWriter::new(out); + + let reader: lsm::LSMReader> = + lsm::LSMReader::open(&sorted_stream_path(config, name).to_owned()); + + for (LSMKey(drr), payload) in reader { + out.write_record_and_payload_reader(&drr, &mut io::Cursor::new(&payload)) + .context(format!("drr {:?}", drr_debug(drr)))?; + } + + Ok(()) +} + +use bindings::*; + +use serde::{Deserialize, Serialize, Serializer}; + +/// sort such that +/// BEGIN +/// FREEOBJECTS (firstobj=0, numobjs=0) +/// OBJECT_RANGE by firstobj / 32 +/// FREEOBJECTS, OBJECT by object_id +/// FREE, WRITE by (object_id, offset_in_object) +/// FREEOBJECTS (firstobj=X, numobjs=0) +/// END +#[derive(Serialize, Deserialize)] +struct LSMKey(dmu_replay_record); +use std::cmp::Ordering; + +impl LSMKey { + unsafe fn lower_obj_id(&self) -> Option { + let u = &self.0.drr_u; + let obj_id = match self.0.drr_type { + dmu_replay_record_DRR_BEGIN => return None, // TOOD + dmu_replay_record_DRR_END => return None, // TODO + dmu_replay_record_DRR_OBJECT_RANGE => u.drr_object_range.drr_firstobj, + dmu_replay_record_DRR_OBJECT => u.drr_object.drr_object, + dmu_replay_record_DRR_FREEOBJECTS => u.drr_freeobjects.drr_firstobj, + dmu_replay_record_DRR_WRITE => u.drr_write.drr_object, + dmu_replay_record_DRR_FREE => u.drr_free.drr_object, + _ => unimplemented!(), // TODO + }; + Some(obj_id) + } + + unsafe fn offset_in_object_if_any(&self) -> Option { + let u = &self.0.drr_u; + match self.0.drr_type { + dmu_replay_record_DRR_BEGIN => None, + dmu_replay_record_DRR_END => None, + dmu_replay_record_DRR_OBJECT_RANGE => None, + dmu_replay_record_DRR_OBJECT => None, + dmu_replay_record_DRR_FREEOBJECTS => None, + dmu_replay_record_DRR_WRITE => Some(u.drr_write.drr_offset), + dmu_replay_record_DRR_FREE => Some(u.drr_free.drr_offset), + _ => unimplemented!(), // TODO + } + } + + unsafe fn length_in_object_if_any(&self) -> Option { + let u = &self.0.drr_u; + match self.0.drr_type { + dmu_replay_record_DRR_BEGIN => None, + dmu_replay_record_DRR_END => None, + dmu_replay_record_DRR_OBJECT_RANGE => None, + dmu_replay_record_DRR_OBJECT => None, + dmu_replay_record_DRR_FREEOBJECTS => None, + dmu_replay_record_DRR_WRITE => Some(u.drr_write.drr_logical_size), + dmu_replay_record_DRR_FREE => Some(u.drr_free.drr_length), + _ => unimplemented!(), // TODO + } + } +} + +impl PartialEq for LSMKey { + fn eq(&self, o: &LSMKey) -> bool { + self.cmp(o) == Ordering::Equal + } +} + +impl Eq for LSMKey {} + +impl PartialOrd for LSMKey { + fn partial_cmp(&self, other: &LSMKey) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for LSMKey { + fn cmp(&self, o: &LSMKey) -> Ordering { + unsafe { + let diff_it = || { + format!( + "{}", + difference::Changeset::new( + &format!("{:#?}", drr_debug(&self.0)), + &format!("{:#?}", drr_debug(&o.0)), + "\n" + ) + ) + }; + + const begin_end_type_ordering: &'_ [(usize, dmu_replay_record__bindgen_ty_1)] = &[ + (0, dmu_replay_record_DRR_BEGIN), + (1, dmu_replay_record_DRR_OBJECT_RANGE), + (1, dmu_replay_record_DRR_FREEOBJECTS), + (1, dmu_replay_record_DRR_OBJECT), + (1, dmu_replay_record_DRR_FREE), + (1, dmu_replay_record_DRR_WRITE), + (2, dmu_replay_record_DRR_END), + ]; + let type_ordering_find_prio = + |req_ty, type_ordering: &[(usize, dmu_replay_record__bindgen_ty_1)]| { + for (prio, ty) in type_ordering { + if *ty == req_ty { + return Some(*prio); + } + } + return None; + }; + + let s_ty_prio = + type_ordering_find_prio(self.0.drr_type, begin_end_type_ordering).unwrap(); // FIXME + let o_ty_prio = type_ordering_find_prio(o.0.drr_type, begin_end_type_ordering).unwrap(); // FIXME + if s_ty_prio.cmp(&o_ty_prio) != Ordering::Equal { + return s_ty_prio.cmp(&o_ty_prio); + } + + // handle all special cases that do not fit into OBJECT_RANGE + // 1. FREEOBJECTS(firstobj=0, numobjs=0) + { + let is_freeobjects_0_0 = |drr: &dmu_replay_record| { + if drr.drr_type == dmu_replay_record_DRR_FREEOBJECTS + && unsafe { + drr.drr_u.drr_freeobjects.drr_firstobj == 0 + && drr.drr_u.drr_freeobjects.drr_numobjs == 0 + } + { + 0 + } else { + 1 + } + }; + if is_freeobjects_0_0(&self.0).cmp(&is_freeobjects_0_0(&o.0)) != Ordering::Equal { + return is_freeobjects_0_0(&self.0).cmp(&is_freeobjects_0_0(&o.0)); + } else if is_freeobjects_0_0(&self.0) == 0 { + return Ordering::Equal; + } + } + // 1. FREEOBJECTS(firstobj=X, numobjs=0) + { + let is_freeobjects_x_0 = |drr: &dmu_replay_record| { + if drr.drr_type == dmu_replay_record_DRR_FREEOBJECTS + && unsafe { drr.drr_u.drr_freeobjects.drr_numobjs == 0 } + { + let fo = drr.drr_u.drr_freeobjects.drr_firstobj; + assert!(fo > 0); // see below + fo + } else { + assert_ne!(drr.drr_type, dmu_replay_record_DRR_END); + // all other records lower than FREEOBJECTS(firstobj=X, numobjs=0) + 0 + } + }; + if is_freeobjects_x_0(&self.0).cmp(&is_freeobjects_x_0(&o.0)) != Ordering::Equal { + return is_freeobjects_x_0(&self.0).cmp(&is_freeobjects_x_0(&o.0)); + } + } + + let self_lobjid = self + .lower_obj_id() + .expect("expected record type to have object id"); + let other_lobjid = o + .lower_obj_id() + .expect("expected record type to have object id"); + + // group all records by OBJECT_RANGE + let self_or = (self_lobjid / 32); + let othr_or = (other_lobjid / 32); + if self_or.cmp(&othr_or) != Ordering::Equal { + return self_or.cmp(&othr_or); + } + + // within an OBJECT_RANGE order by the following cohorts: + // 1 2 3 + // OBJECT_RANGE < (FREEOBJECTS,OBJECT) < (WRITE,FREE) + const by_object_range_ordering: &'_ [(usize, dmu_replay_record__bindgen_ty_1)] = &[ + (1, dmu_replay_record_DRR_OBJECT_RANGE), + (2, dmu_replay_record_DRR_FREEOBJECTS), + (2, dmu_replay_record_DRR_OBJECT), + (3, dmu_replay_record_DRR_FREE), + (3, dmu_replay_record_DRR_WRITE), + ]; + let s_ty_prio = + type_ordering_find_prio(self.0.drr_type, by_object_range_ordering).unwrap(); // FIXME + let o_ty_prio = + type_ordering_find_prio(o.0.drr_type, by_object_range_ordering).unwrap(); // FIXME + if s_ty_prio.cmp(&o_ty_prio) != Ordering::Equal { + return s_ty_prio.cmp(&o_ty_prio); + } + + // within each cohort, it is sufficient to order by firstobj + if self_lobjid.cmp(&other_lobjid) != Ordering::Equal { + return self_lobjid.cmp(&other_lobjid); + } + + // within cohort 3, order by offset, then FREE < WRITE, then length + assert_eq!(s_ty_prio, o_ty_prio, "equality, see above"); + if s_ty_prio == 3 { + let s_offset = self + .offset_in_object_if_any() + .ok_or_else(|| { + format_err!("record type must have offset in object:\n{}", diff_it()) + }) + .unwrap(); + let o_offset = o + .offset_in_object_if_any() + .ok_or_else(|| { + format_err!("record type must have offset in object:\n{}", diff_it()) + }) + .unwrap(); + + if s_offset.cmp(&o_offset) != Ordering::Equal { + return s_offset.cmp(&o_offset); + } + + // within cohort 3: if offset is same, FREE < WRITE + const cohort_3_type_ordering: &'_ [(usize, dmu_replay_record__bindgen_ty_1)] = &[ + (1, dmu_replay_record_DRR_FREE), + (2, dmu_replay_record_DRR_WRITE), + ]; + let s_ty_prio = + type_ordering_find_prio(self.0.drr_type, cohort_3_type_ordering).unwrap(); // FIXME + let o_ty_prio = + type_ordering_find_prio(o.0.drr_type, cohort_3_type_ordering).unwrap(); // FIXME + if s_ty_prio.cmp(&o_ty_prio) != Ordering::Equal { + return s_ty_prio.cmp(&o_ty_prio); + } + + // same offset and same type (FREE / WRITE): let the length win + let s_length = self + .length_in_object_if_any() + .ok_or_else(|| { + format_err!("record type must have length in object:\n{}", diff_it()) + }) + .unwrap(); + let o_length = o + .length_in_object_if_any() + .ok_or_else(|| { + format_err!("record type must have length in object:\n{}", diff_it()) + }) + .unwrap(); + + if s_length.cmp(&o_length) != Ordering::Equal { + return s_length.cmp(&o_length); + } + + // FALLTHROUGH + } + + // we have no more discriminators (that we know of at the time of writing) + // => if the records are not equal, the stream is considered invalid + if format!("{:?}", drr_debug(&self.0)) == format!("{:?}", drr_debug(&o.0)) { + // FIXME 111!!!! + return Ordering::Equal; + } else { + panic!("unexpected record:\n{}", diff_it()); + } + } + } +} diff --git a/zquash/src/lsm/mod.rs b/zquash/src/lsm/mod.rs new file mode 100644 index 0000000..7e41f03 --- /dev/null +++ b/zquash/src/lsm/mod.rs @@ -0,0 +1,4 @@ +mod lsm; +mod lsm_srv; +mod object_merge; +pub use lsm_srv::*; diff --git a/zquash/src/lsm/object_merge.rs b/zquash/src/lsm/object_merge.rs new file mode 100644 index 0000000..273e056 --- /dev/null +++ b/zquash/src/lsm/object_merge.rs @@ -0,0 +1,456 @@ +use std::collections::VecDeque; +use std::fmt; + +#[derive(Debug, Clone)] +pub enum KnowledgeKind { + Free, + Occupied(D), + End, +} + +impl KnowledgeKind { + fn is_end(&self) -> bool { + if let KnowledgeKind::End = self { + true + } else { + false + } + } +} + +impl Copy for KnowledgeKind<()> {} + +#[derive(Debug, Clone)] +pub struct Knowledge { + pub from: u64, + pub len: u64, + pub kind: KnowledgeKind, +} + +impl KnowledgeKind { + fn without_data(&self) -> KnowledgeKind<()> { + match self { + KnowledgeKind::End => KnowledgeKind::End, + KnowledgeKind::Free => KnowledgeKind::Free, + KnowledgeKind::Occupied(_) => KnowledgeKind::Occupied(()), + } + } +} + +impl Copy for Knowledge<()> {} + +impl Knowledge { + fn without_data(&self) -> Knowledge<()> { + Knowledge { + from: self.from, + len: self.len, + kind: self.kind.without_data(), + } + } +} + +impl PartialEq for KnowledgeKind<()> { + fn eq(&self, other: &Self) -> bool { + use KnowledgeKind::*; + match (self, other) { + (Free, Free) | (End, End) | (Occupied(()), Occupied(())) => true, + _ => false, + } + } +} + +#[derive(Debug)] +enum TrimResult { + Noop, + Trimmed(D), +} + +impl TrimResult { + fn into_trimmed(self) -> Option { + match self { + TrimResult::Noop => None, + TrimResult::Trimmed(d) => Some(d), + } + } +} + +impl Knowledge { + // Returns what was trimmed-off knowledge if the kind of knowledge can be trimmed (=split) + // or an error if it cannot be trimmed. + fn try_trim_front(&mut self, new_front: u64) -> Result>, ()> { + if new_front <= self.from { + return Ok(TrimResult::Noop); + } + assert!(self.from <= new_front); + assert!((new_front - self.from) <= self.len); + match &mut self.kind { + KnowledgeKind::Free | KnowledgeKind::End => { + let orig_front = self.from; + let trimmed_off_len = new_front - self.from; + self.len -= trimmed_off_len; + self.from = new_front; + Ok(TrimResult::Trimmed(Knowledge { + from: orig_front, + len: trimmed_off_len, + kind: self.kind.clone(), // cheap because this kind doesn't have data + })) + } + KnowledgeKind::Occupied(d) => { + if new_front == self.from + self.len { + let mut orig_d = D::default(); + std::mem::swap(d, &mut orig_d); + let k = Knowledge { + from: self.from, + len: self.len, + kind: KnowledgeKind::Occupied(orig_d), + }; + self.len = 0; + self.from = new_front; + Ok(TrimResult::Trimmed(k)) + } else { + Err(()) + } + } + } + + } +} + +pub struct ObjectMergeHelper { + emit_from: u64, + level_last_insert: Vec>>, + knowledge: Vec>>, +} + +impl ObjectMergeHelper +where + D: Clone + Default + fmt::Debug, +{ + pub fn new(num_levels: usize) -> Self { + ObjectMergeHelper { + emit_from: 0, + level_last_insert: vec![None; num_levels], + knowledge: vec![VecDeque::new(); num_levels], + } + } + + fn find_action(&mut self, out: &mut VecDeque>) -> Option<()> { + for deq in &self.knowledge { + if deq.is_empty() { + return None; + } + } + // all deqs have at least one element + + dbg!(&self.knowledge); + use itertools::Itertools; + let (min_offset, min_end) = { + let offsets = self + .knowledge + .iter() + .filter_map(|deq| deq.front()) + .flat_map(|k| vec![k.from, k.from + k.len]) + .sorted() + .dedup() + .take(2) + .collect::>(); + assert!( + offsets.len() == 2 || (offsets.len() == 1 && offsets[0] == std::u64::MAX), + "{:?}", + offsets + ); + if offsets.len() == 1 { + return None; + } + (offsets[0], offsets[1]) + }; + + if min_end - min_offset == 0 { + return None; + } + + // determine what to emit for this range + // and trim all unrequired knowledge + // note that data cannot be trimmed, but FREE can + let emit_range = dbg!((min_offset, min_end)); + + let knowledge_len = self.knowledge.len(); + let mut knowledge_in_emit_range = self + .knowledge + .iter_mut() + .flat_map(|(deq)| { + deq.iter_mut().filter_map(move |k: &mut Knowledge<_>| { + // range contains begin, but not necessarily end + if k.from >= min_offset { + Some(k) + } else { + None + } + }) + }) + .collect::>(); + assert!(knowledge_in_emit_range.len() > 0); + + // trim all to the end of emit range, and determine the winner + let winner_candidates = dbg!(knowledge_in_emit_range) + .iter_mut() + // NOTE: this map has a side-effect + .map(|k| { + dbg!(dbg!(k).try_trim_front(min_end).unwrap()) // TODO nicer errors + }) + .filter_map(|trim_res| trim_res.into_trimmed()) + .collect::>(); + + // the winner is the highest level that is not a Knowledge::End sentinel + + let winner = winner_candidates + .into_iter() + .filter(|k| !k.kind.is_end()) + .next(); + let winner = match winner { + Some(w) => w, + None => return Some(()), + }; + + // cleanup deq + self.knowledge.iter_mut().for_each(|deq| { + if deq.len() > 0 { + if deq[0].len == 0 { + deq.pop_front(); + } else if deq[0].from <= min_end { + assert!(deq[0].from >= min_end); + } + } + }); + + // check invariant: we have dropped all knowledge before the end of emit_range + self.knowledge.iter().enumerate().for_each(|(level, deq)| { + debug_assert!( + deq.iter().all(|k| (k.from + k.len) >= min_end), + "{:?}", + level + ); + debug_assert!( + deq.len() <= 1, + "level = {:?} deq.len() = {:?}\ndeq = {:#?}", + level, + deq.len(), + deq + ); + }); + + // emit winner + assert!( + self.emit_from <= winner.from, + "impl error: winner must have offset >= already emitted\n{:?}\n{:?}", + self.emit_from, + winner + ); + self.emit_from = winner.from + winner.len; + out.push_back(dbg!(winner).clone()); + + Some(()) + + } + + fn repeat_find_action(&mut self, out: &mut VecDeque>) { + loop { + if self.find_action(out).is_none() { + break; + } + } + } + + fn check_track_and_normalize_insert_at_level( + &mut self, + level: usize, + candidate: &mut Knowledge, + ) { + let mut this_insert = candidate.without_data(); + + assert!( + this_insert.from.checked_add(this_insert.len).is_some(), + "level=({:?}) + offset({:?}) exceed u64 space", + level, + this_insert.from + ); + assert!(level < self.knowledge.len(), "level number out of bounds"); + + let last_insert = &self.level_last_insert[level]; + let last_insert = match last_insert { + Some(i) => i, + None => { + self.level_last_insert[level] = Some(this_insert); + return; + } + }; + + if last_insert.from + last_insert.len > this_insert.from { + // Due to the traversal algorithm used by ZFS (dmu_traverse.c), it is possible that overlapping FREE + // records are emitted (the following excerpt from zstreamdump has omissions, but remains in original order) + // + // OBJECT object = 1 type = 21 bonustype = 0 blksz = 512 bonuslen = 0 dn_slots = 1 raw_bonuslen = 0 flags = 0 maxblkid = 0 indblkshift = 17 nlevels = 1 nblkptr = 3 + // FREE object = 1 offset = 512 length = -1 + // WRITE object = 1 type = 21 checksum type = 7 compression type = 2 + // flags = 0 offset = 0 logical_size = 512 compressed_size = 512 payload_size = 512 props = 8200000000 salt = 0000000000000000 iv = 000000000000000000000000 mac = c3d24b5f100f4c236567115ccdfcc17a + // FREE object = 1 offset = 512 length = 1024 + // + // That second FREE is the effect of only the first of 3 blktpr_t being used to represent the contents of object 1 + // i.e. object 1 consists only of 1 x blksz = 1 x 512 byte in this case + // the dmu_traverse.c code visits the other 2 blkptr_t, though, and issues FREE + // + // LSMKey discriminates FREE records first by offset, then by their length, ascending. + // Hence, we will observe the second FREE before the first FREE when merging. + let is_free_to_end_overlapping_only_with_last_free = last_insert.kind + == KnowledgeKind::Free + && this_insert.kind == KnowledgeKind::Free + && this_insert.from == last_insert.from; + if is_free_to_end_overlapping_only_with_last_free { + // ok, see comment above + // just patch up its range + let new_from = last_insert.from + last_insert.len; + assert!(new_from > this_insert.from); + let subtracted_len = new_from - this_insert.from; + candidate.from = new_from; + candidate.len -= subtracted_len; + this_insert = candidate.without_data(); + } else { + panic!( + "level already observed insert {:?}, cannot insert before it {:?}", + last_insert, this_insert + ); + } + } + + self.level_last_insert[level] = Some(this_insert); + } + + pub fn insert_free( + &mut self, + out: &mut VecDeque>, + level: usize, + from: u64, + len: u64, + ) { + let mut k = Knowledge { + from, + len, + kind: KnowledgeKind::Free, + }; + self.check_track_and_normalize_insert_at_level(level, &mut k); + self.knowledge[level].push_back(k); + self.repeat_find_action(out) + } + + pub fn insert_write( + &mut self, + out: &mut VecDeque>, + level: usize, + from: u64, + len: u64, + data: D, + ) { + let mut k = Knowledge { + from, + len, + kind: KnowledgeKind::Occupied(data), + }; + self.check_track_and_normalize_insert_at_level(level, &mut k); + self.knowledge[level].push_back(k); + self.repeat_find_action(out) + } + + pub fn insert_end(&mut self, out: &mut VecDeque>, level: usize) { + let from = std::cmp::max( + self.level_last_insert[level] + .map(|i| i.from + i.len) + .unwrap_or(0), + self.knowledge[level] + .iter() + .map(|k| k.from + k.len) + .max() + .unwrap_or(0), + ); + let len = std::u64::MAX - from; + let mut k = Knowledge { + from, + len, + kind: KnowledgeKind::End, + }; + self.check_track_and_normalize_insert_at_level(level, &mut k); + self.knowledge[level].push_back(k); + self.repeat_find_action(out) + } +} + +#[cfg(test)] +mod helper_tests { + + use super::*; + #[test] + fn basics_2_levels() { + let mut h: ObjectMergeHelper = ObjectMergeHelper::new(2); + let mut out: VecDeque> = VecDeque::new(); + h.insert_write(&mut out, 0, 2, 3, 23); + h.insert_free(&mut out, 1, 0, 10); + h.insert_end(&mut out, 0); + h.insert_end(&mut out, 1); + let out = Vec::from(out); + assert_eq!( + out, + vec![ + Knowledge { + from: 0, + len: 2, + kind: KnowledgeKind::Free + }, + Knowledge { + from: 2, + len: 3, + kind: KnowledgeKind::Occupied(23 as usize), + }, + Knowledge { + from: 5, + len: 5, + kind: KnowledgeKind::Free + }, + ] + ); + } + + #[test] + fn gaps() { + let mut h = ObjectMergeHelper::new(2); + let mut out = VecDeque::new(); + + h.insert_write(&mut out, 0, 0, 2, 23); + h.insert_free(&mut out, 1, 3, 1); // gap at [2,3) + h.insert_write(&mut out, 0, 5, 2, 42); // gap at [4,5) + + h.insert_end(&mut out, 0); + h.insert_end(&mut out, 1); + let out = Vec::from(out); + + assert_eq!( + out, + vec![ + Knowledge { + from: 0, + len: 2, + kind: KnowledgeKind::Occupied(23 as usize), + }, + Knowledge { + from: 3, + len: 1, + kind: KnowledgeKind::Free, + }, + Knowledge { + from: 5, + len: 2, + kind: KnowledgeKind::Occupied(42), + } + ] + ); + } + +} \ No newline at end of file diff --git a/zquash/src/main.rs b/zquash/src/main.rs new file mode 100644 index 0000000..2ebaadc --- /dev/null +++ b/zquash/src/main.rs @@ -0,0 +1,108 @@ +use std::path::PathBuf; +use structopt::StructOpt; + +use std::fs::File; + +use quicli::prelude::*; + +use bindings::*; +use failure::{format_err, Error, ResultExt}; +use std::collections::BTreeMap; +use std::fmt; +use std::io::{self, BufWriter}; + +use ::zquash::lsm; + +// Add cool slogan for your app here, e.g.: +/// Squash adjacent zfs send streams together +#[derive(StructOpt)] +struct Cli { + // Quick and easy logging setup you get for free with quicli + #[structopt(flatten)] + verbosity: Verbosity, + + #[structopt(subcommand)] + command: Command, +} + +#[derive(StructOpt)] +#[allow(non_camel_case_types)] +enum Command { + load { + stream: PathBuf, + name: Option, + }, + squash { + a_to_b: String, + b_to_c: String, + target: String, + }, + write { + stream: String, + target: Option, + }, +} + +fn main() -> CliResult { + let args = Cli::from_args(); + args.verbosity.setup_env_logger("cli")?; + + match args.command { + Command::load { stream, name } => { + dotenv::dotenv().ok(); + use std::fs; + + let mut input = File::open(stream.clone()).context("open input stream file")?; + let mut input = io::BufReader::new(input); + let name = match name { + Some(n) => n, + None => match stream.file_name().unwrap().to_str() { + Some(n) => n.to_string(), + None => panic!("cannot stream name from stream file name"), + }, + }; + + let root: PathBuf = std::env::var("ZS_LSM_ROOT") + .context("ZS_LSM_ROOT not set")? + .into(); + + let cfg = lsm::LSMSrvConfig { root_dir: root }; + lsm::read_stream(&cfg, &mut input, name).context("read stream")?; + } + Command::squash { + a_to_b, + b_to_c, + target, + } => { + dotenv::dotenv().ok(); + use std::fs; + + let root: PathBuf = std::env::var("ZS_LSM_ROOT") + .context("ZS_LSM_ROOT not set")? + .into(); + + let cfg = lsm::LSMSrvConfig { root_dir: root }; + unsafe { lsm::merge_streams(&cfg, &[a_to_b.as_ref(), b_to_c.as_ref()], target)? }; + } + Command::write { stream, target } => { + dotenv::dotenv().ok(); + + let root: PathBuf = std::env::var("ZS_LSM_ROOT") + .context("ZS_LSM_ROOT not set")? + .into(); + + let mut target: Box = match target { + Some(p) => { + Box::new(File::create(p.clone()).context(format!("create output file {:?}", p))?) + } + None => Box::new(std::io::stdout()), + }; + + let cfg = lsm::LSMSrvConfig { root_dir: root }; + + unsafe { lsm::write_stream(&cfg, stream, &mut target) }?; + } + } + + Ok(()) +} diff --git a/zquash/src/split_tree.rs b/zquash/src/split_tree.rs new file mode 100644 index 0000000..6a8a313 --- /dev/null +++ b/zquash/src/split_tree.rs @@ -0,0 +1,445 @@ +use std::collections::BTreeMap; +use Occ::*; + +#[derive(Clone, PartialEq, Eq)] +pub enum Occ { + Unknown, + Free, + Occupied(T), +} + +impl Occ { + pub fn is_unknown(&self) -> bool { + match self { + Unknown => true, + _ => false, + } + } + + pub fn is_free(&self) -> bool { + match self { + Free => true, + _ => false, + } + } + + pub fn is_occupied(&self) -> bool { + match self { + Occupied(_) => true, + _ => false, + } + } + + pub fn occupied(&self) -> Option<&T> { + match self { + Occupied(o) => Some(o), + _ => None, + } + } + + pub fn occupied_mut(&mut self) -> Option<&mut T> { + match self { + Occupied(o) => Some(o), + _ => None, + } + } + + pub fn to_empty_t(&self) -> Occ<()> { + match self { + Unknown => Unknown, + Free => Free, + Occupied(_) => Occupied(()), + } + } +} + +use std::fmt; + +impl fmt::Debug for Occ { + fn fmt(&self, o: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match self { + Occupied(_) => "Occupied", + Free => "Free", + Unknown => "Unknown", + }; + write!(o, "{}", s) + } +} + +type Offset = u64; + +#[derive(Debug)] +pub struct SplitTree { + m: BTreeMap>, +} + +impl SplitTree { + pub fn new() -> Self { + let mut m = BTreeMap::new(); + m.insert(0, Unknown); + SplitTree { m } + } + + pub fn insert(&mut self, offset: u64, length: u64, content: T) { + //Get state behind insertion + let behind_insertion_offset = offset + length; + let (behind_insertion_definition_offset_ref, behind_insertion_ref) = self + .m + .range(0..=behind_insertion_offset) + .last() + .expect("always at least one split"); + + let (behind_insertion_definition_offset, behind_insertion) = ( + *behind_insertion_definition_offset_ref, + behind_insertion_ref.to_empty_t(), + ); + + //Get state at start of insert + let (start_insert_definition_offset_ref, start_insert_ref) = self + .m + .range(0..=offset) + .last() + .expect("always at least one split"); + //Insert may not start in the middle of occupied space + match start_insert_ref { + Occupied(_) => { + assert!(*start_insert_definition_offset_ref == offset); + } + _ => {} + } + + //Remove all entries in insertion range + let indices_to_remove = self + .m + .range(offset..behind_insertion_offset) + .map(|(k, _)| *k) + .collect::>(); + for k in indices_to_remove { + self.m + .remove(&k) + .expect("This index was just queried, it should be there"); + } + + //Insert write + self.m.insert(offset, Occupied(content)); + + //Write state at end + if behind_insertion_definition_offset != behind_insertion_offset { + let with_t = match behind_insertion { + Occupied(_) => { + panic!("may not insert inside an existing occupied block"); + } + Free => Free, + Unknown => Unknown, + }; + self.m.insert(behind_insertion_offset, with_t); + } + } + + pub fn free(&mut self, offset: u64, length: u64) { + assert!(length > 0); + //Get state behind free + let behind_free_offset = offset + .checked_add(length) + .expect("offset + length overflows address space"); + let (behind_free_definition_offset_ref, behind_free_ref) = self + .m + .range(0..=behind_free_offset) + .last() + .expect("always at least one split"); + + let (behind_free_definition_offset, behind_free) = ( + *behind_free_definition_offset_ref, + behind_free_ref.to_empty_t(), + ); + + //Get state at start of free + let (start_free_definition_offset_ref, start_free_ref) = self + .m + .range(0..=offset) + .last() + .expect("always at least one split"); + //Free may not start in the middle of occupied space + match start_free_ref { + Occupied(_) => { + assert!(*start_free_definition_offset_ref == offset); + } + _ => {} + } + + //Remove all entries in free range + let indices_to_remove = self + .m + .range(offset..behind_free_offset) + .map(|(k, _)| *k) + .collect::>(); + for k in indices_to_remove { + self.m + .remove(&k) + .expect("this index was just queried, it should be there"); + } + + //Insert free + self.m.insert(offset, Free); + + //Write state at end + if behind_free_definition_offset != behind_free_offset { + let with_t = match behind_free { + Occupied(_) => { + panic!("may not free part of occupied block"); + } + Free => Free, + Unknown => Unknown, + }; + let replaced = self.m.insert(behind_free_offset, with_t); + if let Some(_) = replaced { + panic!("it was checked that this does not replace anything:\noffset={}\nlength={}\nbehind_free_offset={}\nbehind_free_definition_offset={}\n{:#?}", offset, length, behind_free_offset, behind_free_definition_offset, self); + } + } + } + + pub fn free_to_end(&mut self, offset: u64) { + use std::ops::Bound::*; + //Get state at start of free + let (start_free_definition_offset_ref, start_free_ref) = self + .m + .range(0..=offset) + .last() + .expect("always at least one split"); + //Free may not start in the middle of occupied space + match start_free_ref { + Occupied(_) => { + assert!(*start_free_definition_offset_ref == offset); + } + _ => {} + } + + //Remove all entries in free range + let indices_to_remove = self + .m + .range((Included(offset), Unbounded)) + .map(|(k, _)| *k) + .collect::>(); + for k in indices_to_remove { + self.m + .remove(&k) + .expect("this index was just queried, it should be there"); + } + + //Insert free + self.m.insert(offset, Free); + } + + pub fn iter<'a>(&'a self) -> impl Iterator)> + 'a { + let sentinel = Some((&std::u64::MAX, &Occ::Unknown)).into_iter(); + let shifted = self.m.iter().skip(1).chain(sentinel); + self.m + .iter() + .zip(shifted) + .map(|((offset, occ), (next_offset, _))| { + assert!(offset <= next_offset); + (*offset, (*next_offset - *offset), occ) + }) + } + + pub fn iter_occupied<'a>(&'a self) -> impl Iterator + 'a { + self.iter().filter_map(|(o, l, occ)| { + if let Some(occ) = occ.occupied() { + Some((o, l, occ)) + } else { + None + } + }) + } + + pub fn get_mut(&mut self, addr: u64) -> (&u64, &mut Occ) { + self.m + .range_mut(0..=addr) + .last() + .expect("always at least one split") + } + + pub fn get(&self, addr: u64) -> (&u64, &Occ) { + self.m + .range(0..=addr) + .last() + .expect("always at least one split") + } + +} + +#[cfg(test)] +mod tests { + use super::*; + + impl SplitTree { + fn get_occ_at_index(&self, offset: u64) -> &Occ { + let (_, occ_ref) = self + .m + .range(0..=offset) + .last() + .expect("always at least one split"); + + occ_ref + } + } + + fn check_unecessary_splits(t: &SplitTree) { + use std::ops::Bound::*; + let mut last_entry: Option> = None; + for (_, v) in t.m.range((Included(&0), Unbounded)) { + if let Some(last_entry) = last_entry { + match (last_entry, v) { + (Unknown, Unknown) => panic!("unnecessary Unknown"), + (Free, Free) => panic!("unneccecary Free"), + _ => {} + } + } + last_entry = Some(v.clone()); + } + } + #[test] + fn insert() { + let mut t = SplitTree::new(); + t.insert(10, 10, ()); + t.insert(20, 10, ()); + t.insert(35, 10, ()); + let t = dbg!(t); + assert!(t.get_occ_at_index(9).is_unknown()); + assert!(t.get_occ_at_index(10).is_occupied()); + assert!(t.get_occ_at_index(20).is_occupied()); + assert!(t.get_occ_at_index(30).is_unknown()); + assert!(t.get_occ_at_index(34).is_unknown()); + assert!(t.get_occ_at_index(35).is_occupied()); + assert!(t.get_occ_at_index(44).is_occupied()); + assert!(t.get_occ_at_index(45).is_unknown()); + check_unecessary_splits(&t); + } + + #[test] + fn free() { + let mut t = SplitTree::new(); + t.insert(10, 10, ()); + t.insert(20, 10, ()); + t.insert(35, 10, ()); + t.free(10, 10); + t.free(33, 12); + let t = dbg!(t); + assert!(t.get_occ_at_index(9).is_unknown()); + assert!(t.get_occ_at_index(10).is_free()); + assert!(t.get_occ_at_index(19).is_free()); + assert!(t.get_occ_at_index(20).is_occupied()); + assert!(t.get_occ_at_index(30).is_unknown()); + assert!(t.get_occ_at_index(32).is_unknown()); + assert!(t.get_occ_at_index(33).is_free()); + assert!(t.get_occ_at_index(44).is_free()); + assert!(t.get_occ_at_index(45).is_unknown()); + check_unecessary_splits(&t); + } + + #[test] + fn overwrite() { + let mut t = SplitTree::new(); + t.insert(10, 10, 1); + t.insert(10, 10, 2); + let occ = t.get_occ_at_index(15); + match occ { + Occupied(2) => {} + _ => panic!("Overwrite did not change data"), + } + } + + #[test] + fn iterator() { + let mut t = SplitTree::new(); + t.insert(10, 10, 1); + t.insert(20, 10, 1); + t.free(30, 10); + let mut it = t.iter(); + assert!(it.next().unwrap().2.is_unknown()); + assert!(it.next().unwrap().2.is_occupied()); + assert!(it.next().unwrap().2.is_occupied()); + assert!(it.next().unwrap().2.is_free()); + assert!(it.next().unwrap().2.is_unknown()); + assert!(it.next().is_none()); + } + + #[should_panic] + #[test] + fn double_insert_panics() { + let mut t = SplitTree::new(); + t.insert(10, 10, ()); + t.insert(5, 10, ()); + } + + #[should_panic] + #[test] + fn double_insert_panics2() { + let mut t = SplitTree::new(); + t.insert(10, 10, ()); + t.insert(15, 10, ()); + } + + #[should_panic] + #[test] + fn free_part_of_occupied_panics() { + let mut t = SplitTree::new(); + t.insert(10, 10, ()); + t.free(5, 10); + } + + #[should_panic] + #[test] + fn free_part_of_occupied_panics2() { + let mut t = SplitTree::new(); + t.insert(10, 10, ()); + t.free(15, 10); + } + + #[should_panic] + #[test] + fn free_part_of_occupied_panics3() { + let mut t = SplitTree::new(); + t.insert(10, 10, ()); + t.free(15, 20); + } + + #[should_panic] + #[test] + fn free_part_of_occupied_panics4() { + let mut t = SplitTree::new(); + t.insert(10, 20, ()); + t.free(15, 5); + } + + #[test] + #[should_panic] + fn free_u64_max_panics() { + let mut t = SplitTree::new(); + t.insert(10, 20, ()); + t.insert(30, 20, ()); + t.free(30, std::u64::MAX); + } + + #[test] + fn free_to_end() { + let mut t = SplitTree::new(); + t.insert(10, 20, ()); + t.insert(30, 20, ()); + t.free_to_end(30); + assert!(t.get_occ_at_index(29).is_occupied()); + assert!(t.get_occ_at_index(30).is_free()); + assert!(t.get_occ_at_index(std::u64::MAX).is_free()); + } + + #[test] + fn free_everything() { + let mut t = SplitTree::new(); + t.insert(10, 20, ()); + t.insert(30, 20, ()); + t.free_to_end(0); + assert!(t.get_occ_at_index(0).is_free()); + assert!(t.get_occ_at_index(30).is_free()); + assert!(t.get_occ_at_index(std::u64::MAX).is_free()); + } +}