diff --git a/NOTICE.txt b/NOTICE.txt index 9b98364d2ab6..a27486d2a5d9 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21,6 +21,10 @@ This product includes software from the mman-win32 project * Copyright https://code.google.com/p/mman-win32/ * Licensed under the MIT License; +This product includes software from the Fast Static Symbol Table (FSST) project (MIT) + * Copyright (c) 2018-2020 CWI, TU Munich, FSU Jena + * https://github.com/cwida/fsst + This product includes software from the LevelDB project * Copyright (c) 2011 The LevelDB Authors. All rights reserved. * Use of this source code is governed by a BSD-style license that can be diff --git a/bench-fsst-onpair/.gitignore b/bench-fsst-onpair/.gitignore new file mode 100644 index 000000000000..338fa47820d5 --- /dev/null +++ b/bench-fsst-onpair/.gitignore @@ -0,0 +1,2 @@ +/target +/corpora diff --git a/bench-fsst-onpair/Cargo.lock b/bench-fsst-onpair/Cargo.lock new file mode 100644 index 000000000000..e1c8fe5f0143 --- /dev/null +++ b/bench-fsst-onpair/Cargo.lock @@ -0,0 +1,919 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrow" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ord" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" + +[[package]] +name = "arrow-select" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bench-fsst-onpair" +version = "0.1.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "fsst-rs", + "onpair", + "tpchgen", + "tpchgen-arrow", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fsst-rs" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b13ac798afc0d9194eb4efefef8b9332efbd80b43f302a968cb8cb23b9d5360" +dependencies = [ + "rustc-hash", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "onpair" +version = "0.1.1" +source = "git+https://github.com/spiraldb/onpair?rev=f5f4bfe1fb7221c42a9f8261c269bdf5bd31052b#f5f4bfe1fb7221c42a9f8261c269bdf5bd31052b" +dependencies = [ + "hashbrown 0.16.1", + "rand", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tpchgen" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e26c6d047694183517044ea72d2e0337b05b43d40809f0958f603a7e5a286855" + +[[package]] +name = "tpchgen-arrow" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64c19d707b783f219883549268da8ea98f91e7aaec3b626563044a21c2b37aa" +dependencies = [ + "arrow", + "tpchgen", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/bench-fsst-onpair/Cargo.toml b/bench-fsst-onpair/Cargo.toml new file mode 100644 index 000000000000..c7f19c6f446d --- /dev/null +++ b/bench-fsst-onpair/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "bench-fsst-onpair" +version = "0.1.0" +edition = "2024" +publish = false + +[[bin]] +name = "bench-fsst-onpair" +path = "src/main.rs" + +[dependencies] +# A published OnPair implementation (the algorithm of arXiv:2508.02280), pinned +# for reproducibility. Git dep => Cargo passes --cap-lints allow, so its +# deny(warnings) won't fail our build on a newer rustc. +onpair = { git = "https://github.com/spiraldb/onpair", rev = "f5f4bfe1fb7221c42a9f8261c269bdf5bd31052b" } + +# FSST reference codec (same algorithm as the Arrow C++ PR #48232 vendors). +fsst-rs = "0.5.11" + +# In-process TPC-H data generation (no external files needed). +tpchgen = "3.0.0" +tpchgen-arrow = "3.0.0" +arrow-array = "59.1" +arrow-schema = "59.1" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 diff --git a/bench-fsst-onpair/src/main.rs b/bench-fsst-onpair/src/main.rs new file mode 100644 index 000000000000..7d7e6ef06223 --- /dev/null +++ b/bench-fsst-onpair/src/main.rs @@ -0,0 +1,462 @@ +//! Apples-to-apples comparison of FSST vs OnPair16 (the paper's 16-byte-max-token +//! variant, arXiv:2508.02280) at 12- and 16-bit dictionary sizes, on the same +//! string corpora. +//! +//! Both codecs run in a single Rust process so encode/decode throughput is +//! measured under one harness. Pin to a single core with `taskset -c 0`. +//! +//! Corpora: +//! * TPC-H string columns (o_comment, p_name, l_comment, c_comment), +//! generated in-process via tpchgen at scale factor 1. +//! * ClickBench: real `hits.parquet`-style data if ONPAIR_BENCH_PARQUET is +//! set (+ optional ONPAIR_BENCH_COLUMN), else a synthetic URL corpus. +//! +//! Size accounting (raw codec output, no downstream integer compression): +//! * OnPair = dict bytes + dict offsets(u32) + codes(u16) + row offsets(u32) +//! * FSST = symbol table + symbol lengths + code bytes + row offsets(u32) +//! +//! Both count an (n+1) u32 row-offset vector so the comparison is fair. + +use std::hint::black_box; +use std::mem::MaybeUninit; +use std::time::Instant; + +use arrow_array::cast::AsArray; +use fsst::Compressor; +use onpair::{Config, MaxDictBits, Threshold, compress as onpair_compress}; +use tpchgen::generators::{ + CustomerGenerator, LineItemGenerator, OrderGenerator, PartGenerator, SupplierGenerator, +}; +use tpchgen_arrow::{ + CustomerArrow, LineItemArrow, OrderArrow, PartArrow, RecordBatchIterator, SupplierArrow, +}; + +const BATCH_SIZE: usize = 8192 * 8; +/// Every corpus is truncated/generated to exactly this many rows for a fair +/// equal-N comparison. +const TARGET_ROWS: usize = 500_000; +const ENCODE_ITERS: usize = 3; +const DECODE_ITERS: usize = 10; + +/// A packed string corpus: concatenated bytes + (n+1) u64 offsets. +struct Corpus { + name: String, + bytes: Vec, + offsets: Vec, + /// Per-row slices, precomputed for FSST training/compression. + n_rows: usize, +} + +impl Corpus { + fn new(name: impl Into, bytes: Vec, offsets: Vec) -> Self { + let n_rows = offsets.len() - 1; + Corpus { name: name.into(), bytes, offsets, n_rows } + } + fn raw_bytes(&self) -> usize { + self.bytes.len() + } + fn rows(&self) -> Vec<&[u8]> { + (0..self.n_rows) + .map(|i| &self.bytes[self.offsets[i] as usize..self.offsets[i + 1] as usize]) + .collect() + } +} + +fn median(mut v: Vec) -> f64 { + v.sort_by(|a, b| a.partial_cmp(b).unwrap()); + v[v.len() / 2] +} + +fn mib(bytes: usize) -> f64 { + bytes as f64 / (1024.0 * 1024.0) +} + +// ───────────────────────────── OnPair ───────────────────────────── + +struct Measured { + label: String, + compressed_bytes: usize, + encode_mibs: f64, + decode_mibs: f64, +} + +fn run_onpair(c: &Corpus, bits: u8, threshold: f64) -> Measured { + let cfg = Config { + max_dict_bits: MaxDictBits::new(bits).unwrap(), + threshold: Threshold::new(threshold).unwrap(), + seed: Some(42), + }; + // Compressed size + a live column for decode timing. + let col = onpair_compress(&c.bytes, &c.offsets, cfg).unwrap(); + let compressed = col.dict.bytes().len() + + col.dict.offsets().len() * 4 + + col.codes.len() * 2 + + col.row_offsets.len() * 4; + + // Encode throughput: full train + compress, median of ENCODE_ITERS. + let mut enc = Vec::with_capacity(ENCODE_ITERS); + for _ in 0..ENCODE_ITERS { + let t = Instant::now(); + let out = onpair_compress(black_box(&c.bytes), black_box(&c.offsets), cfg).unwrap(); + let dt = t.elapsed().as_secs_f64(); + black_box(&out); + enc.push(mib(c.raw_bytes()) / dt); + } + + // Decode throughput: whole-column decompress_into. + let cap = col.view().decoded_len() + 16; // + tail padding + // Correctness: whole-column decode must reconstruct the concatenated input. + { + let mut buf: Vec> = vec![MaybeUninit::uninit(); cap]; + let n = unsafe { col.view().decompress_into(&mut buf) }; + let decoded: &[u8] = unsafe { std::slice::from_raw_parts(buf.as_ptr() as *const u8, n) }; + assert_eq!(n, c.raw_bytes(), "OnPair{bits} decoded len mismatch on {}", c.name); + assert!(decoded == c.bytes.as_slice(), "OnPair{bits} roundtrip mismatch on {}", c.name); + } + let mut dec = Vec::with_capacity(DECODE_ITERS); + for _ in 0..DECODE_ITERS { + let mut buf: Vec> = vec![MaybeUninit::uninit(); cap]; + let t = Instant::now(); + // SAFETY: buf sized to decoded_len()+padding; view from a trusted column. + let n = unsafe { col.view().decompress_into(&mut buf) }; + let dt = t.elapsed().as_secs_f64(); + black_box(&buf[..n]); + dec.push(mib(c.raw_bytes()) / dt); + } + + Measured { + label: format!("OnPair{bits}"), + compressed_bytes: compressed, + encode_mibs: median(enc), + decode_mibs: median(dec), + } +} + +// ───────────────────────────── FSST ───────────────────────────── + +/// Train + compress every row into one concatenated code buffer with +/// (n+1) u32 offsets. Returns (compressor, codes, offsets). +fn fsst_encode(rows: &Vec<&[u8]>) -> (Compressor, Vec, Vec) { + let compressor = Compressor::train(rows); + let total: usize = rows.iter().map(|r| r.len()).sum(); + let mut codes: Vec = Vec::with_capacity(2 * total + 8 * rows.len() + 16); + let mut offsets: Vec = Vec::with_capacity(rows.len() + 1); + offsets.push(0); + let mut scratch: Vec = Vec::with_capacity(1024); + for r in rows { + scratch.clear(); + // FSST worst case: 2 bytes per input byte + a few for escapes. + let need = 2 * r.len() + 16; + if scratch.capacity() < need { + scratch.reserve(need - scratch.capacity()); + } + // SAFETY: scratch has capacity for the FSST worst-case output of `r`. + unsafe { compressor.compress_into(r, &mut scratch) }; + codes.extend_from_slice(&scratch); + offsets.push(codes.len() as u32); + } + (compressor, codes, offsets) +} + +fn run_fsst(c: &Corpus) -> Measured { + let rows = c.rows(); + + let (compressor, codes, offsets) = fsst_encode(&rows); + let compressed = std::mem::size_of_val(compressor.symbol_table()) + + std::mem::size_of_val(compressor.symbol_lengths()) + + codes.len() + + offsets.len() * 4; + + // Encode throughput. + let mut enc = Vec::with_capacity(ENCODE_ITERS); + for _ in 0..ENCODE_ITERS { + let t = Instant::now(); + let out = fsst_encode(black_box(&rows)); + let dt = t.elapsed().as_secs_f64(); + black_box(&out); + enc.push(mib(c.raw_bytes()) / dt); + } + + // Decode throughput: decompress the whole concatenated code stream at once + // (FSST codes are context-free, so this reconstructs concatenated plaintext). + let decompressor = compressor.decompressor(); + let cap = c.raw_bytes() + 16; + // Correctness: decoding the concatenated code stream reconstructs the input. + { + let mut buf: Vec> = vec![MaybeUninit::uninit(); cap]; + let n = decompressor.decompress_into(&codes, &mut buf); + let decoded: &[u8] = unsafe { std::slice::from_raw_parts(buf.as_ptr() as *const u8, n) }; + assert_eq!(n, c.raw_bytes(), "FSST decoded len mismatch on {}", c.name); + assert!(decoded == c.bytes.as_slice(), "FSST roundtrip mismatch on {}", c.name); + } + let mut dec = Vec::with_capacity(DECODE_ITERS); + for _ in 0..DECODE_ITERS { + let mut buf: Vec> = vec![MaybeUninit::uninit(); cap]; + let t = Instant::now(); + let n = decompressor.decompress_into(black_box(&codes), &mut buf); + let dt = t.elapsed().as_secs_f64(); + black_box(&buf[..n]); + dec.push(mib(c.raw_bytes()) / dt); + } + + Measured { + label: "FSST".to_string(), + compressed_bytes: compressed, + encode_mibs: median(enc), + decode_mibs: median(dec), + } +} + +// ───────────────────────────── Corpora ───────────────────────────── + +/// Load any TPC-H string column, dispatching to its table generator by the +/// column-name prefix (o_/l_/c_/p_/s_). +fn tpch_column(col: &str) -> Corpus { + // Scale factor per table so it yields >= TARGET_ROWS rows (rows/SF at SF1: + // lineitem 6.0M, orders 1.5M, customer 150k, part 200k, supplier 10k), then + // truncate to exactly TARGET_ROWS. + let sf: f64 = match col.split('_').next().unwrap() { + "l" | "o" => 1.0, + "c" => 4.0, // 150k * 4 = 600k + "p" => 3.0, // 200k * 3 = 600k + "s" => 50.0, // 10k * 50 = 500k + _ => 1.0, + }; + let idx_of = |schema: &arrow_schema::Schema| { + schema.fields().iter().position(|f| f.name() == col).unwrap_or_else(|| { + panic!("column {col} not found in table schema") + }) + }; + let (bytes, offsets) = match col.split('_').next().unwrap() { + "l" => { + let it = LineItemArrow::new(LineItemGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + "o" => { + let it = OrderArrow::new(OrderGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + "c" => { + let it = CustomerArrow::new(CustomerGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + "p" => { + let it = PartArrow::new(PartGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + "s" => { + let it = SupplierArrow::new(SupplierGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + other => panic!("unknown table prefix {other} for column {col}"), + }; + Corpus::new(format!("tpch/{col}"), bytes, offsets) +} + +fn collect(batches: I, idx: usize) -> (Vec, Vec) +where + I: Iterator, +{ + let mut bytes = Vec::new(); + let mut offsets: Vec = vec![0]; + 'outer: for batch in batches { + let arr = batch.column(idx).as_string_view(); + for v in arr.iter() { + let s = v.unwrap_or("").as_bytes(); + bytes.extend_from_slice(s); + offsets.push(bytes.len() as u64); + if offsets.len() > TARGET_ROWS { + break 'outer; + } + } + } + (bytes, offsets) +} + +fn clickbench_corpus() -> Corpus { + if let Ok(path) = std::env::var("ONPAIR_BENCH_PARQUET") { + if let Some((bytes, offsets, colname)) = read_parquet(&path) { + return Corpus::new(format!("clickbench/{colname}"), bytes, offsets); + } + eprintln!("warning: could not read {path}, falling back to synthetic"); + } + let (bytes, offsets) = synthetic_clickbench_urls(TARGET_ROWS); + Corpus::new("clickbench/synthetic-urls", bytes, offsets) +} + +fn read_parquet(_path: &str) -> Option<(Vec, Vec, String)> { + // Only wired when ONPAIR_BENCH_PARQUET is set; requires the `parquet` crate. + // Left unimplemented to keep the default build light; synthetic is used. + None +} + +fn synthetic_clickbench_urls(n: usize) -> (Vec, Vec) { + const HOSTS: &[&str] = &[ + "https://www.yandex.ru", "https://www.google.com", "https://news.ycombinator.com", + "https://www.example.com", "https://docs.example.org", "https://api.example.net", + "http://m.yandex.ru", "https://maps.example.com", "https://shop.example.com", + "ftp://files.example.com", + ]; + const PATHS: &[&str] = &[ + "/", "/page", "/news", "/search?q=", "/profile", "/login", "/api/v1/data", + "/static/asset.png", "/blog/post-", "/feed.xml", "/sitemap.xml", "/users/", + "/admin/dashboard", "/categories/electronics", "/cart/checkout", + ]; + const TAILS: &[&str] = &["", "alpha", "beta", "gamma", "delta", "001", "002", "003"]; + let mut bytes = Vec::new(); + let mut offsets: Vec = vec![0]; + let mut x = 0x9E3779B97F4A7C15u64; + for _ in 0..n { + x = x.wrapping_add(0x9E3779B97F4A7C15); + let h = HOSTS[(x as usize) % HOSTS.len()]; + let p = PATHS[((x >> 16) as usize) % PATHS.len()]; + let t = TAILS[((x >> 32) as usize) % TAILS.len()]; + let num = (x >> 48) as u16; + let s = format!("{h}{p}{t}{num}"); + bytes.extend_from_slice(s.as_bytes()); + offsets.push(bytes.len() as u64); + } + (bytes, offsets) +} + +/// Synthetic JSON "event" objects: a fixed schema (repeated keys + structural +/// punctuation) with per-row varying values. High-cardinality overall (unique +/// ids/timestamps), but saturated with shared <=16-byte fragments — the shape a +/// Variant/JSON column takes. Newline-free per row. +fn synthetic_variant_json(n: usize) -> (Vec, Vec) { + const EVENTS: &[&str] = &["click", "view", "purchase", "signup", "logout", "search"]; + const COUNTRIES: &[&str] = &["US", "GB", "DE", "FR", "JP", "IN", "BR", "CA"]; + const TIERS: &[&str] = &["free", "pro", "enterprise"]; + let mut bytes = Vec::new(); + let mut offsets: Vec = vec![0]; + let mut x = 0x1234_5678_9ABC_DEF0u64; + let mut next = || { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x + }; + for _ in 0..n { + let uid = next() % 10_000_000; + let ev = EVENTS[(next() as usize) % EVENTS.len()]; + let country = COUNTRIES[(next() as usize) % COUNTRIES.len()]; + let tier = TIERS[(next() as usize) % TIERS.len()]; + let hh = next() % 24; + let mm = next() % 60; + let ss = next() % 60; + let day = 1 + next() % 28; + let amount = (next() % 100000) as f64 / 100.0; + let session = next(); + let s = format!( + "{{\"user_id\":{uid},\"event\":\"{ev}\",\"ts\":\"2024-03-{day:02}T{hh:02}:{mm:02}:{ss:02}Z\",\ +\"country\":\"{country}\",\"tier\":\"{tier}\",\"amount\":{amount:.2},\"session\":\"{session:016x}\"}}" + ); + bytes.extend_from_slice(s.as_bytes()); + offsets.push(bytes.len() as u64); + } + (bytes, offsets) +} + +// ───────────────────────────── main ───────────────────────────── + +/// Write each corpus as a newline-delimited .txt (one row per line) into `dir`, +/// so the C++ harness reads byte-identical inputs. All these columns are +/// newline-free (TPC-H comments/names, synthetic URLs), so line-delimiting is +/// lossless here. +fn dump_corpora(dir: &str) { + use std::io::Write; + std::fs::create_dir_all(dir).expect("create dump dir"); + // A spread across data shapes: free text, multi-word names/types, + // high-cardinality addresses, patterned IDs, and low-cardinality enums. + let tpch_cols = [ + // free text + "o_comment", "l_comment", "c_comment", "p_comment", "s_comment", + // multi-word names / types + "p_name", "p_type", "c_name", "s_name", + // high-cardinality addresses + "c_address", "s_address", + // patterned IDs / numbers + "o_clerk", "c_phone", + // low-cardinality enums / small vocab + "o_orderpriority", "l_shipmode", "p_brand", "p_container", "c_mktsegment", + ]; + let mut corpora: Vec = tpch_cols.iter().map(|c| tpch_column(c)).collect(); + corpora.push(clickbench_corpus()); + { + let (bytes, offsets) = synthetic_variant_json(TARGET_ROWS); + corpora.push(Corpus::new("variant/json-events", bytes, offsets)); + } + for c in &corpora { + let fname = c.name.replace('/', "_"); + let path = format!("{dir}/{fname}.txt"); + let f = std::fs::File::create(&path).expect("create file"); + let mut w = std::io::BufWriter::new(f); + for i in 0..c.n_rows { + let s = &c.bytes[c.offsets[i] as usize..c.offsets[i + 1] as usize]; + assert!(!s.contains(&b'\n'), "row contains newline in {}", c.name); + w.write_all(s).unwrap(); + w.write_all(b"\n").unwrap(); + } + w.flush().unwrap(); + eprintln!("[dump] {path}: {} rows, {:.2} MiB", c.n_rows, mib(c.raw_bytes())); + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + if let Some(pos) = args.iter().position(|a| a == "--dump-corpora") { + let dir = args.get(pos + 1).map(String::as_str).unwrap_or("corpora"); + dump_corpora(dir); + return; + } + + // TPC-H uses threshold 0.2 (matches onpair's tpch bench); ClickBench 0.5. + let corpora: Vec<(Corpus, f64)> = vec![ + (tpch_column("o_comment"), 0.2), + (tpch_column("l_comment"), 0.2), + (tpch_column("c_comment"), 0.2), + (tpch_column("p_name"), 0.2), + (clickbench_corpus(), 0.5), + ]; + + println!( + "{:<26} {:>10} {:>10} {:>9} {:>9} {:>9} {:>9} {:>9} {:>9}", + "corpus", "rows", "raw MiB", + "ratio", "enc MiB/s", "dec MiB/s", "", "", "" + ); + println!("{}", "─".repeat(120)); + + for (c, threshold) in &corpora { + let fsst = run_fsst(c); + let op12 = run_onpair(c, 12, *threshold); + let op16 = run_onpair(c, 16, *threshold); + + println!( + "{:<26} {:>10} {:>10.2}", + c.name, c.n_rows, mib(c.raw_bytes()) + ); + for m in [&fsst, &op12, &op16] { + let ratio = c.raw_bytes() as f64 / m.compressed_bytes as f64; + println!( + " {:<24} {:>10} {:>10.2} {:>8.3}x {:>9.1} {:>9.1}", + m.label, "", mib(m.compressed_bytes), + ratio, m.encode_mibs, m.decode_mibs + ); + } + // Head-to-head deltas: OnPair16 vs FSST. + let ratio_fsst = c.raw_bytes() as f64 / fsst.compressed_bytes as f64; + let ratio_op16 = c.raw_bytes() as f64 / op16.compressed_bytes as f64; + println!( + " → OnPair16 vs FSST: ratio {:+.1}%, encode {:+.1}%, decode {:+.1}%", + (ratio_op16 / ratio_fsst - 1.0) * 100.0, + (op16.encode_mibs / fsst.encode_mibs - 1.0) * 100.0, + (op16.decode_mibs / fsst.decode_mibs - 1.0) * 100.0, + ); + println!(); + } +} diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 4ced2a66bf58..dddbf2271bb7 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -49,6 +49,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES Boost Brotli BZip2 + fsst c-ares gflags glog @@ -72,6 +73,10 @@ set(ARROW_THIRDPARTY_DEPENDENCIES ZLIB zstd) +set(fsst_SOURCE + "BUNDLED" + CACHE STRING "Source of fsst dependency") + # For backward compatibility. We use "BOOST_SOURCE" if "Boost_SOURCE" # isn't specified and "BOOST_SOURCE" is specified. # We renamed "BOOST" dependency name to "Boost" in 3.0.0 because @@ -183,6 +188,8 @@ macro(build_dependency DEPENDENCY_NAME) build_brotli() elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2") build_bzip2() + elseif("${DEPENDENCY_NAME}" STREQUAL "fsst") + build_fsst() elseif("${DEPENDENCY_NAME}" STREQUAL "c-ares") build_cares() elseif("${DEPENDENCY_NAME}" STREQUAL "gflags") @@ -382,6 +389,7 @@ endif() if(ARROW_PARQUET) set(ARROW_WITH_RAPIDJSON ON) set(ARROW_WITH_THRIFT ON) + set(ARROW_WITH_FSST ON) endif() if(ARROW_WITH_THRIFT) @@ -2604,6 +2612,28 @@ if(ARROW_USE_XSIMD) endif() endif() +function(build_fsst) + message(STATUS "Configuring vendored FSST sources") + + set(ARROW_FSST_INCLUDE_DIR + "${ARROW_SOURCE_DIR}/thirdparty/fsst" + PARENT_SCOPE) + set(ARROW_FSST_SOURCES + "${ARROW_SOURCE_DIR}/thirdparty/fsst/libfsst.cpp;${ARROW_SOURCE_DIR}/thirdparty/fsst/fsst_avx512.cpp" + PARENT_SCOPE) + set(FSST_VENDORED + TRUE + PARENT_SCOPE) +endfunction() + +if(ARROW_WITH_FSST) + if(NOT fsst_SOURCE STREQUAL "BUNDLED") + message(FATAL_ERROR "FSST must currently be built from source. Set fsst_SOURCE=BUNDLED." + ) + endif() + resolve_dependency(fsst IS_RUNTIME_DEPENDENCY FALSE) +endif() + macro(build_zlib) message(STATUS "Building ZLIB from source") diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index dc7d40d2a386..a38705cadf86 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -42,6 +42,10 @@ function(ADD_PARQUET_TEST REL_TEST_NAME) ${ARGN}) set(TEST_ARGUMENTS PREFIX "parquet" LABELS "parquet-tests") + set(_PARQUET_TEST_EXTRA_ARGS ${ARG_UNPARSED_ARGUMENTS}) + if(PARQUET_TEST_EXTRA_INCLUDES) + list(APPEND _PARQUET_TEST_EXTRA_ARGS EXTRA_INCLUDES ${PARQUET_TEST_EXTRA_INCLUDES}) + endif() if(ARROW_TEST_LINKAGE STREQUAL "static") add_test_case(${REL_TEST_NAME} @@ -49,14 +53,14 @@ function(ADD_PARQUET_TEST REL_TEST_NAME) parquet_static ${PARQUET_TEST_LINK_LIBS} ${TEST_ARGUMENTS} - ${ARG_UNPARSED_ARGUMENTS}) + ${_PARQUET_TEST_EXTRA_ARGS}) else() add_test_case(${REL_TEST_NAME} STATIC_LINK_LIBS parquet_shared ${PARQUET_TEST_LINK_LIBS} ${TEST_ARGUMENTS} - ${ARG_UNPARSED_ARGUMENTS}) + ${_PARQUET_TEST_EXTRA_ARGS}) endif() endfunction() @@ -134,6 +138,9 @@ elseif(NOT MSVC) list(APPEND PARQUET_TEST_LINK_LIBS ${CMAKE_DL_LIBS}) endif() +set(PARQUET_TEST_EXTRA_INCLUDES) +set(PARQUET_PRIVATE_INCLUDE_DIRS) + # # Generated Thrift sources set(PARQUET_THRIFT_SOURCE_DIR "${ARROW_SOURCE_DIR}/src/generated/") @@ -191,6 +198,28 @@ set(PARQUET_SRCS stream_writer.cc types.cc) +if(DEFINED ARROW_FSST_SOURCES) + list(APPEND PARQUET_SRCS ${ARROW_FSST_SOURCES}) +endif() +if(DEFINED ARROW_FSST_INCLUDE_DIR) + list(APPEND PARQUET_PRIVATE_INCLUDE_DIRS ${ARROW_FSST_INCLUDE_DIR}) + list(APPEND PARQUET_TEST_EXTRA_INCLUDES ${ARROW_FSST_INCLUDE_DIR}) +endif() +if(DEFINED ARROW_FSST_SOURCES) + set_property(SOURCE ${ARROW_FSST_SOURCES} + APPEND + PROPERTY COMPILE_OPTIONS + "$<$,$>:-Wno-error=shorten-64-to-32;-Wno-shorten-64-to-32>" + "$<$,$,$>:-Wno-error=missing-declarations;-Wno-missing-declarations>" + "$<$:/wd4244>") + set_property(SOURCE ${ARROW_FSST_SOURCES} + APPEND + PROPERTY COMPILE_OPTIONS + "$<$,$>>:-include>" + "$<$,$>>:${CMAKE_CURRENT_SOURCE_DIR}/fsst_compat.h>" + ) +endif() + if(ARROW_HAVE_RUNTIME_AVX2) # AVX2 is used as a proxy for BMI2. list(APPEND PARQUET_SRCS level_comparison_avx2.cc level_conversion_bmi2.cc) @@ -306,6 +335,15 @@ add_arrow_lib(parquet STATIC_INSTALL_INTERFACE_LIBS ${PARQUET_STATIC_INSTALL_INTERFACE_LIBS}) +if(PARQUET_PRIVATE_INCLUDE_DIRS) + foreach(_parquet_target parquet_objlib parquet_shared parquet_static) + if(TARGET ${_parquet_target}) + target_include_directories(${_parquet_target} + PRIVATE ${PARQUET_PRIVATE_INCLUDE_DIRS}) + endif() + endforeach() +endif() + if(WIN32 AND NOT (ARROW_TEST_LINKAGE STREQUAL "static")) add_library(parquet_test_support STATIC "${PARQUET_THRIFT_SOURCE_DIR}/parquet_types.cpp") diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc index 8ecb774022f0..59c3b9934a24 100644 --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc @@ -862,7 +862,8 @@ class ColumnReaderImplBase { case Encoding::RLE: case Encoding::DELTA_BINARY_PACKED: case Encoding::DELTA_BYTE_ARRAY: - case Encoding::DELTA_LENGTH_BYTE_ARRAY: { + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + case Encoding::FSST: { auto decoder = MakeTypedDecoder(encoding, descr_, pool_); current_decoder_ = decoder.get(); decoders_[static_cast(encoding)] = std::move(decoder); diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index d0a857dd22ac..03105e4fea4d 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include #include @@ -48,6 +50,7 @@ #include "arrow/util/ubsan.h" #include "arrow/visit_data_inline.h" +#include "fsst.h" // NOLINT(build/include_subdir) #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" @@ -2307,6 +2310,253 @@ class ByteStreamSplitDecoder : public ByteStreamSplitDecoderBase { + public: + using Base = DecoderImpl; + using Base::num_values_; + + explicit FsstDecoder(const ColumnDescriptor* descr, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) + : DecoderImpl(descr, Encoding::FSST), pool_(pool) {} + + void SetData(int num_values, const uint8_t* data, int len) override { + const auto header_size = static_cast(sizeof(fsst_decoder_t)); + if (len < header_size) { + throw ParquetException("FSST page too small to contain decoder header"); + } + num_values_ = num_values; + memcpy(&decoder_, data, sizeof(fsst_decoder_t)); + next_ = data + header_size; + remaining_bytes_ = len - header_size; + decode_buffer_size_ = 0; + } + + int Decode(ByteArray* buffer, int max_values) override { + max_values = std::min(max_values, num_values_); + if (max_values == 0) { + return 0; + } + + const int64_t estimated_output_size = + decode_buffer_size_ + + static_cast(remaining_bytes_) * kMaxDecompressionExpansion; + EnsureDecodeBuffer(estimated_output_size); + + int decoded = 0; + + while (decoded < max_values) { + if (ARROW_PREDICT_FALSE(remaining_bytes_ < static_cast(kLengthPrefixSize))) { + throw ParquetException("FSST data truncated before length prefix"); + } + + const uint32_t compressed_len = SafeLoadAs(next_); + next_ += kLengthPrefixSize; + remaining_bytes_ -= static_cast(kLengthPrefixSize); + + if (ARROW_PREDICT_FALSE(compressed_len > static_cast(remaining_bytes_))) { + throw ParquetException("FSST compressed length exceeds available data"); + } + + const uint8_t* compressed_ptr = next_; + next_ += compressed_len; + remaining_bytes_ -= static_cast(compressed_len); + + uint8_t* value_ptr = nullptr; + const size_t decompressed_len = + DecompressValue(compressed_ptr, compressed_len, &value_ptr); + + buffer[decoded].ptr = value_ptr; + buffer[decoded].len = static_cast(decompressed_len); + decode_buffer_size_ += static_cast(decompressed_len); + ++decoded; + } + + num_values_ -= decoded; + return decoded; + } + + int DecodeSpaced(ByteArray* buffer, int num_values, int null_count, + const uint8_t* valid_bits, int64_t valid_bits_offset) override { + if (null_count == 0) { + return Decode(buffer, num_values); + } + + const int values_to_decode = num_values - null_count; + temp_values_.resize(values_to_decode); + const int decoded = Decode(temp_values_.data(), values_to_decode); + if (ARROW_PREDICT_FALSE(decoded != values_to_decode)) { + throw ParquetException("Expected to decode ", values_to_decode, + " values but decoded ", decoded, " values."); + } + + int value_index = 0; + for (int i = 0; i < num_values; ++i) { + if (bit_util::GetBit(valid_bits, valid_bits_offset + i)) { + buffer[i] = temp_values_[value_index++]; + } else { + buffer[i].ptr = nullptr; + buffer[i].len = 0; + } + } + + return value_index; + } + + int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::Accumulator* builder) override { + int values_decoded = 0; + PARQUET_THROW_NOT_OK(DecodeArrowDense(num_values, null_count, valid_bits, + valid_bits_offset, builder, &values_decoded)); + return values_decoded; + } + + int DecodeArrow( + int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::DictAccumulator* builder) override { + int values_decoded = 0; + PARQUET_THROW_NOT_OK(DecodeArrowDict(num_values, null_count, valid_bits, + valid_bits_offset, builder, &values_decoded)); + return values_decoded; + } + + private: + Status DecodeArrowDense(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::Accumulator* out, + int* out_values_decoded) { + const int values_to_decode = num_values - null_count; + temp_values_.resize(values_to_decode); + + const int decoded = Decode(temp_values_.data(), values_to_decode); + if (ARROW_PREDICT_FALSE(decoded != values_to_decode)) { + throw ParquetException("Expected to decode ", values_to_decode, + " values but decoded ", decoded, " values."); + } + + auto visit_binary_helper = [&](auto* helper) { + auto* values_ptr = temp_values_.data(); + int value_index = 0; + + RETURN_NOT_OK( + VisitBitRuns(valid_bits, valid_bits_offset, num_values, + [&](int64_t position, int64_t run_length, bool is_valid) { + if (is_valid) { + for (int64_t i = 0; i < run_length; ++i) { + const auto& value = values_ptr[value_index++]; + RETURN_NOT_OK(helper->AppendValue( + value.ptr, static_cast(value.len))); + } + } else { + RETURN_NOT_OK(helper->AppendNulls(run_length)); + } + return Status::OK(); + })); + + *out_values_decoded = decoded; + return Status::OK(); + }; + + return DispatchArrowBinaryHelper( + out, num_values, /*estimated_data_length=*/{}, visit_binary_helper); + } + + Status DecodeArrowDict(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::DictAccumulator* builder, + int* out_values_decoded) { + const int values_to_decode = num_values - null_count; + temp_values_.resize(values_to_decode); + + const int decoded = Decode(temp_values_.data(), values_to_decode); + if (ARROW_PREDICT_FALSE(decoded != values_to_decode)) { + throw ParquetException("Expected to decode ", values_to_decode, + " values but decoded ", decoded, " values."); + } + + RETURN_NOT_OK(builder->Reserve(num_values)); + + int value_index = 0; + RETURN_NOT_OK(VisitBitRuns(valid_bits, valid_bits_offset, num_values, + [&](int64_t position, int64_t run_length, bool is_valid) { + if (is_valid) { + for (int64_t i = 0; i < run_length; ++i) { + const auto& value = temp_values_[value_index++]; + RETURN_NOT_OK(builder->Append( + value.ptr, static_cast(value.len))); + } + } else { + RETURN_NOT_OK(builder->AppendNulls(run_length)); + } + return Status::OK(); + })); + + *out_values_decoded = decoded; + return Status::OK(); + } + + uint8_t* EnsureDecodeBuffer(int64_t capacity) { + const int64_t min_capacity = std::max(capacity, kInitialDecodeBufferSize); + const int64_t target = ::arrow::bit_util::NextPower2(min_capacity); + + if (!decode_buffer_) { + PARQUET_ASSIGN_OR_THROW(decode_buffer_, + ::arrow::AllocateResizableBuffer(target, pool_)); + } else if (decode_buffer_->size() < target) { + PARQUET_THROW_NOT_OK(decode_buffer_->Resize(target, false)); + } + return decode_buffer_->mutable_data(); + } + + size_t DecompressValue(const uint8_t* compressed_ptr, uint32_t compressed_len, + uint8_t** value_ptr) { + EnsureDecodeBuffer(decode_buffer_size_ + OutputUpperBound(compressed_len)); + + while (true) { + uint8_t* destination = decode_buffer_->mutable_data() + decode_buffer_size_; + const size_t available = + static_cast(decode_buffer_->size() - decode_buffer_size_); + + const size_t decompressed = fsst_decompress(&decoder_, compressed_len, + compressed_ptr, available, destination); + + if (decompressed > 0 || compressed_len == 0) { + *value_ptr = destination; + return decompressed; + } + + int64_t new_capacity = + std::max(decode_buffer_->size() * 2, + decode_buffer_size_ + OutputUpperBound(compressed_len)); + if (new_capacity <= decode_buffer_->size()) { + throw ParquetException("FSST decompression failed"); + } + EnsureDecodeBuffer(new_capacity); + } + } + + static int64_t OutputUpperBound(uint32_t compressed_len) { + const int64_t expanded = + static_cast(compressed_len) * kMaxDecompressionExpansion; + return std::max(expanded, kInitialDecodeBufferSize); + } + + static constexpr size_t kLengthPrefixSize = sizeof(uint32_t); + static constexpr int64_t kInitialDecodeBufferSize = 1024; + static constexpr int64_t kMaxDecompressionExpansion = 8; + + fsst_decoder_t decoder_{}; + const uint8_t* next_ = nullptr; + int remaining_bytes_ = 0; + ::arrow::MemoryPool* pool_; + std::shared_ptr<::arrow::ResizableBuffer> decode_buffer_; + int64_t decode_buffer_size_ = 0; + std::vector temp_values_; +}; + } // namespace // ---------------------------------------------------------------------- @@ -2373,6 +2623,13 @@ std::unique_ptr MakeDecoder(Type::type type_num, Encoding::type encodin throw ParquetException( "DELTA_BYTE_ARRAY only supports BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY"); } + } else if (encoding == Encoding::FSST) { + switch (type_num) { + case Type::BYTE_ARRAY: + return std::make_unique(descr, pool); + default: + throw ParquetException("FSST encoding only supports BYTE_ARRAY"); + } } else if (encoding == Encoding::DELTA_LENGTH_BYTE_ARRAY) { if (type_num == Type::BYTE_ARRAY) { return std::make_unique(descr, pool); diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index f9367555d97f..e0a2f1ec8269 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -18,8 +18,11 @@ #include "parquet/encoding.h" #include +#include #include #include +#include +#include #include #include #include @@ -44,6 +47,7 @@ #include "arrow/util/ubsan.h" #include "arrow/visit_data_inline.h" +#include "fsst.h" // NOLINT(build/include_subdir) #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" @@ -101,7 +105,7 @@ class EncoderImpl : virtual public Encoder { MemoryPool* memory_pool() const override { return pool_; } int64_t ReportUnencodedDataBytes() override { - if (descr_->physical_type() != Type::BYTE_ARRAY) { + if (descr_ != nullptr && descr_->physical_type() != Type::BYTE_ARRAY) { throw ParquetException("ReportUnencodedDataBytes is only supported for BYTE_ARRAY"); } int64_t bytes = unencoded_byte_array_data_bytes_; @@ -1737,6 +1741,182 @@ std::shared_ptr RleBooleanEncoder::FlushValues() { return buffer; } +// ---------------------------------------------------------------------- + +class FsstEncoder : public EncoderImpl, virtual public TypedEncoder { + public: + explicit FsstEncoder(const ColumnDescriptor* descr, MemoryPool* pool) + : EncoderImpl(descr, Encoding::FSST, pool), + encoder_(nullptr), + unencoded_values_() {} + + ~FsstEncoder() override { + if (encoder_) { + fsst_destroy(encoder_); + } + } + + int64_t EstimatedDataEncodedSize() override { + const int64_t total_size = pending_unencoded_bytes_; + const double scaled = static_cast(total_size) * compression_ratio_hint_; + const int64_t estimated_payload = static_cast(std::ceil(scaled)); + return static_cast(sizeof(fsst_decoder_t)) + + std::max(0, estimated_payload); + } + + std::shared_ptr FlushValues() override { + if (unencoded_values_.empty()) { + PARQUET_ASSIGN_OR_THROW(auto buffer, AllocateBuffer(0, pool_)); + return buffer; + } + + BuildSymbolTable(); + + const int64_t total_input_size = pending_unencoded_bytes_; + + const int64_t decoder_bytes = static_cast(sizeof(fsst_decoder_t)); + const int64_t length_prefix_bytes = static_cast(unencoded_values_.size()) * + static_cast(sizeof(uint32_t)); + const int64_t estimated_buffer_size = decoder_bytes + + total_input_size * kFsstCompressionExpansion + + length_prefix_bytes; + + PARQUET_ASSIGN_OR_THROW(auto output_buffer, + AllocateResizableBuffer(estimated_buffer_size, pool_)); + uint8_t* out_ptr = output_buffer->mutable_data(); + + fsst_decoder_t decoder = fsst_decoder(encoder_); + memcpy(out_ptr, &decoder, sizeof(fsst_decoder_t)); + out_ptr += sizeof(fsst_decoder_t); + + int64_t total_output_size = sizeof(fsst_decoder_t); + std::vector out_lengths(1); + std::vector out_ptrs(1); + + for (size_t i = 0; i < unencoded_values_.size(); ++i) { + const int64_t remaining_capacity = + estimated_buffer_size - total_output_size - sizeof(uint32_t); + if (ARROW_PREDICT_FALSE(remaining_capacity <= 0)) { + throw ParquetException("FSST compression buffer exhausted"); + } + size_t available = static_cast(remaining_capacity); + size_t input_length = static_cast(unencoded_values_[i].len); + const unsigned char* input_ptr = unencoded_values_[i].ptr; + size_t num_compressed = + fsst_compress(encoder_, 1, &input_length, &input_ptr, available, + out_ptr + sizeof(uint32_t), out_lengths.data(), out_ptrs.data()); + + if (num_compressed == 0) { + throw ParquetException("FSST compression buffer too small"); + } + + uint32_t len = static_cast(out_lengths[0]); + memcpy(out_ptr, &len, sizeof(uint32_t)); + out_ptr += sizeof(uint32_t) + out_lengths[0]; + total_output_size += sizeof(uint32_t) + out_lengths[0]; + } + + PARQUET_THROW_NOT_OK(output_buffer->Resize(total_output_size)); + UpdateCompressionStats(total_input_size, total_output_size); + if (encoder_ != nullptr) { + fsst_destroy(encoder_); + encoder_ = nullptr; + } + unencoded_values_.clear(); + pending_unencoded_bytes_ = 0; + unencoded_byte_array_data_bytes_ = 0; + return output_buffer; + } + + using TypedEncoder::Put; + + void Put(const ByteArray* src, int num_values) override { + for (int i = 0; i < num_values; ++i) { + unencoded_values_.push_back(src[i]); + const int64_t length = static_cast(src[i].len); + unencoded_byte_array_data_bytes_ += length; + pending_unencoded_bytes_ += length; + } + } + + void Put(const ::arrow::Array& values) override { + AssertVarLengthBinary(values); + + PARQUET_THROW_NOT_OK(::arrow::VisitArraySpanInline<::arrow::BinaryType>( + *values.data(), + [&](::std::string_view view) { + if (ARROW_PREDICT_FALSE(view.size() > kMaxByteArraySize)) { + return Status::Invalid("Parquet cannot store strings with size 2GB or more"); + } + ByteArray val; + val.len = static_cast(view.size()); + val.ptr = reinterpret_cast(view.data()); + unencoded_values_.push_back(val); + const int64_t length = static_cast(val.len); + unencoded_byte_array_data_bytes_ += length; + pending_unencoded_bytes_ += length; + return Status::OK(); + }, + []() { return Status::OK(); })); + } + + void PutSpaced(const ByteArray* src, int num_values, const uint8_t* valid_bits, + int64_t valid_bits_offset) override { + if (valid_bits != NULLPTR) { + PARQUET_ASSIGN_OR_THROW( + auto buffer, ::arrow::AllocateBuffer(num_values * sizeof(ByteArray), pool_)); + auto buffer_ptr = reinterpret_cast(buffer->mutable_data()); + int num_valid_values = ::arrow::util::internal::SpacedCompress( + src, num_values, valid_bits, valid_bits_offset, buffer_ptr); + Put(buffer_ptr, num_valid_values); + } else { + Put(src, num_values); + } + } + + private: + void BuildSymbolTable() { + if (encoder_ != nullptr) { + fsst_destroy(encoder_); + encoder_ = nullptr; + } + + if (unencoded_values_.empty()) { + return; + } + + std::vector input_lengths; + std::vector input_ptrs; + + for (const auto& val : unencoded_values_) { + input_lengths.push_back(val.len); + input_ptrs.push_back(val.ptr); + } + + encoder_ = + fsst_create(unencoded_values_.size(), input_lengths.data(), input_ptrs.data(), 0); + + if (!encoder_) { + throw ParquetException("Failed to create FSST encoder"); + } + } + + void UpdateCompressionStats(int64_t input_bytes, int64_t payload_bytes) { + const double ratio = static_cast(std::max(0, payload_bytes)) / + static_cast(std::max(int64_t{1}, input_bytes)); + compression_ratio_hint_ = std::max(kMinimumCompressionRatio, ratio); + } + + static constexpr double kDefaultCompressionRatio = 1.0; + static constexpr double kMinimumCompressionRatio = 0.01; + // Allocate worst-case 4x expansion. + static constexpr int64_t kFsstCompressionExpansion = 4; + fsst_encoder_t* encoder_; + std::vector unencoded_values_; + double compression_ratio_hint_ = kDefaultCompressionRatio; + int64_t pending_unencoded_bytes_ = 0; +}; + } // namespace // ---------------------------------------------------------------------- @@ -1821,6 +2001,13 @@ std::unique_ptr MakeEncoder(Type::type type_num, Encoding::type encodin default: throw ParquetException("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY"); } + } else if (encoding == Encoding::FSST) { + switch (type_num) { + case Type::BYTE_ARRAY: + return std::make_unique(descr, pool); + default: + throw ParquetException("FSST encoding only supports BYTE_ARRAY"); + } } else if (encoding == Encoding::RLE) { switch (type_num) { case Type::BOOLEAN: diff --git a/cpp/src/parquet/encoding_test.cc b/cpp/src/parquet/encoding_test.cc index 66a3f7647fa8..38f21d112c5a 100644 --- a/cpp/src/parquet/encoding_test.cc +++ b/cpp/src/parquet/encoding_test.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -28,7 +29,9 @@ #include "arrow/array/builder_binary.h" #include "arrow/array/builder_dict.h" #include "arrow/array/concatenate.h" +#include "arrow/buffer.h" #include "arrow/compute/cast.h" +#include "arrow/io/memory.h" #include "arrow/testing/gtest_util.h" #include "arrow/testing/random.h" #include "arrow/testing/util.h" @@ -41,7 +44,13 @@ #include "arrow/util/endian.h" #include "arrow/util/span.h" #include "arrow/util/string.h" +#include "fsst.h" // NOLINT(build/include_subdir) +#include "parquet/column_page.h" +#include "parquet/column_reader.h" #include "parquet/encoding.h" +#include "parquet/exception.h" +#include "parquet/file_reader.h" +#include "parquet/file_writer.h" #include "parquet/platform.h" #include "parquet/schema.h" #include "parquet/test_util.h" @@ -2593,4 +2602,305 @@ TEST(DeltaByteArrayEncodingAdHoc, ArrowDirectPut) { } } +// ---------------------------------------------------------------------- + +TEST(TestFsstEncoding, BasicRoundTrip) { + ::arrow::random::RandomArrayGenerator rag(0); + constexpr int64_t kNumValues = 2048; + constexpr int64_t kNumUnique = 128; + constexpr int32_t kMinLength = 4; + constexpr int32_t kMaxLength = 48; + + auto values = + rag.BinaryWithRepeats(kNumValues, kNumUnique, kMinLength, kMaxLength, 0.0); + + auto encoder = MakeTypedEncoder(Encoding::FSST); + ASSERT_NO_THROW(encoder->Put(*values)); + auto encoded = encoder->FlushValues(); + ASSERT_NE(nullptr, encoded); + ASSERT_GT(encoded->size(), 0); + + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(values->length()), encoded->data(), + static_cast(encoded->size())); + + std::vector decoded(values->length()); + ASSERT_EQ(static_cast(values->length()), + decoder->Decode(decoded.data(), static_cast(decoded.size()))); + + const auto& binary = checked_cast(*values); + for (int64_t i = 0; i < values->length(); ++i) { + auto view = binary.GetView(i); + ASSERT_EQ(view.size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(decoded[i].ptr, view.data(), view.size())) << i; + } +} + +TEST(TestFsstEncoding, FlushEmptyReturnsEmptyBuffer) { + auto encoder = MakeTypedEncoder(Encoding::FSST); + auto buffer = encoder->FlushValues(); + ASSERT_NE(nullptr, buffer); + ASSERT_EQ(0, buffer->size()); +} + +TEST(TestFsstEncoding, ReportUnencodedBytesResetsCounter) { + ::arrow::random::RandomArrayGenerator rag(42); + auto values = rag.String(256, 1, 24, 0.0); + + auto encoder = MakeTypedEncoder(Encoding::FSST); + ASSERT_NO_THROW(encoder->Put(*values)); + + const auto& binary = checked_cast(*values); + const int64_t expected = binary.total_values_length(); + EXPECT_EQ(expected, encoder->ReportUnencodedDataBytes()); + EXPECT_EQ(0, encoder->ReportUnencodedDataBytes()); + + auto encoded = encoder->FlushValues(); + ASSERT_NE(nullptr, encoded); + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(values->length()), encoded->data(), + static_cast(encoded->size())); + + std::vector decoded(values->length()); + ASSERT_EQ(static_cast(values->length()), + decoder->Decode(decoded.data(), static_cast(decoded.size()))); + + for (int64_t i = 0; i < values->length(); ++i) { + auto view = binary.GetView(i); + ASSERT_EQ(view.size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(decoded[i].ptr, view.data(), view.size())) << i; + } + + EXPECT_EQ(0, encoder->ReportUnencodedDataBytes()); +} + +TEST(TestFsstEncoding, MultiPageRoundTrip) { + constexpr int64_t kStringsPerPage = 10000; + constexpr int64_t kNumPages = 2; + constexpr int64_t kTotalValues = kStringsPerPage * kNumPages; + constexpr int64_t kPageSizeBytes = 512 * 1024; + + std::vector page1; + std::vector page2; + page1.reserve(kStringsPerPage); + page2.reserve(kStringsPerPage); + + for (int64_t i = 0; i < kStringsPerPage; ++i) { + page1.push_back("https://www.example.com/user/profile/page" + std::to_string(i) + + "/settings/preferences/advanced/options"); + page2.push_back("ftp://ftp.downloads.org/pub/software/release" + std::to_string(i) + + "/package/installer/binaries/archive"); + } + + std::vector expected; + expected.reserve(kTotalValues); + expected.insert(expected.end(), page1.begin(), page1.end()); + expected.insert(expected.end(), page2.begin(), page2.end()); + + ASSERT_OK_AND_ASSIGN(auto output_stream, ::arrow::io::BufferOutputStream::Create()); + + schema::NodeVector fields; + fields.push_back(schema::PrimitiveNode::Make("url", Repetition::REQUIRED, + Type::BYTE_ARRAY, ConvertedType::UTF8)); + auto parquet_schema = std::static_pointer_cast( + schema::GroupNode::Make("schema", Repetition::REQUIRED, fields)); + + auto writer_props = WriterProperties::Builder() + .encoding(Encoding::FSST) + ->data_pagesize(kPageSizeBytes) + ->build(); + + std::unique_ptr writer; + ASSERT_NO_THROW( + writer = ParquetFileWriter::Open(output_stream, parquet_schema, writer_props)); + ASSERT_NE(nullptr, writer); + auto* row_group_writer = writer->AppendRowGroup(); + ASSERT_NE(nullptr, row_group_writer); + auto* column_writer = + static_cast*>(row_group_writer->NextColumn()); + ASSERT_NE(nullptr, column_writer); + + auto write_page = [&](const std::vector& source) { + std::vector batch; + batch.reserve(source.size()); + for (const auto& value : source) { + batch.emplace_back(value); + } + column_writer->WriteBatch(static_cast(batch.size()), nullptr, nullptr, + batch.data()); + }; + + write_page(page1); + write_page(page2); + + column_writer->Close(); + row_group_writer->Close(); + ASSERT_NO_THROW(writer->Close()); + + ASSERT_OK_AND_ASSIGN(auto buffer, output_stream->Finish()); + + auto make_reader = [&buffer]() -> std::unique_ptr { + auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(buffer); + return ParquetFileReader::Open(buffer_reader); + }; + + std::unique_ptr page_reader_file; + ASSERT_NO_THROW(page_reader_file = make_reader()); + ASSERT_NE(nullptr, page_reader_file); + auto page_row_group = page_reader_file->RowGroup(0); + auto page_reader = page_row_group->GetColumnPageReader(0); + + int data_page_count = 0; + while (page_reader->NextPage() != nullptr) { + ++data_page_count; + } + EXPECT_GT(data_page_count, 1); + + std::unique_ptr reader; + ASSERT_NO_THROW(reader = make_reader()); + ASSERT_NE(nullptr, reader); + auto row_group_reader = reader->RowGroup(0); + auto column_reader = std::static_pointer_cast>( + row_group_reader->Column(0)); + + std::vector decoded(kTotalValues); + int64_t values_read = 0; + while (values_read < kTotalValues) { + int64_t batch_length = std::min(1024, kTotalValues - values_read); + int64_t batch_read = 0; + column_reader->ReadBatch(batch_length, nullptr, nullptr, decoded.data() + values_read, + &batch_read); + ASSERT_GT(batch_read, 0); + values_read += batch_read; + } + ASSERT_EQ(values_read, kTotalValues); + + for (int64_t i = 0; i < kTotalValues; ++i) { + ASSERT_EQ(expected[i].size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(expected[i].data(), decoded[i].ptr, expected[i].size())) << i; + } +} + +TEST(TestFsstEncoding, MultipleFlushesProduceIndependentPages) { + ::arrow::random::RandomArrayGenerator rag(99); + auto encoder = MakeTypedEncoder(Encoding::FSST); + + const std::vector> batches = { + {64, 3, 12}, {512, 2, 24}, {17, 1, 8}}; + + for (const auto& [size, min_len, max_len] : batches) { + auto batch = rag.String(size, min_len, max_len, 0.0); + + ASSERT_NO_THROW(encoder->Put(*batch)); + auto buffer = encoder->FlushValues(); + ASSERT_NE(nullptr, buffer); + ASSERT_GT(buffer->size(), 0); + + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(batch->length()), buffer->data(), + static_cast(buffer->size())); + + std::vector decoded(batch->length()); + ASSERT_EQ(static_cast(batch->length()), + decoder->Decode(decoded.data(), static_cast(decoded.size()))); + + const auto& binary = checked_cast(*batch); + for (int64_t i = 0; i < batch->length(); ++i) { + auto view = binary.GetView(i); + ASSERT_EQ(view.size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(decoded[i].ptr, view.data(), view.size())) << i; + } + + encoder->ReportUnencodedDataBytes(); + } +} + +TEST(TestFsstEncoding, DecodeRejectsOverstatedLength) { + ::arrow::random::RandomArrayGenerator rag(123); + auto values = rag.String(64, 3, 32, 0.0); + + auto encoder = MakeTypedEncoder(Encoding::FSST); + ASSERT_NO_THROW(encoder->Put(*values)); + auto encoded = encoder->FlushValues(); + ASSERT_NE(nullptr, encoded); + ASSERT_GT(encoded->size(), + static_cast(sizeof(fsst_decoder_t) + sizeof(uint32_t))); + + ASSERT_OK_AND_ASSIGN(auto corrupted, + ::arrow::AllocateBuffer(encoded->size(), default_memory_pool())); + std::memcpy(corrupted->mutable_data(), encoded->data(), encoded->size()); + + auto* length_ptr = + reinterpret_cast(corrupted->mutable_data() + sizeof(fsst_decoder_t)); + *length_ptr = static_cast(encoded->size()); + + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(values->length()), corrupted->data(), + static_cast(corrupted->size())); + + std::vector decoded(values->length()); + EXPECT_THROW(decoder->Decode(decoded.data(), static_cast(decoded.size())), + ParquetException); +} + +TEST(TestFsstEncoding, SetDataRejectsShortHeader) { + auto decoder = MakeTypedDecoder(Encoding::FSST); + + const int num_values = 1; + const int header_bytes = static_cast(sizeof(fsst_decoder_t)); + std::vector truncated(static_cast(header_bytes - 1), 0); + + EXPECT_THROW( + decoder->SetData(num_values, truncated.data(), static_cast(truncated.size())), + ParquetException); +} + +TEST(TestFsstEncoding, HeavyNullsDecodeSpaced) { + ::arrow::random::RandomArrayGenerator rag(321); + constexpr int64_t kNumValues = 4096; + constexpr double kNullProb = 0.85; + + auto values = rag.String(kNumValues, 1, 24, kNullProb); + + auto encoder = MakeTypedEncoder(Encoding::FSST); + ASSERT_NO_THROW(encoder->Put(*values)); + auto encoded = encoder->FlushValues(); + ASSERT_NE(nullptr, encoded); + + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(values->length()), encoded->data(), + static_cast(encoded->size())); + + const auto& binary = checked_cast(*values); + std::vector decoded(values->length()); + std::vector valid_bits(::arrow::bit_util::BytesForBits(values->length()), 0); + + ::arrow::internal::BitmapWriter writer(valid_bits.data(), 0, values->length()); + for (int64_t i = 0; i < values->length(); ++i) { + if (!binary.IsNull(i)) { + writer.Set(); + } else { + writer.Clear(); + } + writer.Next(); + } + writer.Finish(); + + const int null_count = static_cast(binary.null_count()); + ASSERT_EQ(static_cast(values->length() - null_count), + decoder->DecodeSpaced(decoded.data(), static_cast(values->length()), + null_count, valid_bits.data(), 0)); + + for (int64_t i = 0; i < values->length(); ++i) { + if (binary.IsNull(i)) { + EXPECT_EQ(nullptr, decoded[i].ptr); + EXPECT_EQ(0u, decoded[i].len); + continue; + } + auto view = binary.GetView(i); + ASSERT_EQ(view.size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(decoded[i].ptr, view.data(), view.size())) << i; + } +} + } // namespace parquet::test diff --git a/cpp/src/parquet/fsst_compat.h b/cpp/src/parquet/fsst_compat.h new file mode 100644 index 000000000000..1c3ef3fae0b5 --- /dev/null +++ b/cpp/src/parquet/fsst_compat.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +// Provide minimal compatibility shims so third-party FSST sources +// can be compiled with the compilers Arrow supports. + +#if defined(_WIN32) && !defined(_MSC_VER) +# include + +// MinGW does not provide __cpuidex, but FSST only needs the CPUID +// leaf/sub-leaf variant that __cpuid_count implements. +static inline void arrow_fsst_cpuidex(int info[4], int function_id, int subfunction_id) { + __cpuid_count(function_id, subfunction_id, info[0], info[1], info[2], info[3]); +} +# define __cpuidex(info, function_id, subfunction_id) \ + arrow_fsst_cpuidex(info, function_id, subfunction_id) +#endif diff --git a/cpp/src/parquet/onpair/bench_common.h b/cpp/src/parquet/onpair/bench_common.h new file mode 100644 index 000000000000..f8d1b4e3bbb0 --- /dev/null +++ b/cpp/src/parquet/onpair/bench_common.h @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Pieces shared by the standalone string-encoding benchmarks: the packed corpus +// representation, the size-accounting helpers, and the timing conventions. +// +// This header exists so that fsst_onpair_benchmark.cc (FSST / zstd / lz4 / +// OnPair, no Arrow dependency) and cascade_benchmark.cc (the same codecs plus +// Parquet's own byte-array encodings, which needs libparquet) cannot drift apart +// on how bytes are counted. A ratio is only comparable across the two binaries +// if every codec in both is charged the same way, so the accounting lives here +// and nowhere else. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +// zstd: declare the small, ABI-stable subset we use so the standalone build +// needs only the installed libzstd (no dev header). Link libzstd.so directly. +extern "C" { +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int level); +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +size_t ZSTD_compressBound(size_t srcSize); +unsigned ZSTD_isError(size_t code); +// lz4 (fast block compressor) - ABI-stable subset; link the installed liblz4. +int LZ4_compressBound(int inputSize); +int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); +int LZ4_decompress_safe(const char* src, char* dst, int compressedSize, int dstCapacity); +} + +namespace bench { + +using Clock = std::chrono::steady_clock; + +constexpr int kEncodeIters = 3; +constexpr int kDecodeIters = 10; + +inline double Mib(size_t bytes) { return static_cast(bytes) / (1024.0 * 1024.0); } + +inline double Median(std::vector v) { + std::sort(v.begin(), v.end()); + return v[v.size() / 2]; +} + +// Bits to store a value in [0, x] (x==0 -> 0 bits). +inline size_t BitWidth(uint64_t x) { + return x == 0 ? 0 : 64 - static_cast(__builtin_clzll(x)); +} +// Bits to index `count` distinct symbols [0, count) (== ceil(log2 count), >=1). +inline size_t IndexBits(size_t count) { + return count <= 1 ? 1 + : (64 - static_cast( + __builtin_clzll(static_cast(count - 1)))); +} +inline size_t BitPackedBytes(size_t n, size_t bits) { return (n * bits + 7) / 8; } + +// A packed corpus: concatenated bytes + (n+1) u32 offsets. +struct Corpus { + std::string name; + std::vector bytes; + std::vector offsets; + size_t n_rows() const { return offsets.size() - 1; } + size_t raw_bytes() const { return bytes.size(); } + size_t max_row_len() const { + size_t m = 0; + for (size_t i = 0; i < n_rows(); ++i) m = std::max(m, offsets[i + 1] - offsets[i]); + return m; + } + // Realistic per-row row-length side array (delta offsets), bit-packed at the + // width of the longest row. Charged to every value-preserving codec (FSST, + // zstd, lz4, OnPair) so the row boundaries are accounted the way a real + // columnar format stores them - not as raw (n+1) u32. + // + // NOT charged to Parquet's own byte-array encodings: PLAIN and the DELTA_* + // family embed their lengths in the encoded payload, so adding this on top + // would count row boundaries twice. + size_t len_array_bytes() const { + return BitPackedBytes(n_rows(), std::max(1, BitWidth(max_row_len()))); + } +}; + +// Read a newline-delimited file (one row per line) into a packed corpus. +inline Corpus ReadCorpus(const std::filesystem::path& path) { + Corpus c; + c.name = path.stem().string(); + std::ifstream in(path, std::ios::binary); + c.offsets.push_back(0); + std::string line; + while (std::getline(in, line)) { + c.bytes.insert(c.bytes.end(), line.begin(), line.end()); + c.offsets.push_back(static_cast(c.bytes.size())); + } + return c; +} + +struct Measured { + std::string label; + size_t compressed_bytes = 0; + double encode_mibs = 0; + double decode_mibs = 0; +}; + +// TPC-H columns train with a 0.2 sample fraction; the URL corpus with 0.5 +// (matching the Rust harness). +inline double ThresholdFor(const std::string& name) { + return name.rfind("tpch_", 0) == 0 ? 0.2 : 0.5; +} + +// Collect the .txt corpora in `dir`, sorted, so both binaries iterate the same +// set in the same order. +inline std::vector CorpusFiles(const std::string& dir) { + std::vector files; + for (const auto& e : std::filesystem::directory_iterator(dir)) { + if (e.path().extension() == ".txt") files.push_back(e.path()); + } + std::sort(files.begin(), files.end()); + return files; +} + +// Resolve the corpus directory: argv[1], then $ONPAIR_BENCH_DIR, then the +// generator's default output path. +inline std::string CorpusDir(int argc, char** argv) { + if (argc > 1) return argv[1]; + if (const char* env = std::getenv("ONPAIR_BENCH_DIR")) return env; + return "bench-fsst-onpair/corpora"; +} + +} // namespace bench diff --git a/cpp/src/parquet/onpair/cascade_benchmark.cc b/cpp/src/parquet/onpair/cascade_benchmark.cc new file mode 100644 index 000000000000..6e088f976945 --- /dev/null +++ b/cpp/src/parquet/onpair/cascade_benchmark.cc @@ -0,0 +1,963 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Does it pay to cascade a generic codec (zstd/lz4) over a string encoding? +// +// The companion benchmark (fsst_onpair_benchmark.cc) measures FSST, OnPair, zstd +// and lz4 as alternatives. This one measures them as *compositions*, which is +// what a Parquet writer actually does: an encoding produces a page payload and a +// page compressor then runs over the whole payload. Two groups: +// +// A. FSST / OnPair followed by zstd(1) or lz4. Note what this costs: the +// generic layer is whole-page, so the per-row random access that is the +// reason to pick FSST or OnPair in the first place is gone -- you must +// inflate the page before touching one row. The question is whether the +// extra ratio pays for that. OnPair is also measured with a byte-aligned +// layout (u16 codes, u32 dictionary offsets) rather than bit-packed, +// because bit-packed streams are close to incompressible and testing only +// the packed layout would understate what a cascade can do. +// +// B. Parquet's own byte-array encodings -- PLAIN, DELTA_LENGTH_BYTE_ARRAY, +// DELTA_BYTE_ARRAY, RLE_DICTIONARY -- alone and under zstd(1)/lz4, driven +// through the real parquet::Encoder/Decoder API rather than reimplemented. +// PLAIN+ZSTD is what Parquet writes for string columns today, so this is +// the baseline a new encoding has to beat; the companion benchmark's +// zstd-over-concatenated-bytes column is a projection of a page, not one. +// +// Accounting matches the companion benchmark so ratios are comparable across the +// two binaries (see bench_common.h), with one deliberate difference: PLAIN and +// the DELTA_* family embed their own lengths, so they are NOT charged the +// separate bit-packed row-length array that FSST/zstd/lz4/OnPair are. Charging +// it would count row boundaries twice. Every number here is therefore the +// complete page payload needed to reconstruct the column. +// +// Decode is timed as "reconstruct the whole column into a contiguous buffer", +// including the generic decompression and the per-page decoder setup. The +// contiguous copy matters for the Parquet encodings: their Decode() hands back +// pointers into the decoder's own buffer, so timing that alone would report PLAIN +// as nearly free rather than as the memcpy it is. +// +// Build (from the Arrow repo root), one line -- needs libparquet, unlike the +// companion benchmark. libarrow is linked by path because the build directory +// carries two sonames and letting -larrow choose warns about the conflict: +// g++ -std=c++17 -O3 -march=native -Icpp/src -Icpp/build-bench/src +// -Icpp/thirdparty/fsst cpp/thirdparty/fsst/libfsst.cpp +// cpp/thirdparty/fsst/fsst_avx512.cpp cpp/src/parquet/onpair/onpair.cc +// cpp/src/parquet/onpair/cascade_benchmark.cc +// cpp/build-bench/release/libparquet.so cpp/build-bench/release/libarrow.so.2300 +// /usr/lib64/libzstd.so.1 /usr/lib64/liblz4.so.1 +// -Wl,-rpath,$PWD/cpp/build-bench/release -o /tmp/cascade_bench +// +// Run: taskset -c 0 /tmp/cascade_bench (run twice, use the 2nd) + +#include +#include +#include +#include +#include +#include +#include + +#include "fsst.h" +#include "parquet/encoding.h" +#include "parquet/onpair/bench_common.h" +#include "parquet/onpair/onpair.h" +#include "parquet/schema.h" +#include "parquet/types.h" + +namespace op = parquet::onpair; + +namespace { + +using namespace bench; // NOLINT(build/namespaces) + +// The generic (page-level) codec layered over an encoding's payload. + +enum class Generic { kNone, kZstd1, kLz4 }; + +const char* GenericSuffix(Generic g) { + switch (g) { + case Generic::kNone: + return ""; + case Generic::kZstd1: + return "+zstd(1)"; + default: + return "+lz4"; + } +} + +std::vector GenericCompress(const uint8_t* src, size_t n, Generic g) { + if (g == Generic::kNone) return std::vector(src, src + n); + std::vector out; + if (g == Generic::kZstd1) { + out.resize(ZSTD_compressBound(n)); + size_t c = ZSTD_compress(out.data(), out.size(), src, n, 1); + if (ZSTD_isError(c)) { + std::fprintf(stderr, "zstd compress error\n"); + std::abort(); + } + out.resize(c); + } else { + out.resize(static_cast(LZ4_compressBound(static_cast(n)))); + int c = LZ4_compress_default(reinterpret_cast(src), + reinterpret_cast(out.data()), + static_cast(n), static_cast(out.size())); + if (c <= 0) { + std::fprintf(stderr, "lz4 compress error\n"); + std::abort(); + } + out.resize(static_cast(c)); + } + return out; +} + +// `raw_size` is the payload's uncompressed length, which a real page header +// carries, so knowing it here is not cheating. +void GenericDecompress(const uint8_t* src, size_t csize, uint8_t* dst, size_t raw_size, + Generic g) { + if (g == Generic::kNone) { + std::memcpy(dst, src, csize); + return; + } + if (g == Generic::kZstd1) { + size_t w = ZSTD_decompress(dst, raw_size, src, csize); + if (ZSTD_isError(w) || w != raw_size) { + std::fprintf(stderr, "zstd decompress error\n"); + std::abort(); + } + return; + } + int w = LZ4_decompress_safe(reinterpret_cast(src), + reinterpret_cast(dst), static_cast(csize), + static_cast(raw_size)); + if (w != static_cast(raw_size)) { + std::fprintf(stderr, "lz4 decompress error\n"); + std::abort(); + } +} + +// A codec under test: `build` produces the encoded page payload from the corpus +// (the encode side, training included), `decode` reconstructs the concatenated +// column bytes from a payload buffer and returns the byte count written. +using BuildFn = std::function()>; +using DecodeFn = std::function; + +// Extra slack after the payload copy: OnPair's packed-code reader over-reads up +// to 4 bytes past the last code, and its dictionary decode over-reads one token. +constexpr size_t kPad = 64; + +// `extra_bytes` is charged on top of the compressed payload without being part +// of it. Only used to reproduce the companion benchmark's zstd/lz4 accounting, +// which charges an uncompressed row-length array alongside a frame that holds +// only the value bytes. +Measured RunCodec(const Corpus& c, const std::string& label, Generic g, const BuildFn& build, + const DecodeFn& decode, size_t extra_bytes = 0) { + Measured m; + m.label = label; + + std::vector payload = build(); + std::vector comp = GenericCompress(payload.data(), payload.size(), g); + m.compressed_bytes = comp.size() + extra_bytes; + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + std::vector p = build(); + std::vector cc = GenericCompress(p.data(), p.size(), g); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(cc.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + const size_t out_cap = c.raw_bytes() + op::kDecodePadding + kPad; + { + std::vector scratch(payload.size() + kPad, 0); + std::vector out(out_cap, 0); + GenericDecompress(comp.data(), comp.size(), scratch.data(), payload.size(), g); + size_t w = decode(scratch.data(), payload.size(), out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "%s roundtrip mismatch on %s (w=%zu raw=%zu)\n", m.label.c_str(), + c.name.c_str(), w, c.raw_bytes()); + std::abort(); + } + } + + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector scratch(payload.size() + kPad, 0); + std::vector out(out_cap, 0); + auto t0 = Clock::now(); + GenericDecompress(comp.data(), comp.size(), scratch.data(), payload.size(), g); + size_t w = decode(scratch.data(), payload.size(), out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// Little-endian fixed-width field helpers for the payload headers. The headers +// are a handful of bytes per page and are charged, so no codec gets a free ride +// on self-description. + +void PutU32(std::vector* v, uint32_t x) { + uint8_t b[4]; + std::memcpy(b, &x, 4); + v->insert(v->end(), b, b + 4); +} +uint32_t GetU32(const uint8_t* p) { + uint32_t x; + std::memcpy(&x, p, 4); + return x; +} + +// Append the bit-packed per-row length array that every value-preserving codec +// is charged for (see Corpus::len_array_bytes). +void AppendLengths(const Corpus& c, std::vector* v) { + size_t bits = std::max(1, BitWidth(c.max_row_len())); + std::vector lens(c.n_rows()); + for (size_t i = 0; i < c.n_rows(); ++i) lens[i] = c.offsets[i + 1] - c.offsets[i]; + std::vector packed = op::PackValues(lens.data(), lens.size(), bits); + packed.resize(BitPackedBytes(c.n_rows(), bits)); + v->insert(v->end(), packed.begin(), packed.end()); +} + +// Byte-aligned analog of the above: lengths at the smallest whole-byte width. +void AppendLengthsByteAligned(const Corpus& c, std::vector* v) { + size_t w = std::max(1, (BitWidth(c.max_row_len()) + 7) / 8); + for (size_t i = 0; i < c.n_rows(); ++i) { + uint32_t len = c.offsets[i + 1] - c.offsets[i]; + for (size_t b = 0; b < w; ++b) v->push_back(static_cast(len >> (8 * b))); + } +} + +// Group A codec 1: FSST +// +// Payload: [u32 table_bytes][u32 stream_bytes][symbol table][code stream][lengths] + +std::vector BuildFsstPayload(const Corpus& c) { + size_t n = c.n_rows(); + std::vector lenIn(n); + std::vector strIn(n); + for (size_t i = 0; i < n; ++i) { + lenIn[i] = c.offsets[i + 1] - c.offsets[i]; + strIn[i] = c.bytes.data() + c.offsets[i]; + } + fsst_encoder_t* enc = fsst_create(n, lenIn.data(), strIn.data(), 0); + + size_t out_cap = 7 * n + 2 * c.raw_bytes() + 16; // fsst.h bound: 7 + 2*len per string + std::vector stream(out_cap); + std::vector lenOut(n); + std::vector strOut(n); + size_t done = fsst_compress(enc, n, lenIn.data(), strIn.data(), out_cap, stream.data(), + lenOut.data(), strOut.data()); + if (done != n) { + std::fprintf(stderr, "FSST: only compressed %zu/%zu rows\n", done, n); + std::abort(); + } + size_t total = 0; + for (size_t i = 0; i < n; ++i) total += lenOut[i]; + + unsigned char table[FSST_MAXHEADER]; + size_t table_bytes = fsst_export(enc, table); + fsst_destroy(enc); + + std::vector v; + v.reserve(8 + table_bytes + total + c.len_array_bytes()); + PutU32(&v, static_cast(table_bytes)); + PutU32(&v, static_cast(total)); + v.insert(v.end(), table, table + table_bytes); + v.insert(v.end(), stream.begin(), stream.begin() + total); + AppendLengths(c, &v); + return v; +} + +size_t DecodeFsstPayload(const uint8_t* p, size_t /*size*/, uint8_t* out, size_t out_cap) { + size_t table_bytes = GetU32(p); + size_t stream_bytes = GetU32(p + 4); + fsst_decoder_t dec; + fsst_import(&dec, p + 8); + return fsst_decompress(&dec, stream_bytes, const_cast(p + 8 + table_bytes), + out_cap, out); +} + +// Group A codec 2: OnPair, bit-packed (the layout Tables 1-3 report) +// +// Payload: [u32 tokens][u32 dict_bytes][u32 codes][u8 code_bits][u8 off_bits][pad*2] +// [dictionary blob][packed dict offsets][packed codes][lengths] + +// Choose the dictionary budget the way the companion benchmark's OnPair-auto +// does: train at every width in 9..16 and keep the one that stores least. +size_t OnPairSize(const op::Column& col, const Corpus& c) { + size_t db = col.dict.logical_bytes(); + return db + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(db))) + + BitPackedBytes(col.codes.size(), IndexBits(col.dict.num_tokens())) + + c.len_array_bytes(); +} + +uint8_t PickOnPairBits(const Corpus& c, double threshold) { + uint8_t best_bits = 9; + size_t best_sz = SIZE_MAX; + for (uint8_t b = 9; b <= 16; ++b) { + op::Config cfg{b, threshold, 42}; + op::Column col = + op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), c.n_rows(), cfg); + size_t sz = OnPairSize(col, c); + if (sz < best_sz) { + best_sz = sz; + best_bits = b; + } + } + return best_bits; +} + +constexpr size_t kOnPairHeader = 16; + +std::vector BuildOnPairPayload(const Corpus& c, const op::Config& cfg, + bool byte_aligned) { + op::Column col = + op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), c.n_rows(), cfg); + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = byte_aligned ? 16 : IndexBits(col.dict.num_tokens()); + size_t off_bits = byte_aligned ? 32 : std::max(1, BitWidth(dict_bytes)); + + std::vector v; + v.reserve(kOnPairHeader + dict_bytes + col.codes.size() * 2 + c.len_array_bytes()); + PutU32(&v, static_cast(col.dict.num_tokens())); + PutU32(&v, static_cast(dict_bytes)); + PutU32(&v, static_cast(col.codes.size())); + v.push_back(static_cast(code_bits)); + v.push_back(static_cast(off_bits)); + v.push_back(0); + v.push_back(0); + + v.insert(v.end(), col.dict.bytes.begin(), col.dict.bytes.begin() + dict_bytes); + + std::vector offs(col.dict.offsets.begin(), col.dict.offsets.end()); + std::vector packed_offs = op::PackValues(offs.data(), offs.size(), off_bits); + packed_offs.resize(BitPackedBytes(offs.size(), off_bits)); + v.insert(v.end(), packed_offs.begin(), packed_offs.end()); + + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed_codes = op::PackValues(cw.data(), cw.size(), code_bits); + packed_codes.resize(BitPackedBytes(cw.size(), code_bits)); + v.insert(v.end(), packed_codes.begin(), packed_codes.end()); + + if (byte_aligned) { + AppendLengthsByteAligned(c, &v); + } else { + AppendLengths(c, &v); + } + return v; +} + +// Rebuilds the dictionary from the payload (a reader must, since the decoder +// needs read-padded token bytes and materialized offsets) and decodes the packed +// code stream in place. +size_t DecodeOnPairPayload(const uint8_t* p, size_t /*size*/, uint8_t* out) { + size_t num_tokens = GetU32(p); + size_t dict_bytes = GetU32(p + 4); + size_t num_codes = GetU32(p + 8); + size_t code_bits = p[12]; + size_t off_bits = p[13]; + + const uint8_t* dict_blob = p + kOnPairHeader; + const uint8_t* packed_offs = dict_blob + dict_bytes; + size_t offs_bytes = BitPackedBytes(num_tokens + 1, off_bits); + const uint8_t* packed_codes = packed_offs + offs_bytes; + + op::CompactDictionary dict; + dict.bytes.resize(dict_bytes + op::kDecodePadding, 0); + std::memcpy(dict.bytes.data(), dict_blob, dict_bytes); + dict.offsets.resize(num_tokens + 1); + for (size_t t = 0; t <= num_tokens; ++t) { + dict.offsets[t] = op::GetBits(packed_offs, t * off_bits, off_bits); + } + dict.RecomputeMaxTokenLen(); + + return op::DecompressPacked(dict, packed_codes, num_codes, code_bits, out); +} + +// Group A codec 3: dictionary-encode, then OnPair the distinct values. +// +// The layout a columnar format uses for repetitive columns: bit-packed references +// into the distinct set, with OnPair run over only the distinct values. Measured +// here as well as in the companion benchmark so that a decode number for it can +// sit in the same table as the Parquet-native pages, which are 4.6-7.5% slower in +// this binary. +// +// Payload: [u32 distinct][u32 rows][u32 distinct_bytes][u8 ref_bits][u8 dlen_bits] +// [pad*2][OnPair payload of the distinct set][packed distinct lengths] +// [packed row references] +// +// Note what is NOT charged: the per-row length array. Row lengths are recovered +// from a row's reference plus the distinct-value lengths, so charging both would +// count boundaries twice. This matches the companion benchmark's accounting. + +// Allocation-free byte-range hash, 8 bytes per step plus a tail and a final +// avalanche -- cheaper than byte-at-a-time for the short values that dominate +// low-cardinality columns. Same function the companion benchmark uses, so both +// binaries build the identical distinct set in the identical order. +inline uint64_t HashBytes(const uint8_t* p, size_t len) { + uint64_t h = 0x9E3779B97F4A7C15ull ^ (static_cast(len) * 0xff51afd7ed558ccdull); + size_t i = 0; + for (; i + 8 <= len; i += 8) { + uint64_t w; + std::memcpy(&w, p + i, 8); + h = (h ^ w) * 0x100000001b3ull; + } + if (i < len) { + uint64_t w = 0; + std::memcpy(&w, p + i, len - i); + h = (h ^ w) * 0x100000001b3ull; + } + h ^= h >> 29; + h *= 0xbf58476d1ce4e5b9ull; + h ^= h >> 32; + return h; +} + +struct DedupSet { + std::vector bytes; // distinct values, concatenated in first-seen order + std::vector offsets; // n_distinct + 1 + std::vector refs; // one per row + size_t n_distinct = 0; +}; + +// Open-addressing (linear-probe) table keyed on the row bytes, assigning ids in +// first-seen order -- the same distinct set and order an unordered_map would +// give, so ratios match, without per-key allocation. +DedupSet BuildDedupSet(const Corpus& c) { + size_t n = c.n_rows(); + DedupSet d; + size_t cap = 1; + while (cap < n * 2) cap <<= 1; // power of two, <=50% load + const uint32_t kEmpty = 0xFFFFFFFFu; + std::vector table(cap, kEmpty); + uint64_t mask = cap - 1; + d.offsets.push_back(0); + d.refs.resize(n); + uint32_t n_distinct = 0; + for (size_t i = 0; i < n; ++i) { + const uint8_t* row = c.bytes.data() + c.offsets[i]; + size_t len = c.offsets[i + 1] - c.offsets[i]; + uint64_t slot = HashBytes(row, len) & mask; + uint32_t id; + for (;;) { + uint32_t cur = table[slot]; + if (cur == kEmpty) { + id = n_distinct++; + table[slot] = id; + d.bytes.insert(d.bytes.end(), row, row + len); + d.offsets.push_back(static_cast(d.bytes.size())); + break; + } + size_t off = d.offsets[cur]; + size_t clen = d.offsets[cur + 1] - off; + if (clen == len && std::memcmp(d.bytes.data() + off, row, len) == 0) { + id = cur; + break; + } + slot = (slot + 1) & mask; // linear probe + } + d.refs[i] = id; + } + d.n_distinct = n_distinct; + return d; +} + +constexpr size_t kDictOnPairHeader = 16; + +std::vector BuildDictOnPairPayload(const Corpus& c, const op::Config& cfg) { + DedupSet d = BuildDedupSet(c); + op::Column col = op::Compress(d.bytes.data(), d.bytes.size(), d.offsets.data(), + d.n_distinct, cfg); + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t off_bits = std::max(1, BitWidth(dict_bytes)); + + size_t dmax = 0; + for (size_t j = 0; j + 1 < d.offsets.size(); ++j) + dmax = std::max(dmax, d.offsets[j + 1] - d.offsets[j]); + size_t dlen_bits = std::max(1, BitWidth(dmax)); + size_t ref_bits = IndexBits(d.n_distinct); + + std::vector v; + PutU32(&v, static_cast(d.n_distinct)); + PutU32(&v, static_cast(c.n_rows())); + PutU32(&v, static_cast(d.bytes.size())); + v.push_back(static_cast(ref_bits)); + v.push_back(static_cast(dlen_bits)); + v.push_back(0); + v.push_back(0); + + PutU32(&v, static_cast(col.dict.num_tokens())); + PutU32(&v, static_cast(dict_bytes)); + PutU32(&v, static_cast(col.codes.size())); + v.push_back(static_cast(code_bits)); + v.push_back(static_cast(off_bits)); + v.push_back(0); + v.push_back(0); + + v.insert(v.end(), col.dict.bytes.begin(), col.dict.bytes.begin() + dict_bytes); + + std::vector offs(col.dict.offsets.begin(), col.dict.offsets.end()); + std::vector packed_offs = op::PackValues(offs.data(), offs.size(), off_bits); + packed_offs.resize(BitPackedBytes(offs.size(), off_bits)); + v.insert(v.end(), packed_offs.begin(), packed_offs.end()); + + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed_codes = op::PackValues(cw.data(), cw.size(), code_bits); + packed_codes.resize(BitPackedBytes(cw.size(), code_bits)); + v.insert(v.end(), packed_codes.begin(), packed_codes.end()); + + std::vector dlens(d.n_distinct); + for (size_t j = 0; j < d.n_distinct; ++j) dlens[j] = d.offsets[j + 1] - d.offsets[j]; + std::vector packed_dlens = op::PackValues(dlens.data(), dlens.size(), dlen_bits); + packed_dlens.resize(BitPackedBytes(dlens.size(), dlen_bits)); + v.insert(v.end(), packed_dlens.begin(), packed_dlens.end()); + + std::vector packed_refs = op::PackValues(d.refs.data(), d.refs.size(), ref_bits); + packed_refs.resize(BitPackedBytes(d.refs.size(), ref_bits)); + v.insert(v.end(), packed_refs.begin(), packed_refs.end()); + return v; +} + +// The dedup cascade's own budget sweep. PickOnPairBits cannot be reused: it scores +// OnPairSize over the whole column, whereas this payload trains OnPair over the +// *distinct set* and then adds packed references and distinct-value lengths, so +// the width that stores the column least is not the width that stores this least. +// +// It scores the assembled payload rather than a size formula on purpose -- the +// payload has five sections, and a parallel formula would drift from +// BuildDictOnPairPayload the first time either one changed. The build is wasted +// work, but this runs in per-corpus setup, not in the timed region. +uint8_t PickDictOnPairBits(const Corpus& c, double threshold) { + uint8_t best_bits = 9; + size_t best_sz = SIZE_MAX; + for (uint8_t b = 9; b <= 16; ++b) { + size_t sz = BuildDictOnPairPayload(c, op::Config{b, threshold, 42}).size(); + if (sz < best_sz) { + best_sz = sz; + best_bits = b; + } + } + return best_bits; +} + +// Materialize the distinct values (one OnPair decode), rebuild their offsets from +// the packed length array, then gather rows by unpacking each reference. Values of +// <=16 bytes take one branchless 128-bit store; `scratch` and `out` are padded. +// +// Unlike the companion benchmark, the offset rebuild is inside the timed region: +// this payload is self-describing, so a reader really does pay it. The difference +// is a prefix sum over the distinct set, which is negligible where dedup wins and +// only matters on all-distinct columns, where dedup loses regardless. +size_t DecodeDictOnPairPayload(const uint8_t* p, uint8_t* out, + std::vector* scratch) { + size_t n_distinct = GetU32(p); + size_t n_rows = GetU32(p + 4); + size_t distinct_bytes = GetU32(p + 8); + size_t ref_bits = p[12]; + size_t dlen_bits = p[13]; + + const uint8_t* q = p + kDictOnPairHeader; + size_t num_tokens = GetU32(q); + size_t dict_bytes = GetU32(q + 4); + size_t num_codes = GetU32(q + 8); + size_t code_bits = q[12]; + size_t off_bits = q[13]; + + const uint8_t* dict_blob = q + kOnPairHeader; + const uint8_t* packed_offs = dict_blob + dict_bytes; + const uint8_t* packed_codes = packed_offs + BitPackedBytes(num_tokens + 1, off_bits); + const uint8_t* packed_dlens = packed_codes + BitPackedBytes(num_codes, code_bits); + const uint8_t* packed_refs = packed_dlens + BitPackedBytes(n_distinct, dlen_bits); + + op::CompactDictionary dict; + dict.bytes.resize(dict_bytes + op::kDecodePadding, 0); + std::memcpy(dict.bytes.data(), dict_blob, dict_bytes); + dict.offsets.resize(num_tokens + 1); + for (size_t t = 0; t <= num_tokens; ++t) { + dict.offsets[t] = op::GetBits(packed_offs, t * off_bits, off_bits); + } + dict.RecomputeMaxTokenLen(); + + if (scratch->size() < distinct_bytes + op::kDecodePadding) { + scratch->assign(distinct_bytes + op::kDecodePadding, 0); + } + op::DecompressPacked(dict, packed_codes, num_codes, code_bits, scratch->data()); + + std::vector doff(n_distinct + 1); + doff[0] = 0; + for (size_t j = 0; j < n_distinct; ++j) { + doff[j + 1] = doff[j] + op::GetBits(packed_dlens, j * dlen_bits, dlen_bits); + } + + const uint8_t* dbuf = scratch->data(); + size_t w = 0, bp = 0; + for (size_t i = 0; i < n_rows; ++i) { + uint32_t id = op::GetBits(packed_refs, bp, ref_bits); + bp += ref_bits; + size_t off = doff[id]; + size_t len = doff[id + 1] - off; + if (len <= 16) { + std::memcpy(out + w, dbuf + off, 16); + } else { + std::memcpy(out + w, dbuf + off, len); + } + w += len; + } + return w; +} + +// Group B: Parquet's own byte-array encodings, through the real encoder/decoder. + +std::shared_ptr ByteArrayDescr() { + auto node = parquet::schema::PrimitiveNode::Make("ba", parquet::Repetition::REQUIRED, + parquet::Type::BYTE_ARRAY); + return std::make_shared(node, /*max_definition_level=*/0, + /*max_repetition_level=*/0); +} + +std::vector CorpusValues(const Corpus& c) { + std::vector vals(c.n_rows()); + for (size_t i = 0; i < c.n_rows(); ++i) { + vals[i] = parquet::ByteArray(c.offsets[i + 1] - c.offsets[i], c.bytes.data() + c.offsets[i]); + } + return vals; +} + +// Copy the decoded ByteArrays into one contiguous buffer. PLAIN and +// DELTA_LENGTH_BYTE_ARRAY hand back pointers into the page, so without this a +// "decode" would be pointer arithmetic and the comparison meaningless. +size_t Gather(const std::vector& vals, uint8_t* out) { + size_t w = 0; + for (const parquet::ByteArray& v : vals) { + std::memcpy(out + w, v.ptr, v.len); + w += v.len; + } + return w; +} + +// PLAIN / DELTA_LENGTH_BYTE_ARRAY / DELTA_BYTE_ARRAY: a single self-describing +// buffer, so nothing is added to it -- these encodings carry their own lengths. +std::vector BuildPqPayload(const Corpus& c, + const std::vector& vals, + parquet::Encoding::type e) { + auto enc = parquet::MakeTypedEncoder(e); + enc->Put(vals.data(), static_cast(c.n_rows())); + auto buf = enc->FlushValues(); + return std::vector(buf->data(), buf->data() + buf->size()); +} + +size_t DecodePqPayload(const Corpus& c, parquet::Encoding::type e, const uint8_t* p, + size_t size, uint8_t* out) { + int n = static_cast(c.n_rows()); + auto dec = parquet::MakeTypedDecoder(e); + dec->SetData(n, p, static_cast(size)); + std::vector vals(n); + int got = dec->Decode(vals.data(), n); + if (got != n) { + std::fprintf(stderr, "parquet decode short read (%d of %d)\n", got, n); + std::abort(); + } + return Gather(vals, out); +} + +// RLE_DICTIONARY is two streams -- a dictionary page and an index page -- so the +// payload concatenates them behind a header, and both are charged. +// +// Payload: [u32 dict_bytes][u32 idx_bytes][u32 num_entries][u32 dict page encoding] +// [dict page][indices] +// +// `dict_page_enc` is PLAIN for the RLE_DICTIONARY rows, which is the only thing a +// writer may emit: a BYTE_ARRAY dictionary page is PLAIN-encoded by spec. Passing +// anything else measures a *hypothetical* format change -- the fair opponent for +// replacing the dictionary page with an OnPair blob, since both ask the same +// question of the spec. It is stored in the header rather than threaded through +// the decode lambda so the payload stays self-describing; the word it occupies +// was already there as padding, so the RLE_DICTIONARY sizes do not move. +constexpr size_t kDictHeader = 16; + +std::vector BuildDictPayload(const Corpus& c, + const std::vector& vals, + const parquet::ColumnDescriptor* descr, + parquet::Encoding::type dict_page_enc) { + auto base = parquet::MakeEncoder(parquet::Type::BYTE_ARRAY, parquet::Encoding::PLAIN, + /*use_dictionary=*/true, descr); + auto* enc = dynamic_cast*>(base.get()); + auto* dict_enc = dynamic_cast*>(base.get()); + enc->Put(vals.data(), static_cast(c.n_rows())); + + size_t dict_bytes = static_cast(dict_enc->dict_encoded_size()); + std::vector dict_page(dict_bytes); + dict_enc->WriteDict(dict_page.data()); + int num_entries = dict_enc->num_entries(); + + std::vector idx(static_cast(enc->EstimatedDataEncodedSize()) + 16); + int idx_bytes = dict_enc->WriteIndices(idx.data(), static_cast(idx.size())); + if (idx_bytes <= 0) { + std::fprintf(stderr, "RLE_DICTIONARY: WriteIndices failed\n"); + std::abort(); + } + + // Re-encode the dictionary entries under another encoding. The detour through + // PLAIN is unavoidable: WriteDict is the only public way to get the entries in + // id order, and that order has to survive or the index stream stops matching. + // It makes this variant's *encode* throughput pessimistic -- a writer for such + // a format would encode the entries once -- but leaves ratio and decode exact. + if (dict_page_enc != parquet::Encoding::PLAIN) { + auto pd = parquet::MakeTypedDecoder(parquet::Encoding::PLAIN); + pd->SetData(num_entries, dict_page.data(), static_cast(dict_bytes)); + std::vector entries(num_entries); + if (pd->Decode(entries.data(), num_entries) != num_entries) { + std::fprintf(stderr, "dictionary page re-encode: short read\n"); + std::abort(); + } + auto re = parquet::MakeTypedEncoder(dict_page_enc); + re->Put(entries.data(), num_entries); + auto buf = re->FlushValues(); + dict_page.assign(buf->data(), buf->data() + buf->size()); + dict_bytes = dict_page.size(); + } + + std::vector v; + v.reserve(kDictHeader + dict_bytes + static_cast(idx_bytes)); + PutU32(&v, static_cast(dict_bytes)); + PutU32(&v, static_cast(idx_bytes)); + PutU32(&v, static_cast(num_entries)); + PutU32(&v, static_cast(dict_page_enc)); + v.insert(v.end(), dict_page.begin(), dict_page.end()); + v.insert(v.end(), idx.begin(), idx.begin() + idx_bytes); + return v; +} + +size_t DecodeDictPayload(const Corpus& c, const parquet::ColumnDescriptor* descr, + const uint8_t* p, size_t /*size*/, uint8_t* out) { + size_t dict_bytes = GetU32(p); + size_t idx_bytes = GetU32(p + 4); + int num_entries = static_cast(GetU32(p + 8)); + auto dict_page_enc = static_cast(GetU32(p + 12)); + const uint8_t* dict_page = p + kDictHeader; + const uint8_t* idx = dict_page + dict_bytes; + + // SetDict below takes any TypedDecoder, so the dictionary page's encoding is + // free to vary while the index path stays identical. + auto dict_dec = parquet::MakeTypedDecoder(dict_page_enc); + dict_dec->SetData(num_entries, dict_page, static_cast(dict_bytes)); + + int n = static_cast(c.n_rows()); + auto dec = parquet::MakeDictDecoder(descr); + dec->SetDict(dict_dec.get()); + dec->SetData(n, idx, static_cast(idx_bytes)); + std::vector vals(n); + int got = dec->Decode(vals.data(), n); + if (got != n) { + std::fprintf(stderr, "RLE_DICTIONARY decode short read (%d of %d)\n", got, n); + std::abort(); + } + return Gather(vals, out); +} + +// zstd / lz4 over the concatenated value bytes, in two accountings. +// +// The companion benchmark charges zstd/lz4 a *separate, uncompressed* bit-packed +// row-length array, because a raw frame of concatenated bytes cannot recover row +// boundaries on its own. A real Parquet page puts the lengths inside the page, so +// the page compressor sees them too -- and on columns of near-constant width that +// array is almost free once compressed. Both are reported here: the split-length +// figure is what Tables 1-3 print, the in-page figure is what Parquet stores. + +std::vector BuildConcatPayload(const Corpus& c, bool with_lengths) { + std::vector v = c.bytes; + if (with_lengths) AppendLengths(c, &v); + return v; +} + +} // namespace + +int main(int argc, char** argv) { + std::string dir = CorpusDir(argc, argv); + std::vector files = CorpusFiles(dir); + if (files.empty()) { + std::fprintf(stderr, "no .txt corpora in %s\n", dir.c_str()); + return 1; + } + auto descr = ByteArrayDescr(); + + std::printf("%-30s %10s %10s %7s %9s %9s\n", "corpus", "rows", "raw MiB", "ratio", + "enc MiB/s", "dec MiB/s"); + std::printf("%s\n", std::string(92, '-').c_str()); + + for (const auto& path : files) { + Corpus c = ReadCorpus(path); + double threshold = ThresholdFor(c.name); + std::vector vals = CorpusValues(c); + const size_t out_cap = c.raw_bytes() + op::kDecodePadding + kPad; + + op::Config cfg{PickOnPairBits(c, threshold), threshold, 42}; + op::Config cfg16{16, threshold, 42}; + op::Config cfg_dop{PickDictOnPairBits(c, threshold), threshold, 42}; + std::vector dop_scratch; + + auto fsst_build = [&] { return BuildFsstPayload(c); }; + auto fsst_decode = [&](const uint8_t* p, size_t s, uint8_t* o) { + return DecodeFsstPayload(p, s, o, out_cap); + }; + auto op_build = [&] { return BuildOnPairPayload(c, cfg, /*byte_aligned=*/false); }; + auto op16_build = [&] { return BuildOnPairPayload(c, cfg16, /*byte_aligned=*/false); }; + auto opba_build = [&] { return BuildOnPairPayload(c, cfg, /*byte_aligned=*/true); }; + auto op_decode = [&](const uint8_t* p, size_t s, uint8_t* o) { + return DecodeOnPairPayload(p, s, o); + }; + auto dop_build = [&] { return BuildDictOnPairPayload(c, cfg16); }; + auto dop_auto_build = [&] { return BuildDictOnPairPayload(c, cfg_dop); }; + auto dop_decode = [&](const uint8_t* p, size_t /*s*/, uint8_t* o) { + return DecodeDictOnPairPayload(p, o, &dop_scratch); + }; + auto concat_build = [&] { return BuildConcatPayload(c, /*with_lengths=*/true); }; + auto bytes_build = [&] { return BuildConcatPayload(c, /*with_lengths=*/false); }; + auto concat_decode = [&](const uint8_t* p, size_t /*s*/, uint8_t* o) { + std::memcpy(o, p, c.raw_bytes()); + return c.raw_bytes(); + }; + auto pq = [&](parquet::Encoding::type e) { + return std::make_pair(BuildFn([&, e] { return BuildPqPayload(c, vals, e); }), + DecodeFn([&, e](const uint8_t* p, size_t s, uint8_t* o) { + return DecodePqPayload(c, e, p, s, o); + })); + }; + auto dict_of = [&](parquet::Encoding::type dpe) { + return std::make_pair( + BuildFn([&, dpe] { return BuildDictPayload(c, vals, descr.get(), dpe); }), + DecodeFn([&](const uint8_t* p, size_t s, uint8_t* o) { + return DecodeDictPayload(c, descr.get(), p, s, o); + })); + }; + auto dict_plain = dict_of(parquet::Encoding::PLAIN); + auto dict_dlba = dict_of(parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY); + const BuildFn& dict_build = dict_plain.first; + const DecodeFn& dict_decode = dict_plain.second; + + std::vector ms; + // Reference points, measured here so every ratio in this table shares one + // accounting. The two [split len] rows reproduce the companion benchmark's + // zstd/lz4 columns exactly (uncompressed row-length array charged on the + // side); the two [len in page] rows are the same codecs with the lengths + // inside the compressed page, which is what Parquet actually writes. FSST + // and OnPair alone already match Tables 1-3, since their payload carries the + // bit-packed length array uncompressed either way. + ms.push_back(RunCodec(c, "zstd(1) [split len]", Generic::kZstd1, bytes_build, concat_decode, + c.len_array_bytes())); + ms.push_back(RunCodec(c, "lz4 [split len]", Generic::kLz4, bytes_build, concat_decode, + c.len_array_bytes())); + ms.push_back( + RunCodec(c, "zstd(1) [len in page]", Generic::kZstd1, concat_build, concat_decode)); + ms.push_back(RunCodec(c, "lz4 [len in page]", Generic::kLz4, concat_build, concat_decode)); + ms.push_back(RunCodec(c, "FSST", Generic::kNone, fsst_build, fsst_decode)); + ms.push_back(RunCodec(c, "OnPair-auto", Generic::kNone, op_build, op_decode)); + ms.push_back(RunCodec(c, "OnPair16", Generic::kNone, op16_build, op_decode)); + ms.push_back(RunCodec(c, "DICT+OnPair", Generic::kNone, dop_build, dop_decode)); + ms.push_back(RunCodec(c, "DICT+OnPair-auto", Generic::kNone, dop_auto_build, dop_decode)); + + // Group A: cascade a generic codec over FSST / OnPair. + ms.push_back(RunCodec(c, "FSST+zstd(1)", Generic::kZstd1, fsst_build, fsst_decode)); + ms.push_back(RunCodec(c, "FSST+lz4", Generic::kLz4, fsst_build, fsst_decode)); + ms.push_back(RunCodec(c, "OnPair-auto+zstd(1)", Generic::kZstd1, op_build, op_decode)); + ms.push_back(RunCodec(c, "OnPair-auto+lz4", Generic::kLz4, op_build, op_decode)); + ms.push_back(RunCodec(c, "OnPair-bytealign", Generic::kNone, opba_build, op_decode)); + ms.push_back( + RunCodec(c, "OnPair-bytealign+zstd(1)", Generic::kZstd1, opba_build, op_decode)); + + // Group B: Parquet's own byte-array encodings, alone and cascaded. + struct PqCase { + const char* label; + parquet::Encoding::type enc; + }; + for (const PqCase& pc : + {PqCase{"PLAIN", parquet::Encoding::PLAIN}, + PqCase{"DELTA_LENGTH_BYTE_ARRAY", parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY}, + PqCase{"DELTA_BYTE_ARRAY", parquet::Encoding::DELTA_BYTE_ARRAY}}) { + auto fns = pq(pc.enc); + for (Generic g : {Generic::kNone, Generic::kZstd1, Generic::kLz4}) { + ms.push_back(RunCodec(c, pc.label + std::string(GenericSuffix(g)), g, fns.first, + fns.second)); + } + } + for (Generic g : {Generic::kNone, Generic::kZstd1, Generic::kLz4}) { + ms.push_back(RunCodec(c, "RLE_DICTIONARY" + std::string(GenericSuffix(g)), g, dict_build, + dict_decode)); + } + // A dictionary page carrying DELTA_LENGTH_BYTE_ARRAY instead of PLAIN. Not a + // page any writer can emit -- the spec fixes BYTE_ARRAY dictionary pages at + // PLAIN -- so these two rows measure a format change, and are the opponent + // DICT+OnPair deserves: both replace the dictionary page's encoding and leave + // the index stream alone. Encode throughput here is pessimistic; see + // BuildDictPayload. + for (Generic g : {Generic::kNone, Generic::kZstd1}) { + ms.push_back(RunCodec(c, "DICT+DLBA" + std::string(GenericSuffix(g)), g, dict_dlba.first, + dict_dlba.second)); + } + + std::printf("%-30s %10zu %10.2f\n", c.name.c_str(), c.n_rows(), Mib(c.raw_bytes())); + auto ratio = [&](const Measured& m) { + return static_cast(c.raw_bytes()) / static_cast(m.compressed_bytes); + }; + for (const Measured& m : ms) { + std::printf(" %-32s %10.2f %8.3fx %9.1f %9.1f\n", m.label.c_str(), + Mib(m.compressed_bytes), ratio(m), m.encode_mibs, m.decode_mibs); + } + + // The two questions this benchmark exists to answer, stated per corpus: + // what the cascade buys over the encoding alone, and how the best + // Parquet-native page compares with OnPair alone. + auto find = [&](const std::string& label) -> const Measured* { + for (const Measured& m : ms) + if (m.label == label) return &m; + return nullptr; + }; + const Measured* fsst_alone = find("FSST"); + const Measured* fsst_z = find("FSST+zstd(1)"); + const Measured* op_alone = find("OnPair-auto"); + const Measured* op_z = find("OnPair-auto+zstd(1)"); + const Measured* opba_z = find("OnPair-bytealign+zstd(1)"); + std::printf(" -> FSST+zstd vs FSST alone: ratio %+.1f%%, decode %+.1f%%\n", + (ratio(*fsst_z) / ratio(*fsst_alone) - 1.0) * 100.0, + (fsst_z->decode_mibs / fsst_alone->decode_mibs - 1.0) * 100.0); + std::printf(" -> OnPair+zstd vs OnPair alone: ratio %+.1f%%, decode %+.1f%%\n", + (ratio(*op_z) / ratio(*op_alone) - 1.0) * 100.0, + (op_z->decode_mibs / op_alone->decode_mibs - 1.0) * 100.0); + std::printf(" -> OnPair byte-aligned+zstd vs bit-packed OnPair alone: ratio %+.1f%%\n", + (ratio(*opba_z) / ratio(*op_alone) - 1.0) * 100.0); + + const Measured* best_pq = nullptr; + for (const Measured& m : ms) { + if (m.label.rfind("PLAIN", 0) != 0 && m.label.rfind("DELTA", 0) != 0 && + m.label.rfind("RLE_DICTIONARY", 0) != 0) { + continue; + } + if (best_pq == nullptr || m.compressed_bytes < best_pq->compressed_bytes) best_pq = &m; + } + std::printf(" -> best Parquet-native (%s): %.3fx; OnPair alone %+.1f%% vs it\n\n", + best_pq->label.c_str(), ratio(*best_pq), + (ratio(*op_alone) / ratio(*best_pq) - 1.0) * 100.0); + std::fflush(stdout); + } + return 0; +} diff --git a/cpp/src/parquet/onpair/fsst16.cc b/cpp/src/parquet/onpair/fsst16.cc new file mode 100644 index 000000000000..6c348ce3d514 --- /dev/null +++ b/cpp/src/parquet/onpair/fsst16.cc @@ -0,0 +1,478 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/onpair/fsst16.h" + +#include +#include +#include + +namespace parquet::fsst16 { +namespace { + +// The first 256 codes are the literal bytes; learned symbols start here. +constexpr size_t kCodeBase = 256; +// Longest run of one row the sample generator takes in a single bite. The +// reference's constant. +constexpr size_t kSampleLine = 512; + +// The reference's integer hash, used for sample selection and round skipping so +// that both pick the same rows it would. +inline uint64_t FsstHash(uint64_t w) { + const uint64_t p = w * 2971215073ull; + return p ^ (p >> 15); +} + +// Byte-range hash for the symbol indexes below: eight bytes a step with a tail, +// then an avalanche. +inline uint64_t HashBytes(const uint8_t* p, size_t len) { + uint64_t h = 0x9E3779B97F4A7C15ull ^ (static_cast(len) * 0xff51afd7ed558ccdull); + size_t i = 0; + for (; i + 8 <= len; i += 8) { + uint64_t w; + std::memcpy(&w, p + i, 8); + h = (h ^ w) * 0x100000001b3ull; + } + if (i < len) { + uint64_t w = 0; + std::memcpy(&w, p + i, len - i); + h = (h ^ w) * 0x100000001b3ull; + } + h ^= h >> 29; + h *= 0xbf58476d1ce4e5b9ull; + h ^= h >> 32; + return h; +} + +struct Sym { + uint8_t b[kMaxSymbolLen]; + uint8_t len; +}; + +// Concatenation, truncated at the length cap exactly as the reference truncates +// at eight bytes rather than rejecting the pair. +inline Sym Concat(const Sym& a, const Sym& b, size_t cap) { + Sym s; + size_t n = std::min(static_cast(a.len) + b.len, cap); + std::memcpy(s.b, a.b, a.len); + if (n > a.len) std::memcpy(s.b + a.len, b.b, n - a.len); + s.len = static_cast(n); + return s; +} + +// Longest-prefix index over the symbol table. +// +// The reference indexes with a two-byte lookup array plus a three-byte-prefix +// hash, both of which lean on a symbol fitting in one machine word. A 16-byte +// symbol does not fit, so this keeps one open-addressed set per symbol length +// and probes them longest-first, skipping lengths the table has none of. It only +// ever runs over the training sample - tens of KiB, five times - so a few probes +// per position costs far less than constraining what a symbol may be. +class SymbolIndex { + public: + explicit SymbolIndex(const std::vector* syms) : syms_(syms), slots_(kSlots, 0) {} + + void Clear() { + std::fill(slots_.begin(), slots_.end(), 0u); + len_present_ = 0; + max_len_ = 1; + } + + /// False if a symbol with these exact bytes is already indexed. + bool Insert(const Sym& s, uint32_t code) { + size_t i = Probe(s.b, s.len); + if (slots_[i] != 0) return false; + slots_[i] = code + 1; + len_present_ |= uint32_t{1} << s.len; + if (s.len > max_len_) max_len_ = s.len; + return true; + } + + /// Code of the longest symbol that prefixes [p, p+n). Falls back to the + /// literal code for the first byte, which is always resident. + uint32_t Find(const uint8_t* p, size_t n) const { + size_t hi = std::min(max_len_, n); + for (size_t len = hi; len >= 2; --len) { + if (((len_present_ >> len) & 1u) == 0) continue; + size_t i = Probe(p, len); + if (slots_[i] != 0) return slots_[i] - 1; + } + return p[0]; + } + + private: + // Twice the 65536-symbol ceiling, so the set never passes half load and the + // probe chains stay short without any rehashing. + static constexpr size_t kSlots = size_t{1} << 17; + + size_t Probe(const uint8_t* p, size_t len) const { + size_t i = HashBytes(p, len) & (kSlots - 1); + while (slots_[i] != 0) { + const Sym& s = (*syms_)[slots_[i] - 1]; + if (s.len == len && std::memcmp(s.b, p, len) == 0) break; + i = (i + 1) & (kSlots - 1); + } + return i; + } + + const std::vector* syms_; + std::vector slots_; // code + 1, or 0 for empty + uint32_t len_present_ = 0; // bit L set when some symbol has length L + uint32_t max_len_ = 1; +}; + +// Adjacent-pair frequencies, sparse (V1). +// +// Keyed by the two codes packed into one word. Occupied slots are listed as they +// are first written so that clearing between rounds and walking the pairs for +// candidate generation both cost the number of distinct pairs rather than the +// table size. +class PairCounts { + public: + void Reset(size_t expected_pairs) { + size_t want = 1024; + const size_t ceiling = size_t{1} << 22; // grows past this only if a column needs it + while (want < 2 * expected_pairs && want < ceiling) want <<= 1; + if (want > slots_) { + Allocate(want); + } else { + Clear(); + } + } + + void Clear() { + for (size_t i : used_) { + keys_[i] = 0; + vals_[i] = 0; + } + used_.clear(); + } + + void Inc(uint32_t c1, uint32_t c2) { + if (2 * used_.size() >= slots_) Grow(); + const uint64_t key = (static_cast(c1) << 16 | c2) + 1; // 0 marks empty + size_t i = Slot(key); + if (keys_[i] == 0) { + keys_[i] = key; + used_.push_back(i); + } + ++vals_[i]; + } + + const std::vector& used() const { return used_; } + uint32_t left(size_t slot) const { return static_cast((keys_[slot] - 1) >> 16); } + uint32_t right(size_t slot) const { return static_cast((keys_[slot] - 1) & 0xFFFF); } + uint32_t count(size_t slot) const { return vals_[slot]; } + + private: + void Allocate(size_t want) { + slots_ = want; + keys_.assign(slots_, 0); + vals_.assign(slots_, 0); + used_.clear(); + } + + size_t Slot(uint64_t key) const { + size_t i = FsstHash(key) & (slots_ - 1); + while (keys_[i] != 0 && keys_[i] != key) i = (i + 1) & (slots_ - 1); + return i; + } + + // Kept at or below half load so probe chains stay short. + void Grow() { + std::vector old_keys = std::move(keys_); + std::vector old_vals = std::move(vals_); + std::vector old_used = std::move(used_); + Allocate(slots_ * 2); + for (size_t o : old_used) { + const size_t i = Slot(old_keys[o]); + keys_[i] = old_keys[o]; + vals_[i] = old_vals[o]; + used_.push_back(i); + } + } + + size_t slots_ = 0; + std::vector keys_; // packed pair + 1 + std::vector vals_; + std::vector used_; +}; + +struct Cand { + Sym sym; + uint64_t score; +}; + +// Candidate set for one round, deduplicating by symbol bytes and summing the +// scores of duplicates, as the reference's candidate set does. +class CandSet { + public: + void Reset(size_t expected) { + size_t want = 16; + while (want < 2 * expected + 16) want <<= 1; + slots_.assign(want, kEmpty); + cands_.clear(); + } + + void AddOrInc(const Sym& s, uint64_t score) { + const size_t mask = slots_.size() - 1; + size_t i = HashBytes(s.b, s.len) & mask; + while (slots_[i] != kEmpty) { + Cand& c = cands_[slots_[i]]; + if (c.sym.len == s.len && std::memcmp(c.sym.b, s.b, s.len) == 0) { + c.score += score; + return; + } + i = (i + 1) & mask; + } + slots_[i] = static_cast(cands_.size()); + cands_.push_back({s, score}); + } + + std::vector& cands() { return cands_; } + + private: + static constexpr uint32_t kEmpty = 0xFFFFFFFFu; + std::vector slots_; + std::vector cands_; +}; + +class Trainer { + public: + explicit Trainer(const Config& cfg) + : cfg_(cfg), + cap_(std::min(std::max(cfg.max_symbol_len, 2), kMaxSymbolLen)), + index_(&syms_) { + count1_.assign(cfg_.max_symbols, 0); + } + + Tokens Run(const uint8_t* data, const uint32_t* offsets, size_t n) { + BuildSample(data, offsets, n); + size_t sample_bytes = 0; + for (const auto& l : lines_) sample_bytes += l.second; + pairs_.Reset(2 * sample_bytes + 16); + + ResetTable(); + + int64_t best_gain = INT64_MIN; + std::vector best_syms = syms_; + std::vector best_count1 = count1_; + + // Five rounds at sample fractions 8, 38, 68, 98, 128; the last measures the + // table it inherits without proposing a new one. + for (size_t frac = 8;; frac += 30) { + std::fill(count1_.begin(), count1_.end(), 0u); + pairs_.Clear(); + const int64_t gain = CompressCount(frac); + if (gain >= best_gain) { + best_gain = gain; + best_syms = syms_; + best_count1 = count1_; + } + if (frac >= 128) break; + MakeTable(frac); + } + + // Rebuild the winning table from its own single-symbol counts, dropping the + // symbols the winning round never actually used. + syms_ = std::move(best_syms); + count1_ = std::move(best_count1); + pairs_.Clear(); + MakeTable(128); + + return Emit(); + } + + private: + // Sample selection, following the reference: take the whole column when it is + // under the target, otherwise fill the target with randomly chosen runs of + // randomly chosen rows. + void BuildSample(const uint8_t* data, const uint32_t* offsets, size_t n) { + const size_t total = n == 0 ? 0 : offsets[n]; + if (total <= cfg_.sample_target) { + for (size_t i = 0; i < n; ++i) { + if (offsets[i + 1] > offsets[i]) { + lines_.emplace_back(data + offsets[i], offsets[i + 1] - offsets[i]); + } + } + return; + } + + std::vector> spans; // (offset in buf, length) + buf_.reserve(cfg_.sample_target + kSampleLine); + uint64_t rnd = FsstHash(cfg_.seed); + while (buf_.size() < cfg_.sample_target) { + rnd = FsstHash(rnd); + size_t row = rnd % n; + while (offsets[row + 1] == offsets[row]) { + if (++row == n) row = 0; + } + const size_t len = offsets[row + 1] - offsets[row]; + const size_t chunks = 1 + (len - 1) / kSampleLine; + rnd = FsstHash(rnd); + const size_t chunk = kSampleLine * (rnd % chunks); + const size_t take = std::min(len - chunk, kSampleLine); + const uint8_t* src = data + offsets[row] + chunk; + spans.emplace_back(buf_.size(), take); + buf_.insert(buf_.end(), src, src + take); + } + // Resolved after the buffer stops growing, since inserting reallocates it. + for (const auto& sp : spans) lines_.emplace_back(buf_.data() + sp.first, sp.second); + } + + void ResetTable() { + syms_.resize(kCodeBase); + for (size_t i = 0; i < kCodeBase; ++i) { + syms_[i].len = 1; + syms_[i].b[0] = static_cast(i); + } + index_.Clear(); + } + + void AddSym(const Sym& s) { + if (s.len < 2) return; // V4: the literals are already resident + const uint32_t code = static_cast(syms_.size()); + syms_.push_back(s); + if (!index_.Insert(s, code)) syms_.pop_back(); + } + + // Round skipping: the reference's per-row draw in 1..128. + static size_t Rnd128(size_t i, size_t frac) { + return 1 + (FsstHash((i + 1) * frac) & 127); + } + + // Compress the sample with the current table, counting single symbols and + // adjacent pairs, and return the gain: bytes saved against storing the sample + // raw, given that every code costs two bytes. + int64_t CompressCount(size_t frac) { + int64_t gain = 0; + for (size_t i = 0; i < lines_.size(); ++i) { + if (frac < 128 && Rnd128(i, frac) > frac) continue; + const uint8_t* cur = lines_[i].first; + const uint8_t* end = cur + lines_[i].second; + if (cur >= end) continue; + const uint8_t* start = cur; + + uint32_t code1 = index_.Find(cur, end - cur); + cur += syms_[code1].len; + gain += static_cast(syms_[code1].len) - 2; + + for (;;) { + // Not extending this symbol is one option, so count it alone. + ++count1_[code1]; + // Taking just its first byte is the other, unless they are the same. + if (syms_[code1].len != 1) ++count1_[*start]; + + if (cur == end) break; + + start = cur; + const uint32_t code2 = index_.Find(cur, end - cur); + cur += syms_[code2].len; + gain += static_cast(cur - start) - 2; + + if (frac < 128) { // the last round proposes nothing, so counts no pairs + pairs_.Inc(code1, code2); + if (cur - start > 1) pairs_.Inc(code1, *start); + } + code1 = code2; + } + } + return gain; + } + + // Clear the table and refill it from the highest-scoring candidates. + void MakeTable(size_t frac) { + const uint64_t min_count = (5 * frac) / 128; + cands_.Reset(syms_.size() + pairs_.used().size()); + + // Every counted symbol is a candidate to keep. + for (size_t p1 = 0; p1 < syms_.size(); ++p1) { + const uint32_t c1 = count1_[p1]; + if (c1 == 0) continue; + const Sym& s1 = syms_[p1]; + // V4: promoting single bytes is the reference's way of holding its escape + // rate down. Kept so scores match, though a length-1 candidate is never + // admitted here. + const uint64_t cnt = (s1.len == 1 ? 8ull : 1ull) * c1; + if (cnt < min_count) continue; + cands_.AddOrInc(s1, cnt * s1.len); + } + + // Every counted pair is a candidate to merge (V2). The last round proposes + // nothing, matching the reference's refusal to grow symbols there. + if (frac < 128) { + for (size_t slot : pairs_.used()) { + const uint32_t cnt = pairs_.count(slot); + if (cnt < min_count) continue; + const uint32_t p1 = pairs_.left(slot); + if (count1_[p1] == 0) continue; + const Sym& s1 = syms_[p1]; + if (s1.len >= cap_) continue; // cannot be extended + const Sym s3 = Concat(s1, syms_[pairs_.right(slot)], cap_); + cands_.AddOrInc(s3, static_cast(cnt) * s3.len); + } + } + + std::vector& cs = cands_.cands(); + // Highest score first; V5 for the tie-break. + std::sort(cs.begin(), cs.end(), [](const Cand& a, const Cand& b) { + if (a.score != b.score) return a.score > b.score; + if (a.sym.len != b.sym.len) return a.sym.len < b.sym.len; + return std::memcmp(a.sym.b, b.sym.b, a.sym.len) < 0; + }); + + ResetTable(); + for (const Cand& c : cs) { + if (syms_.size() >= cfg_.max_symbols) break; + AddSym(c.sym); + } + } + + Tokens Emit() const { + Tokens t; + t.bytes.reserve(kCodeBase + (syms_.size() - kCodeBase) * cap_); + t.offsets.reserve(syms_.size() + 1); + t.offsets.push_back(0); + for (const Sym& s : syms_) { + t.bytes.insert(t.bytes.end(), s.b, s.b + s.len); + t.offsets.push_back(static_cast(t.bytes.size())); + } + return t; + } + + const Config& cfg_; + const size_t cap_; // effective max symbol length + + std::vector buf_; // sample backing store + std::vector> lines_; // sample rows + + std::vector syms_; // 0..255 literals, then learned symbols + SymbolIndex index_; + std::vector count1_; + PairCounts pairs_; + CandSet cands_; +}; + +} // namespace + +Tokens Train(const uint8_t* bytes, const uint32_t* offsets, size_t num_rows, + const Config& cfg) { + Trainer t(cfg); + return t.Run(bytes, offsets, num_rows); +} + +} // namespace parquet::fsst16 diff --git a/cpp/src/parquet/onpair/fsst16.h b/cpp/src/parquet/onpair/fsst16.h new file mode 100644 index 000000000000..4c7358bbd252 --- /dev/null +++ b/cpp/src/parquet/onpair/fsst16.h @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// FSST's symbol-table training algorithm, lifted to a 16-bit code space. +// +// Reference: P. Boncz, T. Neumann, V. Leis, "FSST: Fast Random Access String +// Compression", VLDB 2020. The vendored reference implementation trains a +// 255-symbol table addressed by one output byte, with one code reserved to +// escape a literal byte. This trains the same way but over a table addressed by +// two output bytes, which is what a like-for-like comparison against a +// 16-bit-code dictionary codec needs. +// +// WHY THIS IS A SEPARATE IMPLEMENTATION, NOT A PARAMETER +// +// The reference cannot be widened in place. Two of its structures are tied to +// the narrow code space: +// +// * The pair-frequency counter is a dense code-by-code matrix. It is a few +// hundred KiB at a 9-bit code space and tens of GiB at a 16-bit one, so a +// 16-bit table needs a sparse counter and a candidate-generation loop that +// walks occupied entries instead of the full square. +// * A symbol is stored in a single 64-bit word, which caps it at 8 bytes. A +// 16-byte symbol needs a wider representation, and the byte-at-a-time +// longest-match index built around that word has to be replaced. +// +// The vendored reference is therefore left untouched, so the 8-bit baseline it +// produces stays exactly what it was. +// +// CODE SPACE +// +// Codes 0..255 are the literal bytes; codes 256 and up are learned symbols. +// This is the natural reading of the reference's own escape mechanism at two +// bytes per code: there, escaping a literal costs a code plus the byte, so the +// trainer works hard to keep literals rare. Here a literal costs one code, the +// same as any symbol, so escapes disappear and the 256 single bytes are simply +// always resident. Every emitted code is the same fixed width, so the output +// size is exactly two bytes times the number of codes, and the trainer's +// objective reduces to emitting as few codes as possible. +// +// TRAINING SHAPE PRESERVED FROM THE REFERENCE +// +// Progressive sampling over five rounds at increasing sample fractions; each +// round compresses the sample with the current table while counting single- +// symbol and adjacent-pair frequencies; candidate symbols are the counted +// symbols plus every counted pair concatenated; a candidate's score is its +// count times its length; single-byte candidates are scored eight times up; +// candidates below a round-scaled minimum count are discarded; the table is +// cleared and refilled from the highest-scoring candidates each round; the +// round with the best measured gain is kept and rebuilt from its own counts at +// the end. +// +// DEVIATIONS, and why each is forced or harmless +// +// V1. Sparse pair counter. Open-addressed rather than a dense matrix, for the +// reason above. Counts are 32-bit and do not saturate, where the +// reference's pair counts saturate at twelve bits. A pair frequent enough +// to saturate is selected either way, so this can only change the +// relative order of two already-selected candidates. +// V2. Candidate generation walks the occupied pair entries rather than nesting +// a right-code loop inside a left-code loop. The set of candidates is the +// same; the order in which they are first seen is not, which matters only +// for ties. +// V3. No terminator byte. The reference picks the least frequent byte as a +// terminator, forces it into every table, and refuses to build a +// multi-byte symbol containing it, so that its match loop can read past +// the end of a string. This bounds-checks its match loop instead, which +// removes the special case entirely. +// V4. Single-byte candidates are scored and ranked but never admitted, since +// the 256 literals are already resident. The eight-times promotion is kept +// so the pair-generation path sees the same scores, but it cannot change +// the table. +// V5. Ties in candidate score are broken by shorter-first then lexicographic +// byte order, rather than by the reference's numeric ordering of the +// symbol's packed word. Both are arbitrary; a total order is all that is +// needed for a deterministic table. +// V6. No code renumbering at the end. The reference renumbers so that its +// most frequent symbols land in the range addressable by a single byte; +// with a fixed two-byte code, code order cannot affect the output size. +// +// The trained table is emitted as a token list, so the tokenizer and decoder of +// the 16-bit dictionary codec it is being compared against can consume it +// directly. That is deliberate: the parsing pass and the decode pass are then +// literally the same code for both, and every difference that remains is a +// difference in how the table was chosen. +// +// NOT a production encoder - this is a benchmark artifact. +// +// Little-endian hosts only. + +#pragma once + +#include +#include +#include + +namespace parquet::fsst16 { + +/// Longest symbol the reference can represent, and the cap this trainer allows +/// as an upper bound on `Config::max_symbol_len`. +constexpr size_t kMaxSymbolLen = 16; + +struct Config { + /// Longest symbol the trainer may build. 8 is the reference's own cap and + /// isolates the effect of the wider code; 16 removes that cap so the only + /// remaining difference from a 16-byte dictionary codec is the training. + int max_symbol_len = 8; + /// Bytes of the column the trainer looks at. The reference's default is 16 + /// KiB regardless of column size. + size_t sample_target = size_t{1} << 14; + /// Table ceiling, counting the 256 resident literals. + size_t max_symbols = size_t{1} << 16; + /// Sample-selection seed. The reference's constant. + uint64_t seed = 4637947; +}; + +/// A trained table as a flat token list: the 256 literals in code order first, +/// then the learned symbols. Token id is the code. +struct Tokens { + std::vector bytes; + std::vector offsets; // length num_tokens + 1 + + size_t num_tokens() const { return offsets.empty() ? 0 : offsets.size() - 1; } +}; + +/// Train a table against (bytes, offsets). `offsets` has length num_rows + 1; +/// row i is bytes[offsets[i]..offsets[i+1]]. +Tokens Train(const uint8_t* bytes, const uint32_t* offsets, size_t num_rows, + const Config& cfg); + +} // namespace parquet::fsst16 diff --git a/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc b/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc new file mode 100644 index 000000000000..70a508de3ad9 --- /dev/null +++ b/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc @@ -0,0 +1,1139 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Standalone C++ comparison of FSST (Arrow PR #48232 vendored codec), a C++ +// OnPair implementation, zstd level 1 and lz4, in one process on identical +// string corpora. Reports compression ratio and encode/decode throughput. +// Corpora are produced by the Rust bench-fsst-onpair/ generator (--dump-corpora). +// +// All codecs run single-threaded; pin the process with `taskset -c 0`, and run +// it twice using the warm second run. +// +// Build (from the Arrow repo root), one line: +// g++ -std=c++17 -O3 -march=native -Icpp/src -Icpp/thirdparty/fsst +// cpp/thirdparty/fsst/libfsst.cpp cpp/thirdparty/fsst/fsst_avx512.cpp +// cpp/src/parquet/onpair/onpair.cc cpp/src/parquet/onpair/prefix_plus.cc +// cpp/src/parquet/onpair/fsst_onpair_benchmark.cc +// /usr/lib64/libzstd.so.1 /usr/lib64/liblz4.so.1 -o /tmp/fsst_onpair_bench +// +// Run: taskset -c 0 /tmp/fsst_onpair_bench +// (corpora_dir defaults to $ONPAIR_BENCH_DIR, then ./bench-fsst-onpair/corpora) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fsst.h" +#include "parquet/onpair/bench_common.h" +#include "parquet/onpair/fsst16.h" +#include "parquet/onpair/onpair.h" +#include "parquet/onpair/prefix_plus.h" + +namespace op = parquet::onpair; +namespace f16 = parquet::fsst16; + +namespace { + +// Corpus / Measured / Mib / Median / BitWidth / IndexBits / BitPackedBytes / +// Clock / the iteration counts / ThresholdFor - shared with cascade_benchmark.cc +// so both binaries account bytes identically. +using namespace bench; // NOLINT(build/namespaces) + +// FSST + +// Compress the whole corpus into one packed buffer; returns compressed bytes, +// per-row lengths, and the serialized symbol-table size. +struct FsstEncoded { + std::vector output; // packed compressed bytes + size_t total = 0; // used bytes in `output` + size_t table_bytes = 0; // fsst_export size (symbol table) +}; + +FsstEncoded FsstEncode(const Corpus& c) { + size_t n = c.n_rows(); + std::vector lenIn(n); + std::vector strIn(n); + for (size_t i = 0; i < n; ++i) { + lenIn[i] = c.offsets[i + 1] - c.offsets[i]; + strIn[i] = c.bytes.data() + c.offsets[i]; + } + fsst_encoder_t* enc = fsst_create(n, lenIn.data(), strIn.data(), 0); + + // Conservative per-string bound from fsst.h: 7 + 2*len. + size_t out_cap = 7 * n + 2 * c.raw_bytes() + 16; + FsstEncoded e; + e.output.resize(out_cap); + std::vector lenOut(n); + std::vector strOut(n); + size_t done = fsst_compress(enc, n, lenIn.data(), strIn.data(), out_cap, e.output.data(), + lenOut.data(), strOut.data()); + if (done != n) { + std::fprintf(stderr, "FSST: only compressed %zu/%zu rows\n", done, n); + std::abort(); + } + e.total = 0; + for (size_t i = 0; i < n; ++i) e.total += lenOut[i]; + + unsigned char table[FSST_MAXHEADER]; + e.table_bytes = fsst_export(enc, table); + fsst_destroy(enc); + return e; +} + +Measured RunFsst(const Corpus& c) { + size_t n = c.n_rows(); + FsstEncoded e = FsstEncode(c); + Measured m; + m.label = "FSST"; + m.compressed_bytes = e.table_bytes + e.total + c.len_array_bytes(); + + // Encode throughput (train + compress). + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + FsstEncoded tmp = FsstEncode(c); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.total) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Rebuild a decoder from the packed stream for decode timing. + std::vector lenIn(n); + std::vector strIn(n); + for (size_t i = 0; i < n; ++i) { + lenIn[i] = c.offsets[i + 1] - c.offsets[i]; + strIn[i] = c.bytes.data() + c.offsets[i]; + } + fsst_encoder_t* enc2 = fsst_create(n, lenIn.data(), strIn.data(), 0); + fsst_decoder_t dec = fsst_decoder(enc2); + fsst_destroy(enc2); + + size_t cap = c.raw_bytes() + 16; + // Correctness: decoding the whole packed stream reconstructs the input. + { + std::vector out(cap); + size_t w = fsst_decompress(&dec, e.total, e.output.data(), cap, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "FSST roundtrip mismatch on %s (w=%zu raw=%zu)\n", c.name.c_str(), w, + c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap); + auto t0 = Clock::now(); + size_t w = fsst_decompress(&dec, e.total, e.output.data(), cap, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// zstd (block-compression baseline; no random access) + +// Compresses the whole concatenated corpus in one frame at the given level. +// Unlike FSST/OnPair this gives no per-row random access - it's a reference for +// what a general-purpose block compressor achieves on the same bytes. +Measured RunZstd(const Corpus& c, int level) { + size_t bound = ZSTD_compressBound(c.raw_bytes()); + std::vector comp(bound); + size_t csize = ZSTD_compress(comp.data(), bound, c.bytes.data(), c.raw_bytes(), level); + if (ZSTD_isError(csize)) { + std::fprintf(stderr, "zstd compress error on %s\n", c.name.c_str()); + std::abort(); + } + Measured m; + m.label = "zstd(" + std::to_string(level) + ")"; + // Add (n+1) u32 row offsets so row recovery is accounted for, as with the + // other codecs (zstd's frame decompresses to concatenated plaintext only). + m.compressed_bytes = csize + c.len_array_bytes(); + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + size_t r = ZSTD_compress(comp.data(), bound, c.bytes.data(), c.raw_bytes(), level); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(r) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + size_t cap = c.raw_bytes() + 16; + { + std::vector out(cap); + size_t w = ZSTD_decompress(out.data(), cap, comp.data(), csize); + if (ZSTD_isError(w) || w != c.raw_bytes() || + std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "zstd roundtrip mismatch on %s\n", c.name.c_str()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap); + auto t0 = Clock::now(); + size_t w = ZSTD_decompress(out.data(), cap, comp.data(), csize); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// lz4 (fast block-compression baseline; no random access) + +Measured RunLz4(const Corpus& c) { + int raw = static_cast(c.raw_bytes()); + int bound = LZ4_compressBound(raw); + std::vector comp(bound); + int csize = LZ4_compress_default(reinterpret_cast(c.bytes.data()), comp.data(), raw, + bound); + if (csize <= 0) { + std::fprintf(stderr, "lz4 compress error on %s\n", c.name.c_str()); + std::abort(); + } + Measured m; + m.label = "lz4"; + m.compressed_bytes = static_cast(csize) + c.len_array_bytes(); // + bit-packed lengths + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + int r = LZ4_compress_default(reinterpret_cast(c.bytes.data()), comp.data(), raw, + bound); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(r) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + int cap = raw + 16; + { + std::vector out(cap); + int w = LZ4_decompress_safe(comp.data(), out.data(), csize, cap); + if (w != raw || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "lz4 roundtrip mismatch on %s\n", c.name.c_str()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap); + auto t0 = Clock::now(); + int w = LZ4_decompress_safe(comp.data(), out.data(), csize, cap); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// OnPair + +Measured RunOnPair(const Corpus& c, uint8_t bits, double threshold, size_t* out_tokens = nullptr, + size_t* out_max_len = nullptr) { + op::Config cfg; + cfg.max_dict_bits = bits; + cfg.threshold_fraction = threshold; + cfg.seed = 42; + size_t n = c.n_rows(); + + op::Column col = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + Measured m; + m.label = "OnPair" + std::to_string(bits); + if (out_tokens != nullptr) *out_tokens = col.dict.num_tokens(); + if (out_max_len != nullptr) *out_max_len = col.dict.max_token_len; + // Realistic bit-packed accounting: codes packed at the true code width for the + // trained dictionary (not a fixed u16), dictionary offsets bit-packed, and the + // shared per-row length array (in place of the OnPair code-offset array). + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t codes = BitPackedBytes(col.codes.size(), code_bits); + size_t dict_offsets = + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(dict_bytes))); + m.compressed_bytes = dict_bytes + dict_offsets + codes + c.len_array_bytes(); + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + op::Column tmp = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.codes.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Decode the *packed* code stream (unpack `code_bits` per code + gather), so + // decode pays the real bit-unpacking cost that the packed ratio implies. + size_t cap = op::DecodedLen(col) + op::kDecodePadding; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), code_bits); + { + std::vector out(cap, 0); + size_t w = op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "OnPair%u packed roundtrip mismatch on %s (w=%zu raw=%zu)\n", bits, + c.name.c_str(), w, c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap, 0); + auto t0 = Clock::now(); + size_t w = op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// Bit-packed size of an OnPair column (dict + bit-packed dict offsets + codes at +// the true code width + the shared per-row length array). +size_t OnPairSize(const op::Column& col, const Corpus& c) { + size_t db = col.dict.logical_bytes(); + return db + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(db))) + + BitPackedBytes(col.codes.size(), IndexBits(col.dict.num_tokens())) + c.len_array_bytes(); +} + +// OnPair with the dictionary bit-width chosen per column: try 9..16 and keep the +// width that minimizes bit-packed size, then report that width's ratio/decode. +// This exhaustive full-column sweep is the reliable way to pick the width. A +// cheap "train on a sub-sample and project to full size" picker does NOT +// reproduce it: training is not scale-invariant (the dynamic-threshold controller +// paces against the input size, so a sub-sample yields a differently *shaped* +// dictionary, not a smaller one), and enlarging the sample doesn't fix it - a +// token-gain curve fitted on a sample inherits the same skew. A cheap picker +// therefore needs a verify-against-the-ceiling fail-safe (train at the chosen +// budget and the ceiling, keep whichever stores less), not blind trust. +Measured RunOnPairAuto(const Corpus& c, double threshold) { + size_t n = c.n_rows(); + uint8_t best_bits = 9; + size_t best_sz = SIZE_MAX; + for (uint8_t b = 9; b <= 16; ++b) { + op::Config cfg{b, threshold, 42}; + op::Column col = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + size_t sz = OnPairSize(col, c); + if (sz < best_sz) { best_sz = sz; best_bits = b; } + } + + op::Config cfg{best_bits, threshold, 42}; + op::Column col = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + Measured m; + // Report the *stored* code width = ceil(log2(tokens trained)), which is what + // determines size. It can be < the budget when training saturates first. + size_t stored_bits = IndexBits(col.dict.num_tokens()); + m.label = "OnPair-auto(" + std::to_string(stored_bits) + "b)"; + m.compressed_bytes = OnPairSize(col, c); + + // Encode throughput at the chosen width (a real encoder adds only a cheap + // one-pass width estimate, not a full re-search, so this is representative). + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + op::Column tmp = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.codes.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Packed decode at the chosen stored width. + size_t cap = op::DecodedLen(col) + op::kDecodePadding; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), stored_bits); + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap, 0); + auto t0 = Clock::now(); + size_t w = op::DecompressPacked(col.dict, packed.data(), col.codes.size(), stored_bits, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// FSST's training algorithm at a 16-bit code space +// +// Same page layout, same parsing pass and same decode kernel as OnPair16 - only +// the table is built differently (see fsst16.h). That is the point of the row: +// with the format and both hot loops held identical, the difference between this +// and OnPair16 is attributable to table construction and nothing else, so it is +// charged by OnPairSize exactly as OnPair16 is. +// +// `max_symbol_len` 8 is FSST's own cap, which isolates the effect of the wider +// code; 16 removes the cap so nothing but the training differs from OnPair16. +Measured RunFsst16(const Corpus& c, int max_symbol_len, size_t sample_target = 0, + const char* suffix = "", size_t* out_tokens = nullptr, + size_t* out_max_len = nullptr) { + size_t n = c.n_rows(); + f16::Config cfg; + cfg.max_symbol_len = max_symbol_len; + if (sample_target != 0) cfg.sample_target = sample_target; + + auto build = [&] { + f16::Tokens t = f16::Train(c.bytes.data(), c.offsets.data(), n, cfg); + return op::CompressWithTokens(c.bytes.data(), c.offsets.data(), n, t.bytes, t.offsets); + }; + + op::Column col = build(); + Measured m; + m.label = "FSST16-" + std::to_string(max_symbol_len) + "B" + suffix; + m.compressed_bytes = OnPairSize(col, c); + if (out_tokens != nullptr) *out_tokens = col.dict.num_tokens(); + if (out_max_len != nullptr) *out_max_len = col.dict.max_token_len; + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + op::Column tmp = build(); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.codes.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t cap = op::DecodedLen(col) + op::kDecodePadding; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), code_bits); + { + std::vector out(cap, 0); + size_t w = + op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "FSST16-%dB packed roundtrip mismatch on %s (w=%zu raw=%zu)\n", + max_symbol_len, c.name.c_str(), w, c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap, 0); + auto t0 = Clock::now(); + size_t w = + op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// dedup-then-OnPair +// +// The layout a real columnar format uses for repetitive columns: encode the +// column as bit-packed references into the set of distinct values, and run +// OnPair over only those distinct values. Removes whole-value repetition (which +// OnPair's <=16-byte substring dictionary can't exploit) before compressing the +// residual substring redundancy. Matches the OnPair README's guidance for +// low-cardinality columns. + +inline size_t CeilLog2(size_t x) { + if (x <= 1) return 1; // need >=1 bit even for a 2-value dictionary + return 64 - static_cast(__builtin_clzll(x - 1)); +} + +// Fast allocation-free byte-range hash, consuming 8 bytes per step (with a +// tail) and a final avalanche - far cheaper than a byte-at-a-time FNV for the +// short strings that dominate low-cardinality columns. +inline uint64_t HashBytes(const uint8_t* p, size_t len) { + uint64_t h = 0x9E3779B97F4A7C15ull ^ (static_cast(len) * 0xff51afd7ed558ccdull); + size_t i = 0; + for (; i + 8 <= len; i += 8) { + uint64_t w; + std::memcpy(&w, p + i, 8); + h = (h ^ w) * 0x100000001b3ull; + } + if (i < len) { + uint64_t w = 0; + std::memcpy(&w, p + i, len - i); + h = (h ^ w) * 0x100000001b3ull; + } + h ^= h >> 29; + h *= 0xbf58476d1ce4e5b9ull; + h ^= h >> 32; + return h; +} + +Measured RunOnPairDedup(const Corpus& c, uint8_t bits, double threshold) { + op::Config cfg; + cfg.max_dict_bits = bits; + cfg.threshold_fraction = threshold; + cfg.seed = 42; + size_t n = c.n_rows(); + + // Build the distinct-value set in first-seen order + per-row references. + // Open-addressing (linear-probe) table keyed on the row bytes, assigning ids + // in first-seen order - same distinct set/order as a std::unordered_map would + // give (so ratios are identical) but without per-key node allocation or the + // std::hash + pointer-chase overhead, which dominated encode. + auto build_dedup = [&](std::vector* d_bytes, std::vector* d_offsets, + std::vector* refs) { + size_t cap = 1; + while (cap < n * 2) cap <<= 1; // power-of-two, <=50% load + const uint32_t kEmpty = 0xFFFFFFFFu; + std::vector table(cap, kEmpty); // slot -> distinct id + uint64_t mask = cap - 1; + d_offsets->push_back(0); + refs->resize(n); + uint32_t n_distinct = 0; + for (size_t i = 0; i < n; ++i) { + const uint8_t* row = c.bytes.data() + c.offsets[i]; + size_t len = c.offsets[i + 1] - c.offsets[i]; + uint64_t slot = HashBytes(row, len) & mask; + uint32_t id; + for (;;) { + uint32_t cur = table[slot]; + if (cur == kEmpty) { // new distinct value + id = n_distinct++; + table[slot] = id; + d_bytes->insert(d_bytes->end(), row, row + len); + d_offsets->push_back(static_cast(d_bytes->size())); + break; + } + size_t off = (*d_offsets)[cur]; + size_t clen = (*d_offsets)[cur + 1] - off; + if (clen == len && std::memcmp(d_bytes->data() + off, row, len) == 0) { + id = cur; // seen before + break; + } + slot = (slot + 1) & mask; // linear probe + } + (*refs)[i] = id; + } + return static_cast(n_distinct); + }; + + std::vector d_bytes; + std::vector d_offsets; + std::vector refs; + size_t n_distinct = build_dedup(&d_bytes, &d_offsets, &refs); + + op::Column col = op::Compress(d_bytes.data(), d_bytes.size(), d_offsets.data(), n_distinct, cfg); + + Measured m; + m.label = "OnPair" + std::to_string(bits) + "-dedup"; + // Realistic bit-packed accounting, applied to the OnPair-encoded distinct set + // (dict + true-width codes + bit-packed dict offsets + a distinct-value length + // array) plus the bit-packed per-row reference (index) column. + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t codes = BitPackedBytes(col.codes.size(), code_bits); + size_t dict_offsets = + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(dict_bytes))); + size_t dmax = 0; + for (size_t j = 0; j + 1 < d_offsets.size(); ++j) + dmax = std::max(dmax, d_offsets[j + 1] - d_offsets[j]); + size_t distinct_len_bytes = BitPackedBytes(n_distinct, std::max(1, BitWidth(dmax))); + size_t onpair_bytes = dict_bytes + dict_offsets + codes + distinct_len_bytes; + size_t refs_bytes = BitPackedBytes(n, IndexBits(n_distinct)); // index column + m.compressed_bytes = onpair_bytes + refs_bytes; + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + std::vector db; + std::vector doff; + std::vector rf; + auto t0 = Clock::now(); + size_t nd = build_dedup(&db, &doff, &rf); + op::Column tmp = op::Compress(db.data(), db.size(), doff.data(), nd, cfg); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.codes.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Decode: materialize distinct values once (OnPair decode), then gather rows + // by reference. dbuf[d_offsets[id]..] holds distinct value `id` (decode + // reproduces the concatenated distinct bytes in id order). + // + // Gather uses a branchless fixed-16-byte copy when the value is <=16 bytes + // (the common case for the low-cardinality columns where dedup shines): one + // 128-bit store instead of a variable-length memcpy dispatch. Safe because the + // OnPair decode buffer is read-padded by kDecodePadding(16) and `out` carries + // 16 bytes of write padding; the cursor advances by the true length so the + // over-store is overwritten by the next row (or absorbed by the pad on the + // last). Values >16 bytes fall back to an exact memcpy. + size_t dlen = op::DecodedLen(col); + size_t cap = c.raw_bytes() + 16; + // Pack the distinct-set code stream and the per-row reference (index) column, + // so decode pays the real unpacking cost the packed sizes imply. + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed_codes = op::PackValues(cw.data(), cw.size(), code_bits); + size_t ref_bits = IndexBits(n_distinct); + std::vector packed_refs = op::PackValues(refs.data(), n, ref_bits); + + // Materialize the distinct values (unpack the OnPair code stream), then gather + // each row by unpacking its reference and copying the referenced value. <=16-byte + // values use one branchless 128-bit store (dbuf/out are 16-byte padded). + auto decode = [&](uint8_t* dbuf, uint8_t* out) -> size_t { + op::DecompressPacked(col.dict, packed_codes.data(), col.codes.size(), code_bits, dbuf); + size_t w = 0, bp = 0; + for (size_t i = 0; i < n; ++i) { + uint32_t id = op::GetBits(packed_refs.data(), bp, ref_bits); + bp += ref_bits; + size_t off = d_offsets[id]; + size_t len = d_offsets[id + 1] - off; + const uint8_t* src = dbuf + off; + if (len <= 16) std::memcpy(out + w, src, 16); + else std::memcpy(out + w, src, len); + w += len; + } + return w; + }; + + { + std::vector dbuf(dlen + op::kDecodePadding, 0), out(cap); + size_t w = decode(dbuf.data(), out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "OnPair%u-dedup packed roundtrip mismatch on %s\n", bits, c.name.c_str()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector dbuf(dlen + op::kDecodePadding, 0), out(cap); + auto t0 = Clock::now(); + size_t w = decode(dbuf.data(), out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// FSST+ / OnPair+ (common-prefix extraction, DICT mode) +// +// Y. L. Alexandre, "FSST+: Enhancing String Compression Through Common Prefix +// Extraction," MSc thesis, CWI, 2025. We evaluate the thesis's "DICT FSST+" +// path (the one it recommends for columnar integration, sec 5.3.5): the column +// is dictionary-encoded (distinct values + bit-packed row references), the +// distinct set is sorted, and prefix extraction is applied to that sorted +// dictionary. Sorting the dictionary is free of row-order concerns because the +// references carry the mapping - which is exactly why the thesis notes the +// within-block-sort limitation "does not apply" to DICT FSST+. +// +// The DP cleaving (prefix_plus::CleaveSorted, thesis sec 5.2.2) is shared. FSST+ +// cleaves the FSST-compressed distinct values (prefix/suffix are spans of the +// compressed stream; the escape-255 guard keeps a symbol whole). OnPair+ cleaves +// raw bytes and OnPair-compresses the {shared prefixes + per-value suffixes} as +// one column - a deviation forced by OnPair emitting codes, not a splittable +// byte stream. Both count a bit-packed row-reference (index) column, matching +// OnPair16-dedup, so ratios are directly comparable. + +// Shared distinct-value builder (first-seen order) + per-row references. Same +// open-addressing table as RunOnPairDedup's local build_dedup. +struct Dedup { + std::vector bytes; + std::vector offsets; // n_distinct + 1 + std::vector refs; // n_rows -> distinct id + size_t n_distinct = 0; +}; + +Dedup BuildDedup(const Corpus& c) { + size_t n = c.n_rows(); + Dedup d; + d.offsets.push_back(0); + d.refs.resize(n); + size_t cap = 1; + while (cap < n * 2) cap <<= 1; + const uint32_t kEmpty = 0xFFFFFFFFu; + std::vector table(cap, kEmpty); + uint64_t mask = cap - 1; + uint32_t n_distinct = 0; + for (size_t i = 0; i < n; ++i) { + const uint8_t* row = c.bytes.data() + c.offsets[i]; + size_t len = c.offsets[i + 1] - c.offsets[i]; + uint64_t slot = HashBytes(row, len) & mask; + uint32_t id; + for (;;) { + uint32_t cur = table[slot]; + if (cur == kEmpty) { + id = n_distinct++; + table[slot] = id; + d.bytes.insert(d.bytes.end(), row, row + len); + d.offsets.push_back(static_cast(d.bytes.size())); + break; + } + size_t off = d.offsets[cur]; + size_t clen = d.offsets[cur + 1] - off; + if (clen == len && std::memcmp(d.bytes.data() + off, row, len) == 0) { id = cur; break; } + slot = (slot + 1) & mask; + } + d.refs[i] = id; + } + d.n_distinct = n_distinct; + return d; +} + +namespace pp = parquet::prefix_plus; + +// ---- FSST+ ---------------------------------------------------------------- + +struct FsstPlusEnc { + std::vector comp; // FSST-compressed distinct values (spans by id) + std::vector comp_off; // distinct id -> byte offset into comp + std::vector comp_len; // distinct id -> compressed length + std::vector table; // fsst_export symbol table + size_t table_bytes = 0; + fsst_decoder_t dec{}; + std::vector order; // sorted rank -> distinct id (by compressed bytes) + std::vector sorted_pos; // distinct id -> sorted rank + pp::Cleaving cl; // over the sorted compressed spans + size_t nd = 0; +}; + +FsstPlusEnc EncodeFsstPlus(const Corpus& c, const Dedup& dd) { + FsstPlusEnc e; + size_t nd = dd.n_distinct; + e.nd = nd; + + std::vector lenIn(nd); + std::vector strIn(nd); + for (size_t i = 0; i < nd; ++i) { + lenIn[i] = dd.offsets[i + 1] - dd.offsets[i]; + strIn[i] = dd.bytes.data() + dd.offsets[i]; + } + fsst_encoder_t* enc = fsst_create(nd, lenIn.data(), strIn.data(), 0); + size_t out_cap = 7 * nd + 2 * dd.bytes.size() + 16; + e.comp.assign(out_cap, 0); + std::vector lenOut(nd); + std::vector strOut(nd); + size_t done = fsst_compress(enc, nd, lenIn.data(), strIn.data(), out_cap, e.comp.data(), + lenOut.data(), strOut.data()); + if (done != nd) { + std::fprintf(stderr, "FSST+ compressed %zu/%zu distinct on %s\n", done, nd, c.name.c_str()); + std::abort(); + } + e.comp_off.resize(nd); + e.comp_len.resize(nd); + for (size_t i = 0; i < nd; ++i) { + e.comp_off[i] = static_cast(strOut[i] - e.comp.data()); + e.comp_len[i] = static_cast(lenOut[i]); + } + unsigned char tbl[FSST_MAXHEADER]; + e.table_bytes = fsst_export(enc, tbl); + e.table.assign(tbl, tbl + e.table_bytes); + e.dec = fsst_decoder(enc); + fsst_destroy(enc); + + // Sort distinct ids by their compressed bytes so shared compressed prefixes + // are adjacent (FSST maps equal inputs to equal compressed forms). + e.order.resize(nd); + for (size_t i = 0; i < nd; ++i) e.order[i] = static_cast(i); + const uint8_t* base = e.comp.data(); + std::sort(e.order.begin(), e.order.end(), [&](uint32_t a, uint32_t b) { + size_t la = e.comp_len[a], lb = e.comp_len[b]; + int cmp = std::memcmp(base + e.comp_off[a], base + e.comp_off[b], std::min(la, lb)); + if (cmp != 0) return cmp < 0; + return la < lb; + }); + e.sorted_pos.resize(nd); + for (size_t k = 0; k < nd; ++k) e.sorted_pos[e.order[k]] = static_cast(k); + + std::vector sptr(nd); + std::vector slen(nd); + for (size_t k = 0; k < nd; ++k) { + sptr[k] = base + e.comp_off[e.order[k]]; + slen[k] = e.comp_len[e.order[k]]; + } + e.cl = pp::CleaveSorted(sptr.data(), slen.data(), nd, pp::kMaxPrefix, /*guard_escape255=*/true); + return e; +} + +// Exact FSST+ stored size (thesis sec 3.1 layout) + bit-packed row references. +size_t FsstPlusSize(const FsstPlusEnc& e, const Corpus& c) { + size_t nd = e.nd; + size_t num_blocks = (nd + pp::kBlockSize - 1) / pp::kBlockSize; + size_t bytes = 2 + 4 * num_blocks + 4; // num_blocks + block_start_offsets[] + data_end_offset + for (size_t bstart = 0; bstart < nd; bstart += pp::kBlockSize) { + size_t bn = std::min(pp::kBlockSize, nd - bstart); + bytes += 1 + 2 * bn; // num_strings + suffix_data_area_offsets[] + for (size_t k = bstart; k < bstart + bn; ++k) { + uint32_t p = e.cl.prefix_len[k]; + uint32_t clen = e.comp_len[e.order[k]]; + bytes += 1; // prefix_length + if (p > 0) bytes += 2; // jump_back_offset + bytes += clen - p; // compressed suffix + if (p > 0 && e.cl.chunk_first[k] == k) bytes += p; // shared prefix, stored once + } + } + bytes += e.table_bytes; + return bytes + BitPackedBytes(c.n_rows(), IndexBits(nd)); +} + +Measured RunFsstPlus(const Corpus& c) { + size_t n = c.n_rows(); + Dedup dd = BuildDedup(c); + FsstPlusEnc e = EncodeFsstPlus(c, dd); + Measured m; + m.label = "FSST+"; + m.compressed_bytes = FsstPlusSize(e, c); + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + Dedup d2 = BuildDedup(c); + FsstPlusEnc e2 = EncodeFsstPlus(c, d2); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(e2.nd) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Decode: reconstruct each sorted distinct value from its stored (prefix once + // + own suffix) compressed form and FSST-decode it, then gather rows by ref. + size_t distinct_total = dd.bytes.size(); + size_t cap = c.raw_bytes() + 16; + const uint8_t* base = e.comp.data(); + auto decode = [&](std::vector& dictbuf, std::vector& voff, + uint8_t* out) -> size_t { + voff.assign(e.nd + 1, 0); + std::vector tmp; + tmp.reserve(pp::kMaxPrefix + 256); + size_t w = 0; + for (size_t k = 0; k < e.nd; ++k) { + uint32_t p = e.cl.prefix_len[k]; + uint32_t rep = e.cl.chunk_first[k]; + uint32_t clen = e.comp_len[e.order[k]]; + const uint8_t* cbytes = base + e.comp_off[e.order[k]]; + tmp.clear(); + if (p > 0) { + const uint8_t* rbytes = base + e.comp_off[e.order[rep]]; + tmp.insert(tmp.end(), rbytes, rbytes + p); + } + tmp.insert(tmp.end(), cbytes + p, cbytes + clen); + size_t dl = fsst_decompress(&e.dec, tmp.size(), tmp.data(), dictbuf.size() - w, + dictbuf.data() + w); + w += dl; + voff[k + 1] = static_cast(w); + } + size_t o = 0; + for (size_t i = 0; i < n; ++i) { + uint32_t k = e.sorted_pos[dd.refs[i]]; + size_t off = voff[k], len = voff[k + 1] - off; + std::memcpy(out + o, dictbuf.data() + off, len); + o += len; + } + return o; + }; + + { + std::vector dictbuf(distinct_total + 32, 0), out(cap); + std::vector voff; + size_t w = decode(dictbuf, voff, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "FSST+ roundtrip mismatch on %s (w=%zu raw=%zu)\n", c.name.c_str(), w, + c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector dictbuf(distinct_total + 32, 0), out(cap); + std::vector voff; + auto t0 = Clock::now(); + size_t w = decode(dictbuf, voff, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// ---- OnPair+ -------------------------------------------------------------- + +struct OnPairPlusEnc { + op::Column col; // {shared prefixes + suffixes} as one column + std::vector piece_off; // num_pieces + 1 byte offsets (also decoded boundaries) + size_t num_pieces = 0; + std::vector order, sorted_pos; // distinct id <-> sorted rank (by raw bytes) + pp::Cleaving cl; // over raw sorted values + std::vector prefix_piece; // sorted rank of a chunk rep -> its prefix piece index + size_t n_prefix_pieces = 0; // suffix piece of sorted value k == n_prefix_pieces + k + size_t nd = 0; +}; + +OnPairPlusEnc EncodeOnPairPlus(const Corpus& c, const Dedup& dd, double threshold) { + (void)c; + OnPairPlusEnc e; + size_t nd = dd.n_distinct; + e.nd = nd; + + const uint8_t* base = dd.bytes.data(); + e.order.resize(nd); + for (size_t i = 0; i < nd; ++i) e.order[i] = static_cast(i); + std::sort(e.order.begin(), e.order.end(), [&](uint32_t a, uint32_t b) { + size_t la = dd.offsets[a + 1] - dd.offsets[a], lb = dd.offsets[b + 1] - dd.offsets[b]; + int cmp = std::memcmp(base + dd.offsets[a], base + dd.offsets[b], std::min(la, lb)); + if (cmp != 0) return cmp < 0; + return la < lb; + }); + e.sorted_pos.resize(nd); + for (size_t k = 0; k < nd; ++k) e.sorted_pos[e.order[k]] = static_cast(k); + + std::vector sptr(nd); + std::vector slen(nd); + for (size_t k = 0; k < nd; ++k) { + uint32_t id = e.order[k]; + sptr[k] = base + dd.offsets[id]; + slen[k] = dd.offsets[id + 1] - dd.offsets[id]; + } + e.cl = pp::CleaveSorted(sptr.data(), slen.data(), nd, pp::kMaxPrefix, /*guard_escape255=*/false); + + // Pieces: each chunk's shared prefix once, then every value's suffix. + std::vector pbytes; + std::vector poff; + poff.push_back(0); + e.prefix_piece.assign(nd, 0xFFFFFFFFu); + uint32_t pc = 0; + for (size_t k = 0; k < nd; ++k) { + if (e.cl.prefix_len[k] > 0 && e.cl.chunk_first[k] == k) { + e.prefix_piece[k] = pc++; + pbytes.insert(pbytes.end(), sptr[k], sptr[k] + e.cl.prefix_len[k]); + poff.push_back(static_cast(pbytes.size())); + } + } + e.n_prefix_pieces = pc; + for (size_t k = 0; k < nd; ++k) { + uint32_t p = e.cl.prefix_len[k]; + pbytes.insert(pbytes.end(), sptr[k] + p, sptr[k] + slen[k]); + poff.push_back(static_cast(pbytes.size())); + } + e.num_pieces = poff.size() - 1; + e.piece_off = std::move(poff); + + op::Config cfg{16, threshold, 42}; + e.col = op::Compress(pbytes.data(), pbytes.size(), e.piece_off.data(), e.num_pieces, cfg); + return e; +} + +// OnPair+ stored size: the shared OnPair model (dict + bit-packed offsets + +// codes for prefixes-once + suffixes), a bit-packed piece-boundary array (in +// place of FSST+'s compressed byte spans), the same per-value prefix_length / +// jump-back overhead and block headers as FSST+, and the row-reference column. +size_t OnPairPlusSize(const OnPairPlusEnc& e, const Corpus& c) { + const op::Column& col = e.col; + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t codes = BitPackedBytes(col.codes.size(), code_bits); + size_t dict_offsets = + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(dict_bytes))); + size_t piece_bound = + BitPackedBytes(e.num_pieces, std::max(1, BitWidth(col.codes.size()))); + size_t nd = e.nd; + size_t num_blocks = (nd + pp::kBlockSize - 1) / pp::kBlockSize; + size_t structural = 2 + 4 * num_blocks + 4; + for (size_t bstart = 0; bstart < nd; bstart += pp::kBlockSize) { + size_t bn = std::min(pp::kBlockSize, nd - bstart); + structural += 1; // num_strings + for (size_t k = bstart; k < bstart + bn; ++k) + structural += 1 + (e.cl.prefix_len[k] > 0 ? 2 : 0); + } + size_t refs = BitPackedBytes(c.n_rows(), IndexBits(nd)); + return dict_bytes + dict_offsets + codes + piece_bound + structural + refs; +} + +Measured RunOnPairPlus(const Corpus& c, double threshold) { + size_t n = c.n_rows(); + Dedup dd = BuildDedup(c); + OnPairPlusEnc e = EncodeOnPairPlus(c, dd, threshold); + Measured m; + m.label = "OnPair+"; + m.compressed_bytes = OnPairPlusSize(e, c); + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + Dedup d2 = BuildDedup(c); + OnPairPlusEnc e2 = EncodeOnPairPlus(c, d2, threshold); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(e2.num_pieces) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Decode: OnPair-materialize all pieces (unpacking the code stream), then for + // each sorted value concatenate its prefix piece + suffix piece, and gather + // rows by reference. + size_t piece_total = op::DecodedLen(e.col); + size_t distinct_total = dd.bytes.size(); + size_t cap = c.raw_bytes() + 16; + size_t code_bits = IndexBits(e.col.dict.num_tokens()); + std::vector cw(e.col.codes.begin(), e.col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), code_bits); + + auto decode = [&](std::vector& piecebuf, std::vector& dictbuf, + std::vector& voff, uint8_t* out) -> size_t { + op::DecompressPacked(e.col.dict, packed.data(), e.col.codes.size(), code_bits, piecebuf.data()); + voff.assign(e.nd + 1, 0); + size_t w = 0; + for (size_t k = 0; k < e.nd; ++k) { + uint32_t p = e.cl.prefix_len[k]; + if (p > 0) { + uint32_t pi = e.prefix_piece[e.cl.chunk_first[k]]; + std::memcpy(dictbuf.data() + w, piecebuf.data() + e.piece_off[pi], p); + w += p; + } + uint32_t sfx = static_cast(e.n_prefix_pieces) + static_cast(k); + size_t soff = e.piece_off[sfx], slen = e.piece_off[sfx + 1] - soff; + std::memcpy(dictbuf.data() + w, piecebuf.data() + soff, slen); + w += slen; + voff[k + 1] = static_cast(w); + } + size_t o = 0; + for (size_t i = 0; i < n; ++i) { + uint32_t k = e.sorted_pos[dd.refs[i]]; + size_t off = voff[k], len = voff[k + 1] - off; + std::memcpy(out + o, dictbuf.data() + off, len); + o += len; + } + return o; + }; + + { + std::vector piecebuf(piece_total + op::kDecodePadding, 0); + std::vector dictbuf(distinct_total + 32, 0), out(cap); + std::vector voff; + size_t w = decode(piecebuf, dictbuf, voff, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "OnPair+ roundtrip mismatch on %s (w=%zu raw=%zu)\n", c.name.c_str(), w, + c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector piecebuf(piece_total + op::kDecodePadding, 0); + std::vector dictbuf(distinct_total + 32, 0), out(cap); + std::vector voff; + auto t0 = Clock::now(); + size_t w = decode(piecebuf, dictbuf, voff, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +} // namespace + +int main(int argc, char** argv) { + std::string dir = CorpusDir(argc, argv); + std::vector files = CorpusFiles(dir); + if (files.empty()) { + std::fprintf(stderr, "no .txt corpora in %s\n", dir.c_str()); + return 1; + } + + // ONPAIR_BENCH_CORE restricts the run to the FSST-vs-OnPair table set: the two + // FSST16 variants next to the FSST8 baseline and the two OnPair rows, with the + // cascade and dedup codecs skipped so a comparison run does not pay for rows it + // does not report. Same binary and therefore the same accounting either way. + const bool core_only = std::getenv("ONPAIR_BENCH_CORE") != nullptr; + + std::printf("%-26s %10s %10s %7s %9s %9s\n", "corpus", "rows", "raw MiB", "ratio", + "enc MiB/s", "dec MiB/s"); + std::printf("%s\n", std::string(90, '-').c_str()); + + for (const auto& path : files) { + Corpus c = ReadCorpus(path); + double threshold = ThresholdFor(c.name); + + Measured fsst = RunFsst(c); + size_t t8 = 0, l8 = 0, t16 = 0, l16 = 0, tfull = 0, lfull = 0; + Measured f16_8 = RunFsst16(c, 8, 0, "", &t8, &l8); + Measured f16_16 = RunFsst16(c, 16, 0, "", &t16, &l16); + // Same 16-byte cap, but trained on 4 MiB instead of the reference's fixed 16 + // KiB. The rows above look at 16 KiB no matter how large the column is, which + // caps how many distinct symbols the trainer can even see evidence for; this + // row separates that sampling choice from the training algorithm itself. 4 MiB + // is 256 times the reference sample and enough to fill a 16-bit table, while + // staying bounded enough to time. + Measured f16_m = RunFsst16(c, 16, size_t{4} << 20, "-4M", &tfull, &lfull); + size_t op16_tokens = 0, op16_max_len = 0; + Measured op16 = RunOnPair(c, 16, threshold, &op16_tokens, &op16_max_len); + Measured opauto = RunOnPairAuto(c, threshold); + + std::printf("%-26s %10zu %10.2f\n", c.name.c_str(), c.n_rows(), Mib(c.raw_bytes())); + auto emit = [&](const Measured& m) { + double ratio = static_cast(c.raw_bytes()) / static_cast(m.compressed_bytes); + std::printf(" %-24s %10s %10.2f %7.3fx %9.1f %9.1f\n", m.label.c_str(), "", + Mib(m.compressed_bytes), ratio, m.encode_mibs, m.decode_mibs); + }; + + if (core_only) { + for (const Measured* m : {&fsst, &f16_8, &f16_16, &f16_m, &op16, &opauto}) emit(*m); + std::printf(" -> tables: FSST16-8B %zu tokens/max %zu, FSST16-16B %zu/%zu, " + "FSST16-16B-4M %zu/%zu, OnPair16 %zu/%zu\n\n", + t8, l8, t16, l16, tfull, lfull, op16_tokens, op16_max_len); + continue; + } + Measured zstd1 = RunZstd(c, 1); + Measured lz4 = RunLz4(c); + Measured op16d = RunOnPairDedup(c, 16, threshold); + Measured fsstp = RunFsstPlus(c); + Measured oppl = RunOnPairPlus(c, threshold); + + for (const Measured* m : + {&fsst, &f16_8, &f16_16, &f16_m, &zstd1, &lz4, &op16, &opauto, &op16d, &fsstp, &oppl}) { + emit(*m); + } + double r_fsst = static_cast(c.raw_bytes()) / fsst.compressed_bytes; + double r_zstd = static_cast(c.raw_bytes()) / zstd1.compressed_bytes; + double r_op16 = static_cast(c.raw_bytes()) / op16.compressed_bytes; + std::printf(" -> OnPair16 vs FSST: ratio %+.1f%%, encode %+.1f%%, decode %+.1f%%\n", + (r_op16 / r_fsst - 1.0) * 100.0, (op16.encode_mibs / fsst.encode_mibs - 1.0) * 100.0, + (op16.decode_mibs / fsst.decode_mibs - 1.0) * 100.0); + std::printf(" -> OnPair16 vs zstd(1): ratio %+.1f%%, encode %+.1f%%, decode %+.1f%%\n", + (r_op16 / r_zstd - 1.0) * 100.0, + (op16.encode_mibs / zstd1.encode_mibs - 1.0) * 100.0, + (op16.decode_mibs / zstd1.decode_mibs - 1.0) * 100.0); + double r_op16d = static_cast(c.raw_bytes()) / op16d.compressed_bytes; + std::printf(" -> OnPair16-dedup vs zstd(1): ratio %+.1f%%, decode %+.1f%%\n", + (r_op16d / r_zstd - 1.0) * 100.0, + (op16d.decode_mibs / zstd1.decode_mibs - 1.0) * 100.0); + double r_fsstp = static_cast(c.raw_bytes()) / fsstp.compressed_bytes; + double r_oppl = static_cast(c.raw_bytes()) / oppl.compressed_bytes; + std::printf(" -> FSST+ vs FSST: ratio %+.1f%%; FSST+ vs zstd(1): ratio %+.1f%%\n", + (r_fsstp / r_fsst - 1.0) * 100.0, (r_fsstp / r_zstd - 1.0) * 100.0); + std::printf(" -> OnPair+ vs OnPair16-dedup: ratio %+.1f%%; OnPair+ vs zstd(1): ratio %+.1f%%\n\n", + (r_oppl / r_op16d - 1.0) * 100.0, (r_oppl / r_zstd - 1.0) * 100.0); + } + return 0; +} diff --git a/cpp/src/parquet/onpair/onpair.cc b/cpp/src/parquet/onpair/onpair.cc new file mode 100644 index 000000000000..a71356a40035 --- /dev/null +++ b/cpp/src/parquet/onpair/onpair.cc @@ -0,0 +1,1326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/onpair/onpair.h" + +#include +#include +#include +#include +#include + +// The decode loop stores exactly one token's length per iteration when a +// predicated store is available, which removes the fixed over-copy entirely. This +// is a compile-time guard on purpose: the portable path below is byte-identical +// and only slower, so a build without SVE loses speed and nothing else. +#if defined(__ARM_FEATURE_SVE) +#include +#endif + +namespace parquet::onpair { +namespace { + +constexpr size_t kBucketPrefixLen = 8; +constexpr size_t kPromoteThreshold = 128; + +// Little-endian packing helpers + +/// Pack the low min(len, data_len, 8) bytes of `data` into a little-endian u64; +/// higher bytes read as zero. +inline uint64_t LoadLeU64(const uint8_t* data, size_t data_len, size_t len) { + size_t n = (len >= kBucketPrefixLen && data_len >= kBucketPrefixLen) + ? kBucketPrefixLen + : std::min(len, data_len); + uint64_t v = 0; + std::memcpy(&v, data, n); // little-endian host + return v; +} + +/// Mask of the low len*8 bits in a u64. +inline uint64_t MaskU64(size_t len) { + return len >= 8 ? ~uint64_t{0} : ((uint64_t{1} << (len * 8)) - 1); +} + +/// Count of matching low bytes between two packed suffixes. +inline size_t MatchingLowBytes(uint64_t x) { + return x == 0 ? 8 : (static_cast(__builtin_ctzll(x)) >> 3); +} + +// Flat u64 -> u32 hash table +// +// The tokenizer probes its lookup tables several times per token, so probe cost +// dominates encode time. std::unordered_map is the wrong shape for that: the +// bucket array load and the node load are dependent, so every miss costs two +// serialized cache misses, and at tens of thousands of tokens neither fits in +// cache. Open addressing with the key and value in one 16-byte slot makes the +// common case a single load. Insert order still decides which of two equal keys +// wins, so swapping this in cannot change the tokenization. + +constexpr uint32_t kFlatEmpty = ~uint32_t{0}; + +inline uint64_t MixU64(uint64_t x) { + x ^= x >> 33; + x *= 0xff51afd7ed558ccdULL; + x ^= x >> 33; + x *= 0xc4ceb9fe1a85ec53ULL; + x ^= x >> 33; + return x; +} + +class FlatU64Map { + public: + FlatU64Map() : slots_(kMinSlots), mask_(kMinSlots - 1) {} + + bool empty() const { return size_ == 0; } + + /// Value for `key`, or kFlatEmpty when absent. + uint32_t Find(uint64_t key) const { + size_t i = MixU64(key) & mask_; + for (;;) { + const Slot& s = slots_[i]; + if (s.val == kFlatEmpty) return kFlatEmpty; + if (s.key == key) return s.val; + i = (i + 1) & mask_; + } + } + + /// Insert `key`, or overwrite the value already stored under it. + void Put(uint64_t key, uint32_t val) { + size_t i = MixU64(key) & mask_; + for (;;) { + Slot& s = slots_[i]; + if (s.val == kFlatEmpty) { + s.key = key; + s.val = val; + ++size_; + // Linear probing degrades sharply past half full; grow well before then. + if (size_ * 2 > slots_.size()) Grow(); + return; + } + if (s.key == key) { + s.val = val; + return; + } + i = (i + 1) & mask_; + } + } + + private: + struct Slot { + uint64_t key = 0; + uint32_t val = kFlatEmpty; + }; + static constexpr size_t kMinSlots = 64; + + void Grow() { + std::vector old(slots_.size() * 2); + old.swap(slots_); + mask_ = slots_.size() - 1; + for (const Slot& s : old) { + if (s.val == kFlatEmpty) continue; + size_t i = MixU64(s.key) & mask_; + while (slots_[i].val != kFlatEmpty) i = (i + 1) & mask_; + slots_[i] = s; + } + } + + std::vector slots_; + size_t mask_; + size_t size_ = 0; +}; + +// Flat pair-frequency counter for the training loop +// +// The trainer touches this once per token boundary, so it sits on the same hot +// path as the matcher and wants the same treatment. Deleting a promoted pair is a +// reset to zero rather than a real erase: a caller cannot tell an absent key from +// a zero count, so the two are equivalent, and it keeps linear probing free of +// tombstones (promotions are also rare - at most one per dictionary entry). + +inline uint32_t MixU32(uint32_t x) { + x ^= x >> 16; + x *= 0x7feb352dU; + x ^= x >> 15; + x *= 0x846ca68bU; + x ^= x >> 16; + return x; +} + +class FlatFreqMap { + public: + FlatFreqMap() : slots_(kMinSlots), mask_(kMinSlots - 1) {} + + /// Saturating increment of `key`'s count (absent == 0), returning the new value. + uint8_t Bump(uint32_t key) { + size_t i = MixU32(key) & mask_; + for (;;) { + Slot& s = slots_[i]; + if (s.count == kEmptyCount) { + s.key = key; + s.count = 1; + ++size_; + if (size_ * 2 > slots_.size()) Grow(); + return 1; + } + if (s.key == key) { + if (s.count < 255) ++s.count; + return static_cast(s.count); + } + i = (i + 1) & mask_; + } + } + + /// Forget `key`'s count. Precondition: Bump(key) was called at least once. + void Reset(uint32_t key) { + size_t i = MixU32(key) & mask_; + for (;;) { + Slot& s = slots_[i]; + if (s.count == kEmptyCount) return; + if (s.key == key) { + s.count = 0; + return; + } + i = (i + 1) & mask_; + } + } + + private: + struct Slot { + uint32_t key = 0; + uint16_t count = kEmptyCount; + }; + static constexpr uint16_t kEmptyCount = 0xFFFF; + static constexpr size_t kMinSlots = 1024; + + void Grow() { + std::vector old(slots_.size() * 2); + old.swap(slots_); + mask_ = slots_.size() - 1; + for (const Slot& s : old) { + if (s.count == kEmptyCount) continue; + size_t i = MixU32(s.key) & mask_; + while (slots_[i].count != kEmptyCount) i = (i + 1) & mask_; + slots_[i] = s; + } + } + + std::vector slots_; + size_t mask_; + size_t size_ = 0; +}; + +// Longest-prefix matcher +// Two-tier index per the paper (sec 3.4.1): a hash map for tokens <=8 bytes, and +// 8-byte-prefix buckets (suffixes sorted descending) for 9..16-byte tokens. +// DEVIATION FROM PAPER (D3): the paper's OnPair16 caps each long bucket at 128 +// suffixes (sec 3.4.4, dropping extras); this port instead promotes an over-full +// bucket to a trie (PROMOTE_THRESHOLD), keeping all suffixes. Also, the paper's +// static parsing phase (sec 3.4.3) finalizes long-pattern lookup with a minimal +// perfect hash; this port keeps an ordinary hash table (the paper notes the +// perfect-hash path is Rust-only). Encode-time behavior only. + +// Prefix filter +// +// One byte per possible two-byte prefix of the data: bit (len-1) is set when some +// token of exactly `len` bytes (2..kBucketPrefixLen) starts with those two bytes, +// and bit 0 when some token longer than kBucketPrefixLen does. Length 1 is left +// out, since a single-byte match always exists and is probed anyway. Packing the +// eight live bits into a byte rather than a u16 halves the table to 64 KB. +// +// Folding the prefix into fewer slots would stay correct - the filter only ever +// SKIPS work, so a collision costs a wasted probe and can never change the answer +// - but measurably loses: at 16 KB and below the index arithmetic costs more than +// the smaller footprint saves, because the live prefix set is already small. + +constexpr size_t kPrefixSlots = size_t{1} << 16; +constexpr uint8_t kMaskLongBit = 1; + +struct LongEntry { + uint64_t suffix; + uint8_t slen; + Token token; +}; + +struct TrieNode { + int token = -1; // -1 == none + std::vector> children; +}; + +struct Bucket { + std::vector entries; + int32_t trie_root = -1; // >=0 once promoted +}; + +class LongestPrefixMatcher { + public: + /// Empty matcher pre-loaded with the 256 single-byte tokens (ids 0..255). + static LongestPrefixMatcher New() { + LongestPrefixMatcher m; + for (uint16_t i = 0; i <= 255; ++i) { + m.short_by_len_[1].Put(static_cast(static_cast(i)), i); + } + m.next_id_ = 256; + return m; + } + + /// Build from a complete dictionary: token at index i receives id i. + static LongestPrefixMatcher FromDictionary(const CompactDictionary& dict) { + LongestPrefixMatcher m; + size_t n = dict.num_tokens(); + for (size_t i = 0; i < n; ++i) { + m.InsertInternal(dict.token_ptr(static_cast(i)), dict.token_len(static_cast(i)), + static_cast(i)); + } + m.next_id_ = static_cast(n); + return m; + } + + /// Insert `data` (len bytes) and assign it the next available token id. + Token Insert(const uint8_t* data, size_t len) { + Token id = static_cast(next_id_++); + InsertInternal(data, len, id); + return id; + } + + size_t size() const { return next_id_; } + + /// Longest token whose bytes are a prefix of `data`, with its length. + std::pair FindLongestMatch(const uint8_t* data, size_t data_len) const { + size_t max_len = std::min(data_len, kMaxTokenSize); + uint64_t low64 = LoadLeU64(data, data_len, std::min(max_len, kBucketPrefixLen)); + + // Every token of 2 bytes or more shares its first two bytes with the data, so + // one array read rules out the lengths at which no token can possibly match. + uint32_t present = max_len >= 2 ? prefix_mask_[low64 & 0xFFFF] : 0; + + if (max_len > kBucketPrefixLen && (present & kMaskLongBit) != 0) { + uint32_t bucket = long_map_.Find(low64); + if (bucket != kFlatEmpty) { + const uint8_t* suf = data + kBucketPrefixLen; + size_t suf_len = max_len - kBucketPrefixLen; + const Bucket& b = buckets_[bucket]; + std::pair hit{0, 0}; + bool found; + if (b.trie_root < 0) { + found = SearchLinear(b.entries, LoadLeU64(suf, suf_len, suf_len), suf_len, &hit); + } else { + found = SearchTrie(static_cast(b.trie_root), suf, suf_len, &hit); + } + if (found) { + return {hit.first, kBucketPrefixLen + hit.second}; + } + } + } + + // Descend only through the occupied lengths. Bit (len-1) holds length `len`, + // so clearing bit 0 also drops the long-token bit. + size_t short_max = std::min(max_len, kBucketPrefixLen); + uint32_t cand = present & (((uint32_t{1} << short_max) - 1) & ~uint32_t{1}); + while (cand != 0) { + size_t len = 32 - static_cast(__builtin_clz(cand)); + cand &= ~(uint32_t{1} << (len - 1)); + uint32_t tok = short_by_len_[len].Find(low64 & MaskU64(len)); + if (tok != kFlatEmpty) { + return {static_cast(tok), len}; + } + } + uint32_t one = short_by_len_[1].Find(low64 & 0xFF); + if (one != kFlatEmpty) return {static_cast(one), 1}; + // Precondition: every single-byte token is present, so the probe above hits. + return {static_cast(data[0]), 1}; + } + + private: + // short_by_len_[len] maps the low-`len`-byte packed key to a token, for len 1..8. + FlatU64Map short_by_len_[kBucketPrefixLen + 1]; + FlatU64Map long_map_; // 8-byte prefix -> index into buckets_ + std::vector buckets_; + std::vector pool_; + std::vector prefix_mask_ = std::vector(kPrefixSlots, 0); + uint32_t next_id_ = 0; + + void InsertInternal(const uint8_t* data, size_t len, Token id) { + if (len >= 2) { + uint32_t p = static_cast(data[0]) | (static_cast(data[1]) << 8); + uint32_t bit = len <= kBucketPrefixLen ? (uint32_t{1} << (len - 1)) : kMaskLongBit; + prefix_mask_[p] |= static_cast(bit); + } + if (len <= kBucketPrefixLen) { + uint64_t key = LoadLeU64(data, len, len); + short_by_len_[len].Put(key, id); + return; + } + uint64_t prefix = LoadLeU64(data, len, kBucketPrefixLen); + size_t slen = len - kBucketPrefixLen; + uint64_t suffix = LoadLeU64(data + kBucketPrefixLen, slen, slen); + uint32_t bi = long_map_.Find(prefix); + if (bi == kFlatEmpty) { + bi = static_cast(buckets_.size()); + buckets_.emplace_back(); + long_map_.Put(prefix, bi); + } + Bucket& b = buckets_[bi]; + if (b.trie_root < 0) { + // Keep descending-by-length order so the first linear match is longest. The + // bucket is already ordered, so place the new entry rather than re-sorting + // the whole thing on every insert - this runs inside the training loop. + // Two entries can only share a length if they also share a suffix, i.e. if + // the same token bytes were inserted twice, so where equal lengths land + // relative to each other is not observable. + LongEntry e{suffix, static_cast(slen), id}; + auto by_len_desc = [](const LongEntry& a, const LongEntry& c) { + return a.slen > c.slen; + }; + auto at = std::upper_bound(b.entries.begin(), b.entries.end(), e, by_len_desc); + b.entries.insert(at, e); + if (b.entries.size() > kPromoteThreshold) { + BuildTrie(&b); + } + } else { + uint8_t buf[8]; + std::memcpy(buf, &suffix, 8); + TrieInsert(static_cast(b.trie_root), buf, slen, id); + } + } + + bool SearchLinear(const std::vector& entries, uint64_t val, size_t max_slen, + std::pair* out) const { + for (const LongEntry& e : entries) { + size_t elen = e.slen; + if (elen <= max_slen && MatchingLowBytes(val ^ e.suffix) >= elen) { + *out = {e.token, elen}; + return true; + } + } + return false; + } + + bool SearchTrie(uint32_t root, const uint8_t* suf, size_t suf_len, + std::pair* out) const { + bool have = false; + uint32_t cur = root; + for (size_t pos = 0; pos < suf_len; ++pos) { + uint32_t child; + if (!TrieFindChild(cur, suf[pos], &child)) break; + cur = child; + if (pool_[cur].token >= 0) { + *out = {static_cast(pool_[cur].token), pos + 1}; + have = true; + } + } + return have; + } + + bool TrieFindChild(uint32_t node, uint8_t byte, uint32_t* out) const { + for (const auto& kv : pool_[node].children) { + if (kv.first == byte) { + *out = kv.second; + return true; + } + } + return false; + } + + uint32_t TrieAlloc() { + uint32_t idx = static_cast(pool_.size()); + pool_.emplace_back(); + return idx; + } + + void TrieInsert(uint32_t root, const uint8_t* suf, size_t slen, Token token) { + uint32_t cur = root; + for (size_t i = 0; i < slen; ++i) { + uint32_t child; + if (TrieFindChild(cur, suf[i], &child)) { + cur = child; + } else { + uint32_t new_idx = TrieAlloc(); + pool_[cur].children.emplace_back(suf[i], new_idx); + cur = new_idx; + } + } + pool_[cur].token = static_cast(token); + } + + void BuildTrie(Bucket* b) { + uint32_t root = TrieAlloc(); + for (const LongEntry& e : b->entries) { + uint8_t buf[8]; + std::memcpy(buf, &e.suffix, 8); + TrieInsert(root, buf, e.slen, e.token); + } + b->entries.clear(); + b->entries.shrink_to_fit(); + b->trie_root = static_cast(root); + } +}; + +// Merge-threshold controller - DEVIATION FROM PAPER (D1; see onpair.h) + +class DynamicThresholdController { + public: + DynamicThresholdController(size_t capacity, size_t total_bytes, double scan_fraction) + : capacity_(capacity), + scan_budget_(static_cast(static_cast(total_bytes) * scan_fraction)), + check_interval_(std::max(capacity / 128, 64)), + next_checkpoint_(check_interval_) {} + + uint8_t get() const { return threshold_; } + bool budget_exhausted() const { return bytes_scanned_ > scan_budget_; } + void on_bytes_scanned(size_t n) { bytes_scanned_ += n; } + + void on_entry_created() { + ++entries_created_; + if (entries_created_ >= next_checkpoint_) Rebalance(); + } + + private: + size_t capacity_; + size_t scan_budget_; + size_t check_interval_; + uint8_t threshold_ = 2; + size_t entries_created_ = 0; + size_t bytes_scanned_ = 0; + size_t entries_at_check_ = 0; + size_t bytes_at_check_ = 0; + size_t next_checkpoint_; + + void Rebalance() { + size_t delta_e = entries_created_ - entries_at_check_; + size_t delta_b = bytes_scanned_ - bytes_at_check_; + double recent_rate = + delta_b > 0 ? static_cast(delta_e) / static_cast(delta_b) : 1e9; + size_t e_rem = capacity_ > entries_created_ ? capacity_ - entries_created_ : 1; + size_t b_rem = scan_budget_ > bytes_scanned_ ? scan_budget_ - bytes_scanned_ : 1; + double target_rate = static_cast(e_rem) / static_cast(b_rem); + double ratio = target_rate > 0.0 ? recent_rate / target_rate : 1e9; + + if (ratio > 2.0 && threshold_ < 255) { + ++threshold_; + } else if (ratio < 0.5 && threshold_ > 2) { + --threshold_; + } + entries_at_check_ = entries_created_; + bytes_at_check_ = bytes_scanned_; + next_checkpoint_ = entries_created_ + check_interval_; + } +}; + +// Seeded PRNG for training-sample shuffle - DEVIATION FROM PAPER (D4) + +inline uint64_t SplitMix64(uint64_t* state) { + uint64_t z = (*state += 0x9E3779B97F4A7C15ull); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); +} + +/// Partial Fisher-Yates: randomize the first `k` positions of `order`. +void PartialShuffle(std::vector* order, size_t k, uint64_t seed) { + size_t n = order->size(); + uint64_t state = seed; + size_t limit = std::min(k, n); + for (size_t i = 0; i < limit; ++i) { + size_t span = n - i; + size_t j = i + static_cast(SplitMix64(&state) % span); + std::swap((*order)[i], (*order)[j]); + } +} + +// Dictionary finalization + +/// Sort tokens bytewise-lexicographically, returning fresh (bytes, offsets). +void SortTokens(const std::vector& bytes, const std::vector& offsets, + std::vector* out_bytes, std::vector* out_offsets) { + size_t n = offsets.size() - 1; + auto tok_begin = [&](size_t id) { return bytes.data() + offsets[id]; }; + auto tok_len = [&](size_t id) { return offsets[id + 1] - offsets[id]; }; + + std::vector perm(n); + std::iota(perm.begin(), perm.end(), 0); + std::sort(perm.begin(), perm.end(), [&](size_t a, size_t b) { + size_t la = tok_len(a), lb = tok_len(b); + int cmp = std::memcmp(tok_begin(a), tok_begin(b), std::min(la, lb)); + if (cmp != 0) return cmp < 0; + return la < lb; + }); + + out_bytes->clear(); + out_bytes->reserve(bytes.size()); + out_offsets->clear(); + out_offsets->reserve(n + 1); + out_offsets->push_back(0); + for (size_t old : perm) { + out_bytes->insert(out_bytes->end(), tok_begin(old), tok_begin(old) + tok_len(old)); + out_offsets->push_back(static_cast(out_bytes->size())); + } +} + +/// Append zero padding so the fixed 16-byte over-read of any token is in bounds. +void PadRaw(std::vector* bytes, const std::vector& offsets) { + size_t need = static_cast(offsets.back()) + kMaxTokenSize; + if (bytes->size() < need) bytes->resize(need, 0); +} + +// Dictionary construction / training (paper sec 3.2) + +struct TrainResult { + CompactDictionary dict; + LongestPrefixMatcher lpm; +}; + +TrainResult Train(const uint8_t* data, const uint32_t* offsets, size_t n, + const Config& cfg, EncodeProfile* profile) { + using Clock = std::chrono::steady_clock; + auto t0 = Clock::now(); + size_t dict_capacity = size_t{1} << cfg.max_dict_bits; + + std::vector dict_bytes; + dict_bytes.reserve(dict_capacity * kMaxTokenSize); + std::vector dict_offsets; + dict_offsets.reserve(dict_capacity + 1); + dict_offsets.push_back(0); + for (uint16_t i = 0; i <= 255; ++i) { + dict_bytes.push_back(static_cast(i)); + dict_offsets.push_back(static_cast(dict_bytes.size())); + } + LongestPrefixMatcher lpm = LongestPrefixMatcher::New(); + + size_t total_bytes = n == 0 ? 0 : offsets[n]; + size_t capacity = dict_capacity - 256; + DynamicThresholdController ctrl(capacity, total_bytes, cfg.threshold_fraction); + uint8_t threshold = ctrl.get(); + + std::vector order(n); + std::iota(order.begin(), order.end(), 0u); + // Full Fisher-Yates shuffle of the entire training order (D4 in onpair.h). The + // dynamic byte budget still stops scanning well before the end, so only a + // sample is trained on - but drawing that sample from a *full* shuffle avoids + // skew on sequentially-ordered columns. (The Rust reference crate partial-shuffles + // only ~0.3n rows and leaves them in the slice's TAIL while the trainer reads from + // the head, so on ordered data like Customer#000... it trains mostly on + // low-numbered rows and builds a skewed dictionary. This port already shuffles + // into the head; a full shuffle matches the reference C++ std::shuffle over all + // rows and removes any doubt.) + PartialShuffle(&order, n, cfg.seed); + + FlatFreqMap freq; + + bool full_dictionary = false; + bool budget_exhausted = false; + + for (uint32_t idx : order) { + if (full_dictionary || budget_exhausted) break; + + size_t s_start = offsets[idx]; + size_t s_end = offsets[idx + 1]; + if (s_end == s_start) continue; + const uint8_t* str = data + s_start; + size_t len = s_end - s_start; + + auto [prev_id, prev_len] = lpm.FindLongestMatch(str, len); + size_t pos = prev_len; + + ctrl.on_bytes_scanned(prev_len); + if (ctrl.budget_exhausted()) { + budget_exhausted = true; + break; + } + + while (pos < len) { + auto [curr_id, curr_len] = lpm.FindLongestMatch(str + pos, len - pos); + + ctrl.on_bytes_scanned(curr_len); + if (ctrl.budget_exhausted()) { + budget_exhausted = true; + break; + } + + size_t pair_len = prev_len + curr_len; + if (pair_len <= kMaxTokenSize) { + uint32_t key = (static_cast(prev_id) << 16) | static_cast(curr_id); + uint8_t count = freq.Bump(key); + if (count >= threshold) { + size_t pair_start = pos - prev_len; + Token new_id = lpm.Insert(str + pair_start, pair_len); + dict_bytes.insert(dict_bytes.end(), str + pair_start, str + pos + curr_len); + dict_offsets.push_back(static_cast(dict_bytes.size())); + + if (lpm.size() == dict_capacity) { + full_dictionary = true; + break; + } + ctrl.on_entry_created(); + threshold = ctrl.get(); + + freq.Reset(key); + prev_id = new_id; + prev_len = pair_len; + pos += curr_len; + continue; + } + } + prev_id = curr_id; + prev_len = curr_len; + pos += curr_len; + } + } + + std::vector sorted_bytes; + std::vector sorted_offsets; + auto t1 = Clock::now(); + SortTokens(dict_bytes, dict_offsets, &sorted_bytes, &sorted_offsets); + PadRaw(&sorted_bytes, sorted_offsets); + + CompactDictionary dict; + dict.bytes = std::move(sorted_bytes); + dict.offsets = std::move(sorted_offsets); + dict.RecomputeMaxTokenLen(); + LongestPrefixMatcher final_lpm = LongestPrefixMatcher::FromDictionary(dict); + if (profile != nullptr) { + profile->train_s = std::chrono::duration(t1 - t0).count(); + profile->rebuild_s = std::chrono::duration(Clock::now() - t1).count(); + } + return TrainResult{std::move(dict), std::move(final_lpm)}; +} + +// Parsing: greedy longest-prefix tokenization (paper sec 3.3) + +void EncodeStrings(const uint8_t* data, const uint32_t* offsets, size_t n, + const LongestPrefixMatcher& lpm, std::vector* codes, + std::vector* row_offsets) { + row_offsets->push_back(0); + for (size_t i = 0; i < n; ++i) { + size_t s = offsets[i]; + size_t e = offsets[i + 1]; + size_t pos = s; + while (pos < e) { + auto [tok, mlen] = lpm.FindLongestMatch(data + pos, e - pos); + codes->push_back(tok); + pos += mlen; + } + row_offsets->push_back(static_cast(codes->size())); + } +} + +} // namespace + +// Public API + +Column Compress(const uint8_t* bytes, size_t /*bytes_len*/, const uint32_t* offsets, + size_t num_rows, const Config& cfg, EncodeProfile* profile) { + TrainResult tr = Train(bytes, offsets, num_rows, cfg, profile); + Column col; + col.dict = std::move(tr.dict); + col.codes.reserve(num_rows == 0 ? 0 : offsets[num_rows]); + col.row_offsets.reserve(num_rows + 1); + auto t0 = std::chrono::steady_clock::now(); + EncodeStrings(bytes, offsets, num_rows, tr.lpm, &col.codes, &col.row_offsets); + if (profile != nullptr) { + profile->tokenize_s = + std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + } + return col; +} + +Column CompressWithTokens(const uint8_t* bytes, const uint32_t* offsets, size_t num_rows, + const std::vector& token_bytes, + const std::vector& token_offsets) { + std::vector sorted_bytes; + std::vector sorted_offsets; + SortTokens(token_bytes, token_offsets, &sorted_bytes, &sorted_offsets); + PadRaw(&sorted_bytes, sorted_offsets); + + Column col; + col.dict.bytes = std::move(sorted_bytes); + col.dict.offsets = std::move(sorted_offsets); + col.dict.RecomputeMaxTokenLen(); + LongestPrefixMatcher lpm = LongestPrefixMatcher::FromDictionary(col.dict); + + col.codes.reserve(num_rows == 0 ? 0 : offsets[num_rows]); + col.row_offsets.reserve(num_rows + 1); + EncodeStrings(bytes, offsets, num_rows, lpm, &col.codes, &col.row_offsets); + return col; +} + +size_t DecodedLen(const Column& col) { + size_t sum = 0; + for (uint16_t c : col.codes) sum += col.dict.token_len(c); + return sum; +} + +void StridedDictionary::Build(const CompactDictionary& dict) { + // Align `slots` to a cache line so that a kStride-byte slot, kStride being a + // power of two no larger than a line, never straddles two lines. std::vector + // only promises alignment for its element type, so over-allocate by one line + // and point into `storage` at the first aligned byte. + constexpr size_t kAlign = 64; + const size_t ntokens = dict.num_tokens(); + const size_t slot_bytes = (ntokens * kStride + kAlign - 1) / kAlign * kAlign; + storage.assign(slot_bytes + kAlign, 0); + const size_t misalign = reinterpret_cast(storage.data()) % kAlign; + slots = storage.data() + (misalign == 0 ? 0 : kAlign - misalign); + lens.resize(ntokens); + for (size_t t = 0; t < ntokens; ++t) { + // Tokens are capped at kMaxTokenSize == kStride by training, so a token fills + // at most its own slot and a length always fits in a byte. The zero-fill above + // is what makes the unused tail of a slot well-defined for a fixed-width read. + const size_t len = dict.offsets[t + 1] - dict.offsets[t]; + std::memcpy(slots + t * kStride, dict.bytes.data() + dict.offsets[t], len); + lens[t] = static_cast(len); + } + max_token_len = dict.max_token_len; +} + +namespace { + +// Building the strided view is one pass over the dictionary, so it pays for itself +// only when the code stream is long enough to amortise it. Below this the blob +// kernel runs directly. The crossover measured at roughly one code per token; the +// build-charged rung's worst corpus (a two-token dictionary over a short column) +// sits exactly here and came out level rather than slower. +bool StridedViewWorthBuilding(size_t ncodes, size_t ntokens) { return ncodes >= ntokens; } + +// Shared body of DecompressInto over the strided view, parameterised on the copy +// width for the same reason the packed kernels are. See max_token_len. +template +size_t DecompressIntoStrided(const StridedDictionary& dict, const Column& col, uint8_t* out) { + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + size_t w = 0; + for (uint16_t code : col.codes) { + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + } + return w; +} + +#if defined(__ARM_FEATURE_SVE) +// As above, storing exactly the token's length. One predicate serves both sides: +// the load cannot read past the slot and the store writes no byte it does not own, +// so there is no over-copy and no width dispatch. +size_t DecompressIntoStridedExact(const StridedDictionary& dict, const Column& col, + uint8_t* out) { + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + size_t w = 0; + for (uint16_t code : col.codes) { + const uint8_t* src = slots + size_t{code} * StridedDictionary::kStride; + const uint32_t len = lens[code]; + svbool_t pg = svwhilelt_b8_u32(0u, len); + svst1_u8(pg, out + w, svld1_u8(pg, src)); + w += len; + } + return w; +} +#endif + +// Shared body of DecompressInto over the stored dictionary, for the streams too +// short to earn the strided view. +template +size_t DecompressIntoFixed(const Column& col, uint8_t* out) { + const CompactDictionary& dict = col.dict; + size_t w = 0; + for (uint16_t code : col.codes) { + const uint8_t* src = dict.token_ptr(code); + size_t len = dict.token_len(code); + std::memcpy(out + w, src, kCopy); // fixed over-copy, kCopy >= every token + w += len; + } + return w; +} + +} // namespace + +size_t DecompressInto(const Column& col, uint8_t* out) { + const size_t maxlen = col.dict.max_token_len; + if (!StridedViewWorthBuilding(col.codes.size(), col.dict.num_tokens())) { + if (maxlen <= 4) return DecompressIntoFixed<4>(col, out); + if (maxlen <= 8) return DecompressIntoFixed<8>(col, out); + return DecompressIntoFixed(col, out); + } + StridedDictionary view; + view.Build(col.dict); +#if defined(__ARM_FEATURE_SVE) + return DecompressIntoStridedExact(view, col, out); +#else + if (maxlen <= 4) return DecompressIntoStrided<4>(view, col, out); + if (maxlen <= 8) return DecompressIntoStrided<8>(view, col, out); + return DecompressIntoStrided(view, col, out); +#endif +} + +std::vector PackValues(const uint32_t* vals, size_t n, size_t bits) { + std::vector out((n * bits + 7) / 8 + 4, 0); + size_t bitpos = 0; + for (size_t i = 0; i < n; ++i) { + size_t byte = bitpos >> 3, off = bitpos & 7; + uint32_t w; + std::memcpy(&w, out.data() + byte, 4); + w |= (vals[i] << off); // vals[i] < 2^bits, bits<=25, off<=7 -> fits in u32 + std::memcpy(out.data() + byte, &w, 4); + bitpos += bits; + } + return out; +} + +namespace { + +// What this loop actually waits on, measured by ablation across 30 corpora rather +// than inferred: the random dictionary read. Deleting the gather while keeping the +// unpack and the store nearly doubles throughput (+83%). Deleting the store while +// keeping the gather makes the loop SLOWER (-14%, on every corpus). So the store is +// not the constraint, and storing fewer bytes is not the lever. +// +// This corrects an earlier reading of the same code. The evidence then was that +// (over-copy factor) x (decode MiB/s) came out constant across corpora, which was +// taken to mean a fixed store-bandwidth ceiling. That product is equally constant +// when the loop is bound by tokens retired, because over-copy is +// kCopy / mean_token_len while throughput is mean_token_len x tokens_per_second -- +// the two models are indistinguishable from that measurement, and the ablation +// picks the other one. +// +// Two experiments were rejected under the old reading. Both really did lose, so +// they are recorded here, but the reason was misattributed: +// +// - Unpacking codes a block at a time and prefetching before the gather: -22% +// with the offsets prefetched, -41% with the payload prefetched, worst case +// -65%, on all corpora. Aimed at the right cost, but a prefetch cannot help a +// stream of unpredictable indices arriving one code ahead of its use; it only +// adds a pass and the traffic of prefetches that arrive too late to hide +// anything. +// - A 12-byte copy as two stores: -4 to -6% wherever it applied. One wide store +// beats two narrow ones, which is a store-issue effect and holds regardless of +// what the loop is bound by. +// +// A third was measured and never built: choosing the copy width per block of 32 +// codes rather than per stream. Worth a median 1.00x of store traffic, because +// nearly every block of 32 contains at least one 16-byte token. +// +// The fix is to change what the gather reads. StridedDictionary gives each token a +// fixed slot and puts its length in a dense byte array, so one random line serves a +// token instead of two -- the layout FSST's decoder has always had, and which +// OnPair gave up when it lifted the length cap. See the kernels below. +template +size_t DecompressPackedFixed(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, + size_t bits, uint8_t* out) { + size_t bitpos = 0, w = 0; + const uint32_t mask = (bits >= 32) ? 0xFFFFFFFFu : ((1u << bits) - 1); + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & mask; // unpack the code + bitpos += bits; + const uint8_t* src = dict.token_ptr(static_cast(code)); + size_t len = dict.token_len(static_cast(code)); + std::memcpy(out + w, src, kCopy); // fixed over-copy, kCopy >= every token + w += len; + } + return w; +} + +// As above but with the code width a compile-time constant, so the mask folds to a +// literal and `bitpos += kBits` strength-reduces. Dispatched once per stream, the +// same way the copy width is. +template +size_t DecompressPackedFixedBits(const CompactDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + constexpr uint32_t kMask = (kBits >= 32) ? 0xFFFFFFFFu : ((uint32_t{1} << kBits) - 1); + const uint8_t* offsets_raw = reinterpret_cast(dict.offsets.data()); + const uint8_t* dict_bytes = dict.bytes.data(); + size_t bitpos = 0, w = 0; + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + // offsets[code] and offsets[code + 1] are adjacent u32s, so one 8-byte load + // yields the token's start and end together. token_ptr/token_len would issue + // two loads for what is almost always a single cache line. The payload still + // lives elsewhere, which is the second random line the strided view removes. + uint64_t pair; + std::memcpy(&pair, offsets_raw + size_t{code} * sizeof(uint32_t), sizeof(pair)); + const uint32_t start = static_cast(pair); + const size_t len = static_cast(pair >> 32) - start; + std::memcpy(out + w, dict_bytes + start, kCopy); + w += len; + } + return w; +} + +// The same loop against the strided view: one random line per token, and the length +// read from a dense byte array small enough to stay resident. The bit-unpack +// prologue is unchanged, so this differs from the kernel above in the gather alone. +template +size_t DecompressStridedBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + constexpr uint32_t kMask = (kBits >= 32) ? 0xFFFFFFFFu : ((uint32_t{1} << kBits) - 1); + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + size_t bitpos = 0, w = 0; + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + } + return w; +} + +#if defined(__ARM_FEATURE_SVE) +// And with a predicated store, which is where most of the remaining gain is. One +// predicate covers the load and the store, so the loop reads only the token's own +// bytes and writes only the bytes it owns: the fixed over-copy is gone, and with it +// the reason to dispatch on max_token_len at all. +template +size_t DecompressStridedExactBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + constexpr uint32_t kMask = (kBits >= 32) ? 0xFFFFFFFFu : ((uint32_t{1} << kBits) - 1); + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + size_t bitpos = 0, w = 0; + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + const uint8_t* src = slots + size_t{code} * StridedDictionary::kStride; + const uint32_t len = lens[code]; + // kStride <= the SVE minimum vector length of 16 bytes, so this predicate never + // needs more lanes than the hardware has. + svbool_t pg = svwhilelt_b8_u32(0u, len); + svst1_u8(pg, out + w, svld1_u8(pg, src)); + w += len; + } + return w; +} +#endif + +// Which decode form to build. Set on the command line to measure one against +// another with nothing else in the build changed. +// +// 0 the per-code loops above, no unroll at all +// 1 groups of one phase period (below); at a 16-bit width the period is a single +// code, so this leaves those columns on the per-code loop +// 2 groups of at least ONPAIR_GROUP_CODES codes at every width, 16 included +#ifndef ONPAIR_GROUP_UNROLL +#define ONPAIR_GROUP_UNROLL 2 +#endif + +// 0 asks for the phase period itself, the smallest group that makes every offset +// and shift constant. A nonzero value asks for at least that many codes, rounded up +// to a whole period, so the request is a floor rather than the group emitted: four +// emits eight codes at the widths whose period is eight and four at 14 and 16, where +// the request is taken literally. +// +// Four is measured, not chosen for its arithmetic. Sweeping the request over 2, 4, 8 +// and 16 only moves the widths where the request is not rounded up, and a 16-bit width +// is the only one of those where the group size is genuinely free. What happens there +// depends on the build, which is the more interesting result of the sweep. With the +// predicated store the answer is a step: two codes gains nothing at all and four gains +// the lot. On the portable build it is a slope, two already gaining about half of what +// eight gains, close to the (1 - 1/n) curve a loop whose group saving is per-code +// cursor arithmetic would follow. So the group form is removing scalar work in one +// build and covering dictionary-read latency in the other, and neither reading +// generalizes to the other. Four is the smallest request that captures the bulk of the +// gain on both. +// +// Above four the sweep stops choosing. Four and eight differ by well under a per cent +// at the only widths where they differ at all, and the two builds disagree on which +// way: with the predicated store four is ahead on 12 of the 15 16-bit columns, and on +// the portable build eight is ahead, in both portable sweeps. This is a tie broken on +// code size and on the shorter remainder a smaller group leaves after its last whole +// group, which matters on a short page and which the corpora here are all too long to +// show. +#if ONPAIR_GROUP_UNROLL >= 2 +#ifndef ONPAIR_GROUP_CODES +#define ONPAIR_GROUP_CODES 4 +#endif +constexpr size_t kGroupCodesRequest = ONPAIR_GROUP_CODES; +#else +constexpr size_t kGroupCodesRequest = 0; +#endif + +// Group unroll on the bit cursor's phase period. +// +// At a fixed code width the (byte offset, intra-byte shift) pair a code is read +// with repeats with period 8 / gcd(kBits, 8) codes: 8 at the widths coprime with 8, +// 4 at 10 and 14, 2 at 12, 1 at 16. Unrolling by that period turns every offset and +// every shift into a compile-time constant and advances the stream pointer once per +// group instead of once per code, so the cursor arithmetic the loops above do per +// code disappears. +// +// Any whole multiple of the period has the same property, so asking for a fixed +// number of codes and rounding up to a whole period is constant-addressed at every +// width. That matters because the period is one code at a 16-bit width -- there is +// no phase to fold -- and a trained 16-bit code space lands there on half the +// corpora, so unrolling by the period alone leaves exactly the largest-dictionary +// columns on the per-code loop. Asking for a group instead unrolls those columns on +// the strength of having several independent gathers in flight rather than on folded +// addressing. +// +// Sorting the measured gain by code width rather than by period separates the two +// effects. The 16-bit columns gain with no phase to fold at all, which is the part +// that is not addressing; the 15-bit columns, the worst phase case, gain about three +// times as much at the same instruction count per code, which is the part that is. +// Both terms are real and neither accounts for the whole. Width and dictionary size +// move together, though, so this does not fully separate cheaper addressing from a +// dictionary that misses L1. +// +// Nothing else changes: the same single fused pass, the same gather, the same +// store, no staging buffer, no prefetch, no added memory traffic. That is what +// separates this from the block-wise unpack recorded above as a loss, which paid an +// extra pass and 64 prefetches per block for the same codes. +// +// The reason to expect anything here is that the ablation ladder above does not +// actually isolate the gather. Deleting the gather deletes its two loads and their +// address arithmetic as well, so that rung bounds the gather plus its instructions, +// not the gather alone -- the same confound as the store-bandwidth reading it +// replaced. This change removes instructions while leaving the gather byte for +// byte identical, so it separates the two where neither ablation could. +// +// A group's last code is read by a 4-byte load at a constant offset, which runs at +// most 3 bytes past the group at every width and group size used here. PackValues +// carries 4 zero-filled tail bytes and the per-code loops read just as far, so the +// precondition on `packed` is unchanged. +template +struct PackedCodeGroup { + static constexpr size_t Gcd(size_t a, size_t b) { return b == 0 ? a : Gcd(b, a % b); } + static constexpr size_t kPeriod = 8 / Gcd(kBits, 8); + // A group has to consume a whole number of bytes so the next group starts at bit + // offset zero again. That holds for any multiple of the period, so round up. + static constexpr size_t kCodes = + kRequest == 0 ? kPeriod : ((kRequest + kPeriod - 1) / kPeriod) * kPeriod; + static constexpr size_t kBytes = kBits * kCodes / 8; + static_assert(kBytes * 8 == kBits * kCodes, "a group must be a whole number of bytes"); +}; + +template +inline uint32_t GroupCodeAt(const uint8_t* p) { + constexpr size_t kOff = (J * kBits) / 8; + constexpr size_t kShift = (J * kBits) % 8; + static_assert(kBits >= 1 && kBits <= 25, "the mask below would overflow"); + static_assert(kShift + kBits <= 32, "one 4-byte load must cover the code"); + constexpr uint32_t kMask = (uint32_t{1} << kBits) - 1; + uint32_t word; + std::memcpy(&word, p + kOff, sizeof(word)); + return (word >> kShift) & kMask; +} + +// One call per code in the group, in stream order. A fold over the comma operator +// is sequenced left to right, which the callers rely on: the write cursor advances +// by each token's own length in turn. +template +inline void ForEachCodeInGroup(const uint8_t* p, Emit emit, std::index_sequence) { + (emit(GroupCodeAt(p)), ...); +} + +// The strided portable loop, group-unrolled. +template +size_t DecompressStridedGroupBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + using Group = PackedCodeGroup; + // With the phase period asked for and a 16-bit width, the group is a single code: + // no phase to fold, and the form would be the per-code loop with the shift known + // to be zero. It measured indistinguishable from the plain loop, so at a group of + // one this defers rather than emitting the same code a second time. Any request of + // two or more never lands here. + if constexpr (Group::kCodes <= 1) { + return DecompressStridedBits(dict, packed, ncodes, out); + } + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + const uint8_t* p = packed; + size_t w = 0; + auto emit = [&](uint32_t code) { + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + }; + for (size_t g = 0, ngroups = ncodes / Group::kCodes; g < ngroups; ++g) { + ForEachCodeInGroup(p, emit, std::make_index_sequence{}); + p += Group::kBytes; + } + // A whole number of groups consumes a whole number of bytes, so the cursor is + // byte aligned here and the tail starts its own bit offset from zero. + constexpr uint32_t kMask = (uint32_t{1} << kBits) - 1; + size_t bitpos = 0; + for (size_t i = (ncodes / Group::kCodes) * Group::kCodes; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, p + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + } + return w; +} + +#if defined(__ARM_FEATURE_SVE) +// And the predicated-store loop, group-unrolled. Same two changes composed: one +// random line per token, exactly the token's bytes stored, constant addressing. +template +size_t DecompressStridedGroupExactBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + using Group = PackedCodeGroup; + // See above: at a group of one there is no phase to fold, and that form measured + // slower than the plain loop. + if constexpr (Group::kCodes <= 1) { + return DecompressStridedExactBits(dict, packed, ncodes, out); + } + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + const uint8_t* p = packed; + size_t w = 0; + auto emit = [&](uint32_t code) { + const uint32_t len = lens[code]; + svbool_t pg = svwhilelt_b8_u32(0u, len); + svst1_u8(pg, out + w, svld1_u8(pg, slots + size_t{code} * StridedDictionary::kStride)); + w += len; + }; + for (size_t g = 0, ngroups = ncodes / Group::kCodes; g < ngroups; ++g) { + ForEachCodeInGroup(p, emit, std::make_index_sequence{}); + p += Group::kBytes; + } + constexpr uint32_t kMask = (uint32_t{1} << kBits) - 1; + size_t bitpos = 0; + for (size_t i = (ncodes / Group::kCodes) * Group::kCodes; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, p + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + emit(code); + } + return w; +} +#endif + +// Runtime code width, for the widths training cannot produce but the format does +// not forbid. Kept so no input is rejected; never on a measured path. +template +size_t DecompressStridedFixed(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out) { + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + const uint32_t mask = (bits >= 32) ? 0xFFFFFFFFu : ((1u << bits) - 1); + size_t bitpos = 0, w = 0; + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & mask; + bitpos += bits; + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + } + return w; +} + +// Resolve `bits` to a constant for the widths a trained dictionary can produce +// (kMinDictBits..kMaxDictBits), falling back to the runtime-width loop otherwise so +// no input is rejected. +#define ONPAIR_DISPATCH_BITS(bits, CALL, FALLBACK) \ + switch (bits) { \ + case 9: return CALL(9); \ + case 10: return CALL(10); \ + case 11: return CALL(11); \ + case 12: return CALL(12); \ + case 13: return CALL(13); \ + case 14: return CALL(14); \ + case 15: return CALL(15); \ + case 16: return CALL(16); \ + default: return FALLBACK; \ + } + +template +size_t DecompressPackedDispatchBits(const CompactDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out) { +#define ONPAIR_BLOB(B) DecompressPackedFixedBits(dict, packed, ncodes, out) + ONPAIR_DISPATCH_BITS(bits, ONPAIR_BLOB, + DecompressPackedFixed(dict, packed, ncodes, bits, out)) +#undef ONPAIR_BLOB +} + +template +size_t DecompressStridedDispatchBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out) { +#if ONPAIR_GROUP_UNROLL +#define ONPAIR_STRIDED(B) DecompressStridedGroupBits(dict, packed, ncodes, out) +#else +#define ONPAIR_STRIDED(B) DecompressStridedBits(dict, packed, ncodes, out) +#endif + ONPAIR_DISPATCH_BITS(bits, ONPAIR_STRIDED, + DecompressStridedFixed(dict, packed, ncodes, bits, out)) +#undef ONPAIR_STRIDED +} + +// Decode through a view, choosing the exact-length store where the target has one. +size_t DecompressThroughView(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out) { +#if defined(__ARM_FEATURE_SVE) +#if ONPAIR_GROUP_UNROLL +#define ONPAIR_STRIDED_EXACT(B) \ + DecompressStridedGroupExactBits(dict, packed, ncodes, out) +#else +#define ONPAIR_STRIDED_EXACT(B) DecompressStridedExactBits(dict, packed, ncodes, out) +#endif + ONPAIR_DISPATCH_BITS(bits, ONPAIR_STRIDED_EXACT, + DecompressStridedFixed(dict, packed, ncodes, bits, out)) +#undef ONPAIR_STRIDED_EXACT +#else + // Read the width, do not scan for it: an O(tokens) scan here costs 1-3% on + // dictionaries of 20-60k tokens, which is charged to decode for something a + // stored format keeps in its header. See CompactDictionary::max_token_len. + // + // Only widths a single store can carry, for the reason recorded above: a 12-byte + // copy moves 25% fewer bytes than 16 but needs two stores and lost 4-6% wherever + // it applied. Narrowing to 8 is worth 25-28% on the corpora that allow it. + // + // `out` needs kDecodePadding of slack either way, and a slot is zero-filled out + // to kStride, so every width here is in bounds and reads defined bytes. + const size_t maxlen = dict.max_token_len; + if (maxlen <= 4) return DecompressStridedDispatchBits<4>(dict, packed, ncodes, bits, out); + if (maxlen <= 8) return DecompressStridedDispatchBits<8>(dict, packed, ncodes, bits, out); + return DecompressStridedDispatchBits(dict, packed, ncodes, bits, out); +#endif +} + +} // namespace + +size_t DecompressPacked(const StridedDictionary& dict, const uint8_t* packed, size_t ncodes, + size_t bits, uint8_t* out) { + return DecompressThroughView(dict, packed, ncodes, bits, out); +} + +size_t DecompressPacked(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, + size_t bits, uint8_t* out) { + if (!StridedViewWorthBuilding(ncodes, dict.num_tokens())) { + // See DecompressThroughView for why the width is read rather than scanned for, + // and why only 4/8/16 are offered. + const size_t maxlen = dict.max_token_len; + if (maxlen <= 4) return DecompressPackedDispatchBits<4>(dict, packed, ncodes, bits, out); + if (maxlen <= 8) return DecompressPackedDispatchBits<8>(dict, packed, ncodes, bits, out); + return DecompressPackedDispatchBits(dict, packed, ncodes, bits, out); + } + StridedDictionary view; + view.Build(dict); + return DecompressThroughView(view, packed, ncodes, bits, out); +} + +#undef ONPAIR_DISPATCH_BITS + +} // namespace parquet::onpair diff --git a/cpp/src/parquet/onpair/onpair.h b/cpp/src/parquet/onpair/onpair.h new file mode 100644 index 000000000000..1e3af8e6f1d7 --- /dev/null +++ b/cpp/src/parquet/onpair/onpair.h @@ -0,0 +1,243 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// C++ implementation of the OnPair short-string compression codec (encode + +// decode paths only), for a like-for-like comparison against FSST inside Arrow's +// benchmark harness. +// +// Algorithm: F. Gargiulo and R. Venturini, "OnPair: Short Strings Compression +// for Fast Random Access," arXiv:2508.02280, 2025. This implements the paper's +// core scheme: a dictionary of the 256 single bytes plus frequent merged pairs, +// greedy longest-prefix tokenization into u16 codes, and a table-lookup decode. +// +// NOT a production Parquet encoder - this is a benchmark artifact. +// +// Conformant with the paper: the two-tier longest-prefix index (sec 3.4.1, a hash +// map for <=8-byte tokens + 8-byte-prefix buckets with suffixes sorted +// descending), the 16-byte max token of OnPair16 (sec 3.2.2), and the fixed-16-byte +// SIMD gather-copy decode (sec 3.5, Alg. 3) - advance by the token's true length, +// relying on 16-byte source read-padding and output write-padding. +// +// DEVIATIONS FROM THE PAPER (engineering choices; they do not affect the code +// format or correctness, only the trained dictionary and encode-time behavior): +// D1. Merge threshold. The paper (sec 3.2.1) fixes it per dataset as +// max(2, floor(log2(S_MiB))). This port instead uses an adaptive +// controller paced to a byte budget (`DynamicThresholdController`), so the +// trained dictionary differs from the paper's. +// D2. Long-bucket overflow. The paper's OnPair16 caps each bucket at 128 +// suffixes (sec 3.4.4), dropping extras; this port promotes an over-full +// bucket to a trie (`PROMOTE_THRESHOLD`), keeping all suffixes. +// D3. Static perfect-hash LPM. The paper finalizes long-pattern lookup with a +// minimal perfect hash for the read-only parsing phase (sec 3.4.3); this port +// keeps std::unordered_map (the paper notes that path is Rust-only). +// D4. Training-sample selection uses a fixed-seed splitmix64 *full* +// Fisher-Yates shuffle (`PartialShuffle` over all rows). Shuffle *extent*, +// not the RNG choice, is what affects the trained dictionary: a full +// shuffle avoids skew on sequentially-ordered columns (the Rust crate +// partial-shuffles only a tail prefix, skewing patterned data like +// Customer#000…). The exact RNG is not specified by the paper, so output +// is deterministic but not bit-identical to the crate. +// +// Little-endian hosts only. + +#pragma once + +#include +#include +#include +#include + +namespace parquet::onpair { + +/// A dictionary entry id and, equivalently, a code in the code stream. +using Token = uint16_t; + +/// Maximum byte length of any dictionary token, and the fixed width the decoder +/// over-reads per token. +constexpr size_t kMaxTokenSize = 16; + +/// Trailing slack an output buffer needs beyond the decoded length: the decoder +/// over-stores a fixed 16-byte chunk for the final token. +constexpr size_t kDecodePadding = kMaxTokenSize; + +/// Training configuration. Mirrors the reference `Config`. +struct Config { + /// Dictionary-size budget: at most 2^max_dict_bits tokens. Valid range 9..=16. + uint8_t max_dict_bits = 12; + /// Dynamic-threshold byte-sampling fraction, in (0, 1]. + double threshold_fraction = 0.15; + /// Deterministic sampling seed. + uint64_t seed = 42; + + static Config Dict12() { return Config{12, 0.15, 42}; } + static Config Dict16() { return Config{16, 0.15, 42}; } +}; + +/// The token table a code stream indexes into: Arrow-binary layout (flat bytes + +/// u32 offsets). `bytes` is read-padded by kMaxTokenSize so the decoder's fixed +/// 16-byte over-read stays in bounds. +struct CompactDictionary { + std::vector bytes; // read-padded + std::vector offsets; // length num_tokens + 1 + + /// Length of the longest token present, which is what the decoder's gather-copy + /// sizes its fixed copy width from. Often well below kMaxTokenSize: TPC-H + /// c_address tops out at 5 bytes, and copying 16 there moves 8x the bytes it + /// needs to. + /// + /// This must never UNDERSTATE the true maximum -- doing so would make the + /// decoder copy less than a token's length and silently truncate. It therefore + /// defaults to the conservative kMaxTokenSize, so a dictionary that never calls + /// RecomputeMaxTokenLen still decodes correctly and merely forgoes the + /// narrowing. A stored format would carry this in its header rather than + /// recompute it, which is why the decoder reads it instead of scanning: an + /// O(tokens) scan per decode call costs 1-3% on dictionaries of 20-60k tokens. + size_t max_token_len = kMaxTokenSize; + + /// Derive max_token_len from `offsets`. Call after building or replacing them. + void RecomputeMaxTokenLen() { + size_t m = 0; + for (size_t t = 0; t + 1 < offsets.size(); ++t) { + const size_t len = offsets[t + 1] - offsets[t]; + if (len > m) m = len; + } + // An empty dictionary decodes nothing; stay conservative rather than pick a + // width from no evidence. + max_token_len = m == 0 ? kMaxTokenSize : m; + } + + size_t num_tokens() const { return offsets.empty() ? 0 : offsets.size() - 1; } + const uint8_t* token_ptr(Token id) const { return bytes.data() + offsets[id]; } + size_t token_len(Token id) const { return offsets[id + 1] - offsets[id]; } + /// Logical (unpadded) byte size of the dictionary blob. + size_t logical_bytes() const { return offsets.empty() ? 0 : offsets.back(); } +}; + +/// A decode-side view of the dictionary: one cache line per token instead of two. +/// +/// CompactDictionary makes the decoder read two independent random locations per +/// token -- a u32 offsets pair from one array, then the payload from a +/// variable-stride blob. That gather, not the store traffic, is what the decode +/// loop waits on. Giving every token a fixed 16-byte slot and its length a byte in +/// a dense side array collapses the pair into one line, which is the layout FSST's +/// decoder has always used (fixed-stride `symbol[]` plus `len[]`). +/// +/// This is deliberately NOT the stored form. A 16-byte slot plus a length byte is +/// ~17 bytes per token against ~12 for blob-plus-offsets, so serializing it would +/// add hundreds of KiB on a 65k-token dictionary and move every compression ratio. +/// It is built once per column from the stored form and thrown away, so it costs +/// footprint only for the duration of a decode and nothing at all on disk. +struct StridedDictionary { + /// One slot per token. Also the decoder's fixed over-read width, and a power of + /// two <= 64, so a 64-byte-aligned base puts every slot inside a single line. + static constexpr size_t kStride = kMaxTokenSize; + + std::vector storage; ///< backing bytes; `slots` is aligned into this + uint8_t* slots = nullptr; ///< kStride bytes per token, 64-byte aligned + std::vector lens; ///< parallel token lengths, one byte each + /// Same meaning and same conservative default as CompactDictionary's: the + /// portable kernel's fixed copy width comes from it, so it must never + /// understate the true maximum. + size_t max_token_len = kMaxTokenSize; + + size_t num_tokens() const { return lens.size(); } + + /// Populate from a stored dictionary. O(tokens), once per column. + void Build(const CompactDictionary& dict); +}; + +/// A compressed string column. `codes` is the row-concatenated code stream; +/// row k is codes[row_offsets[k] .. row_offsets[k+1]]. +struct Column { + CompactDictionary dict; + std::vector codes; + std::vector row_offsets; // length num_rows + 1 + + size_t num_rows() const { + return row_offsets.empty() ? 0 : row_offsets.size() - 1; + } +}; + +/// Wall-clock seconds spent in each phase of Compress. The three sum to the +/// whole call. Only for attributing encode cost; pass null in a timed run. +struct EncodeProfile { + double train_s = 0; ///< greedy pairing pass over the shuffled sample + double rebuild_s = 0; ///< sort the dictionary, rebuild the matcher over it + double tokenize_s = 0; ///< tokenize every row against the frozen dictionary +}; + +/// Train a dictionary against (bytes, offsets) and greedily tokenize every row. +/// `offsets` has length num_rows + 1; row i is bytes[offsets[i]..offsets[i+1]]. +Column Compress(const uint8_t* bytes, size_t bytes_len, const uint32_t* offsets, + size_t num_rows, const Config& cfg, EncodeProfile* profile = nullptr); + +/// Tokenize every row against a token set trained elsewhere, skipping OnPair's +/// own training. `token_bytes`/`token_offsets` are a raw token list in the same +/// layout as CompactDictionary but without the read padding; the returned +/// column's dictionary is its canonical (sorted, padded) form, so token ids are +/// reassigned and the caller's numbering is not preserved. +/// +/// This exists so an alternative dictionary trainer can be measured against +/// OnPair's with the parsing pass and the decode pass held literally identical. +/// The token set must contain all 256 single bytes, which is what lets both +/// tokenize without an escape mechanism. +Column CompressWithTokens(const uint8_t* bytes, const uint32_t* offsets, size_t num_rows, + const std::vector& token_bytes, + const std::vector& token_offsets); + +/// Exact decoded byte length of the whole column (sum of token lengths). +size_t DecodedLen(const Column& col); + +/// Decode the whole column into `out`, returning bytes written. +/// Precondition: out capacity >= DecodedLen(col) + kDecodePadding. +size_t DecompressInto(const Column& col, uint8_t* out); + +// --- Bit-packed code stream (what a real stored format uses) ---------------- +// The in-memory Column holds u16 codes; on storage the code stream is packed at +// the true code width (ceil(log2 num_tokens)). These pack/unpack it so decode +// pays the real unpacking cost, keeping ratio and decode mutually consistent. + +/// Read `nbits` (<=25) at bit offset `bitpos`, little-endian / LSB-first. +/// `p` must have 4 readable bytes at the containing word. +inline uint32_t GetBits(const uint8_t* p, size_t bitpos, size_t nbits) { + uint32_t w; + std::memcpy(&w, p + (bitpos >> 3), 4); + return (w >> (bitpos & 7)) & (nbits >= 32 ? 0xFFFFFFFFu : ((1u << nbits) - 1)); +} + +/// Pack `n` values (each < 2^bits, bits in 1..=25) LSB-first. The result has 4 +/// trailing pad bytes (so a 4-byte window at the last value is in bounds); the +/// logical size is (n*bits+7)/8. +std::vector PackValues(const uint32_t* vals, size_t n, size_t bits); + +/// Decode a bit-packed code stream: read `bits` per code and gather-copy the +/// token. `packed` needs >=4 trailing pad bytes; `out` >= DecodedLen + padding. +/// +/// Builds a StridedDictionary internally and decodes through it, except when the +/// stream is short enough that the O(tokens) build outweighs what it saves, in +/// which case the blob-and-offsets kernel runs directly. Decode a series of pages +/// against one dictionary through the overload below instead, so the build is paid +/// once rather than per page. +size_t DecompressPacked(const CompactDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out); + +/// Same, against a view built once by the caller. Output is byte-identical to the +/// CompactDictionary overload. +size_t DecompressPacked(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out); + +} // namespace parquet::onpair diff --git a/cpp/src/parquet/onpair/onpair_test_standalone.cc b/cpp/src/parquet/onpair/onpair_test_standalone.cc new file mode 100644 index 000000000000..59342e068cf7 --- /dev/null +++ b/cpp/src/parquet/onpair/onpair_test_standalone.cc @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Standalone roundtrip validation for the OnPair C++ port. No Arrow deps. +// Build: g++ -std=c++17 -O2 -I cpp/src onpair.cc onpair_test_standalone.cc -o t +// (run from repo root, adjust -I to reach parquet/onpair/onpair.h) + +#include +#include +#include +#include +#include + +#include "parquet/onpair/onpair.h" + +namespace op = parquet::onpair; + +namespace { + +int g_failures = 0; + +// Pack a set of rows into (bytes, offsets) and roundtrip through OnPair. +bool Roundtrip(const std::vector& rows, uint8_t bits, const char* name) { + std::vector bytes; + std::vector offsets; + offsets.push_back(0); + for (const auto& r : rows) { + bytes.insert(bytes.end(), r.begin(), r.end()); + offsets.push_back(static_cast(bytes.size())); + } + op::Config cfg; + cfg.max_dict_bits = bits; + cfg.threshold_fraction = 0.5; + cfg.seed = 42; + + op::Column col = op::Compress(bytes.data(), bytes.size(), offsets.data(), + rows.size(), cfg); + + // Whole-column decode. + size_t dlen = op::DecodedLen(col); + std::vector out(dlen + op::kDecodePadding, 0); + size_t w = op::DecompressInto(col, out.data()); + bool ok = (w == bytes.size()) && (std::memcmp(out.data(), bytes.data(), bytes.size()) == 0); + if (!ok) { + std::printf(" FAIL %-22s bits=%2u: decoded %zu vs raw %zu%s\n", name, bits, w, + bytes.size(), (w == bytes.size() ? " (content mismatch)" : "")); + ++g_failures; + return false; + } + // Sanity: every code indexes the dictionary. + for (uint16_t c : col.codes) { + if (c >= col.dict.num_tokens()) { + std::printf(" FAIL %-22s bits=%2u: code out of range\n", name, bits); + ++g_failures; + return false; + } + } + return true; +} + +std::vector SyntheticUrls(size_t n) { + const char* hosts[] = {"https://www.yandex.ru", "https://www.google.com", + "https://news.ycombinator.com", "http://m.yandex.ru", + "ftp://files.example.com"}; + const char* paths[] = {"/", "/search?q=", "/api/v1/data", "/blog/post-", "/users/"}; + std::vector out; + uint64_t x = 0x9E3779B97F4A7C15ull; + for (size_t i = 0; i < n; ++i) { + x += 0x9E3779B97F4A7C15ull; + std::string s = hosts[(x) % 5]; + s += paths[(x >> 16) % 5]; + s += std::to_string(static_cast(x >> 48)); + out.push_back(std::move(s)); + } + return out; +} + +} // namespace + +int main() { + std::printf("OnPair C++ port roundtrip tests\n"); + + for (uint8_t bits = 9; bits <= 16; ++bits) { + // Mixed lengths incl. empty, 1-byte, boundary 8/9, 16, >16. + Roundtrip({"", "a", "ab", "12345678", "123456789", "0123456789abcdef", + "0123456789abcdefGHIJ", "hello world hello world"}, + bits, "mixed_lengths"); + + // Binary with NUL bytes. + std::vector bin; + for (int i = 0; i < 40; ++i) { + std::string s; + for (int j = 0; j < 30; ++j) s.push_back(static_cast((i * 7 + j * 3) & 0xFF)); + bin.push_back(std::move(s)); + } + Roundtrip(bin, bits, "binary_nul"); + + // Homogeneous (heavy merges). + Roundtrip(std::vector(50, std::string(40, 'a')), bits, "homogeneous"); + + // Shared long prefix (exercises long bucket / trie promotion). + std::vector shared; + for (int i = 0; i < 300; ++i) shared.push_back("https://prefix/" + std::to_string(i)); + Roundtrip(shared, bits, "shared_long_prefix"); + + // Synthetic URLs. + Roundtrip(SyntheticUrls(20000), bits, "synthetic_urls"); + + // All-empty and single-empty edge cases. + Roundtrip({"", "", ""}, bits, "all_empty"); + Roundtrip({}, bits, "no_rows"); + } + + if (g_failures == 0) { + std::printf("ALL PASS\n"); + return 0; + } + std::printf("%d FAILURES\n", g_failures); + return 1; +} diff --git a/cpp/src/parquet/onpair/prefix_plus.cc b/cpp/src/parquet/onpair/prefix_plus.cc new file mode 100644 index 000000000000..b9f65ed321b0 --- /dev/null +++ b/cpp/src/parquet/onpair/prefix_plus.cc @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/onpair/prefix_plus.h" + +#include +#include +#include + +namespace parquet::prefix_plus { +namespace { + +// Longest common prefix of two strings, capped at `cap`. With `guard`, trim so +// the returned length never falls between an FSST escape byte (255) and its +// literal: if the last matched byte is 255, drop it (thesis Listing 5.4). +size_t Lcp(const uint8_t* a, size_t la, const uint8_t* b, size_t lb, size_t cap, bool guard) { + size_t m = std::min(std::min(la, lb), cap); + size_t l = 0; + while (l < m && a[l] == b[l]) ++l; + if (guard && l != 0 && a[l - 1] == 255) --l; + return l; +} + +// Thesis DP (sec 5.2.2) for one block: strings are strs[0..bn) / lens[0..bn) +// (already sorted). Writes prefix_len[k] and chunk_first_local[k] for each +// string k in the block; chunk_first_local is a block-local index. Returns the +// number of chunks that carry a non-empty prefix. +size_t CleaveBlock(const uint8_t* const* strs, const size_t* lens, size_t bn, size_t max_prefix, + bool guard, uint32_t* prefix_len, uint32_t* chunk_first_local) { + if (bn == 0) return 0; + + // Consecutive LCPs, then min_lcp[i][j] = shared prefix length of strings i..j + // as the running minimum of adjacent LCPs (standard range-LCP identity). + std::vector lcp(bn > 0 ? bn - 1 : 0); + for (size_t i = 0; i + 1 < bn; ++i) { + lcp[i] = Lcp(strs[i], lens[i], strs[i + 1], lens[i + 1], max_prefix, guard); + } + std::vector> min_lcp(bn, std::vector(bn, 0)); + for (size_t i = 0; i < bn; ++i) { + min_lcp[i][i] = static_cast(std::min(lens[i], max_prefix)); + for (size_t j = i + 1; j < bn; ++j) { + min_lcp[i][j] = std::min(min_lcp[i][j - 1], static_cast(lcp[j - 1])); + } + } + + std::vector len_prefix_sum(bn + 1, 0); + for (size_t i = 0; i < bn; ++i) len_prefix_sum[i + 1] = len_prefix_sum[i] + lens[i]; + + constexpr size_t kInf = std::numeric_limits::max(); + std::vector dp(bn + 1, kInf), prev(bn + 1, 0), pfx(bn + 1, 0); + dp[0] = 0; + for (size_t i = 1; i <= bn; ++i) { + for (size_t j = 0; j < i; ++j) { + if (dp[j] == kInf) continue; + const size_t mcp = min_lcp[j][i - 1]; + const size_t candidates[2] = {0, mcp}; + const int n_cand = mcp > 0 ? 2 : 1; + for (int c = 0; c < n_cand; ++c) { + const size_t p = candidates[c]; + const size_t cnt = i - j; + const size_t per_string_overhead = 1 + (p > 0 ? 2 : 0); // prefix_length [+ jumpback] + const size_t overhead = cnt * per_string_overhead; + const size_t sum_len = len_prefix_sum[i] - len_prefix_sum[j]; + // Store the shared prefix once (p bytes) + all suffixes (sum_len - cnt*p) + // + overhead == overhead + sum_len - (cnt-1)*p. + const size_t cost = dp[j] + overhead + sum_len - (cnt - 1) * p; + if (cost < dp[i]) { + dp[i] = cost; + prev[i] = j; + pfx[i] = p; + } + } + } + } + + // Backtrack into chunks (start_local, prefix_length), then assign per string. + size_t idx = bn; + size_t num_prefix_chunks = 0; + while (idx > 0) { + const size_t start = prev[idx]; + const size_t p = pfx[idx]; + if (p > 0) ++num_prefix_chunks; + for (size_t k = start; k < idx; ++k) { + prefix_len[k] = static_cast(p); + chunk_first_local[k] = static_cast(start); + } + idx = start; + } + return num_prefix_chunks; +} + +} // namespace + +Cleaving CleaveSorted(const uint8_t* const* strs, const size_t* lens, size_t n, size_t max_prefix, + bool guard_escape255) { + if (max_prefix > kMaxPrefix) max_prefix = kMaxPrefix; + Cleaving out; + out.prefix_len.assign(n, 0); + out.chunk_first.assign(n, 0); + out.num_prefix_chunks = 0; + + std::vector local_first(kBlockSize); + for (size_t base = 0; base < n; base += kBlockSize) { + const size_t bn = std::min(kBlockSize, n - base); + out.num_prefix_chunks += + CleaveBlock(strs + base, lens + base, bn, max_prefix, guard_escape255, + out.prefix_len.data() + base, local_first.data()); + // Lift block-local chunk-first indices to global positions. + for (size_t k = 0; k < bn; ++k) { + out.chunk_first[base + k] = static_cast(base) + local_first[k]; + } + } + return out; +} + +} // namespace parquet::prefix_plus diff --git a/cpp/src/parquet/onpair/prefix_plus.h b/cpp/src/parquet/onpair/prefix_plus.h new file mode 100644 index 000000000000..fbd03a6d32cf --- /dev/null +++ b/cpp/src/parquet/onpair/prefix_plus.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Common-prefix extraction ("+") for string codecs, the shared core behind +// FSST+ and OnPair+ in the benchmark harness. +// +// Algorithm: Y. L. Alexandre, "FSST+: Enhancing String Compression Through +// Common Prefix Extraction," MSc thesis, CWI, 2025. Within a sorted block of at +// most 128 strings, an optimal set of "similarity chunks" (ranges sharing a +// prefix) is chosen by dynamic programming (thesis sec 5.2.2): each shared +// prefix is stored once and every string keeps a 1-byte prefix length, an +// optional 2-byte jump-back offset, and its own suffix. The DP minimises total +// stored bytes and runs in O(B^2) per block (B = 128, constant), hence O(N) +// overall. +// +// This header exposes only the codec-agnostic cleaving decision. FSST+ runs it +// over FSST-compressed bytes (with `guard_escape255` so a prefix never splits an +// FSST escape from its literal); OnPair+ runs it over raw bytes. The two codecs' +// assembly and size accounting live in the benchmark, next to the other codecs. +// +// Little-endian hosts only. + +#pragma once + +#include +#include +#include + +namespace parquet::prefix_plus { + +/// Strings per block, and the window within which a prefix may be shared. The +/// thesis picks 128 (sec 5.4.1): a block fits in L1 and the O(B^2) DP stays cheap. +constexpr size_t kBlockSize = 128; + +/// Longest shareable prefix. Bounded by the u8 that stores a string's prefix +/// length in the block (thesis sec 3.1). +constexpr size_t kMaxPrefix = 255; + +/// The chosen cleaving for a whole sorted collection. Indices are positions in +/// the sorted collection (0..n). +struct Cleaving { + /// prefix_len[i] = bytes string i shares with (and borrows from) its chunk. + std::vector prefix_len; + /// chunk_first[i] = index of the first string of string i's chunk; its first + /// prefix_len[i] bytes are the shared prefix, stored once for the chunk. + std::vector chunk_first; + /// Number of similarity chunks with a non-empty shared prefix (diagnostic: + /// equals the count of prefixes actually stored). + size_t num_prefix_chunks = 0; +}; + +/// Run the thesis DP (sec 5.2.2) over the sorted collection `strs`/`lens` +/// (length n), independently per block of kBlockSize. `max_prefix` caps prefix +/// length (<= kMaxPrefix). When `guard_escape255` is set, a candidate prefix is +/// trimmed so it never ends between an FSST escape byte (255) and the literal it +/// escapes -- required when cleaving FSST-compressed bytes, and a no-op for raw +/// bytes. The collection must already be sorted so that strings sharing a prefix +/// are adjacent. +Cleaving CleaveSorted(const uint8_t* const* strs, const size_t* lens, size_t n, + size_t max_prefix, bool guard_escape255); + +} // namespace parquet::prefix_plus diff --git a/cpp/src/parquet/onpair/prefix_plus_test_standalone.cc b/cpp/src/parquet/onpair/prefix_plus_test_standalone.cc new file mode 100644 index 000000000000..f756eea7859b --- /dev/null +++ b/cpp/src/parquet/onpair/prefix_plus_test_standalone.cc @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Standalone validation for the prefix_plus cleaving core. No Arrow deps. +// Build: g++ -std=c++17 -O2 -Icpp/src \ +// cpp/src/parquet/onpair/prefix_plus.cc \ +// cpp/src/parquet/onpair/prefix_plus_test_standalone.cc -o t +// +// Checks the cleaving invariant that makes decode correct: for every string i, +// its first prefix_len[i] bytes equal the first prefix_len[i] bytes of its +// chunk representative chunk_first[i] (so reconstructing prefix++suffix rebuilds +// the string exactly), plus structural bounds and the FSST-escape guard. + +#include +#include +#include +#include +#include +#include + +#include "parquet/onpair/prefix_plus.h" + +namespace pp = parquet::prefix_plus; + +namespace { + +int g_failures = 0; + +// Sort `rows` (as the "+" codecs do before cleaving), run CleaveSorted, and +// verify the reconstruction invariant + bounds. `guard` mirrors the FSST-escape +// path. +bool Check(std::vector rows, bool guard, const char* name) { + std::sort(rows.begin(), rows.end()); + + std::vector ptrs(rows.size()); + std::vector lens(rows.size()); + for (size_t i = 0; i < rows.size(); ++i) { + ptrs[i] = reinterpret_cast(rows[i].data()); + lens[i] = rows[i].size(); + } + + pp::Cleaving cl = + pp::CleaveSorted(ptrs.data(), lens.data(), rows.size(), pp::kMaxPrefix, guard); + + if (cl.prefix_len.size() != rows.size() || cl.chunk_first.size() != rows.size()) { + std::printf(" FAIL %-22s: wrong result size\n", name); + ++g_failures; + return false; + } + + for (size_t i = 0; i < rows.size(); ++i) { + const uint32_t p = cl.prefix_len[i]; + const uint32_t cf = cl.chunk_first[i]; + // Bounds. + if (p > lens[i] || p > pp::kMaxPrefix || cf > i) { + std::printf(" FAIL %-22s: bad cleave at %zu (p=%u cf=%u len=%zu)\n", name, i, p, cf, + lens[i]); + ++g_failures; + return false; + } + // Same block (cleaving never crosses a 128-block boundary). + if (cf / pp::kBlockSize != i / pp::kBlockSize) { + std::printf(" FAIL %-22s: chunk crosses block at %zu\n", name, i); + ++g_failures; + return false; + } + // Reconstruct: prefix (from chunk rep) ++ suffix (own bytes after p) == row. + std::string rebuilt(rows[cf].data(), p); + rebuilt.append(rows[i].data() + p, lens[i] - p); + if (rebuilt != rows[i]) { + std::printf(" FAIL %-22s: reconstruction mismatch at %zu\n", name, i); + ++g_failures; + return false; + } + // FSST-escape guard: a prefix must not end right after an escape byte. + if (guard && p > 0 && static_cast(rows[i][p - 1]) == 255) { + std::printf(" FAIL %-22s: prefix splits escape at %zu\n", name, i); + ++g_failures; + return false; + } + } + return true; +} + +} // namespace + +int main() { + std::printf("prefix_plus cleaving tests\n"); + + // Identical strings (whole string becomes the shared prefix, empty suffixes). + Check(std::vector(200, "Customer#000000042"), false, "identical"); + + // Monotonic shared prefix (the target shape). + { + std::vector v; + for (int i = 1; i <= 500; ++i) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "Customer#%09d", i); + v.emplace_back(buf); + } + Check(v, false, "monotonic_prefix"); + } + + // No shared prefix (distinct first bytes) -> every prefix_len should be 0. + { + std::vector v; + for (int i = 0; i < 100; ++i) v.push_back(std::string(1, static_cast('A' + i % 26)) + + std::to_string(i)); + Check(v, false, "no_shared_prefix"); + } + + // Prefix longer than 255 must be capped (uint8 prefix_length). + Check(std::vector(64, std::string(400, 'z')), false, "over_255_cap"); + + // Empty and near-empty rows. + Check({"", "", "a", "ab", "abc"}, false, "empties"); + + // Multi-block (>128) with grouped prefixes across block boundaries. + { + std::vector v; + for (int g = 0; g < 10; ++g) + for (int i = 0; i < 40; ++i) v.push_back("group" + std::to_string(g) + "/item" + + std::to_string(i)); + Check(v, false, "multi_block"); + } + + // FSST-escape guard: rows containing byte 255 adjacent to shared regions. + { + std::vector v; + for (int i = 0; i < 60; ++i) { + std::string s = "pre"; + s.push_back(static_cast(255)); + s.push_back(static_cast('a' + i % 5)); + s += std::to_string(i); + v.push_back(std::move(s)); + } + Check(v, true, "escape_guard"); + } + + // Single row and empty input. + Check({"lonely"}, false, "single_row"); + Check({}, false, "no_rows"); + + if (g_failures == 0) { + std::printf("ALL PASS\n"); + return 0; + } + std::printf("%d FAILURES\n", g_failures); + return 1; +} diff --git a/cpp/src/parquet/onpair/verify_roundtrip.cc b/cpp/src/parquet/onpair/verify_roundtrip.cc new file mode 100644 index 000000000000..363eb2cbd4f9 --- /dev/null +++ b/cpp/src/parquet/onpair/verify_roundtrip.cc @@ -0,0 +1,371 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Visible round-trip proof for the OnPair port and for the FSST16 trainer it is +// compared against: decode every configuration and check EVERY row equals the +// original bytes, then print a few concrete original -> decoded samples so a +// human can eyeball the recovery. +// +// The FSST16 rows are decoded twice, once through the whole-column path and once +// through the bit-packed path at the code width the trained table actually needs, +// because the packed loop is templated on that width and a table of a few hundred +// tokens exercises a far narrower one than OnPair ever produces. +// +// For OnPair, every dictionary budget in the valid 9..16 range is checked, not +// just 16: OnPair-auto picks a width per column, so verifying only 16 would leave +// the width the benchmarks actually report unverified. The merge threshold comes +// from bench_common.h so the dictionary trained here is the one the benchmarks +// measure. +// +// Exits non-zero if any row of any corpus in any configuration or at any width +// fails to round-trip, so this can gate a benchmark run. +// +// Build (one line): g++ -std=c++17 -O2 -Icpp/src +// cpp/src/parquet/onpair/onpair.cc cpp/src/parquet/onpair/fsst16.cc +// cpp/src/parquet/onpair/verify_roundtrip.cc -o /tmp/verify +// Run: /tmp/verify bench-fsst-onpair/corpora/tpch_l_shipmode.txt [more files...] + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "parquet/onpair/bench_common.h" +#include "parquet/onpair/fsst16.h" +#include "parquet/onpair/onpair.h" + +namespace op = parquet::onpair; +namespace f16 = parquet::fsst16; + +namespace { + +struct Corpus { + std::vector bytes; + std::vector offsets; + size_t rows() const { return offsets.size() - 1; } + std::string row(size_t i) const { + return std::string(reinterpret_cast(bytes.data() + offsets[i]), + offsets[i + 1] - offsets[i]); + } +}; + +Corpus Read(const char* path) { + Corpus c; + std::ifstream in(path, std::ios::binary); + c.offsets.push_back(0); + std::string line; + while (std::getline(in, line)) { + c.bytes.insert(c.bytes.end(), line.begin(), line.end()); + c.offsets.push_back(static_cast(c.bytes.size())); + } + return c; +} + +// Compare a decoded concatenated buffer to the original, row by row. +// Returns the number of mismatching rows and the first mismatching index. +size_t CheckPerRow(const Corpus& c, const uint8_t* decoded, size_t dn, long* first_bad) { + *first_bad = -1; + if (dn != c.bytes.size()) { + *first_bad = 0; + return c.rows(); + } + size_t bad = 0; + for (size_t i = 0; i < c.rows(); ++i) { + size_t off = c.offsets[i], len = c.offsets[i + 1] - off; + if (std::memcmp(decoded + off, c.bytes.data() + off, len) != 0) { + if (*first_bad < 0) *first_bad = static_cast(i); + ++bad; + } + } + return bad; +} + +std::string Trunc(const std::string& s, size_t n = 42) { + return s.size() <= n ? s : s.substr(0, n) + "…"; +} + +void Samples(const Corpus& c, const uint8_t* decoded) { + size_t r = c.rows(); + size_t idx[4] = {0, r / 3, (2 * r) / 3, r - 1}; + for (size_t k = 0; k < 4; ++k) { + size_t i = idx[k]; + size_t off = c.offsets[i], len = c.offsets[i + 1] - off; + std::string dec(reinterpret_cast(decoded + off), len); + std::string orig = c.row(i); + std::printf(" row %-8zu original=%-44s decoded=%-44s %s\n", i, + ("\"" + Trunc(orig) + "\"").c_str(), ("\"" + Trunc(dec) + "\"").c_str(), + orig == dec ? "MATCH" : "*** MISMATCH ***"); + } +} + +// Independent scalar reference decode: exact-length copies straight out of the +// stored dictionary, deliberately the dumbest loop that can be written. It shares +// no code with any shipped kernel, which is the point -- the kernels are checked +// against it rather than against each other. +std::vector ReferenceDecode(const op::CompactDictionary& dict, + const std::vector& codes, size_t n) { + std::vector out; + for (size_t i = 0; i < n; ++i) { + const uint8_t* p = dict.token_ptr(codes[i]); + out.insert(out.end(), p, p + dict.token_len(codes[i])); + } + return out; +} + +// Decode is served by several kernels chosen by target features and by stream +// length, and they must be interchangeable to the byte. Checked here: +// +// whole-column DecompressInto, unpacked u16 codes +// packed DecompressPacked, building its own strided view +// packed, prebuilt DecompressPacked against a view the caller built +// packed, short a prefix short enough to trip the guard that skips the view +// and decodes straight out of the stored dictionary +// packed, tails sixteen consecutive lengths, covering every remainder +// +// The short case matters because nothing else reaches that path: real streams carry +// far more codes than tokens, so the fallback would otherwise never run here. +// +// The tail sweep matters for a different reason. The packed kernels decode a fixed +// group of codes per iteration and hand whatever is left to the per-code loop, so a +// length one code short of a whole group is where a mistake in that handover shows +// up. The short case above cannot be relied on for that: its length is fixed by the +// dictionary size, so which remainder it lands on is an accident. Sweeping sixteen +// consecutive lengths covers every remainder any group size up to sixteen can leave, +// whatever the group size is compiled to be. +bool VerifyDecodePaths(const op::Column& col, const char* tag) { + const size_t ncodes = col.codes.size(); + const size_t ntokens = col.dict.num_tokens(); + size_t bits = 1; + while ((size_t{1} << bits) < ntokens) ++bits; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), bits); + op::StridedDictionary view; + view.Build(col.dict); + + std::vector buf(op::DecodedLen(col) + op::kDecodePadding, 0); + size_t bad = 0; + auto agrees_bytes = [&](const char* what, const uint8_t* want, size_t want_len, + size_t got) { + if (got == want_len && std::memcmp(buf.data(), want, want_len) == 0) return; + std::printf(" %s: disagrees with the scalar reference (%zu vs %zu bytes)\n", what, + got, want_len); + ++bad; + }; + auto agrees = [&](const char* what, const std::vector& want, size_t got) { + agrees_bytes(what, want.data(), want.size(), got); + }; + + const std::vector want_all = ReferenceDecode(col.dict, col.codes, ncodes); + agrees("whole-column", want_all, op::DecompressInto(col, buf.data())); + agrees("packed", want_all, + op::DecompressPacked(col.dict, packed.data(), ncodes, bits, buf.data())); + agrees("packed, prebuilt view", want_all, + op::DecompressPacked(view, packed.data(), ncodes, bits, buf.data())); + + const size_t nshort = std::min(ncodes, ntokens == 0 ? 0 : ntokens - 1); + if (nshort != 0) { + agrees("packed, short stream", ReferenceDecode(col.dict, col.codes, nshort), + op::DecompressPacked(col.dict, packed.data(), nshort, bits, buf.data())); + } + + // Decoding a prefix of the codes yields a prefix of the whole-column bytes, so the + // expected output is want_all truncated -- no need to decode a reference per tail. + // The prebuilt view is used so the guard that falls back to the stored dictionary + // on a short stream cannot skip the kernel under test. + size_t tails = 0, want_len = want_all.size(); + for (size_t k = 1; k <= 16 && k < ncodes; ++k) { + want_len -= col.dict.token_len(col.codes[ncodes - k]); + agrees_bytes("packed, tail", want_all.data(), want_len, + op::DecompressPacked(view, packed.data(), ncodes - k, bits, buf.data())); + ++tails; + } + + std::printf(" %-15s: %zu decode paths agree (%zu tokens, %zub codes, " + "max token %zu) %s\n", + tag, 4 + tails, ntokens, bits, col.dict.max_token_len, + bad == 0 ? "[OK]" : "[FAIL]"); + return bad == 0; +} + +// OnPair (no dedup): compress then whole-column decode. +bool VerifyOnPair(const Corpus& c, uint8_t bits, double threshold, bool show_samples) { + op::Config cfg{bits, threshold, 42}; + op::Column col = op::Compress(c.bytes.data(), c.bytes.size(), c.offsets.data(), c.rows(), cfg); + std::vector out(op::DecodedLen(col) + op::kDecodePadding, 0); + size_t dn = op::DecompressInto(col, out.data()); + long bad_at; + size_t bad = CheckPerRow(c, out.data(), dn, &bad_at); + std::printf(" OnPair%-2u : %zu/%zu rows exact %s\n", bits, c.rows() - bad, c.rows(), + bad == 0 ? "[OK]" : "[FAIL]"); + if (bad != 0) std::printf(" first mismatching row: %ld\n", bad_at); + // Every width gets the four-path check, not just 16: each width instantiates a + // different unpack kernel, so agreement at one width says nothing about another. + char tag[32]; + std::snprintf(tag, sizeof(tag), "OnPair%u paths", static_cast(bits)); + bool paths = VerifyDecodePaths(col, tag); + if (show_samples) Samples(c, out.data()); + return paths && bad == 0; +} + +// FSST's training algorithm at a 16-bit code space, decoded through OnPair's own +// path. The trained token list is handed to the encoder's train-free entry point, +// so the parsing pass and the decode kernel are the same code OnPair16 uses and +// only the table differs. +bool VerifyFsst16(const Corpus& c, int max_symbol_len, size_t sample_target, const char* tag, + bool show_samples) { + f16::Config cfg; + cfg.max_symbol_len = max_symbol_len; + if (sample_target != 0) cfg.sample_target = sample_target; + f16::Tokens t = f16::Train(c.bytes.data(), c.offsets.data(), c.rows(), cfg); + op::Column col = + op::CompressWithTokens(c.bytes.data(), c.offsets.data(), c.rows(), t.bytes, t.offsets); + + std::vector out(op::DecodedLen(col) + op::kDecodePadding, 0); + size_t dn = op::DecompressInto(col, out.data()); + long bad_at; + size_t bad = CheckPerRow(c, out.data(), dn, &bad_at); + + // The width a stored format would pack these codes at, which is what the + // benchmark charges for and therefore what has to decode correctly. + size_t nt = col.dict.num_tokens(); + size_t code_bits = 1; + while ((size_t{1} << code_bits) < nt) ++code_bits; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), code_bits); + std::vector pout(op::DecodedLen(col) + op::kDecodePadding, 0); + size_t pn = + op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, pout.data()); + long pbad_at; + size_t pbad = CheckPerRow(c, pout.data(), pn, &pbad_at); + + std::printf(" %-15s: %zu/%zu rows exact, packed %zu/%zu (%zu tokens, %zub codes, " + "max token %zu) %s\n", + tag, c.rows() - bad, c.rows(), c.rows() - pbad, c.rows(), nt, code_bits, + col.dict.max_token_len, (bad == 0 && pbad == 0) ? "[OK]" : "[FAIL]"); + if (bad != 0) std::printf(" first mismatching row: %ld\n", bad_at); + if (pbad != 0) std::printf(" first mismatching packed row: %ld\n", pbad_at); + if (show_samples) Samples(c, pout.data()); + // An FSST-trained table at an 8-byte cap stays far smaller than OnPair's, so this + // is where the narrow end of the width dispatch gets exercised. + bool fsst_paths = VerifyDecodePaths(col, " ↳ paths"); + return bad == 0 && pbad == 0 && fsst_paths; +} + +// The distinct-value set a dedup cascade OnPairs, plus the per-row ids into it. +// Built once per corpus and reused across widths -- deduplicating 500k rows is +// far more expensive than the training pass being verified. +struct Distinct { + std::vector bytes; + std::vector offsets{0}; + std::vector refs; + size_t count() const { return offsets.size() - 1; } +}; + +Distinct BuildDistinct(const Corpus& c) { + size_t n = c.rows(); + Distinct d; + d.refs.resize(n); + std::unordered_map ids; + for (size_t i = 0; i < n; ++i) { + std::string v = c.row(i); + auto it = ids.find(v); + uint32_t id; + if (it == ids.end()) { + id = static_cast(ids.size()); + d.bytes.insert(d.bytes.end(), v.begin(), v.end()); + d.offsets.push_back(static_cast(d.bytes.size())); + ids.emplace(std::move(v), id); + } else { + id = it->second; + } + d.refs[i] = id; + } + return d; +} + +// OnPair-dedup: OnPair the distinct values, then gather every row back through +// its id. Checks the gather as well as the decode -- a correct dictionary paired +// with a broken reference column still reconstructs the wrong column. +bool VerifyOnPairDedup(const Corpus& c, const Distinct& d, uint8_t bits, double threshold, + bool show_samples) { + op::Config cfg{bits, threshold, 42}; + op::Column col = op::Compress(d.bytes.data(), d.bytes.size(), d.offsets.data(), d.count(), cfg); + std::vector dbuf(op::DecodedLen(col) + op::kDecodePadding, 0); + op::DecompressInto(col, dbuf.data()); // distinct values, concatenated in id order + std::vector out(c.bytes.size() + 16, 0); + size_t w = 0; + for (size_t i = 0; i < c.rows(); ++i) { + uint32_t id = d.refs[i]; + size_t off = d.offsets[id], len = d.offsets[id + 1] - off; + std::memcpy(out.data() + w, dbuf.data() + off, len); + w += len; + } + long bad_at; + size_t bad = CheckPerRow(c, out.data(), w, &bad_at); + std::printf(" OnPair%-2u-dedup : %zu/%zu rows exact (%zu distinct) %s\n", bits, + c.rows() - bad, c.rows(), d.count(), bad == 0 ? "[OK]" : "[FAIL]"); + if (bad != 0) std::printf(" first mismatching row: %ld\n", bad_at); + if (show_samples) Samples(c, out.data()); + // A distinct-value dictionary is far smaller than a whole column's, so its codes + // pack into a narrower width than OnPair ever picks -- this is where the low end + // of the width dispatch gets exercised. + return VerifyDecodePaths(col, " ↳ paths") && bad == 0; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: %s [more...]\n", argv[0]); + return 2; + } + int failures = 0; + size_t rows_checked = 0; + for (int a = 1; a < argc; ++a) { + Corpus c = Read(argv[a]); + if (c.rows() == 0) { + std::fprintf(stderr, "%s: no rows read\n", argv[a]); + ++failures; + continue; + } + // Same rule the benchmarks use, so the trained dictionary matches theirs. + double threshold = bench::ThresholdFor(std::filesystem::path(argv[a]).stem().string()); + std::printf("\n%s (%zu rows, %.2f MiB, threshold %.2f)\n", argv[a], c.rows(), + c.bytes.size() / (1024.0 * 1024.0), threshold); + // FSST16 at its native symbol cap, at OnPair's cap, and at a sample large + // enough to actually fill a 16-bit table. The last one is the config whose + // dictionary is big enough to reach the wide packed-decode loops. Its code + // width follows from the trained table, so these run once, not per budget. + if (!VerifyFsst16(c, 8, 0, "FSST16-8B", false)) ++failures; + if (!VerifyFsst16(c, 16, 0, "FSST16-16B", false)) ++failures; + if (!VerifyFsst16(c, 16, size_t{4} << 20, "FSST16-16B-4M", true)) ++failures; + rows_checked += 6 * c.rows(); // 3 configs, each checked unpacked and packed + Distinct d = BuildDistinct(c); + for (uint8_t bits = 9; bits <= 16; ++bits) { + // Samples are the human-readable proof; print them once, at the width the + // report's fixed-budget rows use, rather than eight times per corpus. + bool show = (bits == 16); + if (!VerifyOnPair(c, bits, threshold, show)) ++failures; + if (!VerifyOnPairDedup(c, d, bits, threshold, show)) ++failures; + rows_checked += 2 * c.rows(); + } + } + std::printf("\n%zu row comparisons, %d failure(s)\n", rows_checked, failures); + return failures == 0 ? 0 : 1; +} diff --git a/cpp/src/parquet/onpair/width_sweep_benchmark.cc b/cpp/src/parquet/onpair/width_sweep_benchmark.cc new file mode 100644 index 000000000000..e1956f4c41c8 --- /dev/null +++ b/cpp/src/parquet/onpair/width_sweep_benchmark.cc @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Does a wider code make OnPair decode faster? Isolate the code width from +// everything else that changes with the dictionary budget. +// +// The obvious way to ask this -- compare OnPair-auto (which picks a budget per +// column) against OnPair16 -- cannot answer it. A narrower budget trains a +// SMALLER DICTIONARY, so it also changes the tokens, the token count per row, and +// the gather-copy width the decoder picks from max_token_len. Those move decode +// far more than the unpacking does, and they move in both directions, so the +// comparison is confounded and its answer is noise. +// +// This benchmark holds the dictionary fixed and varies only the packing width. +// For each training budget it takes the ONE trained dictionary and its ONE code +// stream, then bit-packs those same codes at every width from their true width up +// to 16 and times DecompressPacked at each. Identical tokens, identical code +// sequence, identical output bytes, identical copy width -- the only difference is +// how many bits each code occupies and which DecompressPackedFixedBits<> template +// the dispatch lands on. Storing a code in more bits than it needs is pure waste +// on the ratio axis, so any decode gain is the whole case for a wider code. +// +// It also prints the confounded comparison alongside, so the two can be read +// against each other: the "own width" column across training budgets is what a +// budget sweep sees, and the widen-in-place rows are what the width alone does. +// +// PIN AND QUIESCE: this is a timing benchmark. Run it under `taskset -c 0` with +// core 0 idle -- `ps -eo pid,psr,pcpu | awk '$2==0 && $3>5'` must print nothing. +// +// Build (one line): +// g++ -std=c++17 -O3 -march=native -Icpp/src \ +// cpp/src/parquet/onpair/onpair.cc \ +// cpp/src/parquet/onpair/width_sweep_benchmark.cc \ +// /usr/lib64/libzstd.so.1 /usr/lib64/liblz4.so.1 -o /tmp/width_sweep +// Run: +// taskset -c 0 /tmp/width_sweep /tmp/tmp/corpora30 + +#include +#include +#include +#include +#include + +#include "parquet/onpair/bench_common.h" +#include "parquet/onpair/onpair.h" + +namespace op = parquet::onpair; + +namespace { + +// Training budgets to build a dictionary at. Not all of 9..16: each one is a full +// training pass, and three points spanning the range (floor, middle, ceiling) show +// whether the width effect depends on dictionary size. The widths swept per +// dictionary are exhaustive, since that is the axis under test. +constexpr uint8_t kBudgets[] = {9, 12, 16}; + +struct WidthPoint { + size_t bits = 0; + double decode_mibs = 0; + size_t codes_bytes = 0; // bit-packed code stream, logical size +}; + +struct BudgetResult { + uint8_t budget = 0; + size_t num_tokens = 0; + size_t true_bits = 0; // ceil(log2 num_tokens) -- the width a real page stores + size_t num_codes = 0; + size_t max_token_len = 0; + double bytes_per_token = 0; + std::vector widths; + + const WidthPoint* at(size_t bits) const { + for (const WidthPoint& w : widths) { + if (w.bits == bits) return &w; + } + return nullptr; + } +}; + +// Time DecompressPacked on one (dictionary, code stream, width), the same way +// RunCodec does: fresh output buffer per iteration allocated outside the timed +// region, median of kDecodeIters, throughput over the RAW bytes. Aborts if the +// decode does not reproduce the column exactly -- a width that unpacks to the +// wrong codes would otherwise read as a fast decode. +double TimeDecode(const bench::Corpus& c, const op::CompactDictionary& dict, + const std::vector& packed, size_t num_codes, size_t bits) { + const size_t out_cap = c.raw_bytes() + op::kDecodePadding + 64; + { + std::vector out(out_cap, 0); + size_t w = op::DecompressPacked(dict, packed.data(), num_codes, bits, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "width %zu roundtrip mismatch on %s (w=%zu raw=%zu)\n", bits, + c.name.c_str(), w, c.raw_bytes()); + std::abort(); + } + } + std::vector mibs; + for (int it = 0; it < bench::kDecodeIters; ++it) { + std::vector out(out_cap, 0); + auto t0 = bench::Clock::now(); + size_t w = op::DecompressPacked(dict, packed.data(), num_codes, bits, out.data()); + double dt = std::chrono::duration(bench::Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + mibs.push_back(bench::Mib(c.raw_bytes()) / dt); + } + return bench::Median(std::move(mibs)); +} + +BudgetResult SweepBudget(const bench::Corpus& c, uint8_t budget, double threshold) { + op::Config cfg{budget, threshold, 42}; + op::Column col = + op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), c.n_rows(), cfg); + + BudgetResult r; + r.budget = budget; + r.num_tokens = col.dict.num_tokens(); + r.true_bits = bench::IndexBits(r.num_tokens); + r.num_codes = col.codes.size(); + r.max_token_len = col.dict.max_token_len; + r.bytes_per_token = static_cast(c.raw_bytes()) / static_cast(r.num_codes); + + // Widen once; PackValues takes u32. The same values are re-packed at each width, + // so every width decodes the identical code sequence. + std::vector codes32(col.codes.begin(), col.codes.end()); + + for (size_t bits = r.true_bits; bits <= 16; ++bits) { + std::vector packed = op::PackValues(codes32.data(), codes32.size(), bits); + WidthPoint p; + p.bits = bits; + p.codes_bytes = bench::BitPackedBytes(r.num_codes, bits); + p.decode_mibs = TimeDecode(c, col.dict, packed, r.num_codes, bits); + r.widths.push_back(p); + } + return r; +} + +void PrintCorpus(const bench::Corpus& c, const std::vector& results) { + std::printf("\n%s %zu rows, %.2f MiB raw\n", c.name.c_str(), c.n_rows(), + bench::Mib(c.raw_bytes())); + for (const BudgetResult& r : results) { + std::printf( + " budget %2ub: %6zu tokens, true width %zu b, %zu codes (%.2f raw B/token), " + "copy width %zu\n", + r.budget, r.num_tokens, r.true_bits, r.num_codes, r.bytes_per_token, r.max_token_len); + std::printf(" width :"); + for (const WidthPoint& w : r.widths) std::printf(" %8zu", w.bits); + std::printf("\n MiB/s :"); + for (const WidthPoint& w : r.widths) std::printf(" %8.0f", w.decode_mibs); + std::printf("\n vs true:"); + const WidthPoint* base = r.at(r.true_bits); + for (const WidthPoint& w : r.widths) { + std::printf(" %+7.1f%%", 100.0 * (w.decode_mibs - base->decode_mibs) / base->decode_mibs); + } + std::printf("\n codes :"); + for (const WidthPoint& w : r.widths) { + std::printf(" %+7.1f%%", + 100.0 * (static_cast(w.codes_bytes) - base->codes_bytes) / + base->codes_bytes); + } + std::printf(" (bit-packed code stream, vs true width)\n"); + } +} + +} // namespace + +int main(int argc, char** argv) { + std::vector files = bench::CorpusFiles(bench::CorpusDir(argc, argv)); + if (files.empty()) { + std::fprintf(stderr, "no .txt corpora found\n"); + return 2; + } + std::printf( + "OnPair decode throughput vs CODE WIDTH ALONE -- one trained dictionary per\n" + "budget, its code stream re-packed at each width from its true width to 16.\n" + "Same tokens, same code sequence, same copy width; only the unpacking differs.\n" + "'codes' is what the wider packing costs on the ratio axis.\n" + "%d decode iterations, median. Pin to an idle core.\n", + bench::kDecodeIters); + + // Per-corpus deltas for the summary: going from the true width to 16 bits, and + // to true+1, on each dictionary. + struct Delta { + std::string corpus; + uint8_t budget; + size_t true_bits; + double to_16; + double to_plus1; + double codes_cost_16; + }; + std::vector deltas; + + for (const std::filesystem::path& f : files) { + bench::Corpus c = bench::ReadCorpus(f); + if (c.n_rows() == 0) { + std::fprintf(stderr, "%s: no rows\n", f.c_str()); + return 1; + } + double threshold = bench::ThresholdFor(c.name); + std::vector results; + for (uint8_t b : kBudgets) results.push_back(SweepBudget(c, b, threshold)); + PrintCorpus(c, results); + std::fflush(stdout); + + for (const BudgetResult& r : results) { + if (r.true_bits >= 16) continue; // nothing to widen into + const WidthPoint* base = r.at(r.true_bits); + const WidthPoint* w16 = r.at(16); + const WidthPoint* wp1 = r.at(r.true_bits + 1); + deltas.push_back({c.name, r.budget, r.true_bits, + 100.0 * (w16->decode_mibs - base->decode_mibs) / base->decode_mibs, + 100.0 * (wp1->decode_mibs - base->decode_mibs) / base->decode_mibs, + 100.0 * (static_cast(w16->codes_bytes) - base->codes_bytes) / + base->codes_bytes}); + } + } + + // Summary. The question is whether widening the code buys decode speed, so the + // headline is the sign and size of the true-width -> 16-bit change, against what + // that widening costs on the code stream. + std::printf("\n\n=== Summary: decode change from widening the code, dictionary held fixed ===\n"); + std::printf("%-30s %6s %6s %10s %10s %12s\n", "corpus", "budget", "true b", "->true+1", + "->16 b", "codes at 16b"); + std::vector all16, allp1, cost16; + for (const Delta& d : deltas) { + std::printf("%-30s %5ub %5zub %+9.1f%% %+9.1f%% %+11.1f%%\n", d.corpus.c_str(), d.budget, + d.true_bits, d.to_plus1, d.to_16, d.codes_cost_16); + all16.push_back(d.to_16); + allp1.push_back(d.to_plus1); + cost16.push_back(d.codes_cost_16); + } + if (!all16.empty()) { + auto stats = [](std::vector v, const char* label) { + std::sort(v.begin(), v.end()); + int faster = 0; + for (double x : v) { + if (x > 0) ++faster; + } + std::printf(" %-22s median %+6.1f%% min %+6.1f%% max %+6.1f%% faster on %d/%zu\n", label, + bench::Median(v), v.front(), v.back(), faster, v.size()); + }; + std::printf("\n%zu (corpus, budget) pairs where the true width is below 16:\n", all16.size()); + stats(allp1, "decode, true -> true+1"); + stats(all16, "decode, true -> 16 b"); + std::vector c16 = cost16; + std::sort(c16.begin(), c16.end()); + std::printf(" %-22s median %+6.1f%% min %+6.1f%% max %+6.1f%%\n", "code stream at 16 b", + bench::Median(c16), c16.front(), c16.back()); + } + return 0; +} diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift index e3cc5adb9648..b94868ec3782 100644 --- a/cpp/src/parquet/parquet.thrift +++ b/cpp/src/parquet/parquet.thrift @@ -629,6 +629,13 @@ enum Encoding { Support for INT32, INT64 and FIXED_LEN_BYTE_ARRAY added in 2.11. */ BYTE_STREAM_SPLIT = 9; + + /** Fast Static Symbol Table (FSST) encoding for BYTE_ARRAY data. + FSST compresses strings using a symbol table that maps 1-8 byte sequences + to single-byte codes. It allows random access to compressed data and works + well with dictionary encoding by compressing dictionary values. + */ + FSST = 10; } /** diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc index 7109c3d1c83b..7e5324814b0d 100644 --- a/cpp/src/parquet/types.cc +++ b/cpp/src/parquet/types.cc @@ -259,6 +259,8 @@ std::string EncodingToString(Encoding::type t) { return "RLE_DICTIONARY"; case Encoding::BYTE_STREAM_SPLIT: return "BYTE_STREAM_SPLIT"; + case Encoding::FSST: + return "FSST"; default: return "UNKNOWN"; } diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h index c2040e555fdb..0113ec86aa13 100644 --- a/cpp/src/parquet/types.h +++ b/cpp/src/parquet/types.h @@ -531,8 +531,9 @@ struct Encoding { DELTA_BYTE_ARRAY = 7, RLE_DICTIONARY = 8, BYTE_STREAM_SPLIT = 9, + FSST = 10, // Should always be last element (except UNKNOWN) - UNDEFINED = 10, + UNDEFINED = 11, UNKNOWN = 999 }; }; diff --git a/cpp/thirdparty/fsst/LICENSE b/cpp/thirdparty/fsst/LICENSE new file mode 100644 index 000000000000..0cd2c8c46524 --- /dev/null +++ b/cpp/thirdparty/fsst/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018-2020 CWI, TU Munich, FSU Jena + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cpp/thirdparty/fsst/fsst.h b/cpp/thirdparty/fsst/fsst.h new file mode 100644 index 000000000000..71085d57201d --- /dev/null +++ b/cpp/thirdparty/fsst/fsst.h @@ -0,0 +1,227 @@ +/* + * the API for FSST compression -- (c) Peter Boncz, Viktor Leis and Thomas Neumann (CWI, TU Munich), 2018-2019 + * + * =================================================================================================================================== + * this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): + * + * Copyright 2018-2020, CWI, TU Munich, FSU Jena + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + * IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * You can contact the authors via the FSST source repository : https://github.com/cwida/fsst + * =================================================================================================================================== + * + * FSST: Fast Static Symbol Table compression + * see the paper https://github.com/cwida/fsst/raw/master/fsstcompression.pdf + * + * FSST is a compression scheme focused on string/text data: it can compress strings from distributions with many different values (i.e. + * where dictionary compression will not work well). It allows *random-access* to compressed data: it is not block-based, so individual + * strings can be decompressed without touching the surrounding data in a compressed block. When compared to e.g. lz4 (which is + * block-based), FSST achieves similar decompression speed, (2x) better compression speed and 30% better compression ratio on text. + * + * FSST encodes strings also using a symbol table -- but it works on pieces of the string, as it maps "symbols" (1-8 byte sequences) + * onto "codes" (single-bytes). FSST can also represent a byte as an exception (255 followed by the original byte). Hence, compression + * transforms a sequence of bytes into a (supposedly shorter) sequence of codes or escaped bytes. These shorter byte-sequences could + * be seen as strings again and fit in whatever your program is that manipulates strings. + * + * useful property: FSST ensures that strings that are equal, are also equal in their compressed form. + * + * In this API, strings are considered byte-arrays (byte = unsigned char) and a batch of strings is represented as an array of + * unsigned char* pointers to their starts. A seperate length array (of unsigned int) denotes how many bytes each string consists of. + * + * This representation as unsigned char* pointers tries to assume as little as possible on the memory management of the program + * that calls this API, and is also intended to allow passing strings into this API without copying (even if you use C++ strings). + * + * We optionally support C-style zero-terminated strings (zero appearing only at the end). In this case, the compressed strings are + * also zero-terminated strings. In zero-terminated mode, the zero-byte at the end *is* counted in the string byte-length. + */ +#ifndef FSST_INCLUDED_H +#define FSST_INCLUDED_H + +#ifdef _MSC_VER +#define __restrict__ +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ +#define __ORDER_LITTLE_ENDIAN__ 2 +#include +static inline int __builtin_ctzl(unsigned long long x) { + unsigned long ret; + _BitScanForward64(&ret, x); + return (int)ret; +} +#endif + +#ifdef __cplusplus +#define FSST_FALLTHROUGH [[fallthrough]] +#include +extern "C" { +#else +#define FSST_FALLTHROUGH +#endif + +#include + +/* A compressed string is simply a string of 1-byte codes; except for code 255, which is followed by an uncompressed byte. */ +#define FSST_ESC 255 + +/* Data structure needed for compressing strings - use fsst_duplicate() to create thread-local copies. Use fsst_destroy() to free. */ +typedef void* fsst_encoder_t; /* opaque type - it wraps around a rather large (~900KB) C++ object */ + +/* Data structure needed for decompressing strings - read-only and thus can be shared between multiple decompressing threads. */ +typedef struct { + unsigned long long version; /* version id */ + unsigned char zeroTerminated; /* terminator is a single-byte code that does not appear in longer symbols */ + unsigned char len[255]; /* len[x] is the byte-length of the symbol x (1 < len[x] <= 8). */ + unsigned long long symbol[255]; /* symbol[x] contains in LITTLE_ENDIAN the bytesequence that code x represents (0 <= x < 255). */ +} fsst_decoder_t; + +/* Calibrate a FSST symboltable from a batch of strings (it is best to provide at least 16KB of data). */ +fsst_encoder_t* +fsst_create( + size_t n, /* IN: number of strings in batch to sample from. */ + const size_t lenIn[], /* IN: byte-lengths of the inputs */ + const unsigned char *strIn[], /* IN: string start pointers. */ + int zeroTerminated /* IN: whether input strings are zero-terminated. If so, encoded strings are as well (i.e. symbol[0]=""). */ +); + +/* Create another encoder instance, necessary to do multi-threaded encoding using the same symbol table. */ +fsst_encoder_t* +fsst_duplicate( + fsst_encoder_t *encoder /* IN: the symbol table to duplicate. */ +); + +#define FSST_MAXHEADER (8+1+8+2048+1) /* maxlen of deserialized fsst header, produced/consumed by fsst_export() resp. fsst_import() */ + +/* Space-efficient symbol table serialization (smaller than sizeof(fsst_decoder_t) - by saving on the unused bytes in symbols of len < 8). */ +unsigned int /* OUT: number of bytes written in buf, at most sizeof(fsst_decoder_t) */ +fsst_export( + fsst_encoder_t *encoder, /* IN: the symbol table to dump. */ + unsigned char *buf /* OUT: pointer to a byte-buffer where to serialize this symbol table. */ +); + +/* Deallocate encoder. */ +void +fsst_destroy(fsst_encoder_t*); + +/* Return a decoder structure from serialized format (typically used in a block-, file- or row-group header). */ +unsigned int /* OUT: number of bytes consumed in buf (0 on failure). */ +fsst_import( + fsst_decoder_t *decoder, /* IN: this symbol table will be overwritten. */ + unsigned char const *buf /* IN: pointer to a byte-buffer where fsst_export() serialized this symbol table. */ +); + +/* Return a decoder structure from an encoder. */ +fsst_decoder_t +fsst_decoder( + fsst_encoder_t *encoder +); + +/* Compress a batch of strings (on AVX512 machines best performance is obtained by compressing more than 32KB of string volume). */ +/* The output buffer must be large; at least "conservative space" (7+2*inputlength) for the first string for something to happen. */ +size_t /* OUT: the number of compressed strings (<=n) that fit the output buffer. */ +fsst_compress( + fsst_encoder_t *encoder, /* IN: encoder obtained from fsst_create(). */ + size_t nstrings, /* IN: number of strings in batch to compress. */ + const size_t lenIn[], /* IN: byte-lengths of the inputs */ + const unsigned char *strIn[], /* IN: input string start pointers. */ + size_t outsize, /* IN: byte-length of output buffer. */ + unsigned char *output, /* OUT: memory buffer to put the compressed strings in (one after the other). */ + size_t lenOut[], /* OUT: byte-lengths of the compressed strings. */ + unsigned char *strOut[] /* OUT: output string start pointers. Will all point into [output,output+size). */ +); + +/* Decompress a single string, inlined for speed. */ +inline size_t /* OUT: bytesize of the decompressed string. If > size, the decoded output is truncated to size. */ +fsst_decompress( + const fsst_decoder_t *decoder, /* IN: use this symbol table for compression. */ + size_t lenIn, /* IN: byte-length of compressed string. */ + const unsigned char *strIn, /* IN: compressed string. */ + size_t size, /* IN: byte-length of output buffer. */ + unsigned char *output /* OUT: memory buffer to put the decompressed string in. */ +) { + unsigned char*__restrict__ len = (unsigned char* __restrict__) decoder->len; + unsigned char*__restrict__ strOut = (unsigned char* __restrict__) output; + unsigned long long*__restrict__ symbol = (unsigned long long* __restrict__) decoder->symbol; + size_t code, posOut = 0, posIn = 0; +#ifndef FSST_MUST_ALIGN /* defining on platforms that require aligned memory access may help their performance */ +#define FSST_UNALIGNED_STORE(dst,src) memcpy((unsigned long long*) (dst), &(src), sizeof(unsigned long long)) +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + while (posOut+32 <= size && posIn+4 <= lenIn) { + unsigned int nextBlock, escapeMask; + memcpy(&nextBlock, strIn+posIn, sizeof(unsigned int)); + escapeMask = (nextBlock&0x80808080u)&((((~nextBlock)&0x7F7F7F7Fu)+0x7F7F7F7Fu)^0x80808080u); + if (escapeMask == 0) { + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + } else { + unsigned long firstEscapePos=__builtin_ctzl((unsigned long long) escapeMask)>>3; + switch(firstEscapePos) { /* Duff's device */ + case 3: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + // fall through + case 2: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + // fall through + case 1: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + // fall through + case 0: posIn+=2; strOut[posOut++] = strIn[posIn-1]; /* decompress an escaped byte */ + } + } + } + if (posOut+32 <= size) { // handle the possibly 3 last bytes without a loop + if (posIn+2 <= lenIn) { + strOut[posOut] = strIn[posIn+1]; + if (strIn[posIn] != FSST_ESC) { + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + if (strIn[posIn] != FSST_ESC) { + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + } else { + posIn += 2; strOut[posOut++] = strIn[posIn-1]; + } + } else { + posIn += 2; posOut++; + } + } + if (posIn < lenIn) { // last code cannot be an escape + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + } + } +#else + while (posOut+8 <= size && posIn < lenIn) + if ((code = strIn[posIn++]) < FSST_ESC) { /* symbol compressed as code? */ + FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); /* unaligned memory write */ + posOut += len[code]; + } else { + strOut[posOut] = strIn[posIn]; /* decompress an escaped byte */ + posIn++; posOut++; + } +#endif +#endif + while (posIn < lenIn) + if ((code = strIn[posIn++]) < FSST_ESC) { + size_t posWrite = posOut, endWrite = posOut + len[code]; + unsigned char* __restrict__ symbolPointer = ((unsigned char* __restrict__) &symbol[code]) - posWrite; + if ((posOut = endWrite) > size) endWrite = size; + for(; posWrite < endWrite; posWrite++) /* only write if there is room */ + strOut[posWrite] = symbolPointer[posWrite]; + } else { + if (posOut < size) strOut[posOut] = strIn[posIn]; /* idem */ + posIn++; posOut++; + } + if (posOut >= size && (decoder->zeroTerminated&1)) strOut[size-1] = 0; + return posOut; /* full size of decompressed string (could be >size, then the actually decompressed part) */ +} + +#ifdef __cplusplus +} +#endif +#endif /* FSST_INCLUDED_H */ diff --git a/cpp/thirdparty/fsst/fsst_avx512.cpp b/cpp/thirdparty/fsst/fsst_avx512.cpp new file mode 100644 index 000000000000..150683d2f9ac --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512.cpp @@ -0,0 +1,150 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#include "libfsst.hpp" + +#if defined(__x86_64__) || defined(_M_X64) +#include + +#ifdef _WIN32 +namespace libfsst { +bool fsst_hasAVX512() { + int info[4]; + __cpuidex(info, 0x00000007, 0); + return (info[1]>>16)&1; +} +} // namespace libfsst +#else +#include +namespace libfsst { +bool fsst_hasAVX512() { + int info[4]; + __cpuid_count(0x00000007, 0, info[0], info[1], info[2], info[3]); + return (info[1]>>16)&1; +} +} // namespace libfsst +#endif +#else +namespace libfsst { +bool fsst_hasAVX512() { return false; } +} // namespace libfsst +#endif + +namespace libfsst { + +// BULK COMPRESSION OF STRINGS +// +// In one call of this function, we can compress 512 strings, each of maximum length 511 bytes. +// strings can be shorter than 511 bytes, no problem, but if they are longer we need to cut them up. +// +// In each iteration of the while loop, we find one code in each of the unroll*8 strings, i.e. (8,16,24 or 32) for resp. unroll=1,2,3,4 +// unroll3 performs best on my hardware +// +// In the worst case, each final encoded string occupies 512KB bytes (512*1024; with 1024=512xexception, exception = 2 bytes). +// - hence codeBase is a buffer of 512KB (needs 19 bits jobs), symbolBase of 256KB (needs 18 bits jobs). +// +// 'jobX' controls the encoding of each string and is therefore a u64 with format [out:19][pos:9][end:18][cur:18] (low-to-high bits) +// The field 'pos' tells which string we are processing (0..511). We need this info as strings will complete compressing out-of-order. +// +// Strings will have different lengths, and when a string is finished, we reload from the buffer of 512 input strings. +// This continues until we have less than (8,16,24 or 32; depending on unroll) strings left to process. +// - so 'processed' is the amount of strings we started processing and it is between [480,512]. +// Note that when we quit, there will still be some (<32) strings that we started to process but which are unfinished. +// - so 'unfinished' is that amount. These unfinished strings will be encoded further using the scalar method. +// +// Apart from the coded strings, we return in a output[] array of size 'processed' the job values of the 'finished' strings. +// In the following 'unfinished' slots (processed=finished+unfinished) we output the 'job' values of the unfinished strings. +// +// For the finished strings, we need [out:19] to see the compressed size and [pos:9] to see which string we refer to. +// For the unfinished strings, we need all fields of 'job' to continue the compression with scalar code (see SIMD code in compressBatch). +// +// THIS IS A SEPARATE CODE FILE NOT BECAUSE OF MY LOVE FOR MODULARIZED CODE BUT BECAUSE IT ALLOWS TO COMPILE IT WITH DIFFERENT FLAGS +// in particular, unrolling is crucial for gather/scatter performance, but requires registers. the #define all_* expressions however, +// will be detected to be constants by g++ -O2 and will be precomputed and placed into AVX512 registers - spoiling 9 of them. +// This reduces the effectiveness of unrolling, hence -O2 makes the loop perform worse than -O1 which skips this optimization. +// Assembly inspection confirmed that 3-way unroll with -O1 avoids needless load/stores. + +size_t fsst_compressAVX512(SymbolTable &symbolTable, u8* codeBase, u8* symbolBase, SIMDjob *input, SIMDjob *output, size_t n, size_t unroll) { + size_t processed = 0; + // define some constants (all_x means that all 8 lanes contain 64-bits value X) +#ifdef __AVX512F__ + //__m512i all_suffixLim= _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) symbolTable->suffixLim)); -- for variants b,c + __m512i all_MASK = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) -1)); + __m512i all_PRIME = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) FSST_HASH_PRIME)); + __m512i all_ICL_FREE = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) FSST_ICL_FREE)); +#define all_HASH _mm512_srli_epi64(all_MASK, 64-FSST_HASH_LOG2SIZE) +#define all_ONE _mm512_srli_epi64(all_MASK, 63) +#define all_M19 _mm512_srli_epi64(all_MASK, 45) +#define all_M18 _mm512_srli_epi64(all_MASK, 46) +#define all_M28 _mm512_srli_epi64(all_MASK, 36) +#define all_FFFFFF _mm512_srli_epi64(all_MASK, 40) +#define all_FFFF _mm512_srli_epi64(all_MASK, 48) +#define all_FF _mm512_srli_epi64(all_MASK, 56) + + SIMDjob *inputEnd = input+n; + assert(n >= unroll*8 && n <= 512); // should be close to 512 + __m512i job1, job2, job3, job4; // will contain current jobs, for each unroll 1,2,3,4 + __mmask8 loadmask1 = 255, loadmask2 = 255*(unroll>1), loadmask3 = 255*(unroll>2), loadmask4 = 255*(unroll>3); // 2b loaded new strings bitmask per unroll + u32 delta1 = 8, delta2 = 8*(unroll>1), delta3 = 8*(unroll>2), delta4 = 8*(unroll>3); // #new loads this SIMD iteration per unroll + + if (unroll >= 4) { + while (input+delta1+delta2+delta3+delta4 < inputEnd) { + #include "fsst_avx512_unroll4.inc" + } + } else if (unroll == 3) { + while (input+delta1+delta2+delta3 < inputEnd) { + #include "fsst_avx512_unroll3.inc" + } + } else if (unroll == 2) { + while (input+delta1+delta2 < inputEnd) { + #include "fsst_avx512_unroll2.inc" + } + } else { + while (input+delta1 < inputEnd) { + #include "fsst_avx512_unroll1.inc" + } + } + + // flush the job states of the unfinished strings at the end of output[] + processed = n - (inputEnd - input); + u32 unfinished = 0; + if (unroll > 1) { + if (unroll > 2) { + if (unroll > 3) { + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask4=~loadmask4, job4); + unfinished += _mm_popcnt_u32((int) loadmask4); + } + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask3=~loadmask3, job3); + unfinished += _mm_popcnt_u32((int) loadmask3); + } + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask2=~loadmask2, job2); + unfinished += _mm_popcnt_u32((int) loadmask2); + } + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask1=~loadmask1, job1); +#else + (void) symbolTable; + (void) codeBase; + (void) symbolBase; + (void) input; + (void) output; + (void) n; + (void) unroll; +#endif + return processed; +} +} // namespace libfsst + diff --git a/cpp/thirdparty/fsst/fsst_avx512_unroll1.inc b/cpp/thirdparty/fsst/fsst_avx512_unroll1.inc new file mode 100644 index 000000000000..f4b81c7970dd --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512_unroll1.inc @@ -0,0 +1,57 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; diff --git a/cpp/thirdparty/fsst/fsst_avx512_unroll2.inc b/cpp/thirdparty/fsst/fsst_avx512_unroll2.inc new file mode 100644 index 000000000000..aa33cd7e69c5 --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512_unroll2.inc @@ -0,0 +1,114 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + code2 = _mm512_and_epi64(code2, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + // calculate the amount of lanes in job2 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + delta2 = _mm_popcnt_u32((int) loadmask2); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; + _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; diff --git a/cpp/thirdparty/fsst/fsst_avx512_unroll3.inc b/cpp/thirdparty/fsst/fsst_avx512_unroll3.inc new file mode 100644 index 000000000000..e2057032abd3 --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512_unroll3.inc @@ -0,0 +1,171 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// +// +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E3PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask3=11111111, delta3=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; + job3 = _mm512_mask_expandloadu_epi64(job3, loadmask3, input); input += delta3; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); + __m512i word3 = _mm512_i64gather_epi64(_mm512_srli_epi64(job3, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code3: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code3 = _mm512_i64gather_epi64(_mm512_and_epi64(word3, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + // get the first three bytes of the string. + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); + __m512i pos3 = _mm512_mullo_epi64(_mm512_and_epi64(word3, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT + // hash them into a random number: pos3 = pos3*PRIME; pos3 ^= pos3>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); + pos3 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos3,_mm512_srli_epi64(pos3,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write3 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); + __m512i write3 = _mm512_slli_epi64(_mm512_and_epi64(word3, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl3 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); + pos3 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl3, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); + __mmask8 match3 = _mm512_cmpeq_epi64_mask(symb3, _mm512_and_epi64(word3, pos3)) & _mm512_cmplt_epi64_mask(icl3, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); + code3 = _mm512_mask_mov_epi64(code3, match3, _mm512_srli_epi64(icl3, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); + write3 = _mm512_or_epi64(write3, _mm512_and_epi64(code3, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + code2 = _mm512_and_epi64(code2, all_FFFF); + code3 = _mm512_and_epi64(code3, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job3, all_M19), write3, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job3.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); + job3 = _mm512_add_epi64(job3, _mm512_slli_epi64(_mm512_srli_epi64(code3, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job3.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); + job3 = _mm512_add_epi64(job3, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code3, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) + // test which lanes are done now (job3.cur==job3.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job3 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); + loadmask3 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job3, 46), _mm512_and_epi64(_mm512_srli_epi64(job3, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + // calculate the amount of lanes in job2 that are done + // calculate the amount of lanes in job3 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + delta2 = _mm_popcnt_u32((int) loadmask2); + delta3 = _mm_popcnt_u32((int) loadmask3); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job3.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; + _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; + _mm512_mask_compressstoreu_epi64(output, loadmask3, job3); output += delta3; diff --git a/cpp/thirdparty/fsst/fsst_avx512_unroll4.inc b/cpp/thirdparty/fsst/fsst_avx512_unroll4.inc new file mode 100644 index 000000000000..15cca7c938b4 --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512_unroll4.inc @@ -0,0 +1,228 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// +// +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// +// +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// +// +// +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// +// +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E3PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E4PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// +// +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// +// +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask3=11111111, delta3=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask4=11111111, delta4=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; + job3 = _mm512_mask_expandloadu_epi64(job3, loadmask3, input); input += delta3; + job4 = _mm512_mask_expandloadu_epi64(job4, loadmask4, input); input += delta4; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); + __m512i word3 = _mm512_i64gather_epi64(_mm512_srli_epi64(job3, 46), symbolBase, 1); + __m512i word4 = _mm512_i64gather_epi64(_mm512_srli_epi64(job4, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code3: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code4: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code3 = _mm512_i64gather_epi64(_mm512_and_epi64(word3, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code4 = _mm512_i64gather_epi64(_mm512_and_epi64(word4, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + // get the first three bytes of the string. + // get the first three bytes of the string. + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); + __m512i pos3 = _mm512_mullo_epi64(_mm512_and_epi64(word3, all_FFFFFF), all_PRIME); + __m512i pos4 = _mm512_mullo_epi64(_mm512_and_epi64(word4, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT + // hash them into a random number: pos3 = pos3*PRIME; pos3 ^= pos3>>SHIFT + // hash them into a random number: pos4 = pos4*PRIME; pos4 ^= pos4>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); + pos3 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos3,_mm512_srli_epi64(pos3,FSST_SHIFT)), all_HASH), 4); + pos4 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos4,_mm512_srli_epi64(pos4,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl4 = _mm512_i64gather_epi64(pos4, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write3 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write4 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); + __m512i write3 = _mm512_slli_epi64(_mm512_and_epi64(word3, all_FF), 8); + __m512i write4 = _mm512_slli_epi64(_mm512_and_epi64(word4, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl3 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl4 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb4 = _mm512_i64gather_epi64(pos4, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); + pos3 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl3, all_FF)); + pos4 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl4, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); + __mmask8 match3 = _mm512_cmpeq_epi64_mask(symb3, _mm512_and_epi64(word3, pos3)) & _mm512_cmplt_epi64_mask(icl3, all_ICL_FREE); + __mmask8 match4 = _mm512_cmpeq_epi64_mask(symb4, _mm512_and_epi64(word4, pos4)) & _mm512_cmplt_epi64_mask(icl4, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); + code3 = _mm512_mask_mov_epi64(code3, match3, _mm512_srli_epi64(icl3, 16)); + code4 = _mm512_mask_mov_epi64(code4, match4, _mm512_srli_epi64(icl4, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); + write3 = _mm512_or_epi64(write3, _mm512_and_epi64(code3, all_FF)); + write4 = _mm512_or_epi64(write4, _mm512_and_epi64(code4, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + code2 = _mm512_and_epi64(code2, all_FFFF); + code3 = _mm512_and_epi64(code3, all_FFFF); + code4 = _mm512_and_epi64(code4, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job3, all_M19), write3, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job4, all_M19), write4, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job3.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job4.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); + job3 = _mm512_add_epi64(job3, _mm512_slli_epi64(_mm512_srli_epi64(code3, FSST_LEN_BITS), 46)); + job4 = _mm512_add_epi64(job4, _mm512_slli_epi64(_mm512_srli_epi64(code4, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job3.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job4.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); + job3 = _mm512_add_epi64(job3, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code3, 8), all_ONE))); + job4 = _mm512_add_epi64(job4, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code4, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) + // test which lanes are done now (job3.cur==job3.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job3 register) + // test which lanes are done now (job4.cur==job4.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job4 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); + loadmask3 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job3, 46), _mm512_and_epi64(_mm512_srli_epi64(job3, 28), all_M18)); + loadmask4 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job4, 46), _mm512_and_epi64(_mm512_srli_epi64(job4, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + // calculate the amount of lanes in job2 that are done + // calculate the amount of lanes in job3 that are done + // calculate the amount of lanes in job4 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + delta2 = _mm_popcnt_u32((int) loadmask2); + delta3 = _mm_popcnt_u32((int) loadmask3); + delta4 = _mm_popcnt_u32((int) loadmask4); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job3.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job4.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; + _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; + _mm512_mask_compressstoreu_epi64(output, loadmask3, job3); output += delta3; + _mm512_mask_compressstoreu_epi64(output, loadmask4, job4); output += delta4; diff --git a/cpp/thirdparty/fsst/libfsst.cpp b/cpp/thirdparty/fsst/libfsst.cpp new file mode 100644 index 000000000000..e3ba787b9592 --- /dev/null +++ b/cpp/thirdparty/fsst/libfsst.cpp @@ -0,0 +1,651 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#include "libfsst.hpp" + +namespace libfsst { +Symbol concat(Symbol a, Symbol b) { + Symbol s; + u32 length = a.length()+b.length(); + if (length > Symbol::maxLength) length = Symbol::maxLength; + s.set_code_len(FSST_CODE_MASK, length); + s.store_num((b.load_num() << (8*a.length())) | a.load_num()); + return s; +} +} // namespace libfsst + +namespace std { +template <> +class hash { + public: + size_t operator()(const libfsst::QSymbol& q) const { + uint64_t k = q.symbol.load_num(); + const uint64_t m = 0xc6a4a7935bd1e995; + const int r = 47; + uint64_t h = 0x8445d61a4e774912 ^ (8*m); + k *= m; + k ^= k >> r; + k *= m; + h ^= k; + h *= m; + h ^= h >> r; + h *= m; + h ^= h >> r; + return h; + } +}; +} + +namespace libfsst { +bool isEscapeCode(u16 pos) { return pos < FSST_CODE_BASE; } + +std::ostream& operator<<(std::ostream& out, const Symbol& s) { + for (u32 i=0; i line, const size_t len[], bool zeroTerminated=false) { + SymbolTable *st = new SymbolTable(), *bestTable = new SymbolTable(); + int bestGain = (int) -FSST_SAMPLEMAXSZ; // worst case (everything exception) + size_t sampleFrac = 128; + + // start by determining the terminator. We use the (lowest) most infrequent byte as terminator + st->zeroTerminated = zeroTerminated; + if (zeroTerminated) { + st->terminator = 0; // except in case of zeroTerminated mode, then byte 0 is terminator regardless frequency + } else { + u16 byteHisto[256]; + memset(byteHisto, 0, sizeof(byteHisto)); + for(size_t i=0; iterminator = 256; + while(i-- > 0) { + if (byteHisto[i] > minSize) continue; + st->terminator = i; + minSize = byteHisto[i]; + } + } + assert(st->terminator != 256); + + // a random number between 0 and 128 + auto rnd128 = [&](size_t i) { return 1 + (FSST_HASH((i+1UL)*sampleFrac)&127); }; + + // compress sample, and compute (pair-)frequencies + auto compressCount = [&](SymbolTable *st, Counters &counters) { // returns gain + int gain = 0; + + for(size_t i=0; i sampleFrac) continue; + } + if (cur < end) { + u16 code2 = 255, code1 = st->findLongestSymbol(cur, end); + cur += st->symbols[code1].length(); + gain += (int) (st->symbols[code1].length()-(1+isEscapeCode(code1))); + while (true) { + // count single symbol (i.e. an option is not extending it) + counters.count1Inc(code1); + + // as an alternative, consider just using the next byte.. + if (st->symbols[code1].length() != 1) // .. but do not count single byte symbols doubly + counters.count1Inc(*start); + + if (cur==end) { + break; + } + + // now match a new symbol + start = cur; + if (curhashTabSize-1); + Symbol s = st->hashTab[idx]; + code2 = st->shortCodes[word & 0xFFFF] & FSST_CODE_MASK; + word &= (0xFFFFFFFFFFFFFFFF >> (u8) s.icl); + if ((s.icl < FSST_ICL_FREE) & (s.load_num() == word)) { + code2 = s.code(); + cur += s.length(); + } else if (code2 >= FSST_CODE_BASE) { + cur += 2; + } else { + code2 = st->byteCodes[word & 0xFF] & FSST_CODE_MASK; + cur += 1; + } + } else { + code2 = st->findLongestSymbol(cur, end); + cur += st->symbols[code2].length(); + } + + // compute compressed output size + gain += ((int) (cur-start))-(1+isEscapeCode(code2)); + + if (sampleFrac < 128) { // no need to count pairs in final round + // consider the symbol that is the concatenation of the two last symbols + counters.count2Inc(code1, code2); + + // as an alternative, consider just extending with the next byte.. + if ((cur-start) > 1) // ..but do not count single byte extensions doubly + counters.count2Inc(code1, *start); + } + code1 = code2; + } + } + } + return gain; + }; + + auto makeTable = [&](SymbolTable *st, Counters &counters) { + // hashmap of c (needed because we can generate duplicate candidates) + unordered_set cands; + + // artificially make terminater the most frequent symbol so it gets included + u16 terminator = st->nSymbols?FSST_CODE_BASE:st->terminator; + counters.count1Set(terminator,65535); + + auto addOrInc = [&](unordered_set &cands, Symbol s, u64 count) { + if (count < (5*sampleFrac)/128) return; // improves both compression speed (less candidates), but also quality!! + QSymbol q; + q.symbol = s; + q.gain = count * s.length(); + auto it = cands.find(q); + if (it != cands.end()) { + q.gain += (*it).gain; + cands.erase(*it); + } + cands.insert(q); + }; + + // add candidate symbols based on counted frequency + for (u32 pos1=0; pos1nSymbols; pos1++) { + u32 cnt1 = counters.count1GetNext(pos1); // may advance pos1!! + if (!cnt1) continue; + + // heuristic: promoting single-byte symbols (*8) helps reduce exception rates and increases [de]compression speed + Symbol s1 = st->symbols[pos1]; + addOrInc(cands, s1, ((s1.length()==1)?8LL:1LL)*cnt1); + + if (sampleFrac >= 128 || // last round we do not create new (combined) symbols + s1.length() == Symbol::maxLength || // symbol cannot be extended + s1.val.str[0] == st->terminator) { // multi-byte symbols cannot contain the terminator byte + continue; + } + for (u32 pos2=0; pos2nSymbols; pos2++) { + u32 cnt2 = counters.count2GetNext(pos1, pos2); // may advance pos2!! + if (!cnt2) continue; + + // create a new symbol + Symbol s2 = st->symbols[pos2]; + Symbol s3 = concat(s1, s2); + if (s2.val.str[0] != st->terminator) // multi-byte symbols cannot contain the terminator byte + addOrInc(cands, s3, cnt2); + } + } + + // insert candidates into priority queue (by gain) + auto cmpGn = [](const QSymbol& q1, const QSymbol& q2) { return (q1.gain < q2.gain) || (q1.gain == q2.gain && q1.symbol.load_num() > q2.symbol.load_num()); }; + priority_queue,decltype(cmpGn)> pq(cmpGn); + for (auto& q : cands) + pq.push(q); + + // Create new symbol map using best candidates + st->clear(); + while (st->nSymbols < 255 && !pq.empty()) { + QSymbol q = pq.top(); + pq.pop(); + st->add(q.symbol); + } + }; + + u8 bestCounters[512*sizeof(u16)]; +#ifdef NONOPT_FSST + for(size_t frac : {127, 127, 127, 127, 127, 127, 127, 127, 127, 128}) { + sampleFrac = frac; +#else + for(sampleFrac=8; true; sampleFrac += 30) { +#endif + memset(&counters, 0, sizeof(Counters)); + long gain = compressCount(st, counters); + if (gain >= bestGain) { // a new best solution! + counters.backup1(bestCounters); + *bestTable = *st; bestGain = gain; + } + if (sampleFrac >= 128) break; // we do 5 rounds (sampleFrac=8,38,68,98,128) + makeTable(st, counters); + } + delete st; + counters.restore1(bestCounters); + makeTable(bestTable, counters); + bestTable->finalize(zeroTerminated); // renumber codes for more efficient compression + return bestTable; +} + +#ifndef NONOPT_FSST +static inline size_t compressSIMD(SymbolTable &symbolTable, u8* symbolBase, size_t nlines, const size_t len[], const u8* line[], size_t size, u8* dst, size_t lenOut[], u8* strOut[], int unroll) { + size_t curLine = 0, inOff = 0, outOff = 0, batchPos = 0, empty = 0, budget = size; + u8 *lim = dst + size, *codeBase = symbolBase + (1<<18); // 512KB temp space for compressing 512 strings + SIMDjob input[512]; // combined offsets of input strings (cur,end), and string #id (pos) and output (dst) pointer + SIMDjob output[512]; // output are (pos:9,dst:19) end pointers (compute compressed length from this) + size_t jobLine[512]; // for which line in the input sequence was this job (needed because we may split a line into multiple jobs) + + while (curLine < nlines && outOff <= (1<<19)) { + size_t prevLine = curLine, chunk, curOff = 0; + + // bail out if the output buffer cannot hold the compressed next string fully + if (((len[curLine]-curOff)*2 + 7) > budget) break; // see below for the +7 + else budget -= (len[curLine]-curOff)*2; + + strOut[curLine] = (u8*) 0; + lenOut[curLine] = 0; + + do { + do { + chunk = len[curLine] - curOff; + if (chunk > 511) { + chunk = 511; // large strings need to be chopped up into segments of 511 bytes + } + // create a job in this batch + SIMDjob job; + job.cur = inOff; + job.end = job.cur + chunk; + job.pos = batchPos; + job.out = outOff; + + // worst case estimate for compressed size (+7 is for the scatter that writes extra 7 zeros) + outOff += 7 + 2*(size_t)(job.end - job.cur); // note, total size needed is 512*(511*2+7) bytes. + if (outOff > (1<<19)) break; // simdbuf may get full, stop before this chunk + + // register job in this batch + input[batchPos] = job; + jobLine[batchPos] = curLine; + + if (chunk == 0) { + empty++; // detect empty chunks -- SIMD code cannot handle empty strings, so they need to be filtered out + } else { + // copy string chunk into temp buffer + memcpy(symbolBase + inOff, line[curLine] + curOff, chunk); + inOff += chunk; + curOff += chunk; + symbolBase[inOff++] = (u8) symbolTable.terminator; // write an extra char at the end that will not be encoded + } + if (++batchPos == 512) break; + } while(curOff < len[curLine]); + + if ((batchPos == 512) || (outOff > (1<<19)) || (++curLine >= nlines) || (((len[curLine])*2 + 7) > budget)) { // cannot accumulate more? + if (batchPos-empty >= 32) { // if we have enough work, fire off fsst_compressAVX512 (32 is due to max 4x8 unrolling) + // radix-sort jobs on length (longest string first) + // -- this provides best load balancing and allows to skip empty jobs at the end + u16 sortpos[513]; + memset(sortpos, 0, sizeof(sortpos)); + + // calculate length histo + for(size_t i=0; i> (u8) s.icl); + if ((s.icl < FSST_ICL_FREE) && s.load_num() == word) { + *out++ = (u8) s.code(); cur += s.length(); + } else { + // could be a 2-byte or 1-byte code, or miss + // handle everything with predication + *out = (u8) code; + out += 1+((code&FSST_CODE_BASE)>>8); + cur += (code>>FSST_LEN_BITS); + } + } + job.out = out - codeBase; + } + // postprocess job info + job.cur = 0; + job.end = job.out - input[job.pos].out; // misuse .end field as compressed size + job.out = input[job.pos].out; // reset offset to start of encoded string + input[job.pos] = job; + } + + // copy out the result data + for(size_t i=0; i> (u8) s.icl); + if ((s.icl < FSST_ICL_FREE) && s.load_num() == word) { + *out++ = (u8) s.code(); cur += s.length(); + } else if (avoidBranch) { + // could be a 2-byte or 1-byte code, or miss + // handle everything with predication + *out = (u8) code; + out += 1+((code&FSST_CODE_BASE)>>8); + cur += (code>>FSST_LEN_BITS); + } else if ((u8) code < byteLim) { + // 2 byte code after checking there is no longer pattern + *out++ = (u8) code; cur += 2; + } else { + // 1 byte code or miss. + *out = (u8) code; + out += 1+((code&FSST_CODE_BASE)>>8); // predicated - tested with a branch, that was always worse + cur++; + } + } + } + }; + + for(curLine=0; curLine 511) { + chunk = 511; // we need to compress in chunks of 511 in order to be byte-compatible with simd-compressed FSST + } + if ((2*chunk+7) > (size_t) (lim-out)) { + return curLine; // out of memory + } + // copy the string to the 511-byte buffer + memcpy(buf, cur, chunk); + buf[chunk] = (u8) symbolTable.terminator; + cur = buf; + end = cur + chunk; + + // based on symboltable stats, choose a variant that is nice to the branch predictor + if (noSuffixOpt) { + compressVariant(true,false); + } else if (avoidBranch) { + compressVariant(false,true); + } else { + compressVariant(false, false); + } + } while((curOff += chunk) < lenIn[curLine]); + lenOut[curLine] = (size_t) (out - strOut[curLine]); + } + return curLine; +} + +#define FSST_SAMPLELINE ((size_t) 512) + +// quickly select a uniformly random set of lines such that we have between [FSST_SAMPLETARGET,FSST_SAMPLEMAXSZ) string bytes +vector makeSample(u8* sampleBuf, const u8* strIn[], const size_t **lenRef, size_t nlines) { + size_t totSize = 0; + const size_t *lenIn = *lenRef; + vector sample; + + for(size_t i=0; i sample = makeSample(sampleBuf, strIn, &sampleLen, n?n:1); // careful handling of input to get a right-size and representative sample + Encoder *encoder = new Encoder(); + encoder->symbolTable = shared_ptr(buildSymbolTable(encoder->counters, sample, sampleLen, zeroTerminated)); + if (sampleLen != lenIn) delete[] sampleLen; + delete[] sampleBuf; + return (fsst_encoder_t*) encoder; +} + +/* create another encoder instance, necessary to do multi-threaded encoding using the same symbol table */ +extern "C" fsst_encoder_t* fsst_duplicate(fsst_encoder_t *encoder) { + Encoder *e = new Encoder(); + e->symbolTable = ((Encoder*)encoder)->symbolTable; // it is a shared_ptr + return (fsst_encoder_t*) e; +} + +// export a symbol table in compact format. +extern "C" u32 fsst_export(fsst_encoder_t *encoder, u8 *buf) { + Encoder *e = (Encoder*) encoder; + // In ->version there is a versionnr, but we hide also suffixLim/terminator/nSymbols there. + // This is sufficient in principle to *reconstruct* a fsst_encoder_t from a fsst_decoder_t + // (such functionality could be useful to append compressed data to an existing block). + // + // However, the hash function in the encoder hash table is endian-sensitive, and given its + // 'lossy perfect' hashing scheme is *unable* to contain other-endian-produced symbol tables. + // Doing a endian-conversion during hashing will be slow and self-defeating. + // + // Overall, we could support reconstructing an encoder for incremental compression, but + // should enforce equal-endianness. Bit of a bummer. Not going there now. + // + // The version field is now there just for future-proofness, but not used yet + + // version allows keeping track of fsst versions, track endianness, and encoder reconstruction + u64 version = (FSST_VERSION << 32) | // version is 24 bits, most significant byte is 0 + (((u64) e->symbolTable->suffixLim) << 24) | + (((u64) e->symbolTable->terminator) << 16) | + (((u64) e->symbolTable->nSymbols) << 8) | + FSST_ENDIAN_MARKER; // least significant byte is nonzero + + version = swap64_if_be(version); // ensure version is little-endian encoded + + /* do not assume unaligned reads here */ + memcpy(buf, &version, 8); + buf[8] = e->symbolTable->zeroTerminated; + for(u32 i=0; i<8; i++) + buf[9+i] = (u8) e->symbolTable->lenHisto[i]; + u32 pos = 17; + + // emit only the used bytes of the symbols + for(u32 i = e->symbolTable->zeroTerminated; i < e->symbolTable->nSymbols; i++) + for(u32 j = 0; j < e->symbolTable->symbols[i].length(); j++) + buf[pos++] = e->symbolTable->symbols[i].val.str[j]; // serialize used symbol bytes + + return pos; // length of what was serialized +} + +#define FSST_CORRUPT 32774747032022883 /* 7-byte number in little endian containing "corrupt" */ + +extern "C" u32 fsst_import(fsst_decoder_t *decoder, u8 const *buf) { + u64 version = 0; + u32 code, pos = 17; + u8 lenHisto[8]; + + // version field (first 8 bytes) is now there just for future-proofness, unused still (skipped) + memcpy(&version, buf, 8); + version = swap64_if_be(version); // version is always little-endian encoded + + if ((version>>32) != FSST_VERSION) return 0; + decoder->zeroTerminated = buf[8]&1; + memcpy(lenHisto, buf+9, 8); + + // in case of zero-terminated, first symbol is "" (zero always, may be overwritten) + decoder->len[0] = 1; + decoder->symbol[0] = 0; + + // we use lenHisto[0] as 1-byte symbol run length (at the end) + code = decoder->zeroTerminated; + if (decoder->zeroTerminated) lenHisto[0]--; // if zeroTerminated, then symbol "" aka 1-byte code=0, is not stored at the end + + // now get all symbols from the buffer + for(u32 l=1; l<=8; l++) { /* l = 1,2,3,4,5,6,7,8 */ + for(u32 i=0; i < lenHisto[(l&7) /* 1,2,3,4,5,6,7,0 */]; i++, code++) { + decoder->len[code] = (l&7)+1; /* len = 2,3,4,5,6,7,8,1 */ + decoder->symbol[code] = 0; + for(u32 j=0; jlen[code]; j++) + ((u8*) &decoder->symbol[code])[j] = buf[pos++]; // note this enforces 'little endian' symbols + } + } + if (decoder->zeroTerminated) lenHisto[0]++; + + // fill unused symbols with text "corrupt". Gives a chance to detect corrupted code sequences (if there are unused symbols). + while(code<255) { + decoder->symbol[code] = FSST_CORRUPT; + decoder->len[code++] = 8; + } + return pos; +} + +// runtime check for simd +inline size_t _compressImpl(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) { +#ifndef NONOPT_FSST + if (simd && fsst_hasAVX512()) + return compressSIMD(*e->symbolTable, e->simdbuf, nlines, lenIn, strIn, size, output, lenOut, strOut, simd); +#endif + (void) simd; + return compressBulk(*e->symbolTable, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch); +} +size_t compressImpl(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) { + return _compressImpl(e, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch, simd); +} + +// adaptive choosing of scalar compression method based on symbol length histogram +inline size_t _compressAuto(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], int simd) { + bool avoidBranch = false, noSuffixOpt = false; + if (100*e->symbolTable->lenHisto[1] > 65*e->symbolTable->nSymbols && 100*e->symbolTable->suffixLim > 95*e->symbolTable->lenHisto[1]) { + noSuffixOpt = true; + } else if ((e->symbolTable->lenHisto[0] > 24 && e->symbolTable->lenHisto[0] < 92) && + (e->symbolTable->lenHisto[0] < 43 || e->symbolTable->lenHisto[6] + e->symbolTable->lenHisto[7] < 29) && + (e->symbolTable->lenHisto[0] < 72 || e->symbolTable->lenHisto[2] < 72)) { + avoidBranch = true; + } + return _compressImpl(e, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch, simd); +} +size_t compressAuto(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], int simd) { + return _compressAuto(e, nlines, lenIn, strIn, size, output, lenOut, strOut, simd); +} +} // namespace libfsst + +using namespace libfsst; +// the main compression function (everything automatic) +extern "C" size_t fsst_compress(fsst_encoder_t *encoder, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[]) { + // to be faster than scalar, simd needs 64 lines or more of length >=12; or fewer lines, but big ones (totLen > 32KB) + size_t totLen = accumulate(lenIn, lenIn+nlines, 0); + int simd = totLen > nlines*12 && (nlines > 64 || totLen > (size_t) 1<<15); + return _compressAuto((Encoder*) encoder, nlines, lenIn, strIn, size, output, lenOut, strOut, 3*simd); +} + +/* deallocate encoder */ +extern "C" void fsst_destroy(fsst_encoder_t* encoder) { + Encoder *e = (Encoder*) encoder; + delete e; +} + +/* very lazy implementation relying on export and import */ +extern "C" fsst_decoder_t fsst_decoder(fsst_encoder_t *encoder) { + u8 buf[sizeof(fsst_decoder_t)]; + u32 cnt1 = fsst_export(encoder, buf); + fsst_decoder_t decoder; + u32 cnt2 = fsst_import(&decoder, buf); + assert(cnt1 == cnt2); (void) cnt1; (void) cnt2; + return decoder; +} diff --git a/cpp/thirdparty/fsst/libfsst.hpp b/cpp/thirdparty/fsst/libfsst.hpp new file mode 100644 index 000000000000..f61bc0175b63 --- /dev/null +++ b/cpp/thirdparty/fsst/libfsst.hpp @@ -0,0 +1,471 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +#include "fsst.h" // the official FSST API -- also usable by C mortals + +/* unsigned integers */ +namespace libfsst { +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +} // namespace libfsst + +#if UINTPTR_MAX == 0xffffffffU +// We're on a 32-bit platform +#define NONOPT_FSST +#endif + +#define FSST_ENDIAN_MARKER ((u64) 1) +#define FSST_VERSION_20190218 20190218 +#define FSST_VERSION ((u64) FSST_VERSION_20190218) + +// "symbols" are character sequences (up to 8 bytes) +// A symbol is compressed into a "code" of, in principle, one byte. But, we added an exception mechanism: +// byte 255 followed by byte X represents the single-byte symbol X. Its code is 256+X. + +// we represent codes in u16 (not u8). 12 bits code (of which 10 are used), 4 bits length +#define FSST_LEN_BITS 12 +#define FSST_CODE_BITS 9 +#define FSST_CODE_BASE 256UL /* first 256 codes [0,255] are pseudo codes: escaped bytes */ +#define FSST_CODE_MAX (1UL<=8) { + len = 8; + memcpy(val.str, input, 8); + } else { + memcpy(val.str, input, len); + } + set_code_len(FSST_CODE_MAX, len); + } + void set_code_len(u32 code, u32 len) { icl = (len<<28)|(code<<16)|((8-len)*8); } + + u64 load_num() const { return swap64_if_be(val.num); } + void store_num(u64 v) { val.num = swap64_if_be(v); } + + u32 length() const { return (u32) (icl >> 28); } + u16 code() const { return (icl >> 16) & FSST_CODE_MASK; } + u32 ignoredBits() const { return (u32) icl; } + + u8 first() const { assert( length() >= 1); return 0xFF & load_num(); } + u16 first2() const { assert( length() >= 2); return 0xFFFF & load_num(); } + +#define FSST_HASH_LOG2SIZE 10 +#define FSST_HASH_PRIME 2971215073LL +#define FSST_SHIFT 15 +#define FSST_HASH(w) (((w)*FSST_HASH_PRIME)^(((w)*FSST_HASH_PRIME)>>FSST_SHIFT)) + size_t hash() const { size_t v = 0xFFFFFF & load_num(); return FSST_HASH(v); } // hash on the next 3 bytes +}; + +// Symbol that can be put in a queue, ordered on gain +struct QSymbol{ + Symbol symbol; + mutable u32 gain; // mutable because gain value should be ignored in find() on unordered_set of QSymbols + bool operator==(const QSymbol& other) const { return symbol.val.num == other.symbol.val.num && symbol.length() == other.symbol.length(); } +}; + +// we construct FSST symbol tables using a random sample of about 16KB (1<<14) +#define FSST_SAMPLETARGET (1<<14) +#define FSST_SAMPLEMAXSZ ((long) 2*FSST_SAMPLETARGET) + +// two phases of compression, before and after optimize(): +// +// (1) to encode values we probe (and maintain) three datastructures: +// - u16 byteCodes[256] array at the position of the next byte (s.length==1) +// - u16 shortCodes[65536] array at the position of the next twobyte pattern (s.length==2) +// - Symbol hashtable[1024] (keyed by the next three bytes, ie for s.length>2), +// this search will yield a u16 code, it points into Symbol symbols[]. You always find a hit, because the first 256 codes are +// pseudo codes representing a single byte these will become escapes) +// +// (2) when we finished looking for the best symbol table we call optimize() to reshape it: +// - it renumbers the codes by length (first symbols of length 2,3,4,5,6,7,8; then 1 (starting from byteLim are symbols of length 1) +// length 2 codes for which no longer suffix symbol exists (< suffixLim) come first among the 2-byte codes +// (allows shortcut during compression) +// - for each two-byte combination, in all unused slots of shortCodes[], it enters the byteCode[] of the symbol corresponding +// to the first byte (if such a single-byte symbol exists). This allows us to just probe the next two bytes (if there is only one +// byte left in the string, there is still a terminator-byte added during compression) in shortCodes[]. That is, byteCodes[] +// and its codepath is no longer required. This makes compression faster. The reason we use byteCodes[] during symbolTable construction +// is that adding a new code/symbol is expensive (you have to touch shortCodes[] in 256 places). This optimization was +// hence added to make symbolTable construction faster. +// +// this final layout allows for the fastest compression code, only currently present in compressBulk + +// in the hash table, the icl field contains (low-to-high) ignoredBits:16,code:12,length:4 +#define FSST_ICL_FREE ((15<<28)|(((u32)FSST_CODE_MASK)<<16)) // high bits of icl (len=8,code=FSST_CODE_MASK) indicates free bucket + +// ignoredBits is (8-length)*8, which is the amount of high bits to zero in the input word before comparing with the hashtable key +// ..it could of course be computed from len during lookup, but storing it precomputed in some loose bits is faster +// +// the gain field is only used in the symbol queue that sorts symbols on gain + +struct SymbolTable { + static const u32 hashTabSize = 1<> (u8) s.icl)); + return true; + } + bool add(Symbol s) { + assert(FSST_CODE_BASE + nSymbols < FSST_CODE_MAX); + u32 len = s.length(); + s.set_code_len(FSST_CODE_BASE + nSymbols, len); + if (len == 1) { + byteCodes[s.first()] = FSST_CODE_BASE + nSymbols + (1<> ((u8) hashTab[idx].icl)))) { + return (hashTab[idx].icl>>16) & FSST_CODE_MASK; // matched a long symbol + } + if (s.length() >= 2) { + u16 code = shortCodes[s.first2()] & FSST_CODE_MASK; + if (code >= FSST_CODE_BASE) return code; + } + return byteCodes[s.first()] & FSST_CODE_MASK; + } + u16 findLongestSymbol(const u8* cur, const u8* end) const { + return findLongestSymbol(Symbol(cur,end)); // represent the string as a temporary symbol + } + + // rationale for finalize: + // - during symbol table construction, we may create more than 256 codes, but bring it down to max 255 in the last makeTable() + // consequently we needed more than 8 bits during symbol table contruction, but can simplify the codes to single bytes in finalize() + // (this feature is in fact lo longer used, but could still be exploited: symbol construction creates no more than 255 symbols in each pass) + // - we not only reduce the amount of codes to <255, but also *reorder* the symbols and renumber their codes, for higher compression perf. + // we renumber codes so they are grouped by length, to allow optimized scalar string compression (byteLim and suffixLim optimizations). + // - we make the use of byteCode[] no longer necessary by inserting single-byte codes in the free spots of shortCodes[] + // Using shortCodes[] only makes compression faster. When creating the symbolTable, however, using shortCodes[] for the single-byte + // symbols is slow, as each insert touches 256 positions in it. This optimization was added when optimizing symbolTable construction time. + // + // In all, we change the layout and coding, as follows.. + // + // before finalize(): + // - The real symbols are symbols[256..256+nSymbols>. As we may have nSymbols > 255 + // - The first 256 codes are pseudo symbols (all escaped bytes) + // + // after finalize(): + // - table layout is symbols[0..nSymbols>, with nSymbols < 256. + // - Real codes are [0,nSymbols>. 8-th bit not set. + // - Escapes in shortCodes have the 8th bit set (value: 256+255=511). 255 because the code to be emitted is the escape byte 255 + // - symbols are grouped by length: 2,3,4,5,6,7,8, then 1 (single-byte codes last) + // the two-byte codes are split in two sections: + // - first section contains codes for symbols for which there is no longer symbol (no suffix). It allows an early-out during compression + // + // finally, shortCodes[] is modified to also encode all single-byte symbols (hence byteCodes[] is not required on a critical path anymore). + // + void finalize(u8 zeroTerminated) { + assert(nSymbols <= 255); + u8 newCode[256], rsum[8], byteLim = nSymbols - (lenHisto[0] - zeroTerminated); + + // compute running sum of code lengths (starting offsets for each length) + rsum[0] = byteLim; // 1-byte codes are highest + rsum[1] = zeroTerminated; + for(u32 i=1; i<7; i++) + rsum[i+1] = rsum[i] + lenHisto[i]; + + // determine the new code for each symbol, ordered by length (and splitting 2byte symbols into two classes around suffixLim) + suffixLim = rsum[1]; + symbols[newCode[0] = 0] = symbols[256]; // keep symbol 0 in place (for zeroTerminated cases only) + + for(u32 i=zeroTerminated, j=rsum[2]; i 1 && first2 == s2.first2()) // test if symbol k is a suffix of s + opt = 0; + } + newCode[i] = opt?suffixLim++:--j; // symbols without a larger suffix have a code < suffixLim + } else + newCode[i] = rsum[len-1]++; + s1.set_code_len(newCode[i],len); + symbols[newCode[i]] = s1; + } + // renumber the codes in byteCodes[] + for(u32 i=0; i<256; i++) + if ((byteCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) + byteCodes[i] = newCode[(u8) byteCodes[i]] + (1 << FSST_LEN_BITS); + else + byteCodes[i] = 511 + (1 << FSST_LEN_BITS); + + // renumber the codes in shortCodes[] + for(u32 i=0; i<65536; i++) + if ((shortCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) + shortCodes[i] = newCode[(u8) shortCodes[i]] + (shortCodes[i] & (15 << FSST_LEN_BITS)); + else + shortCodes[i] = byteCodes[i&0xFF]; + + // replace the symbols in the hash table + for(u32 i=0; i>8; + } + void count1Inc(u32 pos1) { + if (!count1Low[pos1]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) + count1High[pos1]++; //(0,0)->(1,1)->..->(255,1)->(0,1)->(1,2)->(2,2)->(3,2)..(255,2)->(0,2)->(1,3)->(2,3)... + } + void count2Inc(u32 pos1, u32 pos2) { + if (!count2Low[pos1][pos2]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) + // inc 4-bits high counter with 1<<0 (1) or 1<<4 (16) -- depending on whether pos2 is even or odd, repectively + count2High[pos1][(pos2)>>1] += 1 << (((pos2)&1)<<2); // we take our chances with overflow.. (4K maxval, on a 8K sample) + } + u32 count1GetNext(u32 &pos1) { // note: we will advance pos1 to the next nonzero counter in register range + // read 16-bits single symbol counter, split into two 8-bits numbers (count1Low, count1High), while skipping over zeros + u64 high = fsst_unaligned_load(&count1High[pos1]); // note: this reads 8 subsequent counters [pos1..pos1+7] + + u32 zero = high?(__builtin_ctzl(high)>>3):7UL; // number of zero bytes + high = (high >> (zero << 3)) & 255; // advance to nonzero counter + if (((pos1 += zero) >= FSST_CODE_MAX) || !high) // SKIP! advance pos2 + return 0; // all zero + + u32 low = count1Low[pos1]; + if (low) high--; // high is incremented early and low late, so decrement high (unless low==0) + return (u32) ((high << 8) + low); + } + u32 count2GetNext(u32 pos1, u32 &pos2) { // note: we will advance pos2 to the next nonzero counter in register range + // read 12-bits pairwise symbol counter, split into low 8-bits and high 4-bits number while skipping over zeros + u64 high = fsst_unaligned_load(&count2High[pos1][pos2>>1]); // note: this reads 16 subsequent counters [pos2..pos2+15] + high >>= ((pos2&1) << 2); // odd pos2: ignore the lowest 4 bits & we see only 15 counters + + u32 zero = high?(__builtin_ctzl(high)>>2):(15UL-(pos2&1UL)); // number of zero 4-bits counters + high = (high >> (zero << 2)) & 15; // advance to nonzero counter + if (((pos2 += zero) >= FSST_CODE_MAX) || !high) // SKIP! advance pos2 + return 0UL; // all zero + + u32 low = count2Low[pos1][pos2]; + if (low) high--; // high is incremented early and low late, so decrement high (unless low==0) + return (u32) ((high << 8) + low); + } + void backup1(u8 *buf) { + memcpy(buf, count1High, FSST_CODE_MAX); + memcpy(buf+FSST_CODE_MAX, count1Low, FSST_CODE_MAX); + } + void restore1(u8 *buf) { + memcpy(count1High, buf, FSST_CODE_MAX); + memcpy(count1Low, buf+FSST_CODE_MAX, FSST_CODE_MAX); + } +}; +#endif + + +#define FSST_BUFSZ (3<<19) // 768KB + +// an encoder is a symbolmap plus some bufferspace, needed during map construction as well as compression +struct Encoder { + shared_ptr symbolTable; // symbols, plus metadata and data structures for quick compression (shortCode,hashTab, etc) + union { + Counters counters; // for counting symbol occurences during map construction + u8 simdbuf[FSST_BUFSZ]; // for compression: SIMD string staging area 768KB = 256KB in + 512KB out (worst case for 256KB in) + }; +}; + +// job control integer representable in one 64bits SIMD lane: cur/end=input, out=output, pos=which string (2^9=512 per call) +struct SIMDjob { + u64 out:19,pos:9,end:18,cur:18; // cur/end is input offsets (2^18=256KB), out is output offset (2^19=512KB) +}; + +extern bool +fsst_hasAVX512(); // runtime check for avx512 capability + +extern size_t +fsst_compressAVX512( + SymbolTable &symbolTable, + u8* codeBase, // IN: base address for codes, i.e. compression output (points to simdbuf+256KB) + u8* symbolBase, // IN: base address for string bytes, i.e. compression input (points to simdbuf) + SIMDjob* input, // IN: input array (size n) with job information: what to encode, where to store it. + SIMDjob* output, // OUT: output array (size n) with job information: how much got encoded, end output pointer. + size_t n, // IN: size of arrays input and output (should be max 512) + size_t unroll); // IN: degree of SIMD unrolling + +// C++ fsst-compress function with some more control of how the compression happens (algorithm flavor, simd unroll degree) +size_t compressImpl(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd); +size_t compressAuto(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], int simd); +} // namespace libfsst diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 7ba1f4f876be..ad84ffc98ba2 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -66,6 +66,8 @@ ARROW_CARES_BUILD_VERSION=1.17.2 ARROW_CARES_BUILD_SHA256_CHECKSUM=4803c844ce20ce510ef0eb83f8ea41fa24ecaae9d280c468c582d2bb25b3913d ARROW_CRC32C_BUILD_VERSION=1.1.2 ARROW_CRC32C_BUILD_SHA256_CHECKSUM=ac07840513072b7fcebda6e821068aa04889018f24e10e46181068fb214d7e56 +ARROW_FSST_BUILD_VERSION=89f49c580c6388acf3b6ed2a49e1bfde6c05e616 +ARROW_FSST_BUILD_SHA256_CHECKSUM=5921d2b837800e1c886903d18971994587328b3e5d08d32eb8e80c00890c304d ARROW_GBENCHMARK_BUILD_VERSION=v1.8.3 ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=6bc180a57d23d4d9515519f92b0c83d61b05b5bab188961f36ac7b06b0d9e9ce ARROW_GFLAGS_BUILD_VERSION=v2.2.2 @@ -144,6 +146,7 @@ DEPENDENCIES=( "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz" "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://github.com/c-ares/c-ares/releases/download/cares-${ARROW_CARES_BUILD_VERSION//./_}/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz" "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz" + "ARROW_FSST_URL fsst-${ARROW_FSST_BUILD_VERSION}.tar.gz https://github.com/cwida/fsst/archive/${ARROW_FSST_BUILD_VERSION}.tar.gz" "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz" "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz" "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz" diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 5212dc47e0bd..0544ab3b1f06 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -31,6 +31,7 @@ cpp/src/generated/substrait/* cpp/thirdparty/flatbuffers/include/flatbuffers/base.h cpp/thirdparty/flatbuffers/include/flatbuffers/flatbuffers.h cpp/thirdparty/flatbuffers/include/flatbuffers/stl_emulation.h +cpp/thirdparty/fsst/* dev/requirements*.txt dev/archery/MANIFEST.in dev/archery/requirements*.txt diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 14cd3e363a46..86f9f27305a0 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -1487,6 +1487,7 @@ cdef encoding_name_from_enum(ParquetEncoding encoding_): ParquetEncoding_DELTA_BYTE_ARRAY: 'DELTA_BYTE_ARRAY', ParquetEncoding_RLE_DICTIONARY: 'RLE_DICTIONARY', ParquetEncoding_BYTE_STREAM_SPLIT: 'BYTE_STREAM_SPLIT', + ParquetEncoding_FSST: 'FSST', }.get(encoding_, 'UNKNOWN') @@ -1499,6 +1500,7 @@ cdef encoding_enum_from_name(str encoding_name): 'DELTA_BINARY_PACKED': ParquetEncoding_DELTA_BINARY_PACKED, 'DELTA_LENGTH_BYTE_ARRAY': ParquetEncoding_DELTA_LENGTH_BYTE_ARRAY, 'DELTA_BYTE_ARRAY': ParquetEncoding_DELTA_BYTE_ARRAY, + 'FSST': ParquetEncoding_FSST, 'RLE_DICTIONARY': 'dict', 'PLAIN_DICTIONARY': 'dict', }.get(encoding_name, None) diff --git a/python/pyarrow/includes/libparquet.pxd b/python/pyarrow/includes/libparquet.pxd index 42d48ba050f1..3663b8bcf62b 100644 --- a/python/pyarrow/includes/libparquet.pxd +++ b/python/pyarrow/includes/libparquet.pxd @@ -130,6 +130,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil: ParquetEncoding_RLE_DICTIONARY" parquet::Encoding::RLE_DICTIONARY" ParquetEncoding_BYTE_STREAM_SPLIT \ " parquet::Encoding::BYTE_STREAM_SPLIT" + ParquetEncoding_FSST" parquet::Encoding::FSST" enum ParquetCompression" parquet::Compression::type": ParquetCompression_UNCOMPRESSED" parquet::Compression::UNCOMPRESSED" diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 24cb586c82b3..5fb3cc5b699c 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -821,7 +821,7 @@ def _sanitize_table(table, new_schema, flavor): Can only be used when ``use_dictionary`` is set to False, and cannot be used in combination with ``use_byte_stream_split``. Currently supported values: {'PLAIN', 'BYTE_STREAM_SPLIT', - 'DELTA_BINARY_PACKED', 'DELTA_LENGTH_BYTE_ARRAY', 'DELTA_BYTE_ARRAY'}. + 'DELTA_BINARY_PACKED', 'DELTA_LENGTH_BYTE_ARRAY', 'DELTA_BYTE_ARRAY', 'FSST'}. Certain encodings are only compatible with certain data types. Please refer to the encodings section of `Reading and writing Parquet files `_.