diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2a8db0cb..c2feba7f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.0.4 +current_version = 5.0.0-rc1 commit = False tag = False diff --git a/.gitignore b/.gitignore index cd332d3a..9d954b94 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ cmake-build-release .cache questdb-rs/Cargo.lock include/questdb/ingress/line_sender.gen.h -cython/questdb/ingress/line_sender.pxd \ No newline at end of file +cython/questdb/ingress/line_sender.pxd +profile.out \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 8de0cb5e..2b36780e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required(VERSION 3.15.0) -project(c-questdb-client VERSION 4.0.4) +project(c-questdb-client VERSION 5.0.0) +set(PROJECT_PRE_RELEASE "rc1") set(CPACK_PROJECT_NAME ${PROJECT_NAME}) set(CPACK_PROJECT_VERSION ${PROJECT_VERSION}) @@ -98,6 +99,14 @@ if (QUESTDB_TESTS_AND_EXAMPLES) line_sender_c_example examples/concat.c examples/line_sender_c_example.c) + compile_example( + line_sender_c_example_array_byte_strides + examples/concat.c + examples/line_sender_c_example_array_byte_strides.c) + compile_example( + line_sender_c_example_array_elem_strides + examples/concat.c + examples/line_sender_c_example_array_elem_strides.c) compile_example( line_sender_c_example_auth examples/concat.c @@ -123,6 +132,12 @@ if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_cpp_example examples/line_sender_cpp_example.cpp) + compile_example( + line_sender_cpp_example_array_byte_strides + examples/line_sender_cpp_example_array_byte_strides.cpp) + compile_example( + line_sender_cpp_example_array_elem_strides + examples/line_sender_cpp_example_array_elem_strides.cpp) compile_example( line_sender_cpp_example_auth examples/line_sender_cpp_example_auth.cpp) diff --git a/README.md b/README.md index f4f1931c..d649e63d 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,13 @@ This library makes it easy to insert data into [QuestDB](https://questdb.io/). -This client library implements the [InfluxDB Line Protocol]( +This client library implements the QuestDB's variant of the [InfluxDB Line Protocol]( https://questdb.io/docs/reference/api/ilp/overview/) (ILP) over HTTP and TCP. +When connecting to QuestDB over HTTP, the library will auto-detect the server's +latest supported version and use it. Version 1 is compatible with +the [InfluxDB Database](https://docs.influxdata.com/influxdb/v2/reference/syntax/line-protocol/). + * Implementation is in Rust, with no additional [run-time or link-time dependencies](doc/BUILD.md#pre-requisites-and-dependencies) on the C++ standard library or other libraries. @@ -38,6 +42,21 @@ For an overview and code examples, see the To understand the protocol in more depth, consult the [protocol reference docs](https://questdb.io/docs/reference/api/ilp/overview/). +## Protocol Versions + +The library supports the following ILP protocol versions. + +These protocol versions are supported over both HTTP and TCP. + +If you use HTTP, the library will automatically detect the server's +latest supported protocol version and use it. If you use TCP, you can specify the +`protocol_version=N` parameter when constructing the `Sender` object. + +| Version | Description | Server Comatibility | +| ------- | ------------------------------------------------------- | --------------------- | +| **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | +| **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | + ## Getting Started To get started, read the language-specific guides. diff --git a/ci/compile.yaml b/ci/compile.yaml index aa9e1059..4c892ce4 100644 --- a/ci/compile.yaml +++ b/ci/compile.yaml @@ -4,6 +4,10 @@ steps: rustup default $(toolchain) condition: ne(variables['toolchain'], '') displayName: "Update and set Rust toolchain" + - script: | + python -m pip install --upgrade pip + pip install numpy + displayName: 'Install Python Dependencies' - script: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DQUESTDB_TESTS_AND_EXAMPLES=ON env: JAVA_HOME: $(JAVA_HOME_11_X64) diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index b4e4dc4a..4d7624f5 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -41,7 +41,7 @@ def main(): build_cxx20_dir.glob(f'**/test_line_sender{exe_suffix}'))) system_test_path = pathlib.Path('system_test') / 'test.py' - qdb_v = '8.2.3' # The version of QuestDB we'll test against. + #qdb_v = '8.2.3' # The version of QuestDB we'll test against. run_cmd('cargo', 'test', '--', '--nocapture', cwd='questdb-rs') @@ -51,9 +51,11 @@ def main(): '--', '--nocapture', cwd='questdb-rs') run_cmd('cargo', 'test', '--features=almost-all-features', '--', '--nocapture', cwd='questdb-rs') + run_cmd('cargo', 'test', cwd='questdb-rs-ffi') run_cmd(str(test_line_sender_path)) run_cmd(str(test_line_sender_path_CXX20)) - run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') + #run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') + run_cmd('python3', str(system_test_path), 'run', '--repo', './questdb_nd_arr', '-v') if __name__ == '__main__': diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 1fced6be..03b8d129 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -54,14 +54,25 @@ stages: cd questdb-rs cargo build --examples --features almost-all-features displayName: "Build Rust examples" + ############################# temp for test begin ##################### + - script: | + git clone -b nd_arr --depth 1 https://github.com/questdb/questdb.git ./questdb_nd_arr + displayName: git clone questdb + - task: Maven@3 + displayName: "Compile QuestDB" + inputs: + mavenPOMFile: 'questdb_nd_arr/pom.xml' + jdkVersionOption: '1.11' + options: "-DskipTests -Pbuild-web-console" + ############################# temp for test end ##################### - script: python3 ci/run_all_tests.py env: JAVA_HOME: $(JAVA_HOME_11_X64) displayName: "Tests" - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: ./build - displayName: "Publish build directory" + # - task: PublishBuildArtifacts@1 + # inputs: + # pathToPublish: ./build + # displayName: "Publish build directory" - job: FormatAndLinting displayName: "cargo fmt and clippy" pool: @@ -115,7 +126,7 @@ stages: submodules: false - template: compile.yaml - script: | - git clone --depth 1 https://github.com/questdb/questdb.git + git clone -b nd_arr --depth 1 https://github.com/questdb/questdb.git displayName: git clone questdb - task: Maven@3 displayName: "Compile QuestDB" diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index 0e8a1e85..e21d77d0 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -25,6 +25,7 @@ #include "mock_server.hpp" #include +#include #if defined(PLATFORM_UNIX) # include @@ -190,43 +191,81 @@ bool mock_server::wait_for_data(std::optional wait_timeout_sec) return !!count; } +int32_t bytes_to_int32_le(const std::byte* bytes) +{ + return static_cast( + (bytes[0] << 0) | (bytes[1] << 8) | (bytes[2] << 16) | + (bytes[3] << 24)); +} + size_t mock_server::recv(double wait_timeout_sec) { if (!wait_for_data(wait_timeout_sec)) return 0; - char chunk[1024]; + std::byte chunk[1024]; size_t chunk_len{sizeof(chunk)}; - std::vector accum; + std::vector accum; for (;;) { wait_for_data(); - sock_ssize_t count = - ::recv(_conn_fd, &chunk[0], static_cast(chunk_len), 0); + sock_ssize_t count = ::recv( + _conn_fd, + reinterpret_cast(&chunk[0]), + static_cast(chunk_len), + 0); if (count == -1) throw std::runtime_error{"Bad `recv()`."}; const size_t u_count = static_cast(count); accum.insert(accum.end(), chunk, chunk + u_count); if (accum.size() < 2) continue; - if ((accum[accum.size() - 1] == '\n') && - (accum[accum.size() - 2] != '\\')) + if ((accum[accum.size() - 1] == std::byte('\n')) && + (accum[accum.size() - 2] != std::byte('\\'))) break; } size_t received_count{0}; - const char* head{&accum[0]}; - for (size_t index = 1; index < accum.size(); ++index) + const std::byte* head{&accum[0]}; + size_t index{1}; + while (index < accum.size()) { - const char& last = accum[index]; - const char& prev = accum[index - 1]; - if ((last == '\n') && (prev != '\\')) + const std::byte& last = accum[index]; + const std::byte& prev = accum[index - 1]; + if (last == std::byte('=') && prev == std::byte('=')) + { + index++; + std::byte& binary_type = accum[index]; + if (binary_type == std::byte(16)) // DOUBLE_BINARY_FORMAT_TYPE + index += sizeof(double) + 1; + else if (binary_type == std::byte(14)) // ARRAY_BINARY_FORMAT_TYPE + { + index++; + const std::byte& array_elem_type = accum[index]; + if (array_elem_type == std::byte(10)) + { + index++; + const size_t dims = size_t(accum[index]); + index++; + size_t data_size{sizeof(double)}; + for (size_t i = 0; i < dims; i++) + { + data_size *= bytes_to_int32_le(&accum[index]); + index += sizeof(int32_t); + } + index += data_size; + } + } + continue; + } + else if ((last == std::byte('\n')) && (prev != std::byte('\\'))) { - const char* tail{&last + 1}; - _msgs.emplace_back(head, tail - head); + const std::byte* tail{&last + 1}; + _msgs.emplace_back(head, tail); head = tail; ++received_count; } + index++; } return received_count; } diff --git a/cpp_test/mock_server.hpp b/cpp_test/mock_server.hpp index ba66efb0..8adb6ad7 100644 --- a/cpp_test/mock_server.hpp +++ b/cpp_test/mock_server.hpp @@ -24,13 +24,17 @@ #pragma once +#include #include -#include #include #include #include - #include "build_env.h" +#if __cplusplus < 202002L +# include "questdb/ingress/line_sender.hpp" +#else +# include +#endif #if defined(PLATFORM_UNIX) typedef int socketfd_t; @@ -60,9 +64,14 @@ class mock_server size_t recv(double wait_timeout_sec = 0.1); - const std::vector& msgs() const +#if __cplusplus >= 202002L + using buffer_view = std::span; +#endif + + buffer_view msgs(size_t index) const { - return _msgs; + assert(index < _msgs.size()); + return {_msgs[index].data(), _msgs[index].size()}; } void close(); @@ -75,7 +84,7 @@ class mock_server socketfd_t _listen_fd; socketfd_t _conn_fd; uint16_t _port; - std::vector _msgs; + std::vector> _msgs; }; } // namespace questdb::ingress::test diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 1510aef2..e2507797 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -56,6 +56,68 @@ class on_scope_exit F _f; }; +#if __cplusplus >= 202002L +template +bool operator==(std::span lhs, const char (&rhs)[N]) +{ + constexpr size_t bytelen = N - 1; // Exclude null terminator + const std::span rhs_span{ + reinterpret_cast(rhs), bytelen}; + return lhs.size() == bytelen && std::ranges::equal(lhs, rhs_span); +} + +bool operator==(std::span lhs, const std::string& rhs) +{ + const std::span rhs_span{ + reinterpret_cast(rhs.data()), rhs.size()}; + return lhs.size() == rhs.size() && std::ranges::equal(lhs, rhs_span); +} +#else +template +bool operator==( + const questdb::ingress::buffer_view lhs_view, const char (&rhs)[N]) +{ + constexpr size_t bytelen = N - 1; // Exclude null terminator + const questdb::ingress::buffer_view rhs_view{ + reinterpret_cast(rhs), bytelen}; + return lhs_view == rhs_view; +} + +bool operator==( + const questdb::ingress::buffer_view lhs_view, const std::string& rhs) +{ + const questdb::ingress::buffer_view rhs_view{ + reinterpret_cast(rhs.data()), rhs.size()}; + return lhs_view == rhs_view; +} +#endif + +template +std::string& push_double_arr_to_buffer( + std::string& buffer, + std::array data, + size_t rank, + uintptr_t* shape) +{ + buffer.push_back(14); + buffer.push_back(10); + buffer.push_back(static_cast(rank)); + for (size_t i = 0; i < rank; ++i) + buffer.append( + reinterpret_cast(&shape[i]), sizeof(uint32_t)); + buffer.append( + reinterpret_cast(data.data()), + data.size() * sizeof(double)); + return buffer; +} + +std::string& push_double_to_buffer(std::string& buffer, double data) +{ + buffer.push_back(16); + buffer.append(reinterpret_cast(&data), sizeof(double)); + return buffer; +} + TEST_CASE("line_sender c api basics") { questdb::ingress::test::mock_server server; @@ -65,10 +127,12 @@ TEST_CASE("line_sender c api basics") ::line_sender_error_free(err); }}; ::line_sender_utf8 host = {0, nullptr}; - CHECK(::line_sender_utf8_init(&host, 9, "localhost", &err)); + CHECK(::line_sender_utf8_init(&host, 9, "127.0.0.1", &err)); ::line_sender_opts* opts = ::line_sender_opts_new(::line_sender_protocol_tcp, host, server.port()); CHECK_NE(opts, nullptr); + line_sender_opts_protocol_version( + opts, ::line_sender_protocol_version_2, &err); ::line_sender* sender = ::line_sender_build(opts, &err); line_sender_opts_free(opts); CHECK_NE(sender, nullptr); @@ -90,18 +154,64 @@ TEST_CASE("line_sender c api basics") CHECK(::line_sender_utf8_init(&v1_utf8, 2, "v1", &err)); ::line_sender_column_name f1_name{0, nullptr}; CHECK(::line_sender_column_name_init(&f1_name, 2, "f1", &err)); - ::line_sender_buffer* buffer = line_sender_buffer_new(); + ::line_sender_buffer* buffer = line_sender_buffer_new_for_sender(sender); CHECK(buffer != nullptr); CHECK(::line_sender_buffer_table(buffer, table_name, &err)); CHECK(::line_sender_buffer_symbol(buffer, t1_name, v1_utf8, &err)); CHECK(::line_sender_buffer_column_f64(buffer, f1_name, 0.5, &err)); + + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("a1"); + // 3D array of doubles + size_t rank = 3; + uintptr_t shape[] = {2, 3, 2}; + intptr_t strides[] = {48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + CHECK( + ::line_sender_buffer_column_f64_arr_byte_strides( + buffer, + arr_name, + rank, + shape, + strides, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); + + line_sender_column_name arr_name2 = QDB_COLUMN_NAME_LITERAL("a2"); + intptr_t elem_strides[] = {6, 2, 1}; + CHECK( + ::line_sender_buffer_column_f64_arr_elem_strides( + buffer, + arr_name2, + rank, + shape, + elem_strides, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); - CHECK(::line_sender_buffer_size(buffer) == 27); + CHECK(::line_sender_buffer_size(buffer) == 266); CHECK(::line_sender_flush(sender, buffer, &err)); ::line_sender_buffer_free(buffer); CHECK(server.recv() == 1); - CHECK(server.msgs().front() == "test,t1=v1 f1=0.5 10000000\n"); + std::string expect{"test,t1=v1 f1=="}; + push_double_to_buffer(expect, 0.5).append(",a1=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a2=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(" 10000000\n"); + CHECK(server.msgs(0) == expect); } TEST_CASE("Opts service API tests") @@ -131,8 +241,8 @@ TEST_CASE("Opts service API tests") TEST_CASE("line_sender c++ connect disconnect") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); @@ -141,33 +251,62 @@ TEST_CASE("line_sender c++ connect disconnect") TEST_CASE("line_sender c++ api basics") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ + questdb::ingress::opts opts{ questdb::ingress::protocol::tcp, - std::string("localhost"), + std::string("127.0.0.1"), std::to_string(server.port())}; + opts.protocol_version(questdb::ingress::protocol_version::v2); + questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + // 3D array of doubles + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + std::vector elem_strides{6, 2, 1}; buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") .column("f1", 0.5) + .column("a1", rank, shape, strides, arr_data) + .column("a2", rank, shape, elem_strides, arr_data) .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); - CHECK(buffer.size() == 31); + CHECK(buffer.size() == 270); sender.flush(buffer); CHECK(server.recv() == 1); - CHECK(server.msgs().front() == "test,t1=v1,t2= f1=0.5 10000000\n"); + std::string expect{"test,t1=v1,t2= f1=="}; + push_double_to_buffer(expect, 0.5).append(",a1=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) + .append(",a2=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) + .append(" 10000000\n"); + CHECK(server.msgs(0) == expect); } TEST_CASE("test multiple lines") { questdb::ingress::test::mock_server server; std::string conf_str = - "tcp::addr=localhost:" + std::to_string(server.port()) + ";"; + "tcp::addr=127.0.0.1:" + std::to_string(server.port()) + + ";protocol_version=2;"; questdb::ingress::line_sender sender = questdb::ingress::line_sender::from_conf(conf_str); CHECK_FALSE(sender.must_close()); @@ -175,7 +314,7 @@ TEST_CASE("test multiple lines") CHECK(server.recv() == 0); const auto table_name = "metric1"_tn; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol("t1"_cn, "val1"_utf8) .symbol("t2"_cn, "val2"_utf8) @@ -193,27 +332,26 @@ TEST_CASE("test multiple lines") .at_now(); CHECK(server.recv() == 0); - CHECK(buffer.size() == 137); + CHECK(buffer.size() == 142); sender.flush(buffer); CHECK(server.recv() == 2); + std::string expect{"metric1,t1=val1,t2=val2 f1=t,f2=12345i,f3=="}; + push_double_to_buffer(expect, 10.75) + .append(",f4=\"val3\",f5=\"val4\",f6=\"val5\" 111222233333\n"); + CHECK(server.msgs(0) == expect); CHECK( - server.msgs()[0] == - ("metric1,t1=val1,t2=val2 f1=t,f2=12345i," - "f3=10.75,f4=\"val3\",f5=\"val4\",f6=\"val5\" 111222233333\n")); - CHECK( - server.msgs()[1] == - "metric1,tag3=value\\ 3,tag\\ 4=value:4 field5=f\n"); + server.msgs(1) == "metric1,tag3=value\\ 3,tag\\ 4=value:4 field5=f\n"); } TEST_CASE("State machine testing -- flush without data.") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ + questdb::ingress::line_sender sender{questdb::ingress::opts{ questdb::ingress::protocol::tcp, - std::string_view{"localhost"}, - std::to_string(server.port())}; + std::string_view{"127.0.0.1"}, + std::to_string(server.port())}}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); CHECK(buffer.size() == 0); CHECK_THROWS_WITH_AS( sender.flush(buffer), @@ -226,13 +364,13 @@ TEST_CASE("State machine testing -- flush without data.") TEST_CASE("One symbol only - flush before server accept") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ + questdb::ingress::line_sender sender{questdb::ingress::opts{ questdb::ingress::protocol::tcp, - std::string{"localhost"}, - server.port()}; + std::string{"127.0.0.1"}, + server.port()}}; // Does not raise - this is unlike InfluxDB spec that disallows this. - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test").symbol("t1", std::string{"v1"}).at_now(); CHECK(!sender.must_close()); CHECK(buffer.size() == 11); @@ -243,17 +381,17 @@ TEST_CASE("One symbol only - flush before server accept") // but the server hasn't actually accepted the client connection yet. server.accept(); CHECK(server.recv() == 1); - CHECK(server.msgs()[0] == "test,t1=v1\n"); + CHECK(server.msgs(0) == "test,t1=v1\n"); } TEST_CASE("One column only - server.accept() after flush, before close") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; - // Does not raise - this is unlike InfluxDB spec that disallows this. - questdb::ingress::line_sender_buffer buffer; + // Does not raise - this is unlike the InfluxDB spec that disallows this. + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test").column("t1", "v1").at_now(); CHECK(!sender.must_close()); CHECK(buffer.size() == 13); @@ -263,16 +401,16 @@ TEST_CASE("One column only - server.accept() after flush, before close") sender.close(); CHECK(server.recv() == 1); - CHECK(server.msgs()[0] == "test t1=\"v1\"\n"); + CHECK(server.msgs(0) == "test t1=\"v1\"\n"); } TEST_CASE("Symbol after column") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test").column("t1", "v1"); CHECK_THROWS_AS( @@ -386,55 +524,22 @@ TEST_CASE("Validation of bad chars in key names.") } } -#if __cplusplus >= 202002L -template -bool operator==(std::span lhs, const char (&rhs)[N]) -{ - constexpr size_t bytelen = N - 1; // Exclude null terminator - const std::span rhs_span{ - reinterpret_cast(rhs), bytelen}; - return lhs.size() == bytelen && std::ranges::equal(lhs, rhs_span); -} - -bool operator==(std::span lhs, const std::string& rhs) -{ - const std::span rhs_span{ - reinterpret_cast(rhs.data()), rhs.size()}; - return lhs.size() == rhs.size() && std::ranges::equal(lhs, rhs_span); -} -#else -template -bool operator==( - const questdb::ingress::buffer_view lhs_view, const char (&rhs)[N]) -{ - constexpr size_t bytelen = N - 1; // Exclude null terminator - const questdb::ingress::buffer_view rhs_view{ - reinterpret_cast(rhs), bytelen}; - return lhs_view == rhs_view; -} - -bool operator==( - const questdb::ingress::buffer_view lhs_view, const std::string& rhs) -{ - const questdb::ingress::buffer_view rhs_view{ - reinterpret_cast(rhs.data()), rhs.size()}; - return lhs_view == rhs_view; -} -#endif - TEST_CASE("Buffer move and copy ctor testing") { const size_t init_buf_size = 128; - questdb::ingress::line_sender_buffer buffer1{init_buf_size}; + questdb::ingress::line_sender_buffer buffer1{ + questdb::ingress::protocol_version::v1, init_buf_size}; buffer1.table("buffer1"); CHECK(buffer1.peek() == "buffer1"); - questdb::ingress::line_sender_buffer buffer2{2 * init_buf_size}; + questdb::ingress::line_sender_buffer buffer2{ + questdb::ingress::protocol_version::v1, 2 * init_buf_size}; buffer2.table("buffer2"); CHECK(buffer2.peek() == "buffer2"); - questdb::ingress::line_sender_buffer buffer3{3 * init_buf_size}; + questdb::ingress::line_sender_buffer buffer3{ + questdb::ingress::protocol_version::v1, 3 * init_buf_size}; buffer3.table("buffer3"); CHECK(buffer3.peek() == "buffer3"); @@ -467,13 +572,13 @@ TEST_CASE("Sender move testing.") questdb::ingress::test::mock_server server1; questdb::ingress::test::mock_server server2; - questdb::ingress::utf8_view host{"localhost"}; + questdb::ingress::utf8_view host{"127.0.0.1"}; const questdb::ingress::utf8_view& host_ref = host; - questdb::ingress::line_sender sender1{ - questdb::ingress::protocol::tcp, host_ref, server1.port()}; + questdb::ingress::line_sender sender1{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, host_ref, server1.port()}}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender1.new_buffer(); buffer.table("test").column("t1", "v1").at_now(); server1.close(); @@ -495,8 +600,8 @@ TEST_CASE("Sender move testing.") CHECK_FALSE(sender1.must_close()); CHECK(sender2.must_close()); - questdb::ingress::line_sender sender3{ - questdb::ingress::protocol::tcp, "localhost", server2.port()}; + questdb::ingress::line_sender sender3{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server2.port()}}; CHECK_FALSE(sender3.must_close()); sender3 = std::move(sender2); @@ -507,8 +612,8 @@ TEST_CASE("Bad hostname") { try { - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "dummy_hostname", "9009"}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "dummy_hostname", "9009"}}; CHECK_MESSAGE(false, "Expected exception"); } catch (const questdb::ingress::line_sender_error& se) @@ -529,7 +634,7 @@ TEST_CASE("Bad interface") try { questdb::ingress::opts opts{ - questdb::ingress::protocol::tcp, "localhost", "9009"}; + questdb::ingress::protocol::tcp, "127.0.0.1", "9009"}; opts.bind_interface("dummy_hostname"); questdb::ingress::line_sender sender{opts}; CHECK_MESSAGE(false, "Expected exception"); @@ -551,14 +656,14 @@ TEST_CASE("Bad port") const auto test_bad_port = [](std::string bad_port) { try { - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", bad_port}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", bad_port}}; CHECK_MESSAGE(false, "Expected exception"); } catch (const questdb::ingress::line_sender_error& se) { std::string msg{se.what()}; - std::string exp_msg{"\"localhost:" + bad_port + "\": "}; + std::string exp_msg{"\"127.0.0.1:" + bad_port + "\": "}; CHECK_MESSAGE(msg.find(exp_msg) != std::string::npos, msg); } catch (...) @@ -580,8 +685,8 @@ TEST_CASE("Bad connect") { // Port 1 is generally the tcpmux service which one would // very much expect to never be running. - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", 1}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", 1}}; CHECK_MESSAGE(false, "Expected exception"); } catch (const questdb::ingress::line_sender_error& se) @@ -649,7 +754,7 @@ TEST_CASE("Opts copy ctor, assignment and move testing.") { { questdb::ingress::opts opts1{ - questdb::ingress::protocol::tcp, "localhost", "9009"}; + questdb::ingress::protocol::tcp, "127.0.0.1", "9009"}; questdb::ingress::opts opts2{std::move(opts1)}; } @@ -661,7 +766,7 @@ TEST_CASE("Opts copy ctor, assignment and move testing.") { questdb::ingress::opts opts1{ - questdb::ingress::protocol::tcp, "localhost", "9009"}; + questdb::ingress::protocol::tcp, "127.0.0.1", "9009"}; questdb::ingress::opts opts2{ questdb::ingress::protocol::tcp, "altavista.digital.com", "9009"}; opts1 = std::move(opts2); @@ -679,8 +784,8 @@ TEST_CASE("Opts copy ctor, assignment and move testing.") TEST_CASE("Test timestamp column.") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; const auto now = std::chrono::system_clock::now(); const auto now_micros = @@ -694,7 +799,7 @@ TEST_CASE("Test timestamp column.") const auto now_nanos_ts = questdb::ingress::timestamp_nanos{now_nanos}; const auto now_micros_ts = questdb::ingress::timestamp_micros{now_micros}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test") .column("ts1", questdb::ingress::timestamp_micros{12345}) .column("ts2", now_micros_ts) @@ -715,7 +820,7 @@ TEST_CASE("Test timestamp column.") sender.close(); CHECK(server.recv() == 1); - CHECK(server.msgs()[0] == exp); + CHECK(server.msgs(0) == exp); } TEST_CASE("test timestamp_micros and timestamp_nanos::now()") @@ -741,7 +846,8 @@ TEST_CASE("test timestamp_micros and timestamp_nanos::now()") TEST_CASE("Test Marker") { - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer{ + questdb::ingress::protocol_version::v1}; buffer.clear_marker(); buffer.clear_marker(); @@ -797,25 +903,29 @@ TEST_CASE("Moved View") TEST_CASE("Empty Buffer") { - questdb::ingress::line_sender_buffer b1; + questdb::ingress::line_sender_buffer b1{ + questdb::ingress::protocol_version::v2}; CHECK(b1.size() == 0); questdb::ingress::line_sender_buffer b2{std::move(b1)}; CHECK(b1.size() == 0); CHECK(b2.size() == 0); - questdb::ingress::line_sender_buffer b3; + questdb::ingress::line_sender_buffer b3{ + questdb::ingress::protocol_version::v2}; b3 = std::move(b2); CHECK(b2.size() == 0); CHECK(b3.size() == 0); - questdb::ingress::line_sender_buffer b4; + questdb::ingress::line_sender_buffer b4{ + questdb::ingress::protocol_version::v2}; b4.table("test").symbol("a", "b").at_now(); - questdb::ingress::line_sender_buffer b5; + questdb::ingress::line_sender_buffer b5{ + questdb::ingress::protocol_version::v2}; b5 = std::move(b4); CHECK(b4.size() == 0); CHECK(b5.size() == 9); questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; CHECK_THROWS_WITH_AS( sender.flush(b1), "State error: Bad call to `flush`, should have called `table` instead.", @@ -829,11 +939,11 @@ TEST_CASE("Empty Buffer") TEST_CASE("Opts from conf") { questdb::ingress::opts opts1 = - questdb::ingress::opts::from_conf("tcp::addr=localhost:9009;"); + questdb::ingress::opts::from_conf("tcp::addr=127.0.0.1:9009;"); questdb::ingress::opts opts2 = questdb::ingress::opts::from_conf("tcps::addr=localhost:9009;"); questdb::ingress::opts opts3 = - questdb::ingress::opts::from_conf("https::addr=localhost:9009;"); + questdb::ingress::opts::from_conf("https::addr=127.0.0.1:9009;"); questdb::ingress::opts opts4 = questdb::ingress::opts::from_conf("https::addr=localhost:9009;"); } @@ -841,27 +951,31 @@ TEST_CASE("Opts from conf") TEST_CASE("HTTP basics") { questdb::ingress::opts opts1{ - questdb::ingress::protocol::http, "localhost", 1}; + questdb::ingress::protocol::http, "127.0.0.1", 1}; questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( - "http::addr=localhost:1;username=user;password=pass;request_timeout=" - "5000;retry_timeout=5;"); + "http::addr=127.0.0.1:1;username=user;password=pass;request_timeout=" + "5000;retry_timeout=5;protocol_version=2;"); questdb::ingress::opts opts2{ questdb::ingress::protocol::https, "localhost", "1"}; questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( - "http::addr=localhost:1;token=token;request_min_throughput=1000;retry_" - "timeout=0;"); - opts1.username("user") + "http::addr=127.0.0.1:1;token=token;request_min_throughput=1000;retry_" + "timeout=0;protocol_version=2;"); + opts1.protocol_version(questdb::ingress::protocol_version::v2) + .username("user") .password("pass") .max_buf_size(1000000) .request_timeout(5000) .retry_timeout(5); - opts2.token("token").request_min_throughput(1000).retry_timeout(0); + opts2.protocol_version(questdb::ingress::protocol_version::v2) + .token("token") + .request_min_throughput(1000) + .retry_timeout(0); questdb::ingress::line_sender sender1{opts1}; questdb::ingress::line_sender sender1conf{opts1conf}; questdb::ingress::line_sender sender2{opts2}; questdb::ingress::line_sender sender2conf{opts2conf}; - questdb::ingress::line_sender_buffer b1; + questdb::ingress::line_sender_buffer b1 = sender1.new_buffer(); b1.table("test").symbol("a", "b").at_now(); CHECK_THROWS_AS(sender1.flush(b1), questdb::ingress::line_sender_error); @@ -871,6 +985,85 @@ TEST_CASE("HTTP basics") CHECK_THROWS_AS( questdb::ingress::opts::from_conf( - "http::addr=localhost:1;bind_interface=0.0.0.0;"), + "http::addr=127.0.0.1:1;bind_interface=0.0.0.0;"), questdb::ingress::line_sender_error); -} \ No newline at end of file +} + +TEST_CASE("line sender protocol version default v1 for tcp") +{ + questdb::ingress::test::mock_server server; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, + std::string("127.0.0.1"), + std::to_string(server.port())}}; + CHECK_FALSE(sender.must_close()); + server.accept(); + CHECK(server.recv() == 0); + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table("test") + .symbol("t1", "v1") + .symbol("t2", "") + .column("f1", 0.5) + .at(questdb::ingress::timestamp_nanos{10000000}); + + CHECK(sender.protocol_version() == questdb::ingress::protocol_version::v1); + CHECK(server.recv() == 0); + CHECK(buffer.size() == 31); + sender.flush(buffer); + CHECK(server.recv() == 1); + std::string expect{"test,t1=v1,t2= f1=0.5 10000000\n"}; + CHECK(server.msgs(0) == expect); +} + +TEST_CASE("line sender protocol version v2") +{ + questdb::ingress::test::mock_server server; + questdb::ingress::opts opts{ + questdb::ingress::protocol::tcp, + std::string("127.0.0.1"), + std::to_string(server.port())}; + opts.protocol_version(questdb::ingress::protocol_version::v2); + questdb::ingress::line_sender sender{opts}; + CHECK_FALSE(sender.must_close()); + server.accept(); + CHECK(server.recv() == 0); + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table("test") + .symbol("t1", "v1") + .symbol("t2", "") + .column("f1", 0.5) + .at(questdb::ingress::timestamp_nanos{10000000}); + + CHECK(server.recv() == 0); + CHECK(buffer.size() == 38); + sender.flush(buffer); + CHECK(server.recv() == 1); + std::string expect{"test,t1=v1,t2= f1=="}; + push_double_to_buffer(expect, 0.5).append(" 10000000\n"); + CHECK(server.msgs(0) == expect); +} + +TEST_CASE("Http auto detect line protocol version failed") +{ + try + { + questdb::ingress::opts opts{ + questdb::ingress::protocol::http, "127.0.0.1", 1}; + questdb::ingress::line_sender sender1{opts}; + CHECK_MESSAGE(false, "Expected exception"); + } + catch (const questdb::ingress::line_sender_error& se) + { + std::string msg{se.what()}; + CHECK_MESSAGE( + msg.rfind("Could not detect server's line protocol version", 0) == + 0, + msg); + } + catch (...) + { + CHECK_MESSAGE(false, "Other exception raised."); + } +} diff --git a/doc/DEV_NOTES.md b/doc/DEV_NOTES.md index 66f92c07..7dd5889e 100644 --- a/doc/DEV_NOTES.md +++ b/doc/DEV_NOTES.md @@ -38,15 +38,9 @@ either of the two Rust projects: * [questdb-rs](../questdb-rs/) - Core library * [questdb-rs-ffi](../questdb-rs-ffi/) - C bindings layer. -If you are editing the C functions in the `questdb-rs-ffi` project and what to -see the resulting generated header file, call `cargo build --features gen_h`. - Note that to reduce compile time we don't use cbindgen in the header we ship, which also contains additional formatting and comments. -Similarly, we also support generating Cython bindings via the `gen_cython` -feature. - This generated files should be not be checked in: * `include/questdb/ingress/line_sender.gen.h` * `cython/questdb/ingress/line_sender.pxd` diff --git a/doc/SECURITY.md b/doc/SECURITY.md index ccb00f6b..7b87fd0a 100644 --- a/doc/SECURITY.md +++ b/doc/SECURITY.md @@ -35,7 +35,7 @@ A few important technical details on TLS: are managed centrally. For API usage: -* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/4.0.4/questdb/ingress/struct.SenderBuilder.html#method.auth) - and [`tls`](https://docs.rs/questdb-rs/4.0.4/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. +* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/struct.SenderBuilder.html#method.auth) + and [`tls`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. * C: [examples/line_sender_c_example_auth.c](../examples/line_sender_c_example_auth.c) * C++: [examples/line_sender_cpp_example_auth.cpp](../examples/line_sender_cpp_example_auth.cpp) diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 0f6805cc..01796664 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -9,7 +9,7 @@ static bool example(const char* host, const char* port) line_sender_error* err = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - char* conf_str = concat("tcp::addr=", host, ":", port, ";"); + char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); return false; @@ -25,7 +25,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_array_byte_strides.c b/examples/line_sender_c_example_array_byte_strides.c new file mode 100644 index 00000000..82e076e9 --- /dev/null +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include "concat.h" + +static bool example(const char* host, const char* port) +{ + line_sender_error* err = NULL; + line_sender* sender = NULL; + line_sender_buffer* buffer = NULL; + char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + if (!conf_str) + { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) + goto on_error; + + sender = line_sender_from_conf(conf_str_utf8, &err); + if (!sender) + goto on_error; + + free(conf_str); + conf_str = NULL; + + buffer = line_sender_buffer_new_for_sender(sender); + line_sender_buffer_reserve(buffer, 64 * 1024); + + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders_byte_strides"); + line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); + line_sender_column_name book_col = QDB_COLUMN_NAME_LITERAL("order_book"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 symbol_val = QDB_UTF8_LITERAL("BTC-USD"); + if (!line_sender_buffer_symbol(buffer, symbol_col, symbol_val, &err)) + goto on_error; + + size_t array_rank = 3; + uintptr_t array_shape[] = {2, 3, 2}; + intptr_t array_strides[] = {48, 16, 8}; + + double array_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + if (!line_sender_buffer_column_f64_arr_byte_strides( + buffer, + book_col, + array_rank, + array_shape, + array_strides, + (const uint8_t*)array_data, + sizeof(array_data), + &err)) + goto on_error; + + if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + goto on_error; + + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_close(sender); + return true; + +on_error:; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error: %.*s\n", (int)err_len, err_msg); + free(conf_str); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return false; +} + +int main(int argc, const char* argv[]) +{ + const char* host = (argc >= 2) ? argv[1] : "localhost"; + const char* port = (argc >= 3) ? argv[2] : "9009"; + return !example(host, port); +} diff --git a/examples/line_sender_c_example_array_elem_strides.c b/examples/line_sender_c_example_array_elem_strides.c new file mode 100644 index 00000000..44c81e05 --- /dev/null +++ b/examples/line_sender_c_example_array_elem_strides.c @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include "concat.h" + +static bool example(const char* host, const char* port) +{ + line_sender_error* err = NULL; + line_sender* sender = NULL; + line_sender_buffer* buffer = NULL; + char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + if (!conf_str) + { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) + goto on_error; + + sender = line_sender_from_conf(conf_str_utf8, &err); + if (!sender) + goto on_error; + + free(conf_str); + conf_str = NULL; + + buffer = line_sender_buffer_new_for_sender(sender); + line_sender_buffer_reserve(buffer, 64 * 1024); + + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders_elem_strides"); + line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); + line_sender_column_name book_col = QDB_COLUMN_NAME_LITERAL("order_book"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 symbol_val = QDB_UTF8_LITERAL("BTC-USD"); + if (!line_sender_buffer_symbol(buffer, symbol_col, symbol_val, &err)) + goto on_error; + + size_t array_rank = 3; + uintptr_t array_shape[] = {2, 3, 2}; + intptr_t array_strides[] = {6, 2, 1}; + + double array_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + if (!line_sender_buffer_column_f64_arr_elem_strides( + buffer, + book_col, + array_rank, + array_shape, + array_strides, + (const uint8_t*)array_data, + sizeof(array_data), + &err)) + goto on_error; + + if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + goto on_error; + + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_close(sender); + return true; + +on_error:; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error: %.*s\n", (int)err_len, err_msg); + free(conf_str); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return false; +} + +int main(int argc, const char* argv[]) +{ + const char* host = (argc >= 2) ? argv[1] : "localhost"; + const char* port = (argc >= 3) ? argv[2] : "9009"; + return !example(host, port); +} diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 59a99421..5f4fb8c8 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -11,6 +11,7 @@ static bool example(const char* host, const char* port) line_sender_buffer* buffer = NULL; char* conf_str = concat( "tcp::addr=", host, ":", port, ";" + "protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -30,7 +31,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index e7454d8d..4594afe6 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -11,6 +11,7 @@ static bool example(const char* host, const char* port) line_sender_buffer* buffer = NULL; char* conf_str = concat( "tcps::addr=", host, ":", port, ";" + "protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -30,7 +31,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 6f1d4e43..122d0853 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -9,12 +9,12 @@ int main(int argc, const char* argv[]) line_sender_buffer* buffer = NULL; line_sender_utf8 conf = QDB_UTF8_LITERAL( - "tcp::addr=localhost:9009;"); + "tcp::addr=localhost:9009;protocol_version=2;"); line_sender* sender = line_sender_from_conf(conf, &err); if (!sender) goto on_error; - buffer = line_sender_buffer_new(); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index e67dbe59..75904bc3 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -13,7 +13,7 @@ int main(int argc, const char* argv[]) if (!sender) goto on_error; - buffer = line_sender_buffer_new(); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 427ab705..09a7fe93 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -27,8 +27,8 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); - line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + buffer = line_sender_buffer_new_for_sender(sender); + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_http"); line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index 63151945..058ee223 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -11,6 +11,7 @@ static bool example(const char* ca_path, const char* host, const char* port) line_sender_buffer* buffer = NULL; char* conf_str = concat( "tcps::addr=", host, ":", port, ";", + "protocol_version=2;" "tls_roots=", ca_path, ";", "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" @@ -31,7 +32,7 @@ static bool example(const char* ca_path, const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 2d90812f..97943e74 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -9,7 +9,8 @@ static bool example(std::string_view host, std::string_view port) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=" + std::string{host} + ":" + std::string{port} + ";"); + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -20,9 +21,8 @@ static bool example(std::string_view host, std::string_view port) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) @@ -40,10 +40,7 @@ static bool example(std::string_view host, std::string_view port) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -56,12 +53,11 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + std::cerr << "Usage:\n" + << "line_sender_c_example: [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp new file mode 100644 index 00000000..88de8a39 --- /dev/null +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -0,0 +1,62 @@ +#include +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +static bool array_example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;"); + + const auto table_name = "cpp_market_orders_byte_strides"_tn; + const auto symbol_col = "symbol"_cn; + const auto book_col = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) + .symbol(symbol_col, "BTC-USD"_utf8) + .column(book_col, 3, shape, strides, arr_data) + .at(questdb::ingress::timestamp_nanos::now()); + sender.flush(buffer); + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr << "[ERROR] " << err.what() << std::endl; + return false; + } +} + +int main(int argc, const char* argv[]) +{ + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + + auto port = "9009"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !array_example(host, port); +} diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp new file mode 100644 index 00000000..0065d009 --- /dev/null +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -0,0 +1,62 @@ +#include +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +static bool array_example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;"); + + const auto table_name = "cpp_market_orders_elem_strides"_tn; + const auto symbol_col = "symbol"_cn; + const auto book_col = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{6, 2, 1}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) + .symbol(symbol_col, "BTC-USD"_utf8) + .column(book_col, 3, shape, strides, arr_data) + .at(questdb::ingress::timestamp_nanos::now()); + sender.flush(buffer); + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr << "[ERROR] " << err.what() << std::endl; + return false; + } +} + +int main(int argc, const char* argv[]) +{ + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + + auto port = "9009"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !array_example(host, port); +} diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index 85e0d6e1..9ec2f5e7 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -9,7 +9,8 @@ static bool example(std::string_view host, std::string_view port) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=" + std::string{host} + ":" + std::string{port} + ";" + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -24,9 +25,8 @@ static bool example(std::string_view host, std::string_view port) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) @@ -44,10 +44,7 @@ static bool example(std::string_view host, std::string_view port) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -60,12 +57,11 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + std::cerr << "Usage:\n" + << "line_sender_c_example: [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index f100dd04..3f08d8c7 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -4,14 +4,13 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; -static bool example( - std::string_view host, - std::string_view port) +static bool example(std::string_view host, std::string_view port) { try { auto sender = questdb::ingress::line_sender::from_conf( - "tcps::addr=" + std::string{host} + ":" + std::string{port} + ";" + "tcps::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -26,9 +25,8 @@ static bool example( const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) @@ -46,10 +44,7 @@ static bool example( } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -62,12 +57,11 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" - << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + std::cerr << "Usage:\n" + << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index bb71c6e3..697850e0 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -9,7 +9,7 @@ int main(int argc, const char* argv[]) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=localhost:9009;"); + "tcp::addr=localhost:9009;protocol_version=2;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -20,9 +20,8 @@ int main(int argc, const char* argv[]) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) @@ -40,10 +39,7 @@ int main(int argc, const char* argv[]) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return 1; } diff --git a/examples/line_sender_cpp_example_from_env.cpp b/examples/line_sender_cpp_example_from_env.cpp index 63e99b26..3bf1c02a 100644 --- a/examples/line_sender_cpp_example_from_env.cpp +++ b/examples/line_sender_cpp_example_from_env.cpp @@ -19,9 +19,8 @@ int main(int argc, const char* argv[]) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) @@ -39,10 +38,7 @@ int main(int argc, const char* argv[]) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return 1; } diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 217845a2..1e675935 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -20,9 +20,8 @@ static bool example(std::string_view host, std::string_view port) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) @@ -40,10 +39,7 @@ static bool example(std::string_view host, std::string_view port) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -56,12 +52,11 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + std::cerr << "Usage:\n" + << "line_sender_c_example: [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index c0327e96..4e3d0f10 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -5,19 +5,19 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; static bool example( - std::string_view ca_path, - std::string_view host, - std::string_view port) + std::string_view ca_path, std::string_view host, std::string_view port) { try { auto sender = questdb::ingress::line_sender::from_conf( - "tcps::addr=" + std::string{host} + ":" + std::string{port} + ";" + "tcps::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;" - "tls_roots=" + std::string{ca_path} + ";"); // path to custom `.pem` file. + "tls_roots=" + + std::string{ca_path} + ";"); // path to custom `.pem` file. // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -28,9 +28,8 @@ static bool example( const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) @@ -48,10 +47,7 @@ static bool example( } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -64,13 +60,12 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" - << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" - << " CA_PATH: Certificate authority pem file.\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + std::cerr << "Usage:\n" + << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" + << " CA_PATH: Certificate authority pem file.\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 281b9d78..3efb424b 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -77,6 +77,19 @@ typedef enum line_sender_error_code /** Bad configuration. */ line_sender_error_config_error, + + /** Currently, only arrays with a maximum 32 dimensions are supported. */ + line_sender_error_array_large_dim, + + /** ArrayView internal error, such as failure to get the size of a valid + * dimension. */ + line_sender_error_array_view_internal_error, + + /** Write arrayView to sender buffer error. */ + line_sender_error_array_view_write_to_buffer_error, + + /** Line sender protocol version error. */ + line_sender_error_protocol_version_error, } line_sender_error_code; /** The protocol used to connect with. */ @@ -95,6 +108,24 @@ typedef enum line_sender_protocol line_sender_protocol_https, } line_sender_protocol; +/** The line protocol version used to write data to buffer. */ +typedef enum line_sender_protocol_version +{ + /** + * Version 1 of InfluxDB Line Protocol. + * This version is compatible with the InfluxDB database. + */ + line_sender_protocol_version_1 = 1, + + /** + * Version 2 of InfluxDB Line Protocol. + * Uses a binary format serialization for f64, and supports + * the array data type. + * This version is specific to QuestDB and not compatible with InfluxDB. + */ + line_sender_protocol_version_2 = 2, +} line_sender_protocol_version; + /** Possible sources of the root certificates used to validate the server's * TLS certificate. */ typedef enum line_sender_ca @@ -282,9 +313,14 @@ typedef struct line_sender_buffer line_sender_buffer; /** * Construct a `line_sender_buffer` with a `max_name_len` of `127`, which is * the same as the QuestDB server default. + * You should prefer to use `line_sender_for_sender()` instead, which + * automatically creates a buffer of the same protocol version as the sender. + * This is useful as it can rely on the sender's ability to auto-detect the + * protocol version when communicating over HTTP. */ LINESENDER_API -line_sender_buffer* line_sender_buffer_new(); +line_sender_buffer* line_sender_buffer_new( + line_sender_protocol_version version); /** * Construct a `line_sender_buffer` with a custom maximum length for table @@ -294,7 +330,8 @@ line_sender_buffer* line_sender_buffer_new(); * `line_sender_buffer_new()` instead. */ LINESENDER_API -line_sender_buffer* line_sender_buffer_with_max_name_len(size_t max_name_len); +line_sender_buffer* line_sender_buffer_with_max_name_len( + line_sender_protocol_version version, size_t max_name_len); /** Release the `line_sender_buffer` object. */ LINESENDER_API @@ -461,6 +498,62 @@ bool line_sender_buffer_column_str( line_sender_utf8 value, line_sender_error** err_out); +/** + * Record a multidimensional array of double for the given column. + * + * This API uses BYTE-LEVEL STRIDES where the stride values represent the + * number of bytes between consecutive elements along each dimension. + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] rank Number of dimensions of the array. + * @param[in] shape Array of dimension sizes (length = `rank`). + * Each element must be a positive integer. + * @param[in] strides Array strides. + * @param[in] data_buffer First array element data. + * @param[in] data_buffer_len Bytes length of the array data. + * @param[out] err_out Set to an error object on failure (if non-NULL). + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_f64_arr_byte_strides( + line_sender_buffer* buffer, + line_sender_column_name name, + size_t rank, + const uintptr_t* shape, + const intptr_t* strides, + const uint8_t* data_buffer, + size_t data_buffer_len, + line_sender_error** err_out); + +/** + * Record a multidimensional array of double for the given column. + * + * This function uses ELEMENT-LEVEL STRIDES where the stride values represent + * the number of elements between consecutive elements along each dimension. + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] rank Number of dimensions of the array. + * @param[in] shape Array of dimension sizes (length = `rank`). + * Each element must be a positive integer. + * @param[in] strides Array strides. + * @param[in] data_buffer First array element data. + * @param[in] data_buffer_len Bytes length of the array data. + * @param[out] err_out Set to an error object on failure (if non-NULL). + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_f64_arr_elem_strides( + line_sender_buffer* buffer, + line_sender_column_name name, + size_t rank, + const uintptr_t* shape, + const intptr_t* strides, + const uint8_t* data_buffer, + size_t data_buffer_len, + line_sender_error** err_out); + /** * Record a nanosecond timestamp value for the given column. * @param[in] buffer Line buffer object. @@ -563,6 +656,15 @@ LINESENDER_API bool line_sender_buffer_at_now( line_sender_buffer* buffer, line_sender_error** err_out); +/** + * Check whether the buffer is ready to be flushed. + * If this returns false, the buffer is incomplete and cannot be sent, + * and an error message is set to indicate the problem. + */ +LINESENDER_API +bool line_sender_buffer_check_can_flush( + const line_sender_buffer* buffer, line_sender_error** err_out); + /////////// Connecting, sending and disconnecting. /** @@ -693,6 +795,15 @@ bool line_sender_opts_token_y( line_sender_utf8 token_y, line_sender_error** err_out); +/** + * set the line protocol version. + */ +LINESENDER_API +bool line_sender_opts_protocol_version( + line_sender_opts* opts, + line_sender_protocol_version version, + line_sender_error** err_out); + /** * Configure how long to wait for messages from the QuestDB server during * the TLS handshake and authentication process. @@ -741,6 +852,14 @@ LINESENDER_API bool line_sender_opts_max_buf_size( line_sender_opts* opts, size_t max_buf_size, line_sender_error** err_out); +/** + * Set the maximum length of a table or column name in bytes. + * The default is 127 bytes. + */ +LINESENDER_API +bool line_sender_opts_max_name_len( + line_sender_opts* opts, size_t max_name_len, line_sender_error** err_out); + /** * Set the cumulative duration spent in retries. * The value is in milliseconds, and the default is 10 seconds. @@ -852,6 +971,33 @@ line_sender* line_sender_from_conf( LINESENDER_API line_sender* line_sender_from_env(line_sender_error** err_out); +/** + * Return the sender's protocol version. + * This is either the protocol version that was set explicitly, + * or the one that was auto-detected during the connection process. + * If connecting via TCP and not overridden, the value is V1. + */ +LINESENDER_API +line_sender_protocol_version line_sender_get_protocol_version( + const line_sender* sender); + +/** + * Returns the configured max_name_len, or the default value of 127. + */ +LINESENDER_API +size_t line_sender_get_max_name_len(const line_sender* sender); + +/** + * Construct a `line_sender_buffer` with the sender's + * configured protocol version and other parameters. + * This is equivalent to calling: + * line_sender_buffer_new( + * line_sender_get_protocol_version(sender), + * line_sender_get_max_name_len(sender)) + */ +line_sender_buffer* line_sender_buffer_new_for_sender( + const line_sender* sender); + /** * Tell whether the sender is no longer usable and must be closed. * This happens when there was an earlier failure. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index e118a22f..91fc33d6 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -26,6 +26,7 @@ #include "line_sender.h" +#include #include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #if __cplusplus >= 202002L # include #endif @@ -98,6 +100,15 @@ enum class protocol https, }; +enum class protocol_version +{ + /** InfluxDB Line Protocol v1. */ + v1 = 1, + + /** InfluxDB Line Protocol v2. */ + v2 = 2, +}; + /* Possible sources of the root certificates used to validate the server's TLS * certificate. */ enum class ca @@ -387,13 +398,12 @@ class buffer_view final class line_sender_buffer { public: - explicit line_sender_buffer(size_t init_buf_size = 64 * 1024) noexcept - : line_sender_buffer{init_buf_size, 127} - { - } - - line_sender_buffer(size_t init_buf_size, size_t max_name_len) noexcept + explicit line_sender_buffer( + protocol_version version, + size_t init_buf_size = 64 * 1024, + size_t max_name_len = 127) noexcept : _impl{nullptr} + , _protocol_version{version} , _init_buf_size{init_buf_size} , _max_name_len{max_name_len} { @@ -401,15 +411,19 @@ class line_sender_buffer line_sender_buffer(const line_sender_buffer& other) noexcept : _impl{::line_sender_buffer_clone(other._impl)} + , _protocol_version{other._protocol_version} , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} + { } line_sender_buffer(line_sender_buffer&& other) noexcept : _impl{other._impl} + , _protocol_version{other._protocol_version} , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} + { other._impl = nullptr; } @@ -425,6 +439,7 @@ class line_sender_buffer _impl = nullptr; _init_buf_size = other._init_buf_size; _max_name_len = other._max_name_len; + _protocol_version = other._protocol_version; } return *this; } @@ -437,6 +452,7 @@ class line_sender_buffer _impl = other._impl; _init_buf_size = other._init_buf_size; _max_name_len = other._max_name_len; + _protocol_version = other._protocol_version; other._impl = nullptr; } return *this; @@ -624,6 +640,45 @@ class line_sender_buffer return *this; } + /** + * Record a multidimensional double-precision array for the given column. + * + * @tparam B Strides mode selector: + * - `true` for byte-level strides + * - `false` for element-level strides + * @tparam T Element type (current only `double` is supported). + * @tparam N Number of elements in the flat data array + * + * @param name Column name. + * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). + * @param data Array first element data. Size must match product of + * dimensions. + */ + template + line_sender_buffer& column( + column_name_view name, + const size_t rank, + const std::vector& shape, + const std::vector& strides, + const std::array& data) + { + static_assert( + std::is_same_v, + "Only double types are supported for arrays"); + may_init(); + line_sender_error::wrapped_call( + B ? ::line_sender_buffer_column_f64_arr_byte_strides + : ::line_sender_buffer_column_f64_arr_elem_strides, + _impl, + name._impl, + rank, + shape.data(), + strides.data(), + reinterpret_cast(data.data()), + sizeof(double) * N); + return *this; + } + /** * Record a string value for the given column. * @param name Column name. @@ -756,6 +811,19 @@ class line_sender_buffer line_sender_error::wrapped_call(::line_sender_buffer_at_now, _impl); } + void check_can_flush() const + { + if (!_impl) + { + throw line_sender_error{ + line_sender_error_code::invalid_api_call, + "State error: Bad call to `flush`, should have called `table` " + "instead."}; + } + line_sender_error::wrapped_call( + ::line_sender_buffer_check_can_flush, _impl); + } + ~line_sender_buffer() noexcept { if (_impl) @@ -767,12 +835,16 @@ class line_sender_buffer { if (!_impl) { - _impl = ::line_sender_buffer_with_max_name_len(_max_name_len); + _impl = ::line_sender_buffer_with_max_name_len( + static_cast<::line_sender_protocol_version>( + static_cast(_protocol_version)), + _max_name_len); ::line_sender_buffer_reserve(_impl, _init_buf_size); } } ::line_sender_buffer* _impl; + protocol_version _protocol_version; size_t _init_buf_size; size_t _max_name_len; @@ -785,7 +857,7 @@ class _user_agent static inline ::line_sender_utf8 name() { // Maintained by .bumpversion.cfg - static const char user_agent[] = "questdb/c++/4.0.4"; + static const char user_agent[] = "questdb/c++/5.0.0-rc1"; ::line_sender_utf8 utf8 = ::line_sender_utf8_assert(sizeof(user_agent) - 1, user_agent); return utf8; @@ -834,6 +906,7 @@ class opts * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. * @param[in] port The QuestDB tcp or http port. + * validation. */ opts(protocol protocol, utf8_view host, uint16_t port) noexcept : _impl{::line_sender_opts_new( @@ -1028,6 +1101,17 @@ class opts return *this; } + /** + * The maximum length of a table or column name in bytes. + * The default is 127 bytes. + */ + opts& max_name_len(size_t max_name_len) + { + line_sender_error::wrapped_call( + ::line_sender_opts_max_name_len, _impl, max_name_len); + return *this; + } + /** * Set the cumulative duration spent in retries. * The value is in milliseconds, and the default is 10 seconds. @@ -1071,6 +1155,16 @@ class opts return *this; } + opts& protocol_version(protocol_version version) noexcept + { + const auto c_protocol_version = + static_cast<::line_sender_protocol_version>( + static_cast(version)); + line_sender_error::wrapped_call( + ::line_sender_opts_protocol_version, _impl, c_protocol_version); + return *this; + } + ~opts() noexcept { reset(); @@ -1150,16 +1244,6 @@ class line_sender return {opts::from_env()}; } - line_sender(protocol protocol, utf8_view host, uint16_t port) - : line_sender{opts{protocol, host, port}} - { - } - - line_sender(protocol protocol, utf8_view host, utf8_view port) - : line_sender{opts{protocol, host, port}} - { - } - line_sender(const opts& opts) : _impl{ line_sender_error::wrapped_call(::line_sender_build, opts._impl)} @@ -1187,6 +1271,22 @@ class line_sender return *this; } + questdb::ingress::protocol_version protocol_version() const noexcept + { + ensure_impl(); + return static_cast( + static_cast(::line_sender_get_protocol_version(_impl))); + } + + line_sender_buffer new_buffer(size_t init_buf_size = 64 * 1024) noexcept + { + ensure_impl(); + return line_sender_buffer{ + this->protocol_version(), + init_buf_size, + ::line_sender_get_max_name_len(_impl)}; + } + /** * Send the given buffer of rows to the QuestDB server, clearing the buffer. * @@ -1239,7 +1339,7 @@ class line_sender } else { - line_sender_buffer buffer2{0}; + line_sender_buffer buffer2{this->protocol_version(), 0}; buffer2.may_init(); line_sender_error::wrapped_call( ::line_sender_flush_and_keep, _impl, buffer2._impl); @@ -1276,7 +1376,7 @@ class line_sender } private: - void ensure_impl() + void ensure_impl() const { if (!_impl) throw line_sender_error{ diff --git a/proj b/proj new file mode 100755 index 00000000..d4c80095 --- /dev/null +++ b/proj @@ -0,0 +1,2 @@ +#!/bin/bash +python3 proj.py "$@" diff --git a/proj.bat b/proj.bat new file mode 100644 index 00000000..027f5468 --- /dev/null +++ b/proj.bat @@ -0,0 +1,2 @@ +@echo off +python3 proj.py %* diff --git a/proj.ps1 b/proj.ps1 new file mode 100644 index 00000000..7f9e830e --- /dev/null +++ b/proj.ps1 @@ -0,0 +1 @@ +python3 proj.py $args diff --git a/proj.py b/proj.py new file mode 100755 index 00000000..609df248 --- /dev/null +++ b/proj.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 + +import sys +sys.dont_write_bytecode = True +import pathlib +import shutil +import shlex +import subprocess +import os + + +PROJ_ROOT = pathlib.Path(__file__).parent + + +def _run(*args, env=None, cwd=None): + """ + Log and run a command within the build dir. + On error, exit with child's return code. + """ + args = [str(arg) for arg in args] + cwd = cwd or PROJ_ROOT + sys.stderr.write('[CMD] ') + if env is not None: + env_str = ' '.join(f'{k}={shlex.quote(v)}' for k, v in env.items()) + sys.stderr.write(f'{env_str} ') + env = {**os.environ, **env} + escaped_cmd = ' '.join(shlex.quote(arg) for arg in args) + sys.stderr.write(f'{escaped_cmd}\n') + ret_code = subprocess.run(args, cwd=str(cwd), env=env).returncode + if ret_code != 0: + sys.exit(ret_code) + + +def _rm(path: pathlib.Path, pattern: str): + paths = path.glob(pattern) + for path in paths: + sys.stderr.write(f'[RM] {path}\n') + path.unlink() + + +def _rmtree(path: pathlib.Path): + if not path.exists(): + return + sys.stderr.write(f'[RMTREE] {path}\n') + shutil.rmtree(path, ignore_errors=True) + + +def _has_command(command: str) -> bool: + """ + Check if a command is available in the system. + """ + return shutil.which(command) is not None + + +COMMANDS = [] + + +def command(fn): + COMMANDS.append(fn.__name__) + return fn + + +@command +def clean(): + _rmtree(PROJ_ROOT / 'build') + _rmtree(PROJ_ROOT / 'build_CXX20') + _rmtree(PROJ_ROOT / 'questdb-rs' / 'target') + _rmtree(PROJ_ROOT / 'questdb-rs-ffi' / 'target') + + +@command +def cmake_cxx17(): + _rmtree(PROJ_ROOT / 'build') + cmd = [ + 'cmake', + '-S', '.', + '-B', 'build', + '-DCMAKE_BUILD_TYPE=Release', + '-DQUESTDB_TESTS_AND_EXAMPLES=ON'] + if _has_command('ninja'): + cmd.insert(1, '-G') + cmd.insert(2, 'Ninja') + _run(*cmd) + + +@command +def cmake_cxx20(): + _rmtree(PROJ_ROOT / 'build_CXX20') + cmd = [ + 'cmake', + '-S', '.', + '-B', 'build_CXX20', + '-DCMAKE_BUILD_TYPE=Release', + '-DQUESTDB_TESTS_AND_EXAMPLES=ON', + '-DCMAKE_CXX_STANDARD=20'] + if _has_command('ninja'): + cmd.insert(1, '-G') + cmd.insert(2, 'Ninja') + _run(*cmd) + + +@command +def build_cxx17(): + if not (PROJ_ROOT / 'build').exists(): + cmake_cxx17() + _run('cmake', '--build', 'build') + + +@command +def build_cxx20(): + if not (PROJ_ROOT / 'build_CXX20').exists(): + cmake_cxx20() + _run('cmake', '--build', 'build_CXX20') + + +@command +def build(): + build_cxx17() + build_cxx20() + + +@command +def lint_rust(): + questdb_rs_path = PROJ_ROOT / 'questdb-rs' + questdb_rs_ffi_path = PROJ_ROOT / 'questdb-rs-ffi' + _run('cargo', 'fmt', '--all', '--', '--check', cwd=questdb_rs_path) + _run('cargo', 'clippy', '--all-targets', '--features', 'almost-all-features', '--', '-D', 'warnings', cwd=questdb_rs_path) + _run('cargo', 'fmt', '--all', '--', '--check', cwd=questdb_rs_ffi_path) + _run('cargo', 'clippy', '--all-targets', '--all-features', '--', '-D', 'warnings', cwd=questdb_rs_ffi_path) + + +@command +def lint_cpp(): + try: + _run( + sys.executable, + PROJ_ROOT / 'ci' / 'format_cpp.py', + '--check') + except subprocess.CalledProcessError: + sys.stderr.write('REMINDER: To fix any C++ formatting issues, run: ./proj format_cpp\n') + raise + + +@command +def lint(): + lint_rust() + lint_cpp() + + +@command +def format_rust(): + questdb_rs_path = PROJ_ROOT / 'questdb-rs' + questdb_rs_ffi_path = PROJ_ROOT / 'questdb-rs-ffi' + _run('cargo', 'fmt', '--all', cwd=questdb_rs_path) + _run('cargo', 'fmt', '--all', cwd=questdb_rs_ffi_path) + + +@command +def format_cpp(): + _run( + sys.executable, + PROJ_ROOT / 'ci' / 'format_cpp.py') + + +@command +def test(): + build() + _run( + sys.executable, + PROJ_ROOT / 'ci' / 'run_all_tests.py') + + +@command +def build_latest_questdb(branch='master'): + questdb_path = PROJ_ROOT / 'questdb' + if not questdb_path.exists(): + _run('git', 'clone', 'https://github.com/questdb/questdb.git') + _run('git', 'fetch', 'origin', branch, cwd=questdb_path) + _run('git', 'switch', branch, cwd=questdb_path) + _run('git', 'pull', 'origin', branch, cwd=questdb_path) + _run('git', 'submodule', 'update', '--init', '--recursive', cwd=questdb_path) + _run('mvn', 'clean', 'package', '-DskipTests', '-Pbuild-web-console', cwd=questdb_path) + + +@command +def test_vs_latest_questdb(): + questdb_path = PROJ_ROOT / 'questdb' + if not questdb_path.exists(): + build_latest_questdb() + _run( + sys.executable, + PROJ_ROOT / 'system_test' / 'test.py', + 'run', + '--repo', PROJ_ROOT / 'questdb', + '-v') + + +@command +def all(): + clean() + build() + lint() + test() + test_vs_latest_questdb() + + +def main(): + if len(sys.argv) < 2: + sys.stderr.write('Usage: python3 proj.py \n') + sys.stderr.write('Commands:\n') + for command in COMMANDS: + sys.stderr.write(f' {command}\n') + sys.stderr.write('\n') + sys.exit(0) + fn = sys.argv[1] + args = list(sys.argv)[2:] + globals()[fn](*args) + + +if __name__ == '__main__': + main() diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 41476357..d02045b8 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -26,24 +26,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" -[[package]] -name = "cbindgen" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadd868a2ce9ca38de7eeafdcec9c7065ef89b42b32f0839278d55f35c54d1ff" -dependencies = [ - "heck", - "indexmap", - "log", - "proc-macro2", - "quote", - "serde", - "serde_json", - "syn", - "tempfile", - "toml", -] - [[package]] name = "cc" version = "1.2.17" @@ -87,28 +69,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - [[package]] name = "fnv" version = "1.0.7" @@ -138,18 +98,6 @@ dependencies = [ "wasi 0.14.2+wasi-0.2.4", ] -[[package]] -name = "hashbrown" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "http" version = "1.3.1" @@ -167,16 +115,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" -[[package]] -name = "indexmap" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" -dependencies = [ - "equivalent", - "hashbrown", -] - [[package]] name = "indoc" version = "2.0.6" @@ -195,12 +133,6 @@ version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" -[[package]] -name = "linux-raw-sys" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" - [[package]] name = "log" version = "0.4.27" @@ -266,7 +198,7 @@ dependencies = [ [[package]] name = "questdb-rs" -version = "4.0.4" +version = "5.0.0-rc1" dependencies = [ "base64ct", "dns-lookup", @@ -292,9 +224,8 @@ dependencies = [ [[package]] name = "questdb-rs-ffi" -version = "4.0.4" +version = "5.0.0-rc1" dependencies = [ - "cbindgen", "libc", "questdb-confstr-ffi", "questdb-rs", @@ -359,19 +290,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustix" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.59.0", -] - [[package]] name = "rustls" version = "0.23.25" @@ -495,15 +413,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ - "serde", -] - [[package]] name = "shlex" version = "1.3.0" @@ -546,53 +455,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "tempfile" -version = "3.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" -dependencies = [ - "fastrand", - "getrandom 0.3.2", - "once_cell", - "rustix", - "windows-sys 0.59.0", -] - -[[package]] -name = "toml" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "unicode-ident" version = "1.0.18" @@ -840,15 +702,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" -dependencies = [ - "memchr", -] - [[package]] name = "wit-bindgen-rt" version = "0.39.0" diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index d9d8409c..1d461b5d 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs-ffi" -version = "4.0.4" +version = "5.0.0-rc1" edition = "2021" publish = false @@ -9,13 +9,19 @@ name = "questdb_client" crate-type = ["cdylib", "staticlib"] [dependencies] -questdb-rs = { path = "../questdb-rs", features = [ - "insecure-skip-verify", "tls-native-certs", "ilp-over-http"] } libc = "0.2" questdb-confstr-ffi = { version = "0.1.1", optional = true } -[build-dependencies] -cbindgen = { version = "0.28.0", optional = true, default-features = false } +[dependencies.questdb-rs] +path = "../questdb-rs" +default-features = false +features = [ + "ring-crypto", + "insecure-skip-verify", + "tls-webpki-certs", + "tls-native-certs", + "ilp-over-http" +] [features] # Expose the config parsing C API. @@ -23,7 +29,3 @@ cbindgen = { version = "0.28.0", optional = true, default-features = false } # It is exposed here to avoid having multiple copies of the `questdb-confstr` # crate in the final binary. confstr-ffi = ["dep:questdb-confstr-ffi"] - -# Auto-generate the header. This is for dev-debugging-diffing only. -# A hand-crafted header is easier on the eyes. -gen_h = ["cbindgen"] diff --git a/questdb-rs-ffi/build.rs b/questdb-rs-ffi/build.rs deleted file mode 100644 index 51d6ee1d..00000000 --- a/questdb-rs-ffi/build.rs +++ /dev/null @@ -1,16 +0,0 @@ -#[cfg(feature = "gen_h")] -extern crate cbindgen; - -fn main() -> Result<(), Box> { - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-changed=Cargo.lock"); - - #[cfg(feature = "gen_h")] - { - let crate_dir = std::env::var("CARGO_MANIFEST_DIR")?; - let bindings = cbindgen::generate(crate_dir)?; - bindings.write_to_file("../include/questdb/ingress/line_sender.gen.h"); - } - - Ok(()) -} diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 2db6a917..150b7412 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -34,6 +34,7 @@ use std::slice; use std::str; use questdb::{ + ingress, ingress::{ Buffer, CertificateAuthority, ColumnName, Protocol, Sender, SenderBuilder, TableName, TimestampMicros, TimestampNanos, @@ -41,6 +42,9 @@ use questdb::{ Error, ErrorCode, }; +mod ndarr; +use ndarr::StrideArrayView; + macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { bubble_err_to_c!($err_out, $expression, false) @@ -83,7 +87,7 @@ macro_rules! upd_opts { // already cleaned up object. // To avoid double-freeing, we need to construct a valid "dummy" // object on top of the memory that is still owned by the caller. - let dummy = SenderBuilder::new(Protocol::Tcp, "localhost", 1); + let dummy = SenderBuilder::new(Protocol::Tcp, "127.0.0.1", 1); ptr::write(builder_ref, dummy); return false; } @@ -135,6 +139,18 @@ pub enum line_sender_error_code { /// Bad configuration. line_sender_error_config_error, + + /// Currently, only arrays with a maximum 32 dimensions are supported. + line_sender_error_array_large_dim, + + /// ArrayView internal error, such as failure to get the size of a valid dimension. + line_sender_error_array_view_internal_error, + + /// Write arrayView to sender buffer error. + line_sender_error_array_view_write_to_buffer_error, + + /// Line sender protocol version error. + line_sender_error_protocol_version_error, } impl From for line_sender_error_code { @@ -159,6 +175,18 @@ impl From for line_sender_error_code { line_sender_error_code::line_sender_error_server_flush_error } ErrorCode::ConfigError => line_sender_error_code::line_sender_error_config_error, + ErrorCode::ArrayHasTooManyDims => { + line_sender_error_code::line_sender_error_array_large_dim + } + ErrorCode::ArrayViewError => { + line_sender_error_code::line_sender_error_array_view_internal_error + } + ErrorCode::ArrayWriteToBufferError => { + line_sender_error_code::line_sender_error_array_view_write_to_buffer_error + } + ErrorCode::ProtocolVersionError => { + line_sender_error_code::line_sender_error_protocol_version_error + } } } } @@ -202,6 +230,39 @@ impl From for Protocol { } } +/// The version of InfluxDB Line Protocol used to communicate with the server. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub enum ProtocolVersion { + /// Version 1 of InfluxDB Line Protocol. + /// Full-text protocol. + /// When used over HTTP, it is compatible with the InfluxDB line protocol. + V1 = 1, + + /// Version 2 of InfluxDB Line Protocol. + /// Uses binary format serialization for f64, and supports the array data type. + /// This version is specific to QuestDB and is not compatible with InfluxDB. + V2 = 2, +} + +impl From for ingress::ProtocolVersion { + fn from(version: ProtocolVersion) -> Self { + match version { + ProtocolVersion::V1 => ingress::ProtocolVersion::V1, + ProtocolVersion::V2 => ingress::ProtocolVersion::V2, + } + } +} + +impl From for ProtocolVersion { + fn from(version: ingress::ProtocolVersion) -> Self { + match version { + ingress::ProtocolVersion::V1 => ProtocolVersion::V1, + ingress::ProtocolVersion::V2 => ProtocolVersion::V2, + } + } +} + /// Possible sources of the root certificates used to validate the server's TLS certificate. #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -543,8 +604,10 @@ pub struct line_sender_buffer(Buffer); /// Construct a `line_sender_buffer` with a `max_name_len` of `127`, which is the /// same as the QuestDB server default. #[no_mangle] -pub unsafe extern "C" fn line_sender_buffer_new() -> *mut line_sender_buffer { - let buffer = Buffer::new(); +pub unsafe extern "C" fn line_sender_buffer_new( + version: ProtocolVersion, +) -> *mut line_sender_buffer { + let buffer = Buffer::new(version.into()); Box::into_raw(Box::new(line_sender_buffer(buffer))) } @@ -555,9 +618,10 @@ pub unsafe extern "C" fn line_sender_buffer_new() -> *mut line_sender_buffer { /// call `line_sender_buffer_new()` instead. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_with_max_name_len( + version: ProtocolVersion, max_name_len: size_t, ) -> *mut line_sender_buffer { - let buffer = Buffer::with_max_name_len(max_name_len); + let buffer = Buffer::with_max_name_len(version.into(), max_name_len); Box::into_raw(Box::new(line_sender_buffer(buffer))) } @@ -804,6 +868,102 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( true } +/// Records a float64 multidimensional array with **byte-level strides specification**. +/// +/// The `strides` represent byte offsets between elements along each dimension. +/// +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] rank Array dims. +/// @param[in] shape Array shape. +/// @param[in] strides Array strides, represent byte offsets between elements along each dimension. +/// @param[in] data_buffer Array **first element** data memory ptr. +/// @param[in] data_buffer_len Array data memory length. +/// @param[out] err_out Set on error. +/// # Safety +/// - All pointer parameters must be valid and non-null +/// - shape must point to an array of `rank` integers +/// - data_buffer must point to a buffer of size `data_buffer_len` bytes +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + rank: size_t, + shape: *const usize, + strides: *const isize, + data_buffer: *const u8, + data_buffer_len: size_t, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer_mut(buffer); + let name = name.as_name(); + let view = + match StrideArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len) { + Ok(value) => value, + Err(err) => { + let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); + *err_out = err_ptr; + return false; + } + }; + bubble_err_to_c!( + err_out, + buffer.column_arr::, StrideArrayView<'_, f64, 1>, f64>(name, &view) + ); + true +} + +/// Records a float64 multidimensional array with **element count stride specification**. +/// +/// The `strides` represent element counts between elements along each dimension. +/// +/// converted to byte strides using f64 size +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] rank Array dims. +/// @param[in] shape Array shape. +/// @param[in] strides Array strides, represent element counts between elements along each dimension. +/// @param[in] data_buffer Array **first element** data memory ptr. +/// @param[in] data_buffer_len Array data memory length. +/// @param[out] err_out Set on error. +/// # Safety +/// - All pointer parameters must be valid and non-null +/// - shape must point to an array of `rank` integers +/// - data_buffer must point to a buffer of size `data_buffer_len` bytes +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_elem_strides( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + rank: size_t, + shape: *const usize, + strides: *const isize, + data_buffer: *const u8, + data_buffer_len: size_t, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer_mut(buffer); + let name = name.as_name(); + let view = match StrideArrayView::() as isize }>::new( + rank, + shape, + strides, + data_buffer, + data_buffer_len, + ) { + Ok(value) => value, + Err(err) => { + let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); + *err_out = err_ptr; + return false; + } + }; + bubble_err_to_c!( + err_out, + buffer.column_arr::, StrideArrayView<'_, f64, { std::mem::size_of::() as isize }>, f64>(name, &view) + ); + true +} + /// Record a nanosecond timestamp value for the given column. /// @param[in] buffer Line buffer object. /// @param[in] name Column name. @@ -924,6 +1084,21 @@ pub unsafe extern "C" fn line_sender_buffer_at_now( true } +/** + * Check whether the buffer is ready to be flushed. + * If this returns false, the buffer is incomplete and cannot be sent, + * and an error message is set to indicate the problem. + */ +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_check_can_flush( + buffer: *const line_sender_buffer, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer(buffer); + bubble_err_to_c!(err_out, buffer.check_can_flush()); + true +} + /// Accumulates parameters for a new `line_sender` object. pub struct line_sender_opts(SenderBuilder); @@ -1066,6 +1241,16 @@ pub unsafe extern "C" fn line_sender_opts_token_y( upd_opts!(opts, err_out, token_y, token_y.as_str()) } +/// set the line protocol version. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_protocol_version( + opts: *mut line_sender_opts, + version: ProtocolVersion, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, protocol_version, version.into()) +} + /// Configure how long to wait for messages from the QuestDB server during /// the TLS handshake and authentication process. /// The value is in milliseconds, and the default is 15 seconds. @@ -1131,6 +1316,17 @@ pub unsafe extern "C" fn line_sender_opts_max_buf_size( upd_opts!(opts, err_out, max_buf_size, max_buf_size) } +/// Ser the maximum length of a table or column name in bytes. +/// The default is 127 bytes. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_max_name_len( + opts: *mut line_sender_opts, + max_name_len: size_t, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, max_name_len, max_name_len) +} + /// Set the cumulative duration spent in retries. /// The value is in milliseconds, and the default is 10 seconds. #[no_mangle] @@ -1295,6 +1491,33 @@ unsafe fn unwrap_sender_mut<'a>(sender: *mut line_sender) -> &'a mut Sender { &mut (*sender).0 } +/// Return the sender's protocol version. +/// This is either the protocol version that was set explicitly, +/// or the one that was auto-detected during the connection process. +/// If connecting via TCP and not overridden, the value is V1. +#[no_mangle] +pub unsafe extern "C" fn line_sender_get_protocol_version( + sender: *const line_sender, +) -> ProtocolVersion { + unwrap_sender(sender).protocol_version().into() +} + +#[no_mangle] +pub unsafe extern "C" fn line_sender_get_max_name_len(sender: *const line_sender) -> size_t { + unwrap_sender(sender).max_name_len() +} + +/// Construct a `line_sender_buffer` with a `max_name_len` of `127` and sender's default protocol version +/// which is the same as the QuestDB server default. +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_new_for_sender( + sender: *const line_sender, +) -> *mut line_sender_buffer { + let sender = unwrap_sender(sender); + let buffer = sender.new_buffer(); + Box::into_raw(Box::new(line_sender_buffer(buffer))) +} + /// Tell whether the sender is no longer usable and must be closed. /// This happens when there was an earlier failure. /// This fuction is specific to TCP and is not relevant for HTTP. diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs new file mode 100644 index 00000000..ed3de205 --- /dev/null +++ b/questdb-rs-ffi/src/ndarr.rs @@ -0,0 +1,771 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use questdb::ingress::ArrayElement; +use questdb::ingress::NdArrayView; +use questdb::ingress::MAX_ARRAY_BUFFER_SIZE; +use questdb::Error; +use std::mem::size_of; +use std::slice; + +macro_rules! fmt_error { + ($code:ident, $($arg:tt)*) => { + questdb::Error::new( + questdb::ErrorCode::$code, + format!($($arg)*)) + } +} + +/// A view into a multidimensional array with custom memory strides. +// TODO: We are currently evaluating whether to use StrideArrayView or ndarray's view. +// Current benchmarks show that StrideArrayView's iter implementation underperforms(2x) +// compared to ndarray's. +// We should optimise this implementation to be competitive. +// Unfortunately, the `ndarray` crate does not support negative strides +// which we need to support in this FFI crate for efficient iteration of +// numpy arrays coming from Python without copying the data. +#[derive(Debug)] +pub struct StrideArrayView<'a, T, const N: isize> { + dims: usize, + shape: &'a [usize], + strides: &'a [isize], + data: Option<&'a [u8]>, + _marker: std::marker::PhantomData, +} + +impl NdArrayView for StrideArrayView<'_, T, N> +where + T: ArrayElement, +{ + type Iter<'b> + = RowMajorIter<'b, T, N> + where + Self: 'b, + T: 'b; + + fn ndim(&self) -> usize { + self.dims + } + + fn dim(&self, index: usize) -> Result { + if index >= self.dims { + return Err(fmt_error!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + index, + self.dims + )); + } + Ok(self.shape[index]) + } + + fn as_slice(&self) -> Option<&[T]> { + unsafe { + self.is_c_major().then_some(self.data.map(|data| { + slice::from_raw_parts(data.as_ptr() as *const T, data.len() / size_of::()) + })?) + } + } + + fn iter(&self) -> Self::Iter<'_> { + let mut dim_products = Vec::with_capacity(self.dims); + let mut product = 1; + for &dim in self.shape.iter().rev() { + dim_products.push(product); + product *= dim; + } + dim_products.reverse(); + + // consider minus strides + let base_ptr = match self.data { + None => std::ptr::null(), + Some(data) => { + self.strides + .iter() + .enumerate() + .fold(data.as_ptr(), |ptr, (dim, &stride)| { + let stride_bytes_size = stride * N; + if stride_bytes_size < 0 { + let dim_size = self.shape[dim] as isize; + unsafe { ptr.offset(stride_bytes_size * (dim_size - 1)) } + } else { + ptr + } + }) + } + }; + + RowMajorIter { + base_ptr, + array: self, + dim_products, + current_linear: 0, + total_elements: self.shape.iter().product(), + } + } +} + +impl StrideArrayView<'_, T, N> +where + T: ArrayElement, +{ + /// Creates a new strided array view from raw components (unsafe constructor). + /// + /// # Safety + /// Caller must ensure all the following conditions: + /// - `shape` points to a valid array of at least `dims` elements + /// - `strides` points to a valid array of at least `dims` elements + /// - `data` points to a valid memory block of at least `data_len` bytes + /// - Memory layout must satisfy: + /// 1. `data_len ≥ (shape[0]-1)*abs(strides[0]) + ... + (shape[n-1]-1)*abs(strides[n-1]) + size_of::()` + /// 2. All calculated offsets stay within `[0, data_len - size_of::()]` + /// - Lifetime `'a` must outlive the view's usage + /// - Strides are measured in bytes (not elements) + pub unsafe fn new( + dims: usize, + shape: *const usize, + strides: *const isize, + data: *const u8, + data_len: usize, + ) -> Result { + if dims == 0 { + return Err(fmt_error!( + ArrayViewError, + "Zero-dimensional arrays are not supported", + )); + } + if data_len > MAX_ARRAY_BUFFER_SIZE { + return Err(fmt_error!( + ArrayViewError, + "Array buffer size too big: {}, maximum: {}", + data_len, + MAX_ARRAY_BUFFER_SIZE + )); + } + let shape = slice::from_raw_parts(shape, dims); + let size = shape + .iter() + .try_fold(std::mem::size_of::(), |acc, &dim| { + acc.checked_mul(dim) + .ok_or_else(|| fmt_error!(ArrayViewError, "Array buffer size too big")) + })?; + + if size != data_len { + return Err(fmt_error!( + ArrayViewError, + "Array buffer length mismatch (actual: {}, expected: {})", + data_len, + size + )); + } + let strides = slice::from_raw_parts(strides, dims); + let mut slice = None; + if data_len != 0 { + slice = Some(slice::from_raw_parts(data, data_len)); + } + Ok(Self { + dims, + shape, + strides, + data: slice, + _marker: std::marker::PhantomData::, + }) + } + + /// Verifies if the array follows C-style row-major memory layout. + fn is_c_major(&self) -> bool { + match self.data { + None => false, + Some(data) => { + if data.is_empty() { + return false; + } + + let elem_size = size_of::() as isize; + if self.dims == 1 { + return self.strides[0] * N == elem_size || self.shape[0] == 1; + } + + let mut expected_stride = elem_size; + for (dim, &stride) in self.shape.iter().zip(self.strides).rev() { + if *dim > 1 && stride * N != expected_stride { + return false; + } + expected_stride *= *dim as isize; + } + true + } + } + } +} + +/// Iterator for traversing a stride array in row-major (C-style) order. +pub struct RowMajorIter<'a, T, const N: isize> { + base_ptr: *const u8, + array: &'a StrideArrayView<'a, T, N>, + dim_products: Vec, + current_linear: usize, + total_elements: usize, +} + +impl<'a, T, const N: isize> Iterator for RowMajorIter<'a, T, N> +where + T: ArrayElement, +{ + type Item = &'a T; + fn next(&mut self) -> Option { + if self.current_linear >= self.total_elements { + return None; + } + let mut remaining_index = self.current_linear; + let mut offset = 0; + + for (dim, &dim_factor) in self.dim_products.iter().enumerate() { + let coord = remaining_index / dim_factor; + remaining_index %= dim_factor; + let stride = self.array.strides[dim] * N; + let actual_coord = if stride >= 0 { + coord + } else { + self.array.shape[dim] - 1 - coord + }; + offset += actual_coord * stride.unsigned_abs(); + } + + self.current_linear += 1; + unsafe { + let ptr = self.base_ptr.add(offset); + Some(&*(ptr as *const T)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use questdb::ingress::*; + use questdb::{Error, ErrorCode}; + use std::ptr; + type TestResult = std::result::Result<(), Box>; + + fn to_bytes(data: &[T]) -> Vec { + data.iter() + .flat_map(|x| { + let bytes = unsafe { + std::slice::from_raw_parts(x as *const T as *const u8, size_of::()) + }; + bytes.to_vec() + }) + .collect() + } + + // Duplicated from `questdb::ingress::ndarr::write_array_data` to avoid leaking it to the public API. + pub(crate) fn write_array_data, T>( + array: &A, + buf: &mut [u8], + expect_size: usize, + ) -> Result<(), Error> + where + T: ArrayElement, + { + // When working with contiguous layout, benchmark shows `copy_from_slice` has better performance than + // `std::ptr::copy_nonoverlapping` on both Arm(Macos) and x86(Linux) platform. + // This may because `copy_from_slice` benefits more from compiler. + if let Some(contiguous) = array.as_slice() { + let bytes = unsafe { + slice::from_raw_parts(contiguous.as_ptr() as *const u8, size_of_val(contiguous)) + }; + + if bytes.len() != expect_size { + return Err(fmt_error!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + expect_size, + bytes.len() + )); + } + + if buf.len() < bytes.len() { + return Err(fmt_error!( + ArrayWriteToBufferError, + "Buffer capacity {} < required {}", + buf.len(), + bytes.len() + )); + } + + buf[..bytes.len()].copy_from_slice(bytes); + return Ok(()); + } + + // For non-contiguous memory layouts, direct raw pointer operations are preferred. + let elem_size = size_of::(); + let mut total_len = 0; + for (i, &element) in array.iter().enumerate() { + unsafe { + std::ptr::copy_nonoverlapping( + &element as *const T as *const u8, + buf.as_mut_ptr().add(i * elem_size), + elem_size, + ) + } + total_len += elem_size; + } + if total_len != expect_size { + return Err(fmt_error!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + total_len, + expect_size + )); + } + Ok(()) + } + + #[test] + fn test_buffer_basic_write() -> TestResult { + let elem_size = std::mem::size_of::() as isize; + + let test_data = [1.1, 2.2, 3.3, 4.4]; + let array_view: StrideArrayView<'_, f64, 1> = unsafe { + StrideArrayView::new( + 2, + [2, 2].as_ptr(), + [2 * elem_size, elem_size].as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * elem_size as usize, + ) + }?; + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() + 2u8 + ] + ); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..64], + &[ + 1.1f64.to_ne_bytes(), + 2.2f64.to_le_bytes(), + 3.3f64.to_le_bytes(), + 4.4f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) + } + + #[test] + fn test_buffer_basic_write_with_elem_strides() -> TestResult { + let elem_size = std::mem::size_of::() as isize; + + let test_data = [1.1, 2.2, 3.3, 4.4]; + let array_view: StrideArrayView<'_, f64, 8> = unsafe { + StrideArrayView::new( + 2, + [2, 2].as_ptr(), + [2, 1].as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * elem_size as usize, + ) + }?; + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() + 2u8 + ] + ); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..64], + &[ + 1.1f64.to_ne_bytes(), + 2.2f64.to_le_bytes(), + 3.3f64.to_le_bytes(), + 4.4f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) + } + + #[test] + fn test_stride_array_size_overflow() -> TestResult { + let result = unsafe { + StrideArrayView::::new( + 2, + [u32::MAX as usize, u32::MAX as usize].as_ptr(), + [8, 8].as_ptr(), + ptr::null(), + 0, + ) + }; + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Array buffer size too big")); + Ok(()) + } + + #[test] + fn test_stride_view_length_mismatch() -> TestResult { + let elem_size = size_of::() as isize; + let under_data = [1.1]; + let result: Result, Error> = unsafe { + StrideArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size, elem_size].as_ptr(), + under_data.as_ptr() as *const u8, + under_data.len() * elem_size as usize, + ) + }; + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 8, expected: 16)")); + + let over_data = [1.1, 2.2, 3.3]; + let result: Result, Error> = unsafe { + StrideArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size, elem_size].as_ptr(), + over_data.as_ptr() as *const u8, + over_data.len() * elem_size as usize, + ) + }; + + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 24, expected: 16)")); + Ok(()) + } + + #[test] + fn test_stride_view_length_mismatch_with_elem_strides() -> TestResult { + let elem_size = size_of::() as isize; + let under_data = [1.1]; + let result: Result, Error> = unsafe { + StrideArrayView::new( + 2, + [1, 2].as_ptr(), + [1, 1].as_ptr(), + under_data.as_ptr() as *const u8, + under_data.len() * elem_size as usize, + ) + }; + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 8, expected: 16)")); + + let over_data = [1.1, 2.2, 3.3]; + let result: Result, Error> = unsafe { + StrideArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size, elem_size].as_ptr(), + over_data.as_ptr() as *const u8, + over_data.len() * elem_size as usize, + ) + }; + + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 24, expected: 16)")); + Ok(()) + } + + #[test] + fn test_strided_non_contiguous() -> TestResult { + let elem_size = size_of::() as isize; + let col_major_data = [1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; + let shape = [3usize, 2]; + let strides = [elem_size, shape[0] as isize * elem_size]; + + let array_view: StrideArrayView<'_, f64, 1> = unsafe { + StrideArrayView::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size as usize, + ) + }?; + + assert_eq!(array_view.ndim(), 2); + assert_eq!(array_view.dim(0), Ok(3)); + assert_eq!(array_view.dim(1), Ok(2)); + assert!(array_view.dim(2).is_err()); + assert!(array_view.as_slice().is_none()); + + let mut buffer = vec![0u8; 48]; + write_array_data(&array_view, &mut buffer, 48)?; + + let expected_data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size as usize, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) + } + + #[test] + fn test_strided_non_contiguous_with_elem_strides() -> TestResult { + let elem_size = size_of::() as isize; + let col_major_data = [1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; + let shape = [3usize, 2]; + let strides = [1, shape[0] as isize]; + + let array_view: StrideArrayView<'_, f64, 8> = unsafe { + StrideArrayView::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size as usize, + ) + }?; + + assert_eq!(array_view.ndim(), 2); + assert_eq!(array_view.dim(0), Ok(3)); + assert_eq!(array_view.dim(1), Ok(2)); + assert!(array_view.dim(2).is_err()); + assert!(array_view.as_slice().is_none()); + + let mut buffer = vec![0u8; 48]; + write_array_data(&array_view, &mut buffer, 48)?; + + let expected_data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size as usize, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) + } + + #[test] + fn test_negative_strides() -> TestResult { + let elem_size = size_of::(); + let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; + let view = unsafe { + StrideArrayView::::new( + 2, + &[3usize, 3] as *const usize, + &[-24isize, 8] as *const isize, + (data.as_ptr() as *const u8).add(48), + data.len() * elem_size, + ) + }?; + let collected: Vec<_> = view.iter().copied().collect(); + assert!(view.as_slice().is_none()); + let expected_data = vec![7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 1.0, 2.0, 3.0]; + assert_eq!(collected, expected_data); + let mut buffer = vec![0u8; 72]; + write_array_data(&view, &mut buffer, 72)?; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) + } + + #[test] + fn test_negative_strides_with_elem_strides() -> TestResult { + let elem_size = size_of::(); + let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; + let view = unsafe { + StrideArrayView::::new( + 2, + &[3usize, 3] as *const usize, + &[-3isize, 1] as *const isize, + (data.as_ptr() as *const u8).add(48), + data.len() * elem_size, + ) + }?; + let collected: Vec<_> = view.iter().copied().collect(); + assert!(view.as_slice().is_none()); + let expected_data = vec![7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 1.0, 2.0, 3.0]; + assert_eq!(collected, expected_data); + let mut buffer = vec![0u8; 72]; + write_array_data(&view, &mut buffer, 72)?; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) + } + + #[test] + fn test_basic_edge_cases() -> TestResult { + // empty array + let elem_size = std::mem::size_of::() as isize; + let empty_view: StrideArrayView<'_, f64, 1> = + unsafe { StrideArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0)? }; + assert_eq!(empty_view.ndim(), 2); + assert_eq!(empty_view.dim(0), Ok(0)); + assert_eq!(empty_view.dim(1), Ok(0)); + + // single element array + let single_data = [42.0]; + let single_view: StrideArrayView<'_, f64, 1> = unsafe { + StrideArrayView::new( + 1, + [1].as_ptr(), + [elem_size].as_ptr(), + single_data.as_ptr() as *const u8, + elem_size as usize, + ) + }?; + let mut buf = vec![0u8; 8]; + write_array_data(&single_view, &mut buf, 8).unwrap(); + assert_eq!(buf, 42.0f64.to_ne_bytes()); + Ok(()) + } + + #[test] + fn test_stride_array_view() -> TestResult { + // contiguous layout + let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let shape = [2usize, 3]; + let strides = [ + (shape[1] * size_of::()) as isize, + size_of::() as isize, + ]; + let array = unsafe { + StrideArrayView::::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * size_of::(), + ) + }?; + + assert_eq!(array.ndim(), 2); + assert_eq!(array.dim(0), Ok(2)); + assert_eq!(array.dim(1), Ok(3)); + assert!(array.dim(2).is_err()); + assert!(array.as_slice().is_some()); + let mut buf = vec![0u8; 48]; + write_array_data(&array, &mut buf, 48).unwrap(); + let expected = to_bytes(&test_data); + assert_eq!(buf, expected); + Ok(()) + } + + #[test] + fn test_stride_array_view_strides_with_elem_strides() -> TestResult { + // contiguous layout + let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let shape = [2usize, 3]; + let strides = [shape[1] as isize, 1]; + let array = unsafe { + StrideArrayView::() as isize }>::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * size_of::(), + ) + }?; + + assert_eq!(array.ndim(), 2); + assert_eq!(array.dim(0), Ok(2)); + assert_eq!(array.dim(1), Ok(3)); + assert!(array.dim(2).is_err()); + assert!(array.as_slice().is_some()); + let mut buf = vec![0u8; 48]; + write_array_data(&array, &mut buf, 48).unwrap(); + let expected = to_bytes(&test_data); + assert_eq!(buf, expected); + Ok(()) + } + + #[test] + fn test_stride_minus_non_contiguous_strides_array() -> TestResult { + let test_data = [ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, + ]; + let shape = [2usize, 2]; + let strides = [-8, -2]; + let array = unsafe { + StrideArrayView::() as isize }>::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + test_data.as_ptr().add(11) as *const u8, + 4 * size_of::(), + ) + }?; + + let test_data1 = vec![12.0f64, 10.0, 4.0, 2.0]; + let mut buf = vec![0u8; 32]; + write_array_data(&array, &mut buf, 32).unwrap(); + let expected = to_bytes(&test_data1); + assert_eq!(buf, expected); + Ok(()) + } +} diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index ffc0c092..ff7ec416 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs" -version = "4.0.4" +version = "5.0.0-rc1" edition = "2021" license = "Apache-2.0" description = "QuestDB Client Library for Rust" @@ -23,7 +23,7 @@ socket2 = "0.5.5" dns-lookup = "2.0.4" base64ct = { version = "1.7", features = ["alloc"] } rustls-pemfile = "2.0.0" -ryu = "1.0" +ryu = { version = "1.0" } itoa = "1.0" aws-lc-rs = { version = "1.13", optional = true } ring = { version = "0.17.14", optional = true } @@ -39,7 +39,7 @@ ureq = { version = "3.0.10, <3.1.0", default-features = false, features = ["rust serde_json = { version = "1", optional = true } questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } -no-panic = { version = "0.1", optional = true } +ndarray = { version = "0.16", optional = true } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["ws2def"] } @@ -55,6 +55,7 @@ mio = { version = "1", features = ["os-poll", "net"] } chrono = "0.4.31" tempfile = "3" webpki-roots = "0.26.8" +rstest = "0.25.0" [features] default = ["tls-webpki-certs", "ilp-over-http", "ring-crypto"] @@ -93,12 +94,13 @@ almost-all-features = [ "ring-crypto", "insecure-skip-verify", "json_tests", - "chrono_timestamp" + "chrono_timestamp", + "ndarray" ] [[example]] name = "basic" -required-features = ["chrono_timestamp"] +required-features = ["chrono_timestamp", "ndarray"] [[example]] name = "auth" @@ -110,4 +112,8 @@ required-features = ["chrono_timestamp"] [[example]] name = "http" -required-features = ["ilp-over-http"] +required-features = ["ilp-over-http", "ndarray"] + +[[example]] +name = "protocol_version" +required-features = ["ilp-over-http", "ndarray"] diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 093fd6de..e0ef8c93 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -4,11 +4,26 @@ Official Rust client for [QuestDB](https://questdb.io/), an open-source SQL database designed to process time-series data, faster. The client library is designed for fast ingestion of data into QuestDB via the -InfluxDB Line Protocol (ILP). +InfluxDB Line Protocol (ILP) over either HTTP (recommended) or TCP. * [QuestDB Database docs](https://questdb.io/docs/) * [Docs on InfluxDB Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) +## Protocol Versions + +The library supports the following ILP protocol versions. + +These protocol versions are supported over both HTTP and TCP. + +If you use HTTP, the library will automatically detect the server's +latest supported protocol version and use it. If you use TCP, you can specify the +`protocol_version=N` parameter when constructing the `Sender` object. + +| Version | Description | Server Comatibility | +| ------- | ------------------------------------------------------- | --------------------- | +| **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | +| **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | + ## Quick Start To start using `questdb-rs`, add it as a dependency of your project: @@ -30,7 +45,7 @@ use questdb::{ fn main() -> Result<()> { let mut sender = Sender::from_conf("http::addr=localhost:9000;")?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? @@ -46,7 +61,7 @@ fn main() -> Result<()> { ## Docs Most of the client documentation is on the -[`ingress`](https://docs.rs/questdb-rs/4.0.4/questdb/ingress/) module page. +[`ingress`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/) module page. ## Crate features @@ -74,6 +89,9 @@ These features are opt-in: certificates store. * `insecure-skip-verify`: Allows skipping server certificate validation in TLS (this compromises security). +* `ndarray`: Enables integration with the `ndarray` crate for working with + n-dimensional arrays. Without this feature, you can still send slices, + or integrate custom array types via the `NdArrayView` trait. ## C, C++ and Python APIs diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 636f55e8..98dc34ef 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + #[cfg(feature = "json_tests")] pub mod json_tests { use indoc::indoc; @@ -50,6 +74,8 @@ pub mod json_tests { #[derive(Debug, Serialize, Deserialize)] struct Expected { line: Option, + #[serde(rename = "binaryBase64")] + binary_base64: Option, #[serde(rename = "anyLines")] any_lines: Option>, @@ -95,8 +121,11 @@ pub mod json_tests { indoc! {r#" // This file is auto-generated by build.rs. - use crate::{Result, ingress::{Buffer}}; + use crate::{Result, ingress::{Buffer, ProtocolVersion}}; use crate::tests::{TestResult}; + use base64ct::Base64; + use base64ct::Encoding; + use rstest::rstest; fn matches_any_line(line: &[u8], expected: &[&str]) -> bool { for &exp in expected { @@ -117,14 +146,14 @@ pub mod json_tests { // for line in serde_json::to_string_pretty(&spec).unwrap().split("\n") { // writeln!(output, "/// {}", line)?; // } - writeln!(output, "#[test]")?; + writeln!(output, "#[rstest]")?; writeln!( output, - "fn test_{:03}_{}() -> TestResult {{", + "fn test_{:03}_{}(\n #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion,\n) -> TestResult {{", index, slugify!(&spec.test_name, separator = "_") )?; - writeln!(output, " let mut buffer = Buffer::new();")?; + writeln!(output, " let mut buffer = Buffer::new(version);")?; let (expected, indent) = match &spec.result { Outcome::Success(line) => (Some(line), ""), @@ -168,7 +197,46 @@ pub mod json_tests { } writeln!(output, "{} .at_now()?;", indent)?; if let Some(expected) = expected { - if let Some(ref line) = expected.line { + if let Some(ref base64) = expected.binary_base64 { + writeln!(output, " if version != ProtocolVersion::V1 {{")?; + writeln!( + output, + " let exp = Base64::decode_vec(\"{}\").unwrap();", + base64 + )?; + writeln!( + output, + " assert_eq!(buffer.as_bytes(), exp.as_slice());" + )?; + writeln!(output, " }} else {{")?; + if let Some(ref line) = expected.line { + let exp_ln = format!("{}\n", line); + writeln!(output, " let exp = {:?};", exp_ln)?; + writeln!( + output, + " assert_eq!(buffer.as_bytes(), exp.as_bytes());" + )?; + } else { + // 处理 V1 版本的 any_lines + let any: Vec = expected + .any_lines + .as_ref() + .unwrap() + .iter() + .map(|line| format!("{}\n", line)) + .collect(); + writeln!(output, " let any = [")?; + for line in any.iter() { + writeln!(output, " {:?},", line)?; + } + writeln!(output, " ];")?; + writeln!( + output, + " assert!(matches_any_line(buffer.as_bytes(), &any));" + )?; + } + writeln!(output, " }}")?; + } else if let Some(ref line) = expected.line { let exp_ln = format!("{}\n", line); writeln!(output, " let exp = {:?};", exp_ln)?; writeln!(output, " assert_eq!(buffer.as_bytes(), exp.as_bytes());")?; diff --git a/questdb-rs/examples/auth.rs b/questdb-rs/examples/auth.rs index 2dd6e66d..10eb20be 100644 --- a/questdb-rs/examples/auth.rs +++ b/questdb-rs/examples/auth.rs @@ -1,6 +1,6 @@ use chrono::{TimeZone, Utc}; use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; @@ -17,7 +17,7 @@ fn main() -> Result<()> { ), host, port ))?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; buffer diff --git a/questdb-rs/examples/auth_tls.rs b/questdb-rs/examples/auth_tls.rs index 19225027..8355ebd5 100644 --- a/questdb-rs/examples/auth_tls.rs +++ b/questdb-rs/examples/auth_tls.rs @@ -1,6 +1,6 @@ use chrono::{TimeZone, Utc}; use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; @@ -17,7 +17,7 @@ fn main() -> Result<()> { ), host, port ))?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; buffer diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 6fa665d3..853003b9 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -1,6 +1,7 @@ use chrono::{TimeZone, Utc}; +use ndarray::arr1; use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; @@ -8,7 +9,7 @@ fn main() -> Result<()> { let host: String = std::env::args().nth(1).unwrap_or("localhost".to_string()); let port: &str = &std::env::args().nth(2).unwrap_or("9009".to_string()); let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};"))?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; buffer @@ -17,6 +18,7 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? + .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(designated_timestamp)?; //// If you want to pass the current system timestamp, replace with: diff --git a/questdb-rs/examples/from_conf.rs b/questdb-rs/examples/from_conf.rs index d328c31c..2cdbaed0 100644 --- a/questdb-rs/examples/from_conf.rs +++ b/questdb-rs/examples/from_conf.rs @@ -1,11 +1,11 @@ use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { let mut sender = Sender::from_conf("tcp::addr=localhost:9009;")?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? diff --git a/questdb-rs/examples/from_env.rs b/questdb-rs/examples/from_env.rs index ca338851..8b2b3d8b 100644 --- a/questdb-rs/examples/from_env.rs +++ b/questdb-rs/examples/from_env.rs @@ -1,12 +1,12 @@ use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { // Read configuration string from the `QDB_CLIENT_CONF` environment variable. let mut sender = Sender::from_env()?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index 74b2f3e9..f9e5954e 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -1,17 +1,19 @@ +use ndarray::arr1; use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { let mut sender = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? + .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; Ok(()) diff --git a/questdb-rs/examples/protocol_version.rs b/questdb-rs/examples/protocol_version.rs new file mode 100644 index 00000000..0d184d90 --- /dev/null +++ b/questdb-rs/examples/protocol_version.rs @@ -0,0 +1,35 @@ +use ndarray::arr1; +use questdb::{ + ingress::{Sender, TimestampNanos}, + Result, +}; + +fn main() -> Result<()> { + let mut sender = Sender::from_conf( + "https::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", + )?; + let mut buffer = sender.new_buffer(); + buffer + .table("trades_ilp_v1")? + .symbol("symbol", "ETH-USD")? + .symbol("side", "sell")? + .column_f64("price", 2615.54)? + .column_f64("amount", 0.00044)? + .at(TimestampNanos::now())?; + sender.flush(&mut buffer)?; + + let mut sender2 = Sender::from_conf( + "https::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", + )?; + let mut buffer2 = sender.new_buffer(); + buffer2 + .table("trades_ilp_v2")? + .symbol("symbol", "ETH-USD")? + .symbol("side", "sell")? + .column_f64("price", 2615.54)? + .column_f64("amount", 0.00044)? + .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? + .at(TimestampNanos::now())?; + sender2.flush(&mut buffer2)?; + Ok(()) +} diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index 45f56650..0c32f48a 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + use std::fmt::{Display, Formatter}; macro_rules! fmt { @@ -48,6 +72,18 @@ pub enum ErrorCode { /// Bad configuration. ConfigError, + + /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::MAX_ARRAY_DIMS`] dimensions are supported. + ArrayHasTooManyDims, + + /// Array view internal error. + ArrayViewError, + + /// Array write to buffer error. + ArrayWriteToBufferError, + + /// Validate protocol version error. + ProtocolVersionError, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 323a4a92..68d5e4a6 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -1,4 +1,30 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + use super::conf::ConfigSetting; +use super::MAX_NAME_LEN_DEFAULT; +use crate::error::fmt; use crate::{error, Error}; use base64ct::Base64; use base64ct::Encoding; @@ -16,8 +42,8 @@ use ureq::unversioned::transport::{ Buffers, Connector, LazyBuffers, NextTimeout, Transport, TransportAdapter, }; +use crate::ingress::ProtocolVersion; use ureq::unversioned::*; -use ureq::Error::*; use ureq::{http, Body}; #[derive(PartialEq, Debug, Clone)] @@ -109,6 +135,24 @@ impl HttpHandlerState { Err(err) => (need_retry(Err(err)), response), } } + + pub(crate) fn get_request( + &self, + url: &str, + request_timeout: Duration, + ) -> (bool, Result, ureq::Error>) { + let request = self + .agent + .get(url) + .config() + .timeout_per_call(Some(request_timeout)) + .build(); + let response = request.call(); + match &response { + Ok(res) => (need_retry(Ok(res.status())), response), + Err(err) => (need_retry(Err(err)), response), + } + } } #[derive(Debug)] @@ -236,7 +280,10 @@ fn need_retry(res: Result) -> bool { 599 // Network Connect Timeout Error ) } - Err(err) => matches!(err, Timeout(_) | ConnectionFailed | TooManyRedirects), + Err(err) => matches!( + err, + ureq::Error::Timeout(_) | ureq::Error::ConnectionFailed | ureq::Error::TooManyRedirects + ), } } @@ -383,3 +430,158 @@ pub(super) fn http_send_with_retries( retry_http_send(state, buf, request_timeout, retry_timeout, last_rep) } + +/// Read the server settings from the `/settings` endpoint. +/// This function returns: +/// - A list of supported protocol versions: Default is V1. +/// - The server's max name length: Default is 127. +/// +/// If the server does not support the `/settings` endpoint (404), it returns +/// default values. +pub(super) fn read_server_settings( + state: &HttpHandlerState, + settings_url: &str, +) -> Result<(Vec, usize), Error> { + let default_protocol_version = ProtocolVersion::V1; + + let response = match http_get_with_retries( + state, + settings_url, + *state.config.request_timeout, + Duration::from_secs(1), + ) { + Ok(res) => { + if res.status().is_client_error() || res.status().is_server_error() { + let status = res.status(); + _ = res.into_body().read_to_vec(); + if status.as_u16() == 404 { + return Ok((vec![default_protocol_version], MAX_NAME_LEN_DEFAULT)); + } + return Err(fmt!( + ProtocolVersionError, + "Could not detect server's line protocol version, settings url: {}, status code: {}.", + settings_url, + status + )); + } else { + res + } + } + Err(err) => { + let e = match err { + ureq::Error::StatusCode(code) => { + if code == 404 { + return Ok((vec![default_protocol_version], MAX_NAME_LEN_DEFAULT)); + } else { + fmt!( + ProtocolVersionError, + "Could not detect server's line protocol version, settings url: {}, err: {}.", + settings_url, + err + ) + } + } + e => { + fmt!( + ProtocolVersionError, + "Could not detect server's line protocol version, settings url: {}, err: {}.", + settings_url, + e + ) + } + }; + return Err(e); + } + }; + + let (_, body) = response.into_parts(); + let body_content = body.into_with_config().read_to_string(); + + if let Ok(msg) = body_content { + let json: serde_json::Value = serde_json::from_str(&msg).map_err(|_| { + error::fmt!( + ProtocolVersionError, + "Malformed server response, settings url: {}, err: response is not valid JSON.", + settings_url, + ) + })?; + + let mut support_versions: Vec = vec![]; + if let Some(serde_json::Value::Array(ref values)) = json + .get("config") + .and_then(|v| v.get("line.proto.support.versions")) + { + for value in values.iter() { + if let Some(v) = value.as_u64() { + match v { + 1 => support_versions.push(ProtocolVersion::V1), + 2 => support_versions.push(ProtocolVersion::V2), + _ => {} + } + } + } + } else { + support_versions.push(default_protocol_version); + } + + let max_name_length = json + .get("config") + .and_then(|v| v.get("cairo.max.file.name.length")) + .and_then(|v| v.as_u64()) + .unwrap_or(MAX_NAME_LEN_DEFAULT as u64) as usize; + Ok((support_versions, max_name_length)) + } else { + Err(error::fmt!( + ProtocolVersionError, + "Malformed server response, settings url: {}, err: failed to read response body as UTF-8", settings_url + )) + } +} + +#[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. +fn retry_http_get( + state: &HttpHandlerState, + url: &str, + request_timeout: Duration, + retry_timeout: Duration, + mut last_rep: Result, ureq::Error>, +) -> Result, ureq::Error> { + let mut rng = rand::rng(); + let retry_end = std::time::Instant::now() + retry_timeout; + let mut retry_interval_ms = 10; + let mut need_retry; + loop { + let jitter_ms = rng.random_range(-5i32..5); + let to_sleep_ms = retry_interval_ms + jitter_ms; + let to_sleep = Duration::from_millis(to_sleep_ms as u64); + if (std::time::Instant::now() + to_sleep) > retry_end { + return last_rep; + } + sleep(to_sleep); + if let Ok(last_rep) = last_rep { + // Actively consume the reader to return the connection to the connection pool. + // see https://github.com/algesten/ureq/issues/94 + _ = last_rep.into_body().read_to_vec(); + } + (need_retry, last_rep) = state.get_request(url, request_timeout); + if !need_retry { + return last_rep; + } + retry_interval_ms = (retry_interval_ms * 2).min(1000); + } +} + +#[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. +fn http_get_with_retries( + state: &HttpHandlerState, + url: &str, + request_timeout: Duration, + retry_timeout: Duration, +) -> Result, ureq::Error> { + let (need_retry, last_rep) = state.get_request(url, request_timeout); + if !need_retry || retry_timeout.is_zero() { + return last_rep; + } + + retry_http_get(state, url, request_timeout, retry_timeout, last_rep) +} diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index aed342af..e5f947f0 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -19,7 +19,7 @@ use questdb::{ TimestampNanos}}; fn main() -> Result<()> { let mut sender = Sender::from_conf("http::addr=localhost:9000;")?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? @@ -93,8 +93,8 @@ error as appropriate and continue using it. # Health Check The QuestDB server has a "ping" endpoint you can access to see if it's alive, -and confirm the version of InfluxDB Line Protocol with which you are -interacting: +and confirm the version of the InfluxDB that it is compatible with at a protocol +level. ```shell curl -I http://localhost:9000/ping @@ -256,7 +256,7 @@ with a high-latency network connection. ### Timestamp Column Name -InfluxDB Line Protocol (ILP) does not give a name to the designated timestamp, +The InfluxDB Line Protocol (ILP) does not give a name to the designated timestamp, so if you let this client auto-create the table, it will have the default `timestamp` name. To use a custom name, say `my_ts`, pre-create the table with the desired timestamp column name: @@ -297,9 +297,11 @@ use questdb::ingress::{ TableName, ColumnName, Buffer, + SenderBuilder, TimestampNanos}; # fn main() -> Result<()> { -let mut buffer = Buffer::new(); +let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; +let mut buffer = sender.new_buffer(); let table_name = TableName::new("trades")?; let price_name = ColumnName::new("price")?; buffer.table(table_name)?.column_f64(price_name, 2615.54)?.at(TimestampNanos::now())?; diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 7a496062..dfecc7b6 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,13 +24,14 @@ #![doc = include_str!("mod.md")] +pub use self::ndarr::{ArrayElement, NdArrayView}; pub use self::timestamp::*; - use crate::error::{self, Error, Result}; use crate::gai; use crate::ingress::conf::ConfigSetting; use base64ct::{Base64, Base64UrlUnpadded, Encoding}; use core::time::Duration; +use ndarr::ArrayElementSealed; use rustls::{ClientConnection, RootCertStore, StreamOwned}; use rustls_pki_types::ServerName; use socket2::{Domain, Protocol as SockProtocol, SockAddr, Socket, Type}; @@ -40,6 +41,7 @@ use std::fmt::{Debug, Display, Formatter, Write}; use std::io::{self, BufRead, BufReader, ErrorKind, Write as IoWrite}; use std::ops::Deref; use std::path::PathBuf; +use std::slice::from_raw_parts_mut; use std::str::FromStr; use std::sync::Arc; @@ -55,6 +57,42 @@ use ring::{ signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}, }; +pub(crate) const MAX_NAME_LEN_DEFAULT: usize = 127; + +/// The maximum allowed dimensions for arrays. +pub const MAX_ARRAY_DIMS: usize = 32; + +// TODO: We should probably agree on a significantly +// _smaller_ limit here, since there's no way +// we've ever tested anything that big. +// My gut feeling is that the maximum array buffer should be +// in the order of 100MB or so. +pub const MAX_ARRAY_BUFFER_SIZE: usize = i32::MAX as usize; +pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 + +/// The version of InfluxDB Line Protocol used to communicate with the server. +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum ProtocolVersion { + /// Version 1 of Line Protocol. + /// Full-text protocol. + /// When used over HTTP, this version is compatible with the InfluxDB database. + V1 = 1, + + /// Version 2 of InfluxDB Line Protocol. + /// Uses binary format serialization for f64, and supports the array data type. + /// This version is specific to QuestDB and is not compatible with InfluxDB. + V2 = 2, +} + +impl std::fmt::Display for ProtocolVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProtocolVersion::V1 => write!(f, "v1"), + ProtocolVersion::V2 => write!(f, "v2"), + } + } +} + #[derive(Debug, Copy, Clone)] enum Op { Table = 1, @@ -468,12 +506,15 @@ impl BufferState { /// /// # Example /// -/// ``` +/// ```no_run /// # use questdb::Result; -/// use questdb::ingress::{Buffer, TimestampMicros, TimestampNanos}; +/// # use questdb::ingress::SenderBuilder; /// /// # fn main() -> Result<()> { -/// let mut buffer = Buffer::new(); +/// # let mut sender = SenderBuilder::from_conf("http::addr=localhost:9000;")?.build()?; +/// # use questdb::Result; +/// use questdb::ingress::{Buffer, TimestampMicros, TimestampNanos}; +/// let mut buffer = sender.new_buffer(); /// /// // first row /// buffer @@ -506,6 +547,7 @@ impl BufferState { /// [`column_i64`](Buffer::column_i64), /// [`column_f64`](Buffer::column_f64), /// [`column_str`](Buffer::column_str), +/// [`column_arr`](Buffer::column_arr), /// [`column_ts`](Buffer::column_ts)). /// * Symbols must appear before columns. /// * A row must be terminated with either [`at`](Buffer::at) or @@ -524,6 +566,7 @@ impl BufferState { /// | [`column_i64`](Buffer::column_i64) | [`INTEGER`](https://questdb.io/docs/reference/api/ilp/columnset-types#integer) | /// | [`column_f64`](Buffer::column_f64) | [`FLOAT`](https://questdb.io/docs/reference/api/ilp/columnset-types#float) | /// | [`column_str`](Buffer::column_str) | [`STRING`](https://questdb.io/docs/reference/api/ilp/columnset-types#string) | +/// | [`column_arr`](Buffer::column_arr) | [`ARRAY`](https://questdb.io/docs/reference/api/ilp/columnset-types#array) | /// | [`column_ts`](Buffer::column_ts) | [`TIMESTAMP`](https://questdb.io/docs/reference/api/ilp/columnset-types#timestamp) | /// /// QuestDB supports both `STRING` and `SYMBOL` column types. @@ -555,33 +598,41 @@ pub struct Buffer { state: BufferState, marker: Option<(usize, BufferState)>, max_name_len: usize, + version: ProtocolVersion, } impl Buffer { - /// Construct a `Buffer` with a `max_name_len` of `127`, which is the same as the - /// QuestDB server default. - pub fn new() -> Self { + /// Creates a new [`Buffer`] with default parameters. + /// + /// - Uses the specified protocol version + /// - Sets maximum name length to **127 characters** (QuestDB server default) + /// + /// This is equivalent to [`Sender::new_buffer`] when using the sender's + /// protocol version. For custom name lengths, use [`with_max_name_len`](Self::with_max_name_len) + /// or [`Sender::new_buffer_with_max_name_len`]. + pub fn new(protocol_version: ProtocolVersion) -> Self { + Self::with_max_name_len(protocol_version, MAX_NAME_LEN_DEFAULT) + } + + /// Creates a new [`Buffer`] with a custom maximum name length. + /// + /// - `max_name_len`: Maximum allowed length for table/column names, must match + /// your QuestDB server's `cairo.max.file.name.length` configuration + /// - `protocol_version`: Protocol version to use + /// + /// This is equivalent to [`Sender::new_buffer_with_max_name_len`] when using + /// the sender's protocol version. For the default name length (127), + /// use [`new`](Self::new) or [`Sender::new_buffer`]. + pub fn with_max_name_len(protocol_version: ProtocolVersion, max_name_len: usize) -> Self { Self { output: Vec::new(), state: BufferState::new(), marker: None, - max_name_len: 127, + max_name_len, + version: protocol_version, } } - /// Construct a `Buffer` with a custom maximum length for table and column names. - /// - /// This should match the `cairo.max.file.name.length` setting of the - /// QuestDB instance you're connecting to. - /// - /// If the server does not configure it, the default is `127` and you can simply - /// call [`new`](Buffer::new). - pub fn with_max_name_len(max_name_len: usize) -> Self { - let mut buf = Self::new(); - buf.max_name_len = max_name_len; - buf - } - /// Pre-allocate to ensure the buffer has enough capacity for at least the /// specified additional byte count. This may be rounded up. /// This does not allocate if such additional capacity is already satisfied. @@ -713,11 +764,12 @@ impl Buffer { /// Begin recording a new row for the given table. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// buffer.table("table_name")?; /// # Ok(()) /// # } @@ -725,13 +777,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TableName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// let table_name = TableName::new("table_name")?; /// buffer.table(table_name)?; /// # Ok(()) @@ -762,11 +815,12 @@ impl Buffer { /// Record a symbol for the given column. /// Make sure you record all symbol columns before any other column type. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.symbol("col_name", "value")?; /// # Ok(()) @@ -775,11 +829,12 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let value: String = "value".to_owned(); /// buffer.symbol("col_name", value)?; @@ -789,13 +844,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.symbol(col_name, "value")?; @@ -842,11 +898,12 @@ impl Buffer { /// Record a boolean value for the given column. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_bool("col_name", true)?; /// # Ok(()) @@ -855,13 +912,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_bool(col_name, true)?; @@ -880,11 +938,12 @@ impl Buffer { /// Record an integer value for the given column. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_i64("col_name", 42)?; /// # Ok(()) @@ -893,13 +952,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_i64(col_name, 42); @@ -921,11 +981,12 @@ impl Buffer { /// Record a floating point value for the given column. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_f64("col_name", 3.14)?; /// # Ok(()) @@ -934,13 +995,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_f64(col_name, 3.14)?; @@ -953,18 +1015,25 @@ impl Buffer { Error: From, { self.write_column_key(name)?; - let mut ser = F64Serializer::new(value); - self.output.extend_from_slice(ser.as_str().as_bytes()); + if !matches!(self.version, ProtocolVersion::V1) { + self.output.push(b'='); + self.output.push(DOUBLE_BINARY_FORMAT_TYPE); + self.output.extend_from_slice(&value.to_le_bytes()) + } else { + let mut ser = F64Serializer::new(value); + self.output.extend_from_slice(ser.as_str().as_bytes()) + } Ok(self) } /// Record a string value for the given column. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_str("col_name", "value")?; /// # Ok(()) @@ -973,11 +1042,12 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let value: String = "value".to_owned(); /// buffer.column_str("col_name", value)?; @@ -987,13 +1057,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_str(col_name, "value")?; @@ -1011,14 +1082,122 @@ impl Buffer { Ok(self) } - /// Record a timestamp value for the given column. + /// Record a multidimensional array value for the given column. + /// + /// Supports arrays with up to [`MAX_ARRAY_DIMS`] dimensions. The array elements must + /// be of type `f64`, which is currently the only supported data type. + /// + /// # Examples /// + /// Recording a 2D array using slices: + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let array_2d = vec![vec![1.1, 2.2], vec![3.3, 4.4]]; + /// buffer.column_arr("array_col", &array_2d)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// Recording a 3D array using vectors: + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, ColumnName, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x1")?; + /// let array_3d = vec![vec![vec![42.0; 4]; 3]; 2]; + /// let col_name = ColumnName::new("col1")?; + /// buffer.column_arr(col_name, &array_3d)?; + /// # Ok(()) + /// # } /// ``` + /// + /// # Errors + /// + /// Returns [`Error`] if: + /// - Array dimensions exceed [`MAX_ARRAY_DIMS`] + /// - Failed to get dimension sizes + /// - Column name validation fails + /// - Protocol version v1 is used (arrays require v2+) + #[allow(private_bounds)] + pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> + where + N: TryInto>, + T: NdArrayView, + D: ArrayElement + ArrayElementSealed, + Error: From, + { + if self.version == ProtocolVersion::V1 { + return Err(error::fmt!( + ProtocolVersionError, + "Protocol version v1 does not support array datatype", + )); + } + let ndim = view.ndim(); + if ndim == 0 { + return Err(error::fmt!( + ArrayViewError, + "Zero-dimensional arrays are not supported", + )); + } + + // check dimension less equal than max dims + if MAX_ARRAY_DIMS < ndim { + return Err(error::fmt!( + ArrayHasTooManyDims, + "Array dimension mismatch: expected at most {} dimensions, but got {}", + MAX_ARRAY_DIMS, + ndim + )); + } + + let array_buf_size = check_and_get_array_bytes_size(view)?; + self.write_column_key(name)?; + // binary format flag '=' + self.output.push(b'='); + // binary format entity type + self.output.push(ARRAY_BINARY_FORMAT_TYPE); + // ndarr datatype + self.output.push(D::type_tag()); + // ndarr dims + self.output.push(ndim as u8); + + let dim_header_size = size_of::() * ndim; + self.output.reserve(dim_header_size + array_buf_size); + + for i in 0..ndim { + // ndarr shape + self.output + .extend_from_slice((view.dim(i)? as u32).to_le_bytes().as_slice()); + } + + let index = self.output.len(); + let writeable = + unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), array_buf_size) }; + + // ndarr data + ndarr::write_array_data(view, writeable, array_buf_size)?; + unsafe { self.output.set_len(array_buf_size + index) } + Ok(self) + } + + /// Record a timestamp value for the given column. + /// + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampMicros; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_ts("col_name", TimestampMicros::now())?; /// # Ok(()) @@ -1027,13 +1206,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampMicros; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_ts("col_name", TimestampMicros::new(1659548204354448))?; /// # Ok(()) @@ -1042,14 +1222,15 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampMicros; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_ts(col_name, TimestampMicros::now())?; @@ -1060,7 +1241,7 @@ impl Buffer { /// or you can also pass in a `TimestampNanos`. /// /// Note that both `TimestampMicros` and `TimestampNanos` can be constructed - /// easily from either `chrono::DateTime` and `std::time::SystemTime`. + /// easily from either `std::time::SystemTime` or `chrono::DateTime`. /// /// This last option requires the `chrono_timestamp` feature. pub fn column_ts<'a, N, T>(&mut self, name: N, value: T) -> Result<&mut Self> @@ -1084,12 +1265,13 @@ impl Buffer { /// start recording the next row by calling [Buffer::table] again, or you can send /// the accumulated batch by calling [Sender::flush] or one of its variants. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampNanos; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?.symbol("a", "b")?; /// buffer.at(TimestampNanos::now())?; /// # Ok(()) @@ -1098,13 +1280,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampNanos; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?.symbol("a", "b")?; /// buffer.at(TimestampNanos::new(1659548315647406592))?; /// # Ok(()) @@ -1114,7 +1297,7 @@ impl Buffer { /// You can also pass in a `TimestampMicros`. /// /// Note that both `TimestampMicros` and `TimestampNanos` can be constructed - /// easily from either `chrono::DateTime` and `std::time::SystemTime`. + /// easily from either `std::time::SystemTime` or `chrono::DateTime`. /// pub fn at(&mut self, timestamp: T) -> Result<()> where @@ -1164,11 +1347,12 @@ impl Buffer { /// again, or you can send the accumulated batch by calling [Sender::flush] or one of /// its variants. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?.symbol("a", "b")?; /// buffer.at_now()?; /// # Ok(()) @@ -1183,12 +1367,6 @@ impl Buffer { } } -impl Default for Buffer { - fn default() -> Self { - Self::new() - } -} - /// Connects to a QuestDB instance and inserts data via the ILP protocol. /// /// * To construct an instance, use [`Sender::from_conf`] or the [`SenderBuilder`]. @@ -1199,6 +1377,8 @@ pub struct Sender { handler: ProtocolHandler, connected: bool, max_buf_size: usize, + protocol_version: ProtocolVersion, + max_name_len: usize, } impl std::fmt::Debug for Sender { @@ -1512,7 +1692,8 @@ pub enum Protocol { Tcps, #[cfg(feature = "ilp-over-http")] - /// ILP over HTTP (request-response, InfluxDB-compatible). + /// ILP over HTTP (request-response) + /// Version 1 is compatible with the InfluxDB Line Protocol. Http, #[cfg(feature = "ilp-over-http")] @@ -1648,12 +1829,14 @@ pub struct SenderBuilder { port: ConfigSetting, net_interface: ConfigSetting>, max_buf_size: ConfigSetting, + max_name_len: ConfigSetting, auth_timeout: ConfigSetting, username: ConfigSetting>, password: ConfigSetting>, token: ConfigSetting>, token_x: ConfigSetting>, token_y: ConfigSetting>, + protocol_version: ConfigSetting>, #[cfg(feature = "insecure-skip-verify")] tls_verify: ConfigSetting, @@ -1723,6 +1906,20 @@ impl SenderBuilder { "token_x" => builder.token_x(val)?, "token_y" => builder.token_y(val)?, "bind_interface" => builder.bind_interface(val)?, + "protocol_version" => match val { + "1" => builder.protocol_version(ProtocolVersion::V1)?, + "2" => builder.protocol_version(ProtocolVersion::V2)?, + "auto" => builder, + invalid => { + return Err(error::fmt!( + ConfigError, + "invalid \"protocol_version\" [value={invalid}, allowed-values=[auto, 1, 2]]]\"]" + )) + } + }, + "max_name_len" => { + builder.max_name_len(parse_conf_value(key, val)?)? + } "init_buf_size" => { return Err(error::fmt!( @@ -1822,6 +2019,7 @@ impl SenderBuilder { "retry_timeout" => { builder.retry_timeout(Duration::from_millis(parse_conf_value(key, val)?))? } + // Ignore other parameters. // We don't want to fail on unknown keys as this would require releasing different // library implementations in lock step as soon as a new parameter is added to any of them, @@ -1877,12 +2075,14 @@ impl SenderBuilder { port: ConfigSetting::new_specified(port), net_interface: ConfigSetting::new_default(None), max_buf_size: ConfigSetting::new_default(100 * 1024 * 1024), + max_name_len: ConfigSetting::new_default(MAX_NAME_LEN_DEFAULT), auth_timeout: ConfigSetting::new_default(Duration::from_secs(15)), username: ConfigSetting::new_default(None), password: ConfigSetting::new_default(None), token: ConfigSetting::new_default(None), token_x: ConfigSetting::new_default(None), token_y: ConfigSetting::new_default(None), + protocol_version: ConfigSetting::new_default(None), #[cfg(feature = "insecure-skip-verify")] tls_verify: ConfigSetting::new_default(true), @@ -1955,6 +2155,13 @@ impl SenderBuilder { Ok(self) } + /// Set the line protocol version. + pub fn protocol_version(mut self, protocol_version: ProtocolVersion) -> Result { + self.protocol_version + .set_specified("protocol_version", Some(protocol_version))?; + Ok(self) + } + /// Configure how long to wait for messages from the QuestDB server during /// the TLS handshake and authentication process. This only applies to TCP. /// The default is 15 seconds. @@ -2030,6 +2237,22 @@ impl SenderBuilder { Ok(self) } + /// The maximum length of a table or column name in bytes. + /// Matches the `cairo.max.file.name.length` setting in the server. + /// The default is 127 bytes. + /// If running over HTTP and protocol version 2 is auto-negotiated, this + /// value is picked up from the server. + pub fn max_name_len(mut self, value: usize) -> Result { + if value < 16 { + return Err(error::fmt!( + ConfigError, + "max_name_len must be at least 16 bytes." + )); + } + self.max_name_len.set_specified("max_name_len", value)?; + Ok(self) + } + #[cfg(feature = "ilp-over-http")] /// Set the cumulative duration spent in retries. /// The value is in milliseconds, and the default is 10 seconds. @@ -2122,7 +2345,7 @@ impl SenderBuilder { .map_err(|io_err| map_io_to_socket_err("Could not set SO_KEEPALIVE: ", io_err))?; sock.set_nodelay(true) .map_err(|io_err| map_io_to_socket_err("Could not set TCP_NODELAY: ", io_err))?; - if let Some(ref host) = self.net_interface.deref() { + if let Some(host) = self.net_interface.deref() { let bind_addr = gai::resolve_host(host.as_str())?; sock.bind(&bind_addr).map_err(|io_err| { map_io_to_socket_err( @@ -2374,12 +2597,46 @@ impl SenderBuilder { agent, url, auth, - config: self.http.as_ref().unwrap().clone(), }) } }; + let mut max_name_len = *self.max_name_len; + + let protocol_version = match self.protocol_version.deref() { + Some(v) => *v, + None => match self.protocol { + Protocol::Tcp | Protocol::Tcps => ProtocolVersion::V1, + #[cfg(feature = "ilp-over-http")] + Protocol::Http | Protocol::Https => { + if let ProtocolHandler::Http(http_state) = &handler { + let settings_url = &format!( + "{}://{}:{}/settings", + self.protocol.schema(), + self.host.deref(), + self.port.deref() + ); + let (protocol_versions, server_max_name_len) = + read_server_settings(http_state, settings_url)?; + max_name_len = server_max_name_len; + if protocol_versions.contains(&ProtocolVersion::V2) { + ProtocolVersion::V2 + } else if protocol_versions.contains(&ProtocolVersion::V1) { + ProtocolVersion::V1 + } else { + return Err(error::fmt!( + ProtocolVersionError, + "Server does not support current client" + )); + } + } else { + unreachable!("HTTP handler should be used for HTTP protocol"); + } + } + }, + }; + if auth.is_some() { descr.push_str("auth=on]"); } else { @@ -2391,6 +2648,8 @@ impl SenderBuilder { handler, connected: true, max_buf_size: *self.max_buf_size, + protocol_version, + max_name_len, }; Ok(sender) @@ -2524,7 +2783,6 @@ impl F64Serializer { // This function was taken and customized from the ryu crate. #[cold] - #[cfg_attr(feature = "no-panic", inline)] fn format_nonfinite(&self) -> &'static str { const MANTISSA_MASK: u64 = 0x000fffffffffffff; const SIGN_MASK: u64 = 0x8000000000000000; @@ -2586,6 +2844,15 @@ impl Sender { SenderBuilder::from_env()?.build() } + /// Creates a new [`Buffer`] with default parameters. + /// + /// This initializes a buffer using the sender's protocol version and + /// the QuestDB server's default maximum name length of 127 characters. + /// For custom name lengths, use [`new_buffer_with_max_name_len`](Self::new_buffer_with_max_name_len) + pub fn new_buffer(&self) -> Buffer { + Buffer::with_max_name_len(self.protocol_version, self.max_name_len) + } + #[allow(unused_variables)] fn flush_impl(&mut self, buf: &Buffer, transactional: bool) -> Result<()> { if !self.connected { @@ -2605,6 +2872,8 @@ impl Sender { )); } + self.check_protocol_version(buf.version)?; + let bytes = buf.as_bytes(); if bytes.is_empty() { return Ok(()); @@ -2725,14 +2994,51 @@ impl Sender { pub fn must_close(&self) -> bool { !self.connected } + + /// Return the sender's protocol version. + /// This is either the protocol version that was set explicitly, + /// or the one that was auto-detected during the connection process. + /// If connecting via TCP and not overridden, the value is V1. + pub fn protocol_version(&self) -> ProtocolVersion { + self.protocol_version + } + + /// Return the sender's maxinum name length of any column or table name. + /// This is either set explicitly when constructing the sender, + /// or the default value of 127. + /// When unset and using protocol version 2 over HTTP, the value is read + /// from the server from the `cairo.max.file.name.length` setting in + /// `server.conf` which defaults to 127. + pub fn max_name_len(&self) -> usize { + self.max_name_len + } + + #[inline(always)] + fn check_protocol_version(&self, version: ProtocolVersion) -> Result<()> { + if self.protocol_version != version { + return Err(error::fmt!( + ProtocolVersionError, + "Attempting to send with protocol version {} \ + but the sender is configured to use protocol version {}", + version, + self.protocol_version + )); + } + Ok(()) + } } +pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; +pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; + mod conf; +pub(crate) mod ndarr; mod timestamp; #[cfg(feature = "ilp-over-http")] mod http; +use crate::ingress::ndarr::check_and_get_array_bytes_size; #[cfg(feature = "ilp-over-http")] use http::*; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs new file mode 100644 index 00000000..c793094b --- /dev/null +++ b/questdb-rs/src/ingress/ndarr.rs @@ -0,0 +1,549 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +pub trait NdArrayView +where + T: ArrayElement, +{ + type Iter<'a>: Iterator + where + Self: 'a, + T: 'a; + + /// Returns the number of dimensions (rank) of the array. + fn ndim(&self) -> usize; + + /// Returns the size of the specified dimension. + fn dim(&self, index: usize) -> Result; + + /// Return the array’s data as a slice, if it is c-major-layout. + /// Return `None` otherwise. + fn as_slice(&self) -> Option<&[T]>; + + /// Return an iterator of references to the elements of the array. + /// Iterator element type is `&T`. + fn iter(&self) -> Self::Iter<'_>; +} + +pub(crate) fn write_array_data, T>( + array: &A, + buf: &mut [u8], + expect_size: usize, +) -> Result<(), Error> +where + T: ArrayElement, +{ + // When working with contiguous layout, benchmark shows `copy_from_slice` has better performance than + // `std::ptr::copy_nonoverlapping` on both Arm(Macos) and x86(Linux) platform. + // This may because `copy_from_slice` benefits more from compiler. + if let Some(contiguous) = array.as_slice() { + let bytes = unsafe { + slice::from_raw_parts(contiguous.as_ptr() as *const u8, size_of_val(contiguous)) + }; + + if bytes.len() != expect_size { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + expect_size, + bytes.len() + )); + } + + if buf.len() < bytes.len() { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Buffer capacity {} < required {}", + buf.len(), + bytes.len() + )); + } + + buf[..bytes.len()].copy_from_slice(bytes); + return Ok(()); + } + + // For non-contiguous memory layouts, direct raw pointer operations are preferred. + let elem_size = size_of::(); + let mut total_len = 0; + for (i, &element) in array.iter().enumerate() { + unsafe { + std::ptr::copy_nonoverlapping( + &element as *const T as *const u8, + buf.as_mut_ptr().add(i * elem_size), + elem_size, + ) + } + total_len += elem_size; + } + if total_len != expect_size { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + total_len, + expect_size + )); + } + Ok(()) +} + +pub(crate) fn check_and_get_array_bytes_size, T>( + array: &A, +) -> Result +where + T: ArrayElement, +{ + let mut size = std::mem::size_of::(); + for dim_index in 0..array.ndim() { + let dim = array.dim(dim_index)?; + if dim > MAX_ARRAY_DIM_LEN { + return Err(error::fmt!( + ArrayViewError, + "dimension length out of range: dim {}, dim length {}, max length {}", + dim_index, + dim, + MAX_ARRAY_DIM_LEN + )); + } + // following dimension's length may be zero, so check the size in out of loop + size *= dim; + } + + if size > MAX_ARRAY_BUFFER_SIZE { + return Err(error::fmt!( + ArrayViewError, + "Array buffer size too big: {}, maximum: {}", + size, + MAX_ARRAY_BUFFER_SIZE + )); + } + Ok(size) +} + +/// Marker trait for valid array element types. +/// +/// Implemented for primitive types that can be stored in arrays. +/// Combines type information with data type classification. +pub trait ArrayElement: Copy + 'static {} + +pub(crate) trait ArrayElementSealed { + /// Returns the binary format identifier for array element types compatible + /// with QuestDB's io.questdb.cairo.ColumnType numeric type constants. + fn type_tag() -> u8; +} + +impl ArrayElement for f64 {} + +impl ArrayElementSealed for f64 { + fn type_tag() -> u8 { + 10 // Double + } +} + +/// impl NdArrayView for one dimension vector +impl NdArrayView for Vec { + type Iter<'a> + = std::slice::Iter<'a, T> + where + T: 'a; + + fn ndim(&self) -> usize { + 1 + } + + fn dim(&self, idx: usize) -> Result { + if idx == 0 { + Ok(self.len()) + } else { + Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 1 + )) + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(self.as_slice()) + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter() + } +} + +/// impl NdArrayView for one dimension array +impl NdArrayView for [T; N] { + type Iter<'a> + = std::slice::Iter<'a, T> + where + T: 'a; + + fn ndim(&self) -> usize { + 1 + } + + fn dim(&self, idx: usize) -> Result { + if idx == 0 { + Ok(N) + } else { + Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 1 + )) + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(self) + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter() + } +} + +/// impl NdArrayView for one dimension slice +impl NdArrayView for &[T] { + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a, + T: 'a; + + fn ndim(&self) -> usize { + 1 + } + + fn dim(&self, idx: usize) -> Result { + if idx == 0 { + Ok(self.len()) + } else { + Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 1 + )) + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(self) + } + + fn iter(&self) -> Self::Iter<'_> { + <[T]>::iter(self) + } +} + +/// impl NdArrayView for two dimensions vector +impl NdArrayView for Vec> { + type Iter<'a> + = std::iter::Flatten>> + where + T: 'a; + + fn ndim(&self) -> usize { + 2 + } + + fn dim(&self, idx: usize) -> Result { + match idx { + 0 => Ok(self.len()), + 1 => { + let dim1 = self.first().map_or(0, |v| v.len()); + if self.as_slice().iter().any(|v2| v2.len() != dim1) { + return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + } + Ok(dim1) + } + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 2 + )), + } + } + + fn as_slice(&self) -> Option<&[T]> { + None + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter().flatten() + } +} + +/// impl NdArrayView for two dimensions array +impl NdArrayView for [[T; M]; N] { + type Iter<'a> + = std::iter::Flatten> + where + T: 'a; + + fn ndim(&self) -> usize { + 2 + } + + fn dim(&self, idx: usize) -> Result { + match idx { + 0 => Ok(N), + 1 => Ok(M), + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 2 + )), + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T, N * M) }) + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter().flatten() + } +} + +/// impl NdArrayView for two dimensions slices +impl NdArrayView for &[[T; M]] { + type Iter<'a> + = std::iter::Flatten> + where + Self: 'a, + T: 'a; + + fn ndim(&self) -> usize { + 2 + } + + fn dim(&self, idx: usize) -> Result { + match idx { + 0 => Ok(self.len()), + 1 => Ok(M), + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 2 + )), + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T, self.len() * M) }) + } + + fn iter(&self) -> Self::Iter<'_> { + <[[T; M]]>::iter(self).flatten() + } +} + +/// impl NdArrayView for three dimensions vector +impl NdArrayView for Vec>> { + type Iter<'a> + = std::iter::Flatten>>>> + where + T: 'a; + + fn ndim(&self) -> usize { + 3 + } + + fn dim(&self, idx: usize) -> Result { + match idx { + 0 => Ok(self.len()), + 1 => { + let dim1 = self.first().map_or(0, |v| v.len()); + if self.as_slice().iter().any(|v2| v2.len() != dim1) { + return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + } + Ok(dim1) + } + 2 => { + let dim2 = self + .first() + .and_then(|v2| v2.first()) + .map_or(0, |v3| v3.len()); + + if self + .as_slice() + .iter() + .flat_map(|v2| v2.as_slice().iter()) + .any(|v3| v3.len() != dim2) + { + return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + } + Ok(dim2) + } + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 3 + )), + } + } + + fn as_slice(&self) -> Option<&[T]> { + None + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter().flatten().flatten() + } +} + +/// impl NdArrayView for three dimensions array +impl NdArrayView + for [[[T; M]; N]; L] +{ + type Iter<'a> + = std::iter::Flatten>> + where + T: 'a; + + fn ndim(&self) -> usize { + 3 + } + + fn dim(&self, idx: usize) -> Result { + match idx { + 0 => Ok(L), + 1 => Ok(N), + 2 => Ok(M), + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 3 + )), + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T, L * N * M) }) + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter().flatten().flatten() + } +} + +impl NdArrayView for &[[[T; M]; N]] { + type Iter<'a> + = std::iter::Flatten>> + where + Self: 'a, + T: 'a; + + fn ndim(&self) -> usize { + 3 + } + + fn dim(&self, idx: usize) -> Result { + match idx { + 0 => Ok(self.len()), + 1 => Ok(N), + 2 => Ok(M), + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 3 + )), + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T, self.len() * N * M) }) + } + + fn iter(&self) -> Self::Iter<'_> { + <[[[T; M]; N]]>::iter(self).flatten().flatten() + } +} + +use crate::{error, Error}; +#[cfg(feature = "ndarray")] +use ndarray::{ArrayView, Axis, Dimension}; +use std::slice; + +use super::{MAX_ARRAY_BUFFER_SIZE, MAX_ARRAY_DIM_LEN}; + +#[cfg(feature = "ndarray")] +impl NdArrayView for ArrayView<'_, T, D> +where + T: ArrayElement, + D: Dimension, +{ + type Iter<'a> + = ndarray::iter::Iter<'a, T, D> + where + Self: 'a, + T: 'a; + + fn ndim(&self) -> usize { + self.ndim() + } + + fn dim(&self, index: usize) -> Result { + let len = self.ndim(); + if index < len { + Ok(self.len_of(Axis(index))) + } else { + Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + index, + 3 + )) + } + } + + fn iter(&self) -> Self::Iter<'_> { + self.iter() + } + + fn as_slice(&self) -> Option<&[T]> { + self.as_slice() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_f64_element_type() { + assert_eq!(::type_tag(), 10); + } +} diff --git a/questdb-rs/src/ingress/tests.rs b/questdb-rs/src/ingress/tests.rs index 8d83a7b0..69a8bd6d 100644 --- a/questdb-rs/src/ingress/tests.rs +++ b/questdb-rs/src/ingress/tests.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + use super::*; use crate::ErrorCode; use tempfile::TempDir; @@ -5,9 +29,9 @@ use tempfile::TempDir; #[cfg(feature = "ilp-over-http")] #[test] fn http_simple() { - let builder = SenderBuilder::from_conf("http::addr=localhost;").unwrap(); + let builder = SenderBuilder::from_conf("http::addr=127.0.0.1;").unwrap(); assert_eq!(builder.protocol, Protocol::Http); - assert_specified_eq(&builder.host, "localhost"); + assert_specified_eq(&builder.host, "127.0.0.1"); assert_specified_eq(&builder.port, Protocol::Http.default_port()); assert!(!builder.protocol.tls_enabled()); } @@ -30,10 +54,10 @@ fn https_simple() { #[test] fn tcp_simple() { - let builder = SenderBuilder::from_conf("tcp::addr=localhost;").unwrap(); + let builder = SenderBuilder::from_conf("tcp::addr=127.0.0.1;").unwrap(); assert_eq!(builder.protocol, Protocol::Tcp); assert_specified_eq(&builder.port, Protocol::Tcp.default_port()); - assert_specified_eq(&builder.host, "localhost"); + assert_specified_eq(&builder.host, "127.0.0.1"); assert!(!builder.protocol.tls_enabled()); } @@ -447,12 +471,14 @@ fn connect_timeout_uses_request_timeout() { let builder = SenderBuilder::new(Protocol::Http, "127.0.0.2", "1111") .request_timeout(request_timeout) .unwrap() + .protocol_version(ProtocolVersion::V2) + .unwrap() .retry_timeout(Duration::from_millis(10)) .unwrap() .request_min_throughput(0) .unwrap(); let mut sender = builder.build().unwrap(); - let mut buf = Buffer::new(); + let mut buf = sender.new_buffer(); buf.table("x") .unwrap() .symbol("x", "x") diff --git a/questdb-rs/src/ingress/timestamp.rs b/questdb-rs/src/ingress/timestamp.rs index 93361484..4176a896 100644 --- a/questdb-rs/src/ingress/timestamp.rs +++ b/questdb-rs/src/ingress/timestamp.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + use crate::error; use std::time::{Duration, SystemTime, UNIX_EPOCH}; diff --git a/questdb-rs/src/lib.rs b/questdb-rs/src/lib.rs index dc18bd8c..d46a5333 100644 --- a/questdb-rs/src/lib.rs +++ b/questdb-rs/src/lib.rs @@ -21,7 +21,6 @@ * limitations under the License. * ******************************************************************************/ - #![doc = include_str!("../README.md")] mod error; diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 90ec1222..172267e6 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -22,18 +22,22 @@ * ******************************************************************************/ -use crate::ingress::{Buffer, Protocol, SenderBuilder, TimestampNanos}; +use crate::ingress::{Buffer, Protocol, ProtocolVersion, SenderBuilder, TimestampNanos}; use crate::tests::mock::{certs_dir, HttpResponse, MockServer}; +use crate::tests::{assert_err_contains, TestResult}; use crate::ErrorCode; +use rstest::rstest; use std::io; use std::io::ErrorKind; use std::time::Duration; -use crate::tests::TestResult; - -#[test] -fn test_two_lines() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_two_lines( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? @@ -46,12 +50,8 @@ fn test_two_lines() -> TestResult { .at_now()?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; - - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -63,12 +63,12 @@ fn test_two_lines() -> TestResult { server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush(&mut buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; res?; @@ -77,23 +77,22 @@ fn test_two_lines() -> TestResult { Ok(()) } -#[test] -fn test_text_plain_error() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_text_plain_error( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; - let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -106,26 +105,28 @@ fn test_text_plain_error() -> TestResult { .with_body_str("bad wombat"), )?; - Ok(()) + Ok(server) }); - let res = sender.flush(&mut buffer); - - server_thread.join().unwrap()?; - - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ServerFlushError); - assert_eq!(err.msg(), "Could not flush buffer: bad wombat"); + assert_err_contains( + sender.flush(&mut buffer), + ErrorCode::ServerFlushError, + "Could not flush buffer: bad wombat", + ); assert!(!buffer.is_empty()); + _ = server_thread.join().unwrap()?; Ok(()) } -#[test] -fn test_bad_json_error() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_bad_json_error( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? @@ -133,13 +134,9 @@ fn test_bad_json_error() -> TestResult { .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; - let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -153,12 +150,12 @@ fn test_bad_json_error() -> TestResult { })), )?; - Ok(()) + Ok(server) }); let res = sender.flush_and_keep(&buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; assert!(res.is_err()); let err = res.unwrap_err(); @@ -171,9 +168,13 @@ fn test_bad_json_error() -> TestResult { Ok(()) } -#[test] -fn test_json_error() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_json_error( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? @@ -181,13 +182,9 @@ fn test_json_error() -> TestResult { .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; - let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -204,34 +201,32 @@ fn test_json_error() -> TestResult { })), )?; - Ok(()) + Ok(server) }); - let res = sender.flush_and_keep(&buffer); - - server_thread.join().unwrap()?; - - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ServerFlushError); - assert_eq!( - err.msg(), - "Could not flush buffer: failed to parse line protocol: invalid field format [id: ABC-2, code: invalid, line: 2]" + assert_err_contains( + sender.flush_and_keep(&buffer), + ErrorCode::ServerFlushError, + "Could not flush buffer: failed to parse line protocol: invalid field format [id: ABC-2, code: invalid, line: 2]", ); + _ = server_thread.join().unwrap()?; Ok(()) } -#[test] -fn test_no_connection() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_no_connection( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut sender = SenderBuilder::new(Protocol::Http, "127.0.0.1", 1) + .protocol_version(version)? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - - let mut sender = SenderBuilder::new(Protocol::Http, "127.0.0.1", 1).build()?; let res = sender.flush_and_keep(&buffer); assert!(res.is_err()); let err = res.unwrap_err(); @@ -242,22 +237,22 @@ fn test_no_connection() -> TestResult { Ok(()) } -#[test] -fn test_old_server_without_ilp_http_support() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_old_server_without_ilp_http_support( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; - let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -270,46 +265,41 @@ fn test_old_server_without_ilp_http_support() -> TestResult { .with_body_str("Not Found"), )?; - Ok(()) + Ok(server) }); - let res = sender.flush_and_keep(&buffer); - - server_thread.join().unwrap()?; - - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::HttpNotSupported); - assert_eq!( - err.msg(), - "Could not flush buffer: HTTP endpoint does not support ILP." + assert_err_contains( + sender.flush_and_keep(&buffer), + ErrorCode::HttpNotSupported, + "Could not flush buffer: HTTP endpoint does not support ILP.", ); + _ = server_thread.join().unwrap()?; Ok(()) } -#[test] -fn test_http_basic_auth() -> TestResult { - let mut buffer = Buffer::new(); - buffer - .table("test")? - .symbol("sym", "bol")? - .column_f64("x", 1.0)? - .at_now()?; - +#[rstest] +fn test_http_basic_auth( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { let mut server = MockServer::new()?; let mut sender = server .lsb_http() + .protocol_version(version)? .username("Aladdin")? .password("OpenSesame")? .build()?; + let mut buffer = sender.new_buffer(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); assert_eq!( @@ -320,12 +310,12 @@ fn test_http_basic_auth() -> TestResult { server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush(&mut buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; res?; @@ -334,22 +324,22 @@ fn test_http_basic_auth() -> TestResult { Ok(()) } -#[test] -fn test_unauthenticated() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_unauthenticated( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; - let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -362,42 +352,40 @@ fn test_unauthenticated() -> TestResult { .with_header("WWW-Authenticate", "Basic realm=\"Our Site\""), )?; - Ok(()) + Ok(server) }); - let res = sender.flush(&mut buffer); - - server_thread.join().unwrap()?; - - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::AuthError); - assert_eq!( - err.msg(), - "Could not flush buffer: HTTP endpoint authentication error: Unauthorized [code: 401]" + assert_err_contains( + sender.flush(&mut buffer), + ErrorCode::AuthError, + "Could not flush buffer: HTTP endpoint authentication error: Unauthorized [code: 401]", ); - assert!(!buffer.is_empty()); + _ = server_thread.join().unwrap()?; Ok(()) } -#[test] -fn test_token_auth() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_token_auth( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server + .lsb_http() + .protocol_version(version)? + .token("0123456789")? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().token("0123456789")?.build()?; - let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -406,21 +394,30 @@ fn test_token_auth() -> TestResult { server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush(&mut buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; res?; Ok(()) } -#[test] -fn test_request_timeout() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_request_timeout( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let server = MockServer::new()?; + let request_timeout = Duration::from_millis(50); + let mut sender = server + .lsb_http() + .protocol_version(version)? + .request_timeout(request_timeout)? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? @@ -428,41 +425,35 @@ fn test_request_timeout() -> TestResult { .at_now()?; // Here we use a mock (tcp) server instead and don't send a response back. - let server = MockServer::new()?; - - let request_timeout = Duration::from_millis(50); let time_start = std::time::Instant::now(); - let mut sender = server - .lsb_http() - .request_timeout(request_timeout)? - .build()?; let res = sender.flush_and_keep(&buffer); let time_elapsed = time_start.elapsed(); - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::SocketError); - assert!(err.msg().contains("per call")); + assert_err_contains(res, ErrorCode::SocketError, "per call"); assert!(time_elapsed >= request_timeout); Ok(()) } -#[test] -fn test_tls() -> TestResult { +#[rstest] +fn test_tls( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { let mut ca_path = certs_dir(); ca_path.push("server_rootCA.pem"); + let mut server = MockServer::new()?; + let mut sender = server + .lsb_https() + .tls_roots(ca_path)? + .protocol_version(version)? + .build()?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - - let mut server = MockServer::new()?; - let mut sender = server.lsb_https().tls_roots(ca_path)?.build()?; - - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept_tls_sync()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -471,12 +462,12 @@ fn test_tls() -> TestResult { server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush_and_keep(&buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; // Unpacking the error here allows server errors to bubble first. res?; @@ -484,34 +475,37 @@ fn test_tls() -> TestResult { Ok(()) } -#[test] -fn test_user_agent() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_user_agent( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server + .lsb_http() + .user_agent("wallabies/1.2.99")? + .protocol_version(version)? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().user_agent("wallabies/1.2.99")?.build()?; - - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.header("user-agent"), Some("wallabies/1.2.99")); assert_eq!(req.body(), buffer2.as_bytes()); server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush_and_keep(&buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; // Unpacking the error here allows server errors to bubble first. res?; @@ -519,27 +513,26 @@ fn test_user_agent() -> TestResult { Ok(()) } -#[test] -fn test_two_retries() -> TestResult { +#[rstest] +fn test_two_retries( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { // Note: This also tests that the _same_ connection is being reused, i.e. tests keepalive. - - let mut buffer = Buffer::new(); + let mut server = MockServer::new()?; + let mut sender = server + .lsb_http() + .protocol_version(version)? + .retry_timeout(Duration::from_secs(30))? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - - let mut server = MockServer::new()?; - let mut sender = server - .lsb_http() - .retry_timeout(Duration::from_secs(30))? - .build()?; - - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.body(), buffer2.as_bytes()); @@ -571,12 +564,12 @@ fn test_two_retries() -> TestResult { server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush_and_keep(&buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; // Unpacking the error here allows server errors to bubble first. res?; @@ -584,9 +577,17 @@ fn test_two_retries() -> TestResult { Ok(()) } -#[test] -fn test_one_retry() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_one_retry( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server + .lsb_http() + .retry_timeout(Duration::from_millis(19))? + .protocol_version(version)? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? @@ -594,15 +595,8 @@ fn test_one_retry() -> TestResult { .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?; - let mut sender = server - .lsb_http() - .retry_timeout(Duration::from_millis(19))? - .build()?; - - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.body(), buffer2.as_bytes()); @@ -634,24 +628,27 @@ fn test_one_retry() -> TestResult { }; assert_eq!(err.kind(), ErrorKind::TimedOut); - Ok(()) + Ok(server) }); - let res = sender.flush_and_keep(&buffer); - - server_thread.join().unwrap()?; - - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ServerFlushError); - assert_eq!(err.msg(), "Could not flush buffer: error 2"); + assert_err_contains( + sender.flush_and_keep(&buffer), + ErrorCode::ServerFlushError, + "Could not flush buffer: error 2", + ); + _ = server_thread.join().unwrap()?; Ok(()) } -#[test] -fn test_transactional() -> TestResult { +#[rstest] +fn test_transactional( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; // A buffer with a two tables. - let mut buffer1 = Buffer::new(); + let mut buffer1 = sender.new_buffer(); buffer1 .table("tab1")? .symbol("t1", "v1")? @@ -665,7 +662,7 @@ fn test_transactional() -> TestResult { assert!(!buffer1.transactional()); // A buffer with a single table. - let mut buffer2 = Buffer::new(); + let mut buffer2 = sender.new_buffer(); buffer2 .table("test")? .symbol("t1", "v1")? @@ -674,36 +671,138 @@ fn test_transactional() -> TestResult { let buffer3 = buffer2.clone(); assert!(buffer2.transactional()); - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; - - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.body(), buffer3.as_bytes()); server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); - let res = sender.flush_and_keep_with_flags(&buffer1, true); - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::InvalidApiCall); - assert_eq!( - err.msg(), + assert_err_contains( + sender.flush_and_keep_with_flags(&buffer1, true), + ErrorCode::InvalidApiCall, "Buffer contains lines for multiple tables. \ - Transactional flushes are only supported for buffers containing lines for a single table." + Transactional flushes are only supported for buffers containing lines for a single table.", ); let res = sender.flush_and_keep_with_flags(&buffer2, true); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; // Unpacking the error here allows server errors to bubble first. res?; Ok(()) } + +fn _test_sender_auto_detect_protocol_version( + supported_versions: Option>, + expect_version: ProtocolVersion, +) -> TestResult { + let supported_versions1 = supported_versions.clone(); + let mut server = MockServer::new()? + .configure_settings_response(supported_versions.as_deref().unwrap_or(&[])); + let sender_builder = server.lsb_http(); + + let server_thread = std::thread::spawn(move || -> io::Result { + server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + match supported_versions1 { + None => server.send_http_response_q( + HttpResponse::empty() + .with_status(404, "Not Found") + .with_header("content-type", "text/plain") + .with_body_str("Not Found"), + )?, + Some(_) => server.send_settings_response()?, + } + let exp = &[ + b"test,t1=v1 ", + crate::tests::sender::f64_to_bytes("f1", 0.5, expect_version).as_slice(), + b" 10000000\n", + ] + .concat(); + let req = server.recv_http_q()?; + assert_eq!(req.body(), exp); + server.send_http_response_q(HttpResponse::empty())?; + Ok(server) + }); + + let mut sender = sender_builder.build()?; + assert_eq!(sender.protocol_version(), expect_version); + let mut buffer = sender.new_buffer(); + buffer + .table("test")? + .symbol("t1", "v1")? + .column_f64("f1", 0.5)? + .at(TimestampNanos::new(10000000))?; + let res = sender.flush(&mut buffer); + res?; + _ = server_thread.join().unwrap()?; + Ok(()) +} + +#[test] +fn test_sender_auto_protocol_version_basic() -> TestResult { + _test_sender_auto_detect_protocol_version(Some(vec![1, 2]), ProtocolVersion::V2) +} + +#[test] +fn test_sender_auto_protocol_version_old_server1() -> TestResult { + _test_sender_auto_detect_protocol_version(Some(vec![]), ProtocolVersion::V1) +} + +#[test] +fn test_sender_auto_protocol_version_old_server2() -> TestResult { + _test_sender_auto_detect_protocol_version(None, ProtocolVersion::V1) +} + +#[test] +fn test_sender_auto_protocol_version_only_v1() -> TestResult { + _test_sender_auto_detect_protocol_version(Some(vec![1]), ProtocolVersion::V1) +} + +#[test] +fn test_sender_auto_protocol_version_only_v2() -> TestResult { + _test_sender_auto_detect_protocol_version(Some(vec![2]), ProtocolVersion::V2) +} + +#[test] +fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { + let mut server = MockServer::new()?.configure_settings_response(&[3, 4]); + let sender_builder = server.lsb_http(); + let server_thread = std::thread::spawn(move || -> io::Result { + server.accept()?; + server.send_settings_response()?; + Ok(server) + }); + assert_err_contains( + sender_builder.build(), + ErrorCode::ProtocolVersionError, + "Server does not support current client", + ); + + // We keep the server around til the end of the test to ensure that the response is fully received. + _ = server_thread.join().unwrap()?; + Ok(()) +} + +#[test] +fn test_buffer_protocol_version1_not_support_array() -> TestResult { + let mut buffer = Buffer::new(ProtocolVersion::V1); + let res = buffer + .table("test")? + .symbol("sym", "bol")? + .column_arr("x", &[1.0f64, 2.0]); + assert_err_contains( + res, + ErrorCode::ProtocolVersionError, + "Protocol version v1 does not support array datatype", + ); + Ok(()) +} diff --git a/questdb-rs/src/tests/interop/ilp-client-interop-test.json b/questdb-rs/src/tests/interop/ilp-client-interop-test.json index d3e0e259..0acedad7 100644 --- a/questdb-rs/src/tests/interop/ilp-client-interop-test.json +++ b/questdb-rs/src/tests/interop/ilp-client-interop-test.json @@ -32,6 +32,7 @@ ], "result": { "status": "SUCCESS", + "binaryBase64": "dGVzdF90YWJsZSxzeW1fY29sPXN5bV92YWwgc3RyX2NvbD0iZm9vIGJhciBiYXoiLGxvbmdfY29sPTQyaSxkb3VibGVfY29sPT0QAAAAAABARUAsYm9vbF9jb2w9dAo=", "line": "test_table,sym_col=sym_val str_col=\"foo bar baz\",long_col=42i,double_col=42.5,bool_col=t" } }, @@ -73,6 +74,7 @@ ], "result": { "status": "SUCCESS", + "binaryBase64": "ZG91YmxlcyBkMD09EAAAAAAAAAAALGRtMD09EAAAAAAAAACALGQxPT0QAAAAAAAA8D8sZEUxMDA9PRB9w5QlrUmyVCxkMDAwMDAwMT09EI3ttaD3xrA+LGROMDAwMDAwMT09EI3ttaD3xrC+Cg==", "anyLines": [ "doubles d0=0,dm0=-0,d1=1,dE100=1E+100,d0000001=1E-06,dN0000001=-1E-06", "doubles d0=0.0,dm0=-0.0,d1=1.0,dE100=1e100,d0000001=1e-6,dN0000001=-1e-6" diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 289b41cb..caa407a6 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -36,9 +36,12 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; +use crate::ingress; #[cfg(feature = "ilp-over-http")] use std::io::Write; +use super::ndarr::ArrayColumnTypeTag; + const CLIENT: Token = Token(0); #[derive(Debug)] @@ -50,7 +53,9 @@ pub struct MockServer { tls_conn: Option, pub host: &'static str, pub port: u16, - pub msgs: Vec, + pub msgs: Vec>, + #[cfg(feature = "ilp-over-http")] + settings_response: serde_json::Value, } pub fn certs_dir() -> std::path::PathBuf { @@ -206,6 +211,8 @@ impl MockServer { host: "localhost", port, msgs: Vec::new(), + #[cfg(feature = "ilp-over-http")] + settings_response: serde_json::Value::Null, }) } @@ -302,6 +309,25 @@ impl MockServer { } } + #[cfg(feature = "ilp-over-http")] + pub fn configure_settings_response(mut self, supported_versions: &[u16]) -> Self { + if supported_versions.is_empty() { + self.settings_response = serde_json::json!({"version": "8.1.2"}); + } else { + self.settings_response = serde_json::json!( + {"config":{"release.type":"OSS","release.version":"[DEVELOPMENT]", + "line.proto.support.versions":supported_versions, + "ilp.proto.transports":["tcp", "http"], + "posthog.enabled":false, + "posthog.api.key":null, + "cairo.max.file.name.length":127}, + "preferences.version":0, + "preferences":{}} + ); + } + self + } + #[cfg(feature = "ilp-over-http")] fn do_write(&mut self, buf: &[u8]) -> io::Result { let client = self.client.as_mut().unwrap(); @@ -454,6 +480,15 @@ impl MockServer { Ok(()) } + #[cfg(feature = "ilp-over-http")] + pub fn send_settings_response(&mut self) -> io::Result<()> { + let response = HttpResponse::empty() + .with_status(200, "OK") + .with_body_json(&self.settings_response); + self.send_http_response(response, Some(2.0))?; + Ok(()) + } + #[cfg(feature = "ilp-over-http")] pub fn send_http_response_q(&mut self, response: HttpResponse) -> io::Result<()> { self.send_http_response(response, Some(5.0)) @@ -465,6 +500,14 @@ impl MockServer { let deadline = Instant::now() + Duration::from_secs_f64(wait_timeout_sec); let (pos, method, path) = self.recv_http_method(&mut accum, deadline)?; let (pos, headers) = self.recv_http_headers(pos, &mut accum, deadline)?; + if &method == "GET" { + return Ok(HttpRequest { + method, + path, + headers, + body: vec![], + }); + } let content_length = headers .get("content-length") .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Missing Content-Length"))? @@ -521,15 +564,47 @@ impl MockServer { let mut received_count = 0usize; let mut head = 0usize; - for index in 1..accum.len() { + let binary_length = 0usize; + let mut index = 1; + + while index < accum.len() { let last = accum[index]; let prev = accum[index - 1]; - if (last == b'\n') && (prev != b'\\') { + if last == b'=' && prev == b'=' { + index += 1; + // calc binary length + let binary_type = accum[index]; + if binary_type == ingress::DOUBLE_BINARY_FORMAT_TYPE { + index += size_of::() + 1; + } else if binary_type == ingress::ARRAY_BINARY_FORMAT_TYPE { + index += 1; + let element_type = match ArrayColumnTypeTag::try_from(accum[index]) { + Ok(t) => t, + Err(e) => { + return Err(io::Error::other(e)); + } + }; + let mut elems_size = element_type.size(); + index += 1; + let dims = accum[index] as usize; + index += 1; + for _ in 0..dims { + elems_size *= i32::from_le_bytes( + accum[index..index + size_of::()].try_into().unwrap(), + ) as usize; + index += size_of::(); + } + index += elems_size; + } + } else if (last == b'\n') && (prev != b'\\' && binary_length == 0) { let tail = index + 1; - let msg = std::str::from_utf8(&accum[head..tail]).unwrap(); - self.msgs.push(msg.to_owned()); + let msg = &accum[head..tail]; + self.msgs.push(msg.to_vec()); head = tail; received_count += 1; + index = tail; + } else { + index += 1; } } Ok(received_count) diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index f817287e..c63a65d3 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -21,6 +21,7 @@ * limitations under the License. * ******************************************************************************/ + mod f64_serializer; #[cfg(feature = "ilp-over-http")] @@ -29,6 +30,8 @@ mod http; mod mock; mod sender; +mod ndarr; + #[cfg(feature = "json_tests")] mod json_tests { include!(concat!(env!("OUT_DIR"), "/json_tests.rs")); @@ -36,3 +39,31 @@ mod json_tests { pub type TestError = Box; pub type TestResult = std::result::Result<(), TestError>; + +pub fn assert_err_contains( + result: crate::Result, + expected_code: crate::ErrorCode, + expected_msg_contained: &str, +) { + match result { + Ok(_) => panic!( + "Expected error containing '{}', but got Ok({:?})", + expected_msg_contained, result + ), + Err(e) => { + assert_eq!( + e.code(), + expected_code, + "Expected error code {:?}, but got {:?}", + expected_code, + e.code() + ); + assert!( + e.msg().contains(expected_msg_contained), + "Expected error message to contain {:?}, but got {:?}", + expected_msg_contained, + e.msg() + ); + } + } +} diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs new file mode 100644 index 00000000..659f8ab4 --- /dev/null +++ b/questdb-rs/src/tests/ndarr.rs @@ -0,0 +1,905 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use crate::ingress::{Buffer, NdArrayView, ProtocolVersion, ARRAY_BINARY_FORMAT_TYPE}; +use crate::tests::TestResult; +use crate::ErrorCode; + +#[cfg(feature = "ndarray")] +use crate::ingress::ndarr::write_array_data; +#[cfg(feature = "ndarray")] +use crate::ingress::MAX_ARRAY_DIMS; +#[cfg(feature = "ndarray")] +use ndarray::{arr1, arr2, arr3, s, ArrayD}; +#[cfg(feature = "ndarray")] +use std::iter; + +/// QuestDB column type tags that are supported as array element types. +#[derive(Clone, Copy)] +#[repr(u8)] +pub enum ArrayColumnTypeTag { + Double = 10, +} + +impl ArrayColumnTypeTag { + pub fn size(&self) -> usize { + match self { + ArrayColumnTypeTag::Double => std::mem::size_of::(), + } + } +} + +impl From for u8 { + fn from(tag: ArrayColumnTypeTag) -> Self { + tag as u8 + } +} + +impl TryFrom for ArrayColumnTypeTag { + type Error = String; + + fn try_from(value: u8) -> Result { + match value { + 10 => Ok(ArrayColumnTypeTag::Double), + _ => Err(format!("Unsupported column type tag {} for arrays", value)), + } + } +} + +#[test] +fn test_build_in_1d_array_normal() -> TestResult { + let arr = [1.0f64, 2.0, 3.0, 4.0]; + assert_eq!(arr.ndim(), 1); + assert_eq!(arr.dim(0), Ok(4)); + assert!(arr.dim(1).is_err()); + assert_eq!(NdArrayView::as_slice(&arr), Some(&[1.0, 2.0, 3.0, 4.0][..])); + let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); + assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0]); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [4i32.to_le_bytes()].concat()); + assert_eq!( + &data[28..60], + &[ + 1.0f64.to_ne_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_1d_array_empty() -> TestResult { + let arr: [f64; 0] = []; + assert_eq!(arr.ndim(), 1); + assert_eq!(arr.dim(0), Ok(0)); + assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [0i32.to_le_bytes()].concat()); + Ok(()) +} + +#[test] +fn test_build_in_1d_vec_normal() -> TestResult { + let vec = vec![5.0f64, 6.0, 7.0]; + assert_eq!(vec.ndim(), 1); + assert_eq!(vec.dim(0), Ok(3)); + assert_eq!(NdArrayView::as_slice(&vec), Some(&[5.0, 6.0, 7.0][..])); + let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); + assert_eq!(collected, vec![5.0, 6.0, 7.0]); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [3i32.to_le_bytes()].concat()); + assert_eq!( + &data[28..52], + &[ + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 7.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_1d_vec_empty() -> TestResult { + let vec: Vec = Vec::new(); + assert_eq!(vec.ndim(), 1); + assert_eq!(vec.dim(0), Ok(0)); + assert_eq!(NdArrayView::as_slice(&vec), Some(&[][..])); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [0i32.to_le_bytes()].concat()); + Ok(()) +} + +#[test] +fn test_build_in_1d_slice_normal() -> TestResult { + let data = [10.0f64, 20.0, 30.0, 40.0]; + let slice = &data[1..3]; + assert_eq!(slice.ndim(), 1); + assert_eq!(slice.dim(0), Ok(2)); + assert_eq!(NdArrayView::as_slice(&slice), Some(&[20.0, 30.0][..])); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [2i32.to_le_bytes()].concat()); + assert_eq!( + &data[28..44], + &[20.0f64.to_le_bytes(), 30.0f64.to_le_bytes(),].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_1d_slice_empty() -> TestResult { + let data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let slice = &data[2..2]; + assert_eq!(slice.ndim(), 1); + assert_eq!(slice.dim(0), Ok(0)); + assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [0i32.to_le_bytes()].concat()); + Ok(()) +} + +#[test] +fn test_build_in_2d_array_normal() -> TestResult { + let arr = [[1.0f64, 2.0], [3.0, 4.0], [5.0, 6.0]]; + assert_eq!(arr.ndim(), 2); + assert_eq!(arr.dim(0), Ok(3)); + assert_eq!(arr.dim(1), Ok(2)); + assert_eq!( + NdArrayView::as_slice(&arr), + Some(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0][..]) + ); + let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); + assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("2darray", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [3i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[28..76], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_array_empty() -> TestResult { + let arr: [[f64; 0]; 0] = []; + assert_eq!(arr.ndim(), 2); + assert_eq!(arr.dim(0), Ok(0)); + assert_eq!(arr.dim(1), Ok(0)); + assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("2darray", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [0i32.to_le_bytes(), 0i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_vec_normal() -> TestResult { + let vec = vec![vec![1.0f64, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]]; + assert_eq!(vec.ndim(), 2); + assert_eq!(vec.dim(0), Ok(3)); + assert_eq!(vec.dim(1), Ok(2)); + assert!(NdArrayView::as_slice(&vec).is_none()); + let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); + assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("2darray", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [3i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[28..76], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_vec_irregular_shape() -> TestResult { + let irregular_vec = vec![vec![1.0, 2.0], vec![3.0], vec![4.0, 5.0]]; + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + let result = buffer.column_arr("arr", &irregular_vec); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Irregular array shape")); + Ok(()) +} + +#[test] +fn test_build_in_2d_vec_empty() -> TestResult { + let vec: Vec> = vec![vec![], vec![], vec![]]; + assert_eq!(vec.ndim(), 2); + assert_eq!(vec.dim(0), Ok(3)); + assert_eq!(vec.dim(1), Ok(0)); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("2darray", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [3i32.to_le_bytes(), 0i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_slice_normal() -> TestResult { + let data = [[1.0f64, 2.0], [3.0, 4.0], [5.0, 6.0]]; + let slice = &data[..2]; + assert_eq!(slice.ndim(), 2); + assert_eq!(slice.dim(0), Ok(2)); + assert_eq!(slice.dim(1), Ok(2)); + assert_eq!( + NdArrayView::as_slice(&slice), + Some(&[1.0, 2.0, 3.0, 4.0][..]) + ); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("2darray", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[28..60], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_slice_empty() -> TestResult { + let data = [[1.0f64, 2.0], [3.0, 4.0], [5.0, 6.0]]; + let slice = &data[2..2]; + assert_eq!(slice.ndim(), 2); + assert_eq!(slice.dim(0), Ok(0)); + assert_eq!(slice.dim(1), Ok(2)); + assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("2darray", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [0i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_array_normal() -> TestResult { + let arr = [[[1.0f64, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]; + assert_eq!(arr.ndim(), 3); + assert_eq!(arr.dim(0), Ok(2)); + assert_eq!(arr.dim(1), Ok(2)); + assert_eq!(arr.dim(2), Ok(2)); + assert_eq!( + NdArrayView::as_slice(&arr), + Some(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0][..]) + ); + let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); + assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("3darray", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..96], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 7.0f64.to_le_bytes(), + 8.0f64.to_le_bytes() + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_array_empty() -> TestResult { + let arr: [[[f64; 2]; 0]; 0] = []; + assert_eq!(arr.ndim(), 3); + assert_eq!(arr.dim(0), Ok(0)); + assert_eq!(arr.dim(1), Ok(0)); + assert_eq!(arr.dim(2), Ok(2)); + assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("3darray", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [0i32.to_le_bytes(), 0i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_vec_normal() -> TestResult { + let vec = vec![ + vec![vec![1.0, 2.0, 3.0], vec![4.0, 5.0, 6.0]], + vec![vec![7.0, 8.0, 9.0], vec![10.0, 11.0, 12.0]], + ]; + assert_eq!(vec.ndim(), 3); + assert_eq!(vec.dim(0), Ok(2)); + assert_eq!(vec.dim(1), Ok(2)); + assert_eq!(vec.dim(2), Ok(3)); + assert!(NdArrayView::as_slice(&vec).is_none()); + let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); + assert_eq!( + collected, + vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0] + ); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("3darray", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes(), 3i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..128], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 7.0f64.to_le_bytes(), + 8.0f64.to_le_bytes(), + 9.0f64.to_le_bytes(), + 10.0f64.to_le_bytes(), + 11.0f64.to_le_bytes(), + 12.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_vec_empty() -> TestResult { + let vec: Vec>> = vec![vec![vec![], vec![]], vec![vec![], vec![]]]; + assert_eq!(vec.ndim(), 3); + assert_eq!(vec.dim(0), Ok(2)); + assert_eq!(vec.dim(1), Ok(2)); + assert_eq!(vec.dim(2), Ok(0)); + assert!(NdArrayView::as_slice(&vec).is_none()); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("3darray", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes(), 0i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_vec_irregular_shape() -> TestResult { + let irregular1 = vec![vec![vec![1.0, 2.0], vec![3.0, 4.0]], vec![vec![5.0, 6.0]]]; + + let irregular2 = vec![ + vec![vec![1.0, 2.0], vec![3.0, 4.0, 5.0]], + vec![vec![6.0, 7.0], vec![8.0, 9.0]], + ]; + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + let result = buffer.column_arr("arr", &irregular1); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Irregular array shape")); + + let result = buffer.column_arr("arr", &irregular2); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Irregular array shape")); + Ok(()) +} + +#[test] +fn test_3d_slice_normal() -> TestResult { + let data = [[[1f64, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]; + let slice = &data[..1]; + assert_eq!(slice.ndim(), 3); + assert_eq!(slice.dim(0), Ok(1)); + assert_eq!(slice.dim(1), Ok(2)); + assert_eq!(slice.dim(2), Ok(2)); + assert_eq!( + NdArrayView::as_slice(&slice), + Some(&[1.0, 2.0, 3.0, 4.0][..]) + ); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("3darray", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [1i32.to_le_bytes(), 2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..64], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_3d_slice_empty() -> TestResult { + let data = [[[1f64, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]; + let slice = &data[1..1]; + assert_eq!(slice.ndim(), 3); + assert_eq!(slice.dim(0), Ok(0)); + assert_eq!(slice.dim(1), Ok(2)); + assert_eq!(slice.dim(2), Ok(2)); + assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("3darray", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [0i32.to_le_bytes(), 2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_1d_contiguous_ndarray_buffer() -> TestResult { + let array = arr1(&[1.0, 2.0, 3.0, 4.0]); + let view = array.view(); + let mut buf = vec![0u8; 4 * size_of::()]; + write_array_data(&view, &mut buf[0..], 32)?; + let expected: Vec = array + .iter() + .flat_map(|&x| x.to_ne_bytes().to_vec()) + .collect(); + assert_eq!(buf, expected); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_2d_non_contiguous_ndarray_buffer() -> TestResult { + let array = arr2(&[[1.0, 2.0], [3.0, 4.0]]); + let transposed = array.view().reversed_axes(); + assert!(!transposed.is_standard_layout()); + let mut buf = vec![0u8; 4 * size_of::()]; + write_array_data(&transposed, &mut buf[0..], 32)?; + let expected = [1.0f64, 3.0, 2.0, 4.0] + .iter() + .flat_map(|&x| x.to_ne_bytes()) + .collect::>(); + assert_eq!(buf, expected); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_strided_ndarray_layout() -> TestResult { + let array = arr2(&[ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0], + ]); + let strided_view = array.slice(s![1..;2, 1..;2]); + assert_eq!(strided_view.dim(), (2, 2)); + let mut buf = vec![0u8; 4 * size_of::()]; + write_array_data(&strided_view, &mut buf[0..], 32)?; + + // expect：6.0, 8.0, 14.0, 16.0 + let expected = [6.0f64, 8.0, 14.0, 16.0] + .iter() + .flat_map(|&x| x.to_ne_bytes()) + .collect::>(); + + assert_eq!(buf, expected); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_1d_dimension_ndarray_info() { + let array = arr1(&[1.0, 2.0, 3.0]); + let view = array.view(); + + assert_eq!(NdArrayView::ndim(&view), 1); + assert_eq!(NdArrayView::dim(&view, 0), Ok(3)); + assert!(NdArrayView::dim(&view, 1).is_err()); +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_complex_ndarray_dimensions() { + let array = arr3(&[[[1.0], [2.0]], [[3.0], [4.0]]]); + let view = array.view(); + + assert_eq!(NdArrayView::ndim(&view), 3); + assert_eq!(NdArrayView::dim(&view, 0), Ok(2)); + assert_eq!(NdArrayView::dim(&view, 1), Ok(2)); + assert_eq!(NdArrayView::dim(&view, 2), Ok(1)); +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_buffer_ndarray_write() -> TestResult { + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); + buffer.column_arr("temperature", &array_2d.view())?; + + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes().as_slice(), 2i32.to_le_bytes().as_slice()].concat() + ); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_buffer_write_ndarray_max_dimensions() -> TestResult { + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("nd_test")?; + let shape: Vec = iter::repeat_n(1, MAX_ARRAY_DIMS).collect(); + let array = ArrayD::::zeros(shape.clone()); + buffer.column_arr("max_dim", &array.view())?; + let data = buffer.as_bytes(); + assert_eq!(data[19], MAX_ARRAY_DIMS as u8); + + // 33 dims error + let shape_invalid: Vec<_> = iter::repeat_n(1, MAX_ARRAY_DIMS + 1).collect(); + let array_invalid = ArrayD::::zeros(shape_invalid); + let result = buffer.column_arr("invalid", &array_invalid.view()); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayHasTooManyDims); + Ok(()) +} diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 041e78b5..9b92bfa5 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -26,21 +26,33 @@ use crate::{ ingress::{ Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, }, + tests::assert_err_contains, Error, ErrorCode, }; +use crate::ingress; +#[cfg(feature = "ndarray")] +use crate::ingress::ndarr::write_array_data; +use crate::ingress::ProtocolVersion; use crate::tests::{ mock::{certs_dir, MockServer}, + ndarr::ArrayColumnTypeTag, TestResult, }; - use core::time::Duration; -use std::{io, time::SystemTime}; +#[cfg(feature = "ndarray")] +use ndarray::{arr2, ArrayD}; +use rstest::rstest; +use std::io; + +#[rstest] +fn test_basics( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + use std::time::SystemTime; -#[test] -fn test_basics() -> TestResult { let mut server = MockServer::new()?; - let mut sender = server.lsb_tcp().build()?; + let mut sender = server.lsb_tcp().protocol_version(version)?.build()?; assert!(!sender.must_close()); server.accept()?; @@ -54,7 +66,7 @@ fn test_basics() -> TestResult { let ts_nanos = TimestampNanos::from_systemtime(ts)?; assert_eq!(ts_nanos.as_i64(), ts_nanos_num); - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? @@ -65,32 +77,165 @@ fn test_basics() -> TestResult { .at(ts_nanos)?; assert_eq!(server.recv_q()?, 0); - let exp = format!( - "test,t1=v1 f1=0.5,ts1=12345t,ts2={}t,ts3={}t {}\n", - ts_micros_num, - ts_nanos_num / 1000i64, - ts_nanos_num - ); - let exp_byte = exp.as_bytes(); - assert_eq!(buffer.as_bytes(), exp_byte); + let exp = &[ + "test,t1=v1 ".as_bytes(), + f64_to_bytes("f1", 0.5, version).as_slice(), + format!( + ",ts1=12345t,ts2={}t,ts3={}t {}\n", + ts_micros_num, + ts_nanos_num / 1000i64, + ts_nanos_num + ) + .as_bytes(), + ] + .concat(); + assert_eq!(buffer.as_bytes(), exp); + assert_eq!(buffer.len(), exp.len()); + sender.flush(&mut buffer)?; + assert_eq!(buffer.len(), 0); + assert_eq!(buffer.as_bytes(), b""); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0], *exp); + Ok(()) +} + +#[test] +fn test_array_f64_basic() -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server + .lsb_tcp() + .protocol_version(ProtocolVersion::V2)? + .build()?; + server.accept()?; + + let ts = TimestampNanos::now(); + let mut buffer = sender.new_buffer(); + buffer + .table("my_table")? + .symbol("device", "A001")? + .column_f64("f1", 25.5)? + .column_arr("arr1d", &[1.0, 2.0, 3.0])? + .at(ts)?; + + assert_eq!(server.recv_q()?, 0); + + let exp = &[ + b"my_table,device=A001 ", + f64_to_bytes("f1", 25.5, ProtocolVersion::V2).as_slice(), + b",arr1d=", + b"=", // binary field + &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ArrayColumnTypeTag::Double.into()], + &[1u8], // 1D array + &3u32.to_le_bytes(), // 3 elements + &1.0f64.to_le_bytes(), + &2.0f64.to_le_bytes(), + &3.0f64.to_le_bytes(), + format!(" {}\n", ts.as_i64()).as_bytes(), + ] + .concat(); + + assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; assert_eq!(buffer.len(), 0); assert_eq!(buffer.as_bytes(), b""); assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0].as_bytes(), exp_byte); + assert_eq!(server.msgs[0].as_slice(), exp); Ok(()) } +#[cfg(feature = "ndarray")] #[test] -fn test_max_buf_size() -> TestResult { +fn test_array_f64_for_ndarray() -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server + .lsb_tcp() + .protocol_version(ProtocolVersion::V2)? + .build()?; + server.accept()?; + + let ts = TimestampNanos::now(); + let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); + let array_3d = ArrayD::::ones(vec![2, 3, 4]); + + let mut buffer = sender.new_buffer(); + buffer + .table("my_table")? + .symbol("device", "A001")? + .column_f64("f1", 25.5)? + .column_arr("arr2d", &array_2d.view())? + .column_arr("arr3d", &array_3d.view())? + .at(ts)?; + + assert_eq!(server.recv_q()?, 0); + + let array_header2d = &[ + &[b'='][..], + &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ArrayColumnTypeTag::Double.into()], + &[2u8], + &2i32.to_le_bytes(), + &2i32.to_le_bytes(), + ] + .concat(); + let mut array_data2d = vec![0u8; 4 * size_of::()]; + write_array_data(&array_2d.view(), &mut array_data2d[0..], 32)?; + + let array_header3d = &[ + &[b'='][..], + &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ArrayColumnTypeTag::Double.into()], + &[3u8], + &2i32.to_le_bytes(), + &3i32.to_le_bytes(), + &4i32.to_le_bytes(), + ] + .concat(); + let mut array_data3d = vec![0u8; 24 * size_of::()]; + write_array_data( + &array_3d.view(), + &mut array_data3d[0..], + 24 * size_of::(), + )?; + + let exp = &[ + "my_table,device=A001 ".as_bytes(), + f64_to_bytes("f1", 25.5, ProtocolVersion::V2).as_slice(), + ",arr2d=".as_bytes(), + array_header2d, + array_data2d.as_slice(), + ",arr3d=".as_bytes(), + array_header3d, + array_data3d.as_slice(), + format!(" {}\n", ts.as_i64()).as_bytes(), + ] + .concat(); + + assert_eq!(buffer.as_bytes(), exp); + assert_eq!(buffer.len(), exp.len()); + sender.flush(&mut buffer)?; + assert_eq!(buffer.len(), 0); + assert_eq!(buffer.as_bytes(), b""); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0].as_slice(), exp); + Ok(()) +} + +#[rstest] +fn test_max_buf_size( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { let max = 1024; let mut server = MockServer::new()?; - let mut sender = server.lsb_tcp().max_buf_size(max)?.build()?; + let mut sender = server + .lsb_tcp() + .protocol_version(version)? + .max_buf_size(max)? + .build()?; assert!(!sender.must_close()); server.accept()?; - - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); while buffer.len() < max { buffer @@ -102,16 +247,26 @@ fn test_max_buf_size() -> TestResult { let err = sender.flush(&mut buffer).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidApiCall); - assert_eq!( - err.msg(), - "Could not flush buffer: Buffer size of 1026 exceeds maximum configured allowed size of 1024 bytes." - ); + match version { + ProtocolVersion::V1 => { + assert_eq!( + err.msg(), + "Could not flush buffer: Buffer size of 1026 exceeds maximum configured allowed size of 1024 bytes." + ); + } + ProtocolVersion::V2 => { + assert_eq!( + err.msg(), + "Could not flush buffer: Buffer size of 1025 exceeds maximum configured allowed size of 1024 bytes." + ); + } + } Ok(()) } #[test] fn test_table_name_too_long() -> TestResult { - let mut buffer = Buffer::with_max_name_len(4); + let mut buffer = Buffer::with_max_name_len(ProtocolVersion::V2, 4); let name = "a name too long"; let err = buffer.table(name).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidName); @@ -124,7 +279,7 @@ fn test_table_name_too_long() -> TestResult { #[test] fn test_row_count() -> TestResult { - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); assert_eq!(buffer.row_count(), 0); buffer.table("x")?.symbol("y", "z1")?.at_now()?; @@ -262,9 +417,11 @@ fn test_bad_key( #[test] fn test_timestamp_overloads() -> TestResult { + use std::time::SystemTime; + let tbl_name = TableName::new("tbl_name")?; - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer .table(tbl_name)? .column_ts("a", TimestampMicros::new(12345))? @@ -309,7 +466,7 @@ fn test_chrono_timestamp() -> TestResult { let ts: DateTime = Utc.with_ymd_and_hms(1970, 1, 1, 0, 0, 1).unwrap(); let ts = TimestampNanos::from_datetime(ts)?; - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table(tbl_name)?.column_ts("a", ts)?.at(ts)?; let exp = b"tbl_name a=1000000t 1000000000\n"; @@ -320,7 +477,7 @@ fn test_chrono_timestamp() -> TestResult { macro_rules! column_name_too_long_test_impl { ($column_fn:ident, $value:expr) => {{ - let mut buffer = Buffer::with_max_name_len(4); + let mut buffer = Buffer::with_max_name_len(ProtocolVersion::V2, 4); let name = "a name too long"; let err = buffer.table("tbl")?.$column_fn(name, $value).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidName); @@ -358,17 +515,27 @@ fn test_str_column_name_too_long() -> TestResult { } #[test] -fn test_tls_with_file_ca() -> TestResult { +fn test_arr_column_name_too_long() -> TestResult { + column_name_too_long_test_impl!(column_arr, &[1.0, 2.0, 3.0]) +} + +#[rstest] +fn test_tls_with_file_ca( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { let mut ca_path = certs_dir(); ca_path.push("server_rootCA.pem"); let server = MockServer::new()?; - let lsb = server.lsb_tcps().tls_roots(ca_path)?; + let lsb = server + .lsb_tcps() + .protocol_version(version)? + .tls_roots(ca_path)?; let server_jh = server.accept_tls(); let mut sender = lsb.build()?; let mut server: MockServer = server_jh.join().unwrap()?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? @@ -376,12 +543,17 @@ fn test_tls_with_file_ca() -> TestResult { .at(TimestampNanos::new(10000000))?; assert_eq!(server.recv_q()?, 0); - let exp = b"test,t1=v1 f1=0.5 10000000\n"; + let exp = &[ + "test,t1=v1 ".as_bytes(), + f64_to_bytes("f1", 0.5, version).as_slice(), + " 10000000\n".as_bytes(), + ] + .concat(); assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0].as_bytes(), exp); + assert_eq!(server.msgs[0].as_slice(), exp); Ok(()) } @@ -419,7 +591,7 @@ fn expect_eventual_disconnect(sender: &mut Sender) { let mut retry = || { for _ in 0..1000 { std::thread::sleep(Duration::from_millis(100)); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("test_table")?.symbol("s1", "v1")?.at_now()?; sender.flush(&mut buffer)?; } @@ -435,7 +607,7 @@ fn test_plain_to_tls_server() -> TestResult { let server = MockServer::new()?; let lsb = server.lsb_tcp().auth_timeout(Duration::from_millis(500))?; let server_jh = server.accept_tls(); - let maybe_sender = lsb.build(); + let maybe_sender = lsb.protocol_version(ProtocolVersion::V2)?.build(); let server_err = server_jh.join().unwrap().unwrap_err(); // The server failed to handshake, so disconnected the client. @@ -453,15 +625,20 @@ fn test_plain_to_tls_server() -> TestResult { } #[cfg(feature = "insecure-skip-verify")] -#[test] -fn test_tls_insecure_skip_verify() -> TestResult { +#[rstest] +fn test_tls_insecure_skip_verify( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { let server = MockServer::new()?; - let lsb = server.lsb_tcps().tls_verify(false)?; + let lsb = server + .lsb_tcps() + .protocol_version(version)? + .tls_verify(false)?; let server_jh = server.accept_tls(); let mut sender = lsb.build()?; let mut server: MockServer = server_jh.join().unwrap()?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? @@ -469,12 +646,17 @@ fn test_tls_insecure_skip_verify() -> TestResult { .at(TimestampNanos::new(10000000))?; assert_eq!(server.recv_q()?, 0); - let exp = b"test,t1=v1 f1=0.5 10000000\n"; + let exp = &[ + "test,t1=v1 ".as_bytes(), + f64_to_bytes("f1", 0.5, version).as_slice(), + " 10000000\n".as_bytes(), + ] + .concat(); assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0].as_bytes(), exp); + assert_eq!(server.msgs[0].as_slice(), exp); Ok(()) } @@ -495,3 +677,40 @@ fn bad_uppercase_addr() { assert!(err.code() == ErrorCode::ConfigError); assert!(err.msg() == "Missing \"addr\" parameter in config string"); } + +#[test] +fn tcp_mismatched_buffer_and_sender_version() -> TestResult { + let server = MockServer::new()?; + let mut sender = server + .lsb_tcp() + .protocol_version(ProtocolVersion::V2)? + .build()?; + let mut buffer = Buffer::new(ProtocolVersion::V1); + buffer.table("test")?.symbol("t1", "v1")?.at_now()?; + assert_err_contains( + sender.flush(&mut buffer), + ErrorCode::ProtocolVersionError, + "Attempting to send with protocol version v1 \ + but the sender is configured to use protocol version v2", + ); + Ok(()) +} + +pub(crate) fn f64_to_bytes(name: &str, value: f64, version: ProtocolVersion) -> Vec { + let mut buf = Vec::new(); + buf.extend_from_slice(name.as_bytes()); + buf.push(b'='); + + match version { + ProtocolVersion::V1 => { + let mut ser = crate::ingress::F64Serializer::new(value); + buf.extend_from_slice(ser.as_str().as_bytes()); + } + ProtocolVersion::V2 => { + buf.push(b'='); + buf.push(crate::ingress::DOUBLE_BINARY_FORMAT_TYPE); + buf.extend_from_slice(&value.to_le_bytes()); + } + } + buf +} diff --git a/system_test/fixture.py b/system_test/fixture.py index 40dca629..1f70da13 100644 --- a/system_test/fixture.py +++ b/system_test/fixture.py @@ -23,12 +23,12 @@ ################################################################################ import sys + sys.dont_write_bytecode = True import os import re import pathlib -import textwrap import json import tarfile import shutil @@ -42,11 +42,9 @@ import urllib.error from pprint import pformat - AUTH_TXT = """admin ec-p-256-sha256 fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac # [key/user id] [key type] {keyX keyY}""" - # Valid keys as registered with the QuestDB fixture. AUTH = dict( username="admin", @@ -54,18 +52,17 @@ token_x="fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", token_y="Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac") - CA_PATH = (pathlib.Path(__file__).parent.parent / - 'tls_certs' / 'server_rootCA.pem') + 'tls_certs' / 'server_rootCA.pem') def retry( - predicate_task, - timeout_sec=30, - every=0.05, - msg='Timed out retrying', - backoff_till=5.0, - lead_sleep=0.001): + predicate_task, + timeout_sec=30, + every=0.05, + msg='Timed out retrying', + backoff_till=5.0, + lead_sleep=0.001): """ Repeat task every `interval` until it returns a truthy value or times out. """ @@ -121,8 +118,8 @@ def __init__(self): def list_questdb_releases(max_results=1): url = ( - 'https://api.github.com/repos/questdb/questdb/releases?' + - urllib.parse.urlencode({'per_page': max_results})) + 'https://api.github.com/repos/questdb/questdb/releases?' + + urllib.parse.urlencode({'per_page': max_results})) req = urllib.request.Request( url, headers={ @@ -233,7 +230,7 @@ class QueryError(Exception): class QuestDbFixture: - def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=False): + def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=False, protocol_version=None): self._root_dir = root_dir self.version = _parse_version(self._root_dir.name) self._data_dir = self._root_dir / 'data' @@ -243,7 +240,7 @@ def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=Fals self._conf_path = self._conf_dir / 'server.conf' self._log = None self._proc = None - self.host = 'localhost' + self.host = '127.0.0.1' self.http_server_port = None self.line_tcp_port = None self.pg_port = None @@ -258,6 +255,7 @@ def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=Fals with open(auth_txt_path, 'w', encoding='utf-8') as auth_file: auth_file.write(AUTH_TXT) self.http = http + self.protocol_version = protocol_version def print_log(self): with open(self._log_path, 'r', encoding='utf-8') as log_file: @@ -314,7 +312,7 @@ def check_http_up(): if self._proc.poll() is not None: raise RuntimeError('QuestDB died during startup.') req = urllib.request.Request( - f'http://localhost:{self.http_server_port}/ping', + f'http://127.0.0.1:{self.http_server_port}/ping', method='GET') try: resp = urllib.request.urlopen(req, timeout=1) @@ -351,8 +349,8 @@ def check_http_up(): def http_sql_query(self, sql_query): url = ( - f'http://{self.host}:{self.http_server_port}/exec?' + - urllib.parse.urlencode({'query': sql_query})) + f'http://{self.host}:{self.http_server_port}/exec?' + + urllib.parse.urlencode({'query': sql_query})) buf = None try: resp = urllib.request.urlopen(url, timeout=5) @@ -370,7 +368,7 @@ def http_sql_query(self, sql_query): if 'error' in data: raise QueryError(data['error']) return data - + def query_version(self): try: res = self.http_sql_query('select build') @@ -397,6 +395,7 @@ def retry_check_table( log_ctx=None): sql_query = f"select * from '{table_name}'" http_response_log = [] + def check_table(): try: resp = self.http_sql_query(sql_query) @@ -414,7 +413,8 @@ def check_table(): except TimeoutError as toe: if log: if log_ctx: - log_ctx = f'\n{textwrap.indent(log_ctx, " ")}\n' + log_ctx_str = log_ctx.decode('utf-8', errors='replace') + log_ctx = f'\n{textwrap.indent(log_ctx_str, " ")}\n' sys.stderr.write( f'Timed out after {timeout_sec} seconds ' + f'waiting for query {sql_query!r}. ' + @@ -424,6 +424,26 @@ def check_table(): f'\nQuestDB log:\n') self.print_log() raise toe + + def show_tables(self): + """Return a list of tables in the database.""" + sql_query = "show tables" + try: + resp = self.http_sql_query(sql_query) + return [row[0] for row in resp['dataset']] + except QueryError as qe: + raise qe + + def drop_table(self, table_name): + self.http_sql_query(f"drop table '{table_name}'") + + def drop_all_tables(self): + """Drop all tables in the database.""" + all_tables = self.show_tables() + # if all_tables: + # print(f'Dropping {len(all_tables)} tables: {all_tables!r}') + for table_name in all_tables: + self.drop_table(table_name) def __enter__(self): self.start() @@ -493,7 +513,7 @@ def check_started(): self.listen_port = retry( check_started, timeout_sec=180, # Longer to include time to compile. - msg='Timed out waiting for `tls_proxy` to start.',) + msg='Timed out waiting for `tls_proxy` to start.', ) def connect_to_listening_port(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -518,4 +538,3 @@ def stop(self): if self._log_file: self._log_file.close() self._log_file = None - diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 5d4eca85..0bc73da9 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -37,15 +37,17 @@ """ - -from ast import arg import sys + +import numpy + sys.dont_write_bytecode = True import pathlib import ctypes import os from datetime import datetime +from functools import total_ordering from enum import Enum from ctypes import ( @@ -61,58 +63,107 @@ c_void_p, c_ssize_t) -from typing import Optional, Tuple, Union +from typing import Optional, Union class c_line_sender(ctypes.Structure): pass + class c_line_sender_buffer(ctypes.Structure): pass + c_line_sender_protocol = ctypes.c_int + class Protocol(Enum): TCP = (c_line_sender_protocol(0), 'tcp') TCPS = (c_line_sender_protocol(1), 'tcps') HTTP = (c_line_sender_protocol(2), 'http') HTTPS = (c_line_sender_protocol(3), 'https') + c_line_sender_ca = ctypes.c_int + class CertificateAuthority(Enum): WEBPKI_ROOTS = (c_line_sender_ca(0), 'webpki_roots') OS_ROOTS = (c_line_sender_ca(1), 'os_roots') WEBPKI_AND_OS_ROOTS = (c_line_sender_ca(2), 'webpki_and_os_roots') PEM_FILE = (c_line_sender_ca(3), 'pem_file') + +c_protocol_version = ctypes.c_int + + +@total_ordering +class ProtocolVersion(Enum): + V1 = (c_protocol_version(1), '1') + V2 = (c_protocol_version(2), '2') + + @classmethod + def from_int(cls, value: c_protocol_version): + for member in cls: + if member.value[0].value == value: + return member + raise ValueError(f"invalid protocol version: {value}") + + def __lt__(self, other): + if not isinstance(other, ProtocolVersion): + return NotImplemented + return self.value[0].value < other.value[0].value + + def __eq__(self, other): + if not isinstance(other, ProtocolVersion): + return NotImplemented + return self.value[0].value == other.value[0].value + + class c_line_sender_opts(ctypes.Structure): pass + class c_line_sender_error(ctypes.Structure): pass + c_size_t_p = ctypes.POINTER(c_size_t) +c_ssize_t_p = ctypes.POINTER(c_ssize_t) c_line_sender_p = ctypes.POINTER(c_line_sender) c_line_sender_buffer_p = ctypes.POINTER(c_line_sender_buffer) c_line_sender_opts_p = ctypes.POINTER(c_line_sender_opts) c_line_sender_error_p = ctypes.POINTER(c_line_sender_error) c_line_sender_error_p_p = ctypes.POINTER(c_line_sender_error_p) +c_uint8_p = ctypes.POINTER(c_uint8) + + class c_line_sender_utf8(ctypes.Structure): _fields_ = [("len", c_size_t), ("buf", c_char_p)] + + c_line_sender_utf8_p = ctypes.POINTER(c_line_sender_utf8) + + class c_line_sender_table_name(ctypes.Structure): _fields_ = [("len", c_size_t), ("buf", c_char_p)] + + class line_sender_buffer_view(ctypes.Structure): _fields_ = [("len", c_size_t), - ("buf", ctypes.POINTER(c_uint8))] + ("buf", c_uint8_p)] + c_line_sender_table_name_p = ctypes.POINTER(c_line_sender_table_name) + + class c_line_sender_column_name(ctypes.Structure): _fields_ = [("len", c_size_t), ("buf", c_char_p)] + + c_line_sender_column_name_p = ctypes.POINTER(c_line_sender_column_name) @@ -129,7 +180,7 @@ def _setup_cdll(): 'darwin': 'dylib', 'win32': 'dll'}[sys.platform] dll_path = next( - build_dir.glob(f'**/*questdb_client*.{dll_ext}')) + build_dir.glob(f'**/*questdb_client*.{dll_ext}')) dll = ctypes.CDLL(str(dll_path)) @@ -175,6 +226,7 @@ def set_sig(fn, restype, *argtypes): set_sig( dll.line_sender_buffer_with_max_name_len, c_line_sender_buffer_p, + c_protocol_version, c_size_t) set_sig( dll.line_sender_buffer_free, @@ -237,6 +289,17 @@ def set_sig(fn, restype, *argtypes): c_line_sender_column_name, c_line_sender_utf8, c_line_sender_error_p_p) + set_sig( + dll.line_sender_buffer_column_f64_arr_byte_strides, + c_bool, + c_line_sender_buffer_p, + c_line_sender_column_name, + c_size_t, + c_size_t_p, + c_ssize_t_p, + c_uint8_p, + c_size_t, + c_line_sender_error_p_p) set_sig( dll.line_sender_buffer_column_ts_nanos, c_bool, @@ -268,6 +331,11 @@ def set_sig(fn, restype, *argtypes): c_bool, c_line_sender_buffer_p, c_line_sender_error_p_p) + set_sig( + dll.line_sender_buffer_check_can_flush, + c_bool, + c_line_sender_buffer_p, + c_line_sender_error_p_p) set_sig( dll.line_sender_opts_new, c_line_sender_opts_p, @@ -316,6 +384,12 @@ def set_sig(fn, restype, *argtypes): c_line_sender_opts_p, c_line_sender_utf8, c_line_sender_error_p_p) + set_sig( + dll.line_sender_opts_protocol_version, + c_bool, + c_line_sender_opts_p, + c_protocol_version, + c_line_sender_error_p_p) set_sig( dll.line_sender_opts_auth_timeout, c_bool, @@ -413,6 +487,14 @@ def set_sig(fn, restype, *argtypes): c_line_sender_buffer_p, c_bool, c_line_sender_error_p_p) + set_sig( + dll.line_sender_get_protocol_version, + c_protocol_version, + c_line_sender_p) + set_sig( + dll.line_sender_get_max_name_len, + c_size_t, + c_line_sender_p) return dll @@ -513,6 +595,7 @@ def __init__(self, host, port, protocol=Protocol.TCP): def __getattr__(self, name: str): fn = getattr(_DLL, 'line_sender_opts_' + name) + def wrapper(*args): mapped_args = [ (_utf8(arg) if isinstance(arg, str) else arg) @@ -521,6 +604,7 @@ def wrapper(*args): return _error_wrapped_call(fn, self.impl, *mapped_args) else: return fn(self.impl, *mapped_args) + return wrapper def __del__(self): @@ -533,8 +617,9 @@ def __init__(self, micros: int): class Buffer: - def __init__(self, init_buf_size=65536, max_name_len=127): + def __init__(self, protocol_version: ProtocolVersion, init_buf_size=65536, max_name_len=127, ): self._impl = _DLL.line_sender_buffer_with_max_name_len( + protocol_version.value[0], c_size_t(max_name_len)) _DLL.line_sender_buffer_reserve(self._impl, c_size_t(init_buf_size)) @@ -542,7 +627,7 @@ def __len__(self): return _DLL.line_sender_buffer_size(self._impl) def peek(self) -> str: - # This is a hacky way of doing it because it copies the whole buffer. + #  This is a hacky way of doing it because it copies the whole buffer. # Instead the `buffer` should be made to support the buffer protocol: # https://docs.python.org/3/c-api/buffer.html # This way we would not need to `bytes(..)` the object to keep it alive. @@ -627,6 +712,34 @@ def column( '`bool`, `int`, `float` or `str`.') return self + def column_f64_arr(self, name: str, + rank: int, + shape: tuple[int, ...], + strides: tuple[int, ...], + data: c_void_p, + length: int): + def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POINTER: + arr_type = c_type * len(tpl) + try: + return arr_type(*[c_type(v) for v in tpl]) + except OverflowError as e: + raise ValueError( + f"{name} value exceeds {c_type.__name__} range" + ) from e + + c_shape = _convert_tuple(shape, c_size_t, "shape") + c_strides = _convert_tuple(strides, c_ssize_t, "strides") + _error_wrapped_call( + _DLL.line_sender_buffer_column_f64_arr_byte_strides, + self._impl, + _column_name(name), + c_size_t(rank), + c_shape, + c_strides, + ctypes.cast(data, c_uint8_p), + c_size_t(length) + ) + def at_now(self): _error_wrapped_call( _DLL.line_sender_buffer_at_now, @@ -657,8 +770,8 @@ def _map_value(key, value): return (value, 'on' if value else 'unsafe_off') else: return (value, 'on' if value else 'off') - elif isinstance(value, CertificateAuthority): - return value.value # a tuple of `(c_line_sender_ca, str)` + elif isinstance(value, (CertificateAuthority, ProtocolVersion)): + return value.value # a tuple of `(c enum value, str)` else: return (value, f'{value}') @@ -671,7 +784,11 @@ def __init__( host: str, port: Union[str, int], **kwargs): - + + if protocol in (Protocol.TCPS, Protocol.HTTPS): + if host == '127.0.0.1': + host = 'localhost' # for TLS connections we need a hostname + self._build_mode = build_mode self._impl = None self._conf = [ @@ -679,7 +796,6 @@ def __init__( '::', f'addr={host}:{port};'] self._opts = None - self._buffer = Buffer() opts = _Opts(host, port, protocol) for key, value in kwargs.items(): # Build the config string param pair. @@ -700,9 +816,13 @@ def connect(self): if self._impl: raise SenderError('Already connected') if self._build_mode == BuildMode.CONF: - self._impl = _error_wrapped_call( - _DLL.line_sender_from_conf, - _utf8(self._conf)) + try: + self._impl = _error_wrapped_call( + _DLL.line_sender_from_conf, + _utf8(self._conf)) + except SenderError as e: + raise SenderError( + f'Failed to connect to QuestDB with conf `{self._conf}`: {e}') from e elif self._build_mode == BuildMode.ENV: env_var = 'QDB_CLIENT_CONF' os.environ[env_var] = self._conf @@ -716,12 +836,26 @@ def connect(self): def __enter__(self): self.connect() + self._buffer = Buffer( + protocol_version=self.protocol_version, + max_name_len=self.max_name_len) return self def _check_connected(self): if not self._impl: raise SenderError('Not connected.') + @property + def protocol_version(self): + self._check_connected() + return ProtocolVersion.from_int( + _DLL.line_sender_get_protocol_version(self._impl)) + + @property + def max_name_len(self): + self._check_connected() + return _DLL.line_sender_get_max_name_len(self._impl) + def table(self, table: str): self._buffer.table(table) return self @@ -736,13 +870,21 @@ def column( self._buffer.column(name, value) return self + def column_f64_arr( + self, name: str, + array: numpy.ndarray): + if array.dtype != numpy.float64: + raise ValueError('expect float64 array') + self._buffer.column_f64_arr(name, array.ndim, array.shape, array.strides, array.ctypes.data, array.nbytes) + return self + def at_now(self): self._buffer.at_now() def at(self, timestamp: int): self._buffer.at(timestamp) - def flush(self, buffer: Optional[Buffer]=None, clear=True, transactional=None): + def flush(self, buffer: Optional[Buffer] = None, clear=True, transactional=None): if (buffer is None) and not clear: raise ValueError( 'Clear flag must be True when using internal buffer') diff --git a/system_test/test.py b/system_test/test.py index c28ccd6d..e7d60dbc 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -25,14 +25,16 @@ ################################################################################ import sys + sys.dont_write_bytecode = True import os - import pathlib import math import datetime import argparse import unittest +import itertools +import numpy as np import time import questdb_line_sender as qls import uuid @@ -47,11 +49,13 @@ import subprocess from collections import namedtuple - QDB_FIXTURE: QuestDbFixture = None TLS_PROXY_FIXTURE: TlsProxyFixture = None BUILD_MODE = None +# The first QuestDB version that supports array types. +FIRST_ARRAYS_RELEASE = (8, 3, 1) + def retry_check_table(*args, **kwargs): return QDB_FIXTURE.retry_check_table(*args, **kwargs) @@ -72,7 +76,6 @@ def ns_to_qdb_date(at_ts_ns): token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", token_y="9iYksF4L5mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") - # Bad malformed key AUTH_MALFORMED1 = dict( username="testUser3", @@ -80,7 +83,6 @@ def ns_to_qdb_date(at_ts_ns): token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", token_y="9iYksF4L6mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") - # Another malformed key where the keys invalid base 64. AUTH_MALFORMED2 = dict( username="testUser4", @@ -88,7 +90,6 @@ def ns_to_qdb_date(at_ts_ns): token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk5XEgak", token_y="9iYksF4L6mfmArupv0CMoyVAWjQ4gNIou5noG8") - # All the keys are valid, but the username is wrong. AUTH_MALFORMED3 = dict( username="wrongUser", @@ -100,13 +101,29 @@ def ns_to_qdb_date(at_ts_ns): class TestSender(unittest.TestCase): def _mk_linesender(self): # N.B.: We never connect with TLS here. - auth = AUTH if QDB_FIXTURE.auth else {} + kwargs = AUTH if QDB_FIXTURE.auth else {} + if QDB_FIXTURE.protocol_version: + kwargs["protocol_version"] = QDB_FIXTURE.protocol_version return qls.Sender( BUILD_MODE, qls.Protocol.HTTP if QDB_FIXTURE.http else qls.Protocol.TCP, QDB_FIXTURE.host, QDB_FIXTURE.http_server_port if QDB_FIXTURE.http else QDB_FIXTURE.line_tcp_port, - **auth) + **kwargs) + + @property + def expected_protocol_version(self) -> qls.ProtocolVersion: + """The protocol version that we expect to be handling.""" + if QDB_FIXTURE.protocol_version is None: + if not QDB_FIXTURE.http: + return qls.ProtocolVersion.V1 + + if QDB_FIXTURE.version >= FIRST_ARRAYS_RELEASE: + return qls.ProtocolVersion.V2 + + return qls.ProtocolVersion.V1 + + return QDB_FIXTURE.protocol_version def _expect_eventual_disconnect(self, sender): with self.assertRaisesRegex( @@ -115,9 +132,9 @@ def _expect_eventual_disconnect(self, sender): for _ in range(1000): time.sleep(0.1) (sender - .table(table_name) - .symbol('s1', 'v1') - .at_now()) + .table(table_name) + .symbol('s1', 'v1') + .at_now()) sender.flush() def test_insert_three_rows(self): @@ -126,13 +143,13 @@ def test_insert_three_rows(self): with self._mk_linesender() as sender: for _ in range(3): (sender - .table(table_name) - .symbol('name_a', 'val_a') - .column('name_b', True) - .column('name_c', 42) - .column('name_d', 2.5) - .column('name_e', 'val_b') - .at_now()) + .table(table_name) + .symbol('name_a', 'val_a') + .column('name_b', True) + .column('name_c', 42) + .column('name_d', 2.5) + .column('name_e', 'val_b') + .at_now()) pending = sender.buffer.peek() sender.flush() @@ -161,12 +178,12 @@ def test_repeated_symbol_and_column_names(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('a', 'A') - .symbol('a', 'B') - .column('b', False) - .column('b', 'C') - .at_now()) + .table(table_name) + .symbol('a', 'A') + .symbol('a', 'B') + .column('b', False) + .column('b', 'C') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -188,10 +205,10 @@ def test_same_symbol_and_col_name(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('a', 'A') - .column('a', 'B') - .at_now()) + .table(table_name) + .symbol('a', 'A') + .column('a', 'B') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -209,9 +226,9 @@ def _test_single_symbol_impl(self, sender): pending = None with sender: (sender - .table(table_name) - .symbol('a', 'A') - .at_now()) + .table(table_name) + .symbol('a', 'A') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -232,10 +249,10 @@ def test_two_columns(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .column('a', 'A') - .column('b', 'B') - .at_now()) + .table(table_name) + .column('a', 'A') + .column('b', 'B') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -254,13 +271,13 @@ def test_mismatched_types_across_rows(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .column('a', 1) # LONG - .at_now()) + .table(table_name) + .column('a', 1) # LONG + .at_now()) (sender - .table(table_name) - .symbol('a', 'B') # SYMBOL - .at_now()) + .table(table_name) + .symbol('a', 'B') # SYMBOL + .at_now()) pending = sender.buffer.peek() @@ -278,7 +295,7 @@ def test_mismatched_types_across_rows(self): # If HTTP, the error should cause the whole batch to be ignored. # We assert that the table is empty. with self.assertRaises(TimeoutError): - retry_check_table(table_name, timeout_sec=1, log=False) + retry_check_table(table_name, timeout_sec=0.25, log=False) else: # We only ever get the first row back. resp = retry_check_table(table_name, log_ctx=pending) @@ -293,7 +310,7 @@ def test_mismatched_types_across_rows(self): # The second one is dropped and will not appear in results. with self.assertRaises(TimeoutError): - retry_check_table(table_name, min_rows=2, timeout_sec=1, log=False) + retry_check_table(table_name, min_rows=2, timeout_sec=0.25, log=False) def test_at(self): if QDB_FIXTURE.version <= (6, 0, 7, 1): @@ -304,9 +321,9 @@ def test_at(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('a', 'A') - .at(at_ts_ns)) + .table(table_name) + .symbol('a', 'A') + .at(at_ts_ns)) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) exp_dataset = [['A', ns_to_qdb_date(at_ts_ns)]] @@ -322,9 +339,9 @@ def test_neg_at(self): with self._mk_linesender() as sender: with self.assertRaisesRegex(qls.SenderError, r'.*Timestamp .* is negative.*'): (sender - .table(table_name) - .symbol('a', 'A') - .at(at_ts_ns)) + .table(table_name) + .symbol('a', 'A') + .at(at_ts_ns)) def test_timestamp_col(self): if QDB_FIXTURE.version <= (6, 0, 7, 1): @@ -334,13 +351,13 @@ def test_timestamp_col(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .column('a', qls.TimestampMicros(-1000000)) - .at_now()) + .table(table_name) + .column('a', qls.TimestampMicros(-1000000)) + .at_now()) (sender - .table(table_name) - .column('a', qls.TimestampMicros(1000000)) - .at_now()) + .table(table_name) + .column('a', qls.TimestampMicros(1000000)) + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -353,16 +370,15 @@ def test_timestamp_col(self): scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) - def test_underscores(self): table_name = f'_{uuid.uuid4().hex}_' pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('_a_b_c_', 'A') - .column('_d_e_f_', True) - .at_now()) + .table(table_name) + .symbol('_a_b_c_', 'A') + .column('_d_e_f_', True) + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -422,16 +438,16 @@ def test_floats(self): 1.23456789012, 1000000000000000000000000.0, -1000000000000000000000000.0, - float("nan"), # Converted to `None`. - float("inf"), # Converted to `None`. + float("nan"), # Converted to `None`. + float("inf"), # Converted to `None`. float("-inf")] # Converted to `None`. - # These values below do not round-trip properly: QuestDB limitation. - # 1.2345678901234567, - # 2.2250738585072014e-308, - # -2.2250738585072014e-308, - # 1.7976931348623157e+308, - # -1.7976931348623157e+308] + # These values below do not round-trip properly: QuestDB limitation. + # 1.2345678901234567, + # 2.2250738585072014e-308, + # -2.2250738585072014e-308, + # 1.7976931348623157e+308, + # -1.7976931348623157e+308] table_name = uuid.uuid4().hex pending = None with self._mk_linesender() as sender: @@ -469,9 +485,9 @@ def test_timestamp_column(self): ts = qls.TimestampMicros(3600000000) # One hour past epoch. with self._mk_linesender() as sender: (sender - .table(table_name) - .column('ts1', ts) - .at_now()) + .table(table_name) + .column('ts1', ts) + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -483,6 +499,214 @@ def test_timestamp_column(self): scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) + def test_f64_arr_column(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + array1 = np.array( + [ + [[1.1, 2.2], [3.3, 4.4]], + [[5.5, 6.6], [7.7, 8.8]] + ], + dtype=np.float64 + ) + array2 = array1.T + array3 = array1[::-1, ::-1] + + with self._mk_linesender() as sender: + (sender + .table(table_name) + .column_f64_arr('f64_arr1', array1) + .column_f64_arr('f64_arr2', array2) + .column_f64_arr('f64_arr3', array3) + .at_now()) + + resp = retry_check_table(table_name) + exp_columns = [{'dim': 3, 'elemType': 'DOUBLE', 'name': 'f64_arr1', 'type': 'ARRAY'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'f64_arr2', 'type': 'ARRAY'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'f64_arr3', 'type': 'ARRAY'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(resp['columns'], exp_columns) + expected_data = [[[[[1.1, 2.2], [3.3, 4.4]], [[5.5, 6.6], [7.7, 8.8]]], + [[[1.1, 5.5], [3.3, 7.7]], [[2.2, 6.6], [4.4, 8.8]]], + [[[7.7, 8.8], [5.5, 6.6]], [[3.3, 4.4], [1.1, 2.2]]]]] + scrubbed_data = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_data, expected_data) + + def test_f64_arr_empty(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + empty_array = np.array([], dtype=np.float64).reshape(0, 0, 0) + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('empty', empty_array) + .at_now()) + + resp = retry_check_table(table_name) + exp_columns = [{'dim': 3, 'elemType': 'DOUBLE', 'name': 'empty', 'type': 'ARRAY'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(exp_columns, resp['columns']) + self.assertEqual(resp['dataset'][0][0], []) + + def test_f64_arr_non_contiguous(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + array = np.array([[1.1, 2.2], [3.3, 4.4]], dtype=np.float64)[:, ::2] + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('non_contiguous', array) + .at_now()) + + resp = retry_check_table(table_name) + exp_columns = [{'dim': 2, 'elemType': 'DOUBLE', 'name': 'non_contiguous', 'type': 'ARRAY'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(exp_columns, resp['columns']) + self.assertEqual(resp['dataset'][0][0], [[1.1], [3.3]]) + + def test_f64_arr_zero_dimensional(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + array = np.array(42.0, dtype=np.float64) + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('scalar', array) + .at_now()) + except qls.SenderError as e: + self.assertIn('Zero-dimensional arrays are not supported', str(e)) + + def test_f64_arr_wrong_datatype(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + array = np.array([1, 2], dtype=np.int32) + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('wrong', array) + .at_now()) + except ValueError as e: + self.assertIn('expect float64 array', str(e)) + + def test_f64_arr_mix_dims(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + array_2d = np.array([[1.1, 2.2], [3.3, 4.4]], dtype=np.float64) + array_1d = np.array([1.1], dtype=np.float64) + table_name = uuid.uuid4().hex + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('array', array_2d) + .at_now() + ) + (sender.table(table_name) + .column_f64_arr('array', array_1d) + .at_now() + ) + except qls.SenderError as e: + self.assertIn('cast error from protocol type: DOUBLE[] to column type: DOUBLE[][]', str(e)) + + def test_f64_arr_dims_length_overflow(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + array = np.empty((1 << 29, 0), dtype=np.float64) + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('array', array) + .at_now()) + except qls.SenderError as e: + self.assertIn('dimension length out of range', str(e)) + + def test_f64_arr_max_dims(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + dims = (1,) * 33 + array = np.empty(dims, dtype=np.float64) + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('array', array) + .at_now()) + except qls.SenderError as e: + self.assertIn('Array dimension mismatch: expected at most 32 dimensions, but got 33', str(e)) + + def test_protocol_version_v1(self): + if self.expected_protocol_version >= qls.ProtocolVersion.V2: + self.skipTest('we are only validating the older protocol here') + if QDB_FIXTURE.version <= (6, 1, 2): + self.skipTest('Float issues support') + numbers = [ + 0.0, + -0.0, + 1.0, + -1.0] + + table_name = uuid.uuid4().hex + pending = None + with self._mk_linesender() as sender: + for num in numbers: + sender.table(table_name) + sender.column('n', num) + sender.at_now() + pending = sender.buffer.peek() + + resp = retry_check_table( + table_name, + min_rows=len(numbers), + log_ctx=pending) + exp_columns = [ + {'name': 'n', 'type': 'DOUBLE'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(resp['columns'], exp_columns) + + def massage(num): + if math.isnan(num) or math.isinf(num): + return None + elif num == -0.0: + return 0.0 + else: + return num + + # Comparison excludes timestamp column. + exp_dataset = [[massage(num)] for num in numbers] + scrubbed_dataset = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_dataset, exp_dataset) + + def test_protocol_version_v1_array_unsupported(self): + if self.expected_protocol_version >= qls.ProtocolVersion.V2: + self.skipTest('communicating over a newer protocl that DOES support arrays') + + array1 = np.array( + [ + [[1.1, 2.2], [3.3, 4.4]], + [[5.5, 6.6], [7.7, 8.8]] + ], + dtype=np.float64 + ) + table_name = uuid.uuid4().hex + try: + with self._mk_linesender() as sender: + sender.table(table_name) + sender.column_f64_arr('f64_arr1', array1) + sender.at_now() + except qls.SenderError as e: + self.assertIn('Protocol version v1 does not support array datatype', str(e)) + def _test_example(self, bin_name, table_name, tls=False): if BUILD_MODE != qls.BuildMode.API: self.skipTest('BuildMode.API-only test') @@ -501,7 +725,9 @@ def _test_example(self, bin_name, table_name, tls=False): ca_path = proj.tls_certs_dir / 'server_rootCA.pem' args.append(str(ca_path)) port = TLS_PROXY_FIXTURE.listen_port - args.extend(['localhost', str(port)]) + args.extend(['localhost', str(port)]) + else: + args.extend(['127.0.0.1', str(port)]) subprocess.check_call(args, cwd=bin_path.parent) # Check inserted data. @@ -514,7 +740,11 @@ def _test_example(self, bin_name, table_name, tls=False): {'name': 'timestamp', 'type': 'TIMESTAMP'}] self.assertEqual(resp['columns'], exp_columns) - exp_dataset = [['ETH-USD', 'sell', 2615.54, 0.00044]] # Comparison excludes timestamp column. + exp_dataset = [['ETH-USD', + 'sell', + 2615.54, + 0.00044]] + # Comparison excludes timestamp column. scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) @@ -544,6 +774,54 @@ def test_cpp_tls_example(self): 'cpp_trades_tls_ca', tls=True) + def test_cpp_array_example(self): + self._test_array_example( + 'line_sender_cpp_example_array_byte_strides', + 'cpp_market_orders_byte_strides', ) + self._test_array_example( + 'line_sender_cpp_example_array_elem_strides', + 'cpp_market_orders_elem_strides', ) + + def test_c_array_example(self): + self._test_array_example( + 'line_sender_c_example_array_byte_strides', + 'market_orders_byte_strides', ) + self._test_array_example( + 'line_sender_c_example_array_elem_strides', + 'market_orders_elem_strides', ) + + def _test_array_example(self, bin_name, table_name): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + if QDB_FIXTURE.http: + self.skipTest('TCP-only test') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + if QDB_FIXTURE.auth: + self.skipTest('auth') + + proj = Project() + ext = '.exe' if sys.platform == 'win32' else '' + try: + bin_path = next(proj.build_dir.glob(f'**/{bin_name}{ext}')) + except StopIteration: + raise RuntimeError(f'Could not find {bin_name}{ext} in {proj.build_dir}') + port = QDB_FIXTURE.line_tcp_port + args = [str(bin_path)] + args.extend(['127.0.0.1', str(port)]) + subprocess.check_call(args, cwd=bin_path.parent) + resp = retry_check_table(table_name) + exp_columns = [ + {'name': 'symbol', 'type': 'SYMBOL'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'order_book', 'type': 'ARRAY'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(resp['columns'], exp_columns) + exp_dataset = [['BTC-USD', + [[[48123.5, 2.4], [48124.0, 1.8], [48124.5, 0.9]], + [[48122.5, 3.1], [48122.0, 2.7], [48121.5, 4.3]]]]] + scrubbed_dataset = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_dataset, exp_dataset) + def test_opposite_auth(self): """ We simulate incorrectly connecting either: @@ -552,6 +830,8 @@ def test_opposite_auth(self): """ if QDB_FIXTURE.http: self.skipTest('TCP-only test') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') auth = {} if QDB_FIXTURE.auth else AUTH sender = qls.Sender( BUILD_MODE, @@ -570,9 +850,9 @@ def test_opposite_auth(self): # The sending the first line will not fail. (sender - .table(table_name) - .symbol('s1', 'v1') - .at_now()) + .table(table_name) + .symbol('s1', 'v1') + .at_now()) sender.flush() self._expect_eventual_disconnect(sender) @@ -584,6 +864,9 @@ def test_unrecognized_auth(self): if not QDB_FIXTURE.auth: self.skipTest('No auth') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + sender = qls.Sender( BUILD_MODE, qls.Protocol.TCP, @@ -601,6 +884,9 @@ def test_malformed_auth1(self): if not QDB_FIXTURE.auth: self.skipTest('No auth') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + sender = qls.Sender( BUILD_MODE, qls.Protocol.TCP, @@ -620,6 +906,9 @@ def test_malformed_auth2(self): if not QDB_FIXTURE.auth: self.skipTest('No auth') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + sender = qls.Sender( BUILD_MODE, qls.Protocol.TCP, @@ -639,6 +928,9 @@ def test_malformed_auth3(self): if not QDB_FIXTURE.auth: self.skipTest('No auth') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + sender = qls.Sender( BUILD_MODE, qls.Protocol.TCP, @@ -650,6 +942,10 @@ def test_malformed_auth3(self): self._expect_eventual_disconnect(sender) def test_tls_insecure_skip_verify(self): + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + if QDB_FIXTURE.protocol_version != sorted(list(qls.ProtocolVersion))[-1]: + self.skipTest('Skipping tls test for non-latest protocol version') protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( @@ -662,8 +958,12 @@ def test_tls_insecure_skip_verify(self): self._test_single_symbol_impl(sender) def test_tls_roots(self): + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + if QDB_FIXTURE.protocol_version != sorted(list(qls.ProtocolVersion))[-1]: + self.skipTest('Skipping tls test for non-latest protocol version') protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS - auth = auth=AUTH if QDB_FIXTURE.auth else {} + auth = auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( BUILD_MODE, protocol, @@ -674,12 +974,16 @@ def test_tls_roots(self): self._test_single_symbol_impl(sender) def _test_tls_ca(self, tls_ca): + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + if QDB_FIXTURE.protocol_version != sorted(list(qls.ProtocolVersion))[-1]: + self.skipTest('Skipping tls test for non-latest protocol version') protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS prev_ssl_cert_file = os.environ.get('SSL_CERT_FILE') try: os.environ['SSL_CERT_FILE'] = str( Project().tls_certs_dir / 'server_rootCA.pem') - auth = auth=AUTH if QDB_FIXTURE.auth else {} + auth = auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( BUILD_MODE, protocol, @@ -722,14 +1026,14 @@ def test_http_transactions(self): self.assertIn('error in line 3', str(e)) with self.assertRaises(TimeoutError): - retry_check_table(table_name, timeout_sec=1, log=False) + retry_check_table(table_name, timeout_sec=0.25, log=False) def test_tcp_transactions(self): if QDB_FIXTURE.http: self.skipTest('TCP-only test') if QDB_FIXTURE.version <= (7, 3, 7): self.skipTest('No ILP/HTTP support') - buf = qls.Buffer() + buf = qls.Buffer(self.expected_protocol_version) buf.table('t1').column('c1', 'v1').at(time.time_ns()) with self.assertRaisesRegex(qls.SenderError, r'.*Transactional .* not supported.*'): with self._mk_linesender() as sender: @@ -764,6 +1068,10 @@ def parse_args(): '--unittest-help', action='store_true', help='Show unittest --help') + run_p.add_argument( + '--profile', + action='store_true', + help='Run with cProfile') version_g = run_p.add_mutually_exclusive_group() version_g.add_argument( '--last-n', @@ -803,14 +1111,22 @@ def run_with_existing(args): global QDB_FIXTURE MockFixture = namedtuple( 'MockFixture', - ('host', 'line_tcp_port', 'http_server_port', 'version', 'http')) + ( + 'host', + 'line_tcp_port', + 'http_server_port', + 'version', + 'http', + 'auth', + 'protocol_version')) host, line_tcp_port, http_server_port = args.existing.split(':') QDB_FIXTURE = MockFixture( host, int(line_tcp_port), int(http_server_port), (999, 999, 999), - True) + True, + False) unittest.main() @@ -831,11 +1147,11 @@ def iter_versions(args): if versions_args: versions = { version: ( - 'https://github.com/questdb/questdb/releases/download/' + - version + - '/questdb-' + - version + - '-no-jre-bin.tar.gz') + 'https://github.com/questdb/questdb/releases/download/' + + version + + '/questdb-' + + version + + '-no-jre-bin.tar.gz') for version in versions_args} else: last_n = getattr(args, 'last_n', None) or 1 @@ -852,38 +1168,47 @@ def run_with_fixtures(args): global QDB_FIXTURE global TLS_PROXY_FIXTURE global BUILD_MODE - last_version = None - for questdb_dir in iter_versions(args): - for auth in (False, True): - for http in (False, True): - for build_mode in list(qls.BuildMode): - print(f'Running tests [questdb_dir={questdb_dir}, auth={auth}, http={http}, build_mode={build_mode}]') - if http and last_version <= (7, 3, 7): - print('Skipping ILP/HTTP tests for versions <= 7.3.7') - continue - if http and auth: - print('Skipping auth for ILP/HTTP tests') - continue - QDB_FIXTURE = QuestDbFixture(questdb_dir, auth=auth, http=http) - TLS_PROXY_FIXTURE = None - BUILD_MODE = build_mode - try: - QDB_FIXTURE.start() - # Read the version _after_ a first start so it can rely - # on the live one from the `select build` query. - last_version = QDB_FIXTURE.version - port_to_proxy = QDB_FIXTURE.http_server_port \ - if http else QDB_FIXTURE.line_tcp_port - TLS_PROXY_FIXTURE = TlsProxyFixture(port_to_proxy) - TLS_PROXY_FIXTURE.start() - - test_prog = unittest.TestProgram(exit=False) - if not test_prog.result.wasSuccessful(): - sys.exit(1) - finally: - if TLS_PROXY_FIXTURE: - TLS_PROXY_FIXTURE.stop() - QDB_FIXTURE.stop() + + latest_protocol = sorted(list(qls.ProtocolVersion))[-1] + for questdb_dir, auth in itertools.product(iter_versions(args), (False, True)): + QDB_FIXTURE = QuestDbFixture( + questdb_dir, + auth=auth) + TLS_PROXY_FIXTURE = None + try: + print(f'>>>> STARTING {questdb_dir} [auth={auth}] <<<<') + QDB_FIXTURE.start() + for http, protocol_version, build_mode in itertools.product( + (False, True), # http + [None] + list(qls.ProtocolVersion), # None is for `auto` + list(qls.BuildMode)): + if (build_mode in (qls.BuildMode.API, qls.BuildMode.ENV)) and (protocol_version != latest_protocol): + continue + if http and auth: + continue + if auth and (protocol_version != latest_protocol): + continue + print( + f'Running tests [auth={auth}, http={http}, build_mode={build_mode}, protocol_version={protocol_version}]') + # Read the version _after_ a first start so it can rely + # on the live one from the `select build` query. + BUILD_MODE = build_mode + QDB_FIXTURE.http = http + QDB_FIXTURE.protocol_version = protocol_version + port_to_proxy = QDB_FIXTURE.http_server_port \ + if http else QDB_FIXTURE.line_tcp_port + TLS_PROXY_FIXTURE = TlsProxyFixture(port_to_proxy) + TLS_PROXY_FIXTURE.start() + try: + QDB_FIXTURE.drop_all_tables() + test_prog = unittest.TestProgram(exit=False) + if not test_prog.result.wasSuccessful(): + sys.exit(1) + finally: + if TLS_PROXY_FIXTURE: + TLS_PROXY_FIXTURE.stop() + finally: + QDB_FIXTURE.stop() def run(args, show_help=False): @@ -904,6 +1229,12 @@ def main(): if args.command == 'list': list_releases(args) else: + profile = args.profile + if profile: + sys.argv.remove("--profile") + import cProfile + cProfile.run('main()', filename='profile.out') + return # Repackage args for unittest's own arg parser. sys.argv[:] = sys.argv[:1] + extra_args show_help = getattr(args, 'unittest_help', False)