diff --git a/.env b/.env index b9c7e856094f..3f866fc4c02f 100644 --- a/.env +++ b/.env @@ -84,7 +84,6 @@ R_ORG=rhub R_TAG=latest # Env vars for R builds -R_UPDATE_CLANG=false R_CUSTOM_CCACHE=false ARROW_R_DEV=TRUE R_PRUNE_DEPS=FALSE diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 1e91f6248788..04d974f641ec 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -124,7 +124,7 @@ jobs: run: archery docker push ubuntu-ruby macos: - name: ARM64 macOS 14 GLib & Ruby + name: ARM64 macOS GLib & Ruby runs-on: macos-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 60 diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp index 9fe9d9d1b319..f6e0d3064d59 100644 --- a/c_glib/arrow-glib/reader.cpp +++ b/c_glib/arrow-glib/reader.cpp @@ -668,6 +668,35 @@ garrow_record_batch_file_reader_read_record_batch(GArrowRecordBatchFileReader *r } } +/** + * garrow_record_batch_file_reader_get_metadata: + * @reader: A #GArrowRecordBatchFileReader. + * + * Returns: (nullable) (element-type utf8 utf8) (transfer full): + * The metadata in the footer. + * + * Since: 24.0.0 + */ +GHashTable * +garrow_record_batch_file_reader_get_metadata(GArrowRecordBatchFileReader *reader) +{ + auto arrow_reader = garrow_record_batch_file_reader_get_raw(reader); + auto arrow_metadata = arrow_reader->metadata(); + + if (!arrow_metadata) { + return nullptr; + } + + auto metadata = g_hash_table_new(g_str_hash, g_str_equal); + const auto n = arrow_metadata->size(); + for (int64_t i = 0; i < n; ++i) { + g_hash_table_insert(metadata, + const_cast(arrow_metadata->key(i).c_str()), + const_cast(arrow_metadata->value(i).c_str())); + } + return metadata; +} + struct GArrowFeatherFileReaderPrivate { std::shared_ptr feather_reader; diff --git a/c_glib/arrow-glib/reader.h b/c_glib/arrow-glib/reader.h index 5401aa3bb1fc..1e896fd09fd2 100644 --- a/c_glib/arrow-glib/reader.h +++ b/c_glib/arrow-glib/reader.h @@ -166,6 +166,10 @@ garrow_record_batch_file_reader_read_record_batch(GArrowRecordBatchFileReader *r guint i, GError **error); +GARROW_AVAILABLE_IN_24_0 +GHashTable * +garrow_record_batch_file_reader_get_metadata(GArrowRecordBatchFileReader *reader); + #define GARROW_TYPE_FEATHER_FILE_READER (garrow_feather_file_reader_get_type()) GARROW_AVAILABLE_IN_ALL G_DECLARE_DERIVABLE_TYPE(GArrowFeatherFileReader, diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp index 4228b6091072..0cbd88a769d4 100644 --- a/c_glib/arrow-glib/writer.cpp +++ b/c_glib/arrow-glib/writer.cpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #include #include @@ -288,16 +290,50 @@ GArrowRecordBatchFileWriter * garrow_record_batch_file_writer_new(GArrowOutputStream *sink, GArrowSchema *schema, GError **error) +{ + return garrow_record_batch_file_writer_new_full(sink, schema, nullptr, nullptr, error); +} + +/** + * garrow_record_batch_file_writer_new_full: + * @sink: The output of the writer. + * @schema: The schema of the writer. + * @options: (nullable): The options for serialization. + * @metadata: (nullable) (element-type utf8 utf8): The custom metadata in + * the footer. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (nullable): A newly created #GArrowRecordBatchFileWriter + * or %NULL on error. + * + * Since: 24.0.0 + */ +GArrowRecordBatchFileWriter * +garrow_record_batch_file_writer_new_full(GArrowOutputStream *sink, + GArrowSchema *schema, + GArrowWriteOptions *options, + GHashTable *metadata, + GError **error) { auto arrow_sink = garrow_output_stream_get_raw(sink); auto arrow_schema = garrow_schema_get_raw(schema); + arrow::ipc::IpcWriteOptions arrow_options = arrow::ipc::IpcWriteOptions::Defaults(); + if (options) { + arrow_options = *garrow_write_options_get_raw(options); + } + std::shared_ptr arrow_metadata; + if (metadata) { + arrow_metadata = garrow_internal_hash_table_to_metadata(metadata); + } + std::shared_ptr arrow_writer; - auto arrow_writer_result = arrow::ipc::MakeFileWriter(arrow_sink, arrow_schema); - if (garrow::check(error, arrow_writer_result, "[record-batch-file-writer][open]")) { + auto arrow_writer_result = + arrow::ipc::MakeFileWriter(arrow_sink, arrow_schema, arrow_options, arrow_metadata); + if (garrow::check(error, arrow_writer_result, "[record-batch-file-writer][new]")) { auto arrow_writer = *arrow_writer_result; return garrow_record_batch_file_writer_new_raw(&arrow_writer); } else { - return NULL; + return nullptr; } } diff --git a/c_glib/arrow-glib/writer.h b/c_glib/arrow-glib/writer.h index fc5fe0c2c738..e02da0e30d19 100644 --- a/c_glib/arrow-glib/writer.h +++ b/c_glib/arrow-glib/writer.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -94,6 +95,14 @@ garrow_record_batch_file_writer_new(GArrowOutputStream *sink, GArrowSchema *schema, GError **error); +GARROW_AVAILABLE_IN_24_0 +GArrowRecordBatchFileWriter * +garrow_record_batch_file_writer_new_full(GArrowOutputStream *sink, + GArrowSchema *schema, + GArrowWriteOptions *options, + GHashTable *metadata, + GError **error); + /** * GArrowCSVQuotingStyle: * @GARROW_CSV_QUOTING_STYLE_NEEDED: Only enclose values in quotes which need them. diff --git a/c_glib/test/test-file-writer.rb b/c_glib/test/test-file-writer.rb index 06c9dfa25c7f..41fd00cee4ee 100644 --- a/c_glib/test/test-file-writer.rb +++ b/c_glib/test/test-file-writer.rb @@ -88,4 +88,36 @@ def test_write_table input.close end end + + def test_footer_custom_metadata + tempfile = Tempfile.open("arrow-ipc-file-writer") + output = Arrow::FileOutputStream.new(tempfile.path, false) + + array = build_boolean_array([true, false, true]) + field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) + schema = Arrow::Schema.new([field]) + + options = Arrow::WriteOptions.new + metadata = {"key1" => "value1", "key2" => "value2"} + begin + file_writer = Arrow::RecordBatchFileWriter.new(output, + schema, + options, + metadata) + file_writer.close + assert do + file_writer.closed? + end + ensure + output.close + end + + input = Arrow::MemoryMappedInputStream.new(tempfile.path) + begin + file_reader = Arrow::RecordBatchFileReader.new(input) + assert_equal(metadata, file_reader.metadata) + ensure + input.close + end + end end diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index c0d5a69a94ec..da378eac4302 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -33,9 +33,6 @@ ENV R_PRUNE_DEPS=${r_prune_deps} ARG r_custom_ccache=false ENV R_CUSTOM_CCACHE=${r_custom_ccache} -ARG r_update_clang=false -ENV R_UPDATE_CLANG=${r_update_clang} - ARG tz="UTC" ENV TZ=${tz} diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index b42b444e0b5a..ddeb2acc3b7a 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -85,17 +85,6 @@ else $PACKAGE_MANAGER install -y rsync cmake curl fi -# Update clang version to latest available. -# This is only for rhub/clang20. If we change the base image from rhub/clang20, -# we need to update this part too. -if [ "$R_UPDATE_CLANG" = true ]; then - apt update -y --allow-releaseinfo-change # flag needed for when debian version changes - apt install -y gnupg - curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/llvm.gpg - echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-20 main" > /etc/apt/sources.list.d/llvm20.list - apt update -y --allow-releaseinfo-change # flag needed for when debian version changes - apt install -y clang-20 lld-20 -fi # Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786 Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}' diff --git a/ci/scripts/r_install_system_dependencies.sh b/ci/scripts/r_install_system_dependencies.sh index bc3f5a80d1e8..237e0e9408b3 100755 --- a/ci/scripts/r_install_system_dependencies.sh +++ b/ci/scripts/r_install_system_dependencies.sh @@ -35,17 +35,19 @@ else apt-get update fi -# Install curl and OpenSSL (technically, only needed for S3/GCS support, but -# installing the R curl package fails without it) +# Install curl, OpenSSL, and libuv +# - curl/OpenSSL: technically only needed for S3/GCS support, but +# installing the R curl package fails without it +# - libuv: required by the fs R package (no longer bundles libuv by default) case "$PACKAGE_MANAGER" in apt-get) - apt-get install -y libcurl4-openssl-dev libssl-dev + apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev ;; apk) - $PACKAGE_MANAGER add curl-dev openssl-dev + $PACKAGE_MANAGER add curl-dev openssl-dev libuv-dev ;; *) - $PACKAGE_MANAGER install -y libcurl-devel openssl-devel + $PACKAGE_MANAGER install -y libcurl-devel openssl-devel libuv-devel ;; esac diff --git a/ci/scripts/r_revdepcheck.sh b/ci/scripts/r_revdepcheck.sh index 9949c98d9b16..4205151655e2 100755 --- a/ci/scripts/r_revdepcheck.sh +++ b/ci/scripts/r_revdepcheck.sh @@ -36,6 +36,7 @@ apt install -y \ libbz2-dev \ libc-ares-dev \ libcurl4-openssl-dev \ + libuv1-dev \ libgflags-dev \ libgoogle-glog-dev \ liblz4-dev \ diff --git a/compose.yaml b/compose.yaml index c799059fe254..16b44c2edfd4 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1667,7 +1667,6 @@ services: tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} r_custom_ccache: ${R_CUSTOM_CCACHE} - r_update_clang: ${R_UPDATE_CLANG} shm_size: *shm-size environment: <<: [*common, *sccache] diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ea15bb706691..a77ed39eac81 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -211,17 +211,6 @@ else() set(MSVC_TOOLCHAIN FALSE) endif() -find_package(ClangTools) -find_package(InferTools) -if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" - OR CLANG_TIDY_FOUND - OR INFER_FOUND) - # Generate a Clang compile_commands.json "compilation database" file for use - # with various development tools, such as Vim's YouCompleteMe plugin. - # See http://clang.llvm.org/docs/JSONCompilationDatabase.html - set(CMAKE_EXPORT_COMPILE_COMMANDS 1) -endif() - # Needed for Gandiva. # Use the first Python installation on PATH, not the newest one set(Python3_FIND_STRATEGY "LOCATION") @@ -649,22 +638,6 @@ if(UNIX) VERBATIM) endif(UNIX) -# -# "make infer" target -# - -if(${INFER_FOUND}) - # runs infer capture - add_custom_target(infer ${BUILD_SUPPORT_DIR}/run-infer.sh ${INFER_BIN} - ${CMAKE_BINARY_DIR}/compile_commands.json 1) - # runs infer analyze - add_custom_target(infer-analyze ${BUILD_SUPPORT_DIR}/run-infer.sh ${INFER_BIN} - ${CMAKE_BINARY_DIR}/compile_commands.json 2) - # runs infer report - add_custom_target(infer-report ${BUILD_SUPPORT_DIR}/run-infer.sh ${INFER_BIN} - ${CMAKE_BINARY_DIR}/compile_commands.json 3) -endif() - # # Link targets # diff --git a/cpp/cmake_modules/FindClangTools.cmake b/cpp/cmake_modules/FindClangTools.cmake deleted file mode 100644 index 1364ccbed816..000000000000 --- a/cpp/cmake_modules/FindClangTools.cmake +++ /dev/null @@ -1,122 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find the clang-tidy and clang-format modules -# -# Usage of this module as follows: -# -# find_package(ClangTools) -# -# Variables used by this module which can change the default behaviour and need -# to be set before calling find_package: -# -# CLANG_FORMAT_VERSION - -# The version of clang-format to find. If this is not specified, clang-format -# will not be searched for. -# -# ClangTools_PATH - -# When set, this path is inspected in addition to standard library binary locations -# to find clang-tidy and clang-format -# -# This module defines -# CLANG_TIDY_BIN, The path to the clang tidy binary -# CLANG_TIDY_FOUND, Whether clang tidy was found -# CLANG_FORMAT_BIN, The path to the clang format binary -# CLANG_FORMAT_FOUND, Whether clang format was found - -set(CLANG_TOOLS_SEARCH_PATHS - ${ClangTools_PATH} - $ENV{CLANG_TOOLS_PATH} - /usr/local/bin - /usr/bin - "C:/Program Files/LLVM/bin" # Windows, non-conda - "$ENV{CONDA_PREFIX}/Library/bin" # Windows, conda - "$ENV{CONDA_PREFIX}/bin") # Unix, conda -if(APPLE) - find_program(BREW brew) - if(BREW) - execute_process(COMMAND ${BREW} --prefix "llvm@${ARROW_CLANG_TOOLS_VERSION_MAJOR}" - OUTPUT_VARIABLE CLANG_TOOLS_BREW_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT CLANG_TOOLS_BREW_PREFIX) - execute_process(COMMAND ${BREW} --prefix llvm - OUTPUT_VARIABLE CLANG_TOOLS_BREW_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE) - endif() - if(CLANG_TOOLS_BREW_PREFIX) - list(APPEND CLANG_TOOLS_SEARCH_PATHS "${CLANG_TOOLS_BREW_PREFIX}/bin") - endif() - endif() -endif() - -function(FIND_CLANG_TOOL NAME OUTPUT VERSION_CHECK_PATTERN) - unset(CLANG_TOOL_BIN CACHE) - find_program(CLANG_TOOL_BIN - NAMES ${NAME}-${ARROW_CLANG_TOOLS_VERSION} - ${NAME}-${ARROW_CLANG_TOOLS_VERSION_MAJOR} - PATHS ${CLANG_TOOLS_SEARCH_PATHS} - NO_DEFAULT_PATH) - if(NOT CLANG_TOOL_BIN) - # try searching for non-versioned tool and check the version - find_program(CLANG_TOOL_BIN - NAMES ${NAME} - PATHS ${CLANG_TOOLS_SEARCH_PATHS} - NO_DEFAULT_PATH) - if(CLANG_TOOL_BIN) - unset(CLANG_TOOL_VERSION_MESSAGE) - execute_process(COMMAND ${CLANG_TOOL_BIN} "-version" - OUTPUT_VARIABLE CLANG_TOOL_VERSION_MESSAGE - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT (${CLANG_TOOL_VERSION_MESSAGE} MATCHES ${VERSION_CHECK_PATTERN})) - message(STATUS "${NAME} found, but version did not match \"${VERSION_CHECK_PATTERN}\"" - ) - set(CLANG_TOOL_BIN "CLANG_TOOL_BIN-NOTFOUND") - endif() - endif() - endif() - if(CLANG_TOOL_BIN) - set(${OUTPUT} - ${CLANG_TOOL_BIN} - PARENT_SCOPE) - else() - set(${OUTPUT} - "${OUTPUT}-NOTFOUND" - PARENT_SCOPE) - endif() -endfunction() - -string(REGEX REPLACE "\\." "\\\\." ARROW_CLANG_TOOLS_VERSION_ESCAPED - "${ARROW_CLANG_TOOLS_VERSION}") - -find_clang_tool(clang-tidy CLANG_TIDY_BIN - "LLVM version ${ARROW_CLANG_TOOLS_VERSION_ESCAPED}") -if(CLANG_TIDY_BIN) - set(CLANG_TIDY_FOUND 1) - message(STATUS "clang-tidy found at ${CLANG_TIDY_BIN}") -else() - set(CLANG_TIDY_FOUND 0) - message(STATUS "clang-tidy ${ARROW_CLANG_TOOLS_VERSION} not found") -endif() - -find_clang_tool(clang-format CLANG_FORMAT_BIN - "clang-format version ${ARROW_CLANG_TOOLS_VERSION_ESCAPED}") -if(CLANG_FORMAT_BIN) - set(CLANG_FORMAT_FOUND 1) - message(STATUS "clang-format found at ${CLANG_FORMAT_BIN}") -else() - set(CLANG_FORMAT_FOUND 0) - message(STATUS "clang-format ${ARROW_CLANG_TOOLS_VERSION} not found") -endif() - -find_package_handle_standard_args(ClangTools REQUIRED_VARS CLANG_FORMAT_BIN - CLANG_TIDY_BIN) diff --git a/cpp/cmake_modules/FindInferTools.cmake b/cpp/cmake_modules/FindInferTools.cmake deleted file mode 100644 index c4b65653ae9a..000000000000 --- a/cpp/cmake_modules/FindInferTools.cmake +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find the infer module -# -# Usage of this module as follows: -# -# find_package(InferTools) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# InferTools_PATH - -# When set, this path is inspected instead of standard library binary locations -# to find infer -# -# This module defines -# INFER_BIN, The path to the infer binary -# INFER_FOUND, Whether infer was found - -find_program(INFER_BIN - NAMES infer - PATHS ${InferTools_PATH} - $ENV{INFER_TOOLS_PATH} - /usr/local/bin - /usr/bin - /usr/local/homebrew/bin - /opt/local/bin - NO_DEFAULT_PATH) - -if("${INFER_BIN}" STREQUAL "INFER_BIN-NOTFOUND") - set(INFER_FOUND 0) - message(STATUS "infer not found") -else() - set(INFER_FOUND 1) - message(STATUS "infer found at ${INFER_BIN}") -endif() diff --git a/cpp/src/arrow/array/concatenate.cc b/cpp/src/arrow/array/concatenate.cc index 2b832b977f23..999cbd1c87bf 100644 --- a/cpp/src/arrow/array/concatenate.cc +++ b/cpp/src/arrow/array/concatenate.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -424,7 +425,7 @@ class ConcatenateImpl { out_->buffers.resize(2); for (const auto& in_data : in_) { - for (const auto& buf : util::span(in_data->buffers).subspan(2)) { + for (const auto& buf : std::span(in_data->buffers).subspan(2)) { out_->buffers.push_back(buf); } } diff --git a/cpp/src/arrow/array/data.cc b/cpp/src/arrow/array/data.cc index 1c56a4850622..cb7959174a08 100644 --- a/cpp/src/arrow/array/data.cc +++ b/cpp/src/arrow/array/data.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -102,7 +103,7 @@ bool DictionaryMayHaveLogicalNulls(const ArrayData& data) { namespace { -BufferSpan PackVariadicBuffers(util::span> buffers) { +BufferSpan PackVariadicBuffers(std::span> buffers) { return {const_cast(reinterpret_cast(buffers.data())), static_cast(buffers.size() * sizeof(std::shared_ptr))}; } @@ -322,7 +323,7 @@ void ArraySpan::SetMembers(const ArrayData& data) { if (type_id == Type::STRING_VIEW || type_id == Type::BINARY_VIEW) { // store the span of data buffers in the third buffer - this->buffers[2] = internal::PackVariadicBuffers(util::span(data.buffers).subspan(2)); + this->buffers[2] = internal::PackVariadicBuffers(std::span(data.buffers).subspan(2)); } if (type_id == Type::DICTIONARY) { @@ -679,7 +680,7 @@ std::shared_ptr ArraySpan::ToArrayData() const { return result; } -util::span> ArraySpan::GetVariadicBuffers() const { +std::span> ArraySpan::GetVariadicBuffers() const { DCHECK(HasVariadicBuffers()); return {buffers[2].data_as>(), static_cast(buffers[2].size) / sizeof(std::shared_ptr)}; diff --git a/cpp/src/arrow/array/data.h b/cpp/src/arrow/array/data.h index c6636df9bb30..92308e8a010f 100644 --- a/cpp/src/arrow/array/data.h +++ b/cpp/src/arrow/array/data.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -31,7 +32,6 @@ #include "arrow/type_fwd.h" #include "arrow/util/bit_util.h" #include "arrow/util/macros.h" -#include "arrow/util/span.h" #include "arrow/util/visibility.h" namespace arrow { @@ -481,6 +481,7 @@ struct ARROW_EXPORT ArrayData { std::shared_ptr statistics; }; +/// \class BufferSpan /// \brief A non-owning Buffer reference struct ARROW_EXPORT BufferSpan { // It is the user of this class's responsibility to ensure that @@ -501,6 +502,7 @@ struct ARROW_EXPORT BufferSpan { } }; +/// \class ArraySpan /// \brief EXPERIMENTAL: A non-owning array data container /// /// Unlike ArrayData, this class doesn't own its referenced data type nor data buffers. @@ -577,11 +579,11 @@ struct ARROW_EXPORT ArraySpan { /// this array type /// \return A span of the requested length template - util::span GetSpan(int i, int64_t length) const { + std::span GetSpan(int i, int64_t length) const { const int64_t buffer_length = buffers[i].size / static_cast(sizeof(T)); assert(i > 0 && length + offset <= buffer_length); ARROW_UNUSED(buffer_length); - return util::span(buffers[i].data_as() + this->offset, length); + return std::span(buffers[i].data_as() + this->offset, length); } /// \brief Access a buffer's data as a span @@ -593,11 +595,11 @@ struct ARROW_EXPORT ArraySpan { /// this array type /// \return A span of the requested length template - util::span GetSpan(int i, int64_t length) { + std::span GetSpan(int i, int64_t length) { const int64_t buffer_length = buffers[i].size / static_cast(sizeof(T)); assert(i > 0 && length + offset <= buffer_length); ARROW_UNUSED(buffer_length); - return util::span(buffers[i].mutable_data_as() + this->offset, length); + return std::span(buffers[i].mutable_data_as() + this->offset, length); } inline bool IsNull(int64_t i) const { return !IsValid(i); } @@ -709,7 +711,7 @@ struct ARROW_EXPORT ArraySpan { /// sizeof(shared_ptr). /// /// \see HasVariadicBuffers - util::span> GetVariadicBuffers() const; + std::span> GetVariadicBuffers() const; bool HasVariadicBuffers() const; private: diff --git a/cpp/src/arrow/array/util.cc b/cpp/src/arrow/array/util.cc index 03d8c32c4e36..1c19bd5a5468 100644 --- a/cpp/src/arrow/array/util.cc +++ b/cpp/src/arrow/array/util.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -44,7 +45,6 @@ #include "arrow/util/endian.h" #include "arrow/util/logging_internal.h" #include "arrow/util/sort_internal.h" -#include "arrow/util/span.h" #include "arrow/visit_data_inline.h" #include "arrow/visit_type_inline.h" diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index bd0d00126d51..c1d96375bf09 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -17,6 +17,7 @@ #include "arrow/array/validate.h" +#include #include #include "arrow/array.h" // IWYU pragma: keep @@ -650,9 +651,9 @@ struct ValidateArrayImpl { HexEncode(data, BinaryViewType::kPrefixSize)); }; - util::span views(data.GetValues(1), - static_cast(data.length)); - util::span data_buffers(data.buffers.data() + 2, data.buffers.size() - 2); + std::span views(data.GetValues(1), + static_cast(data.length)); + std::span data_buffers(data.buffers.data() + 2, data.buffers.size() - 2); for (size_t i = 0; i < static_cast(data.length); ++i) { if (data.IsNull(i)) continue; @@ -663,7 +664,7 @@ struct ValidateArrayImpl { } if (views[i].is_inline()) { - auto padding_bytes = util::span(views[i].inlined.data).subspan(views[i].size()); + auto padding_bytes = std::span(views[i].inlined.data).subspan(views[i].size()); for (auto padding_byte : padding_bytes) { if (padding_byte != 0) { return Status::Invalid("View at slot ", i, " was inline with size ", diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index ce909a3ea182..a2c841208273 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,6 @@ #include "arrow/status.h" #include "arrow/type_fwd.h" #include "arrow/util/macros.h" -#include "arrow/util/span.h" #include "arrow/util/visibility.h" namespace arrow { @@ -236,8 +236,8 @@ class ARROW_EXPORT Buffer { /// \brief Return the buffer's data as a span template - util::span span_as() const { - return util::span(data_as(), static_cast(size() / sizeof(T))); + std::span span_as() const { + return std::span(data_as(), static_cast(size() / sizeof(T))); } /// \brief Return a writable pointer to the buffer's data @@ -269,8 +269,8 @@ class ARROW_EXPORT Buffer { /// \brief Return the buffer's mutable data as a span template - util::span mutable_span_as() { - return util::span(mutable_data_as(), static_cast(size() / sizeof(T))); + std::span mutable_span_as() { + return std::span(mutable_data_as(), static_cast(size() / sizeof(T))); } /// \brief Return the device address of the buffer's data diff --git a/cpp/src/arrow/c/bridge.cc b/cpp/src/arrow/c/bridge.cc index dd25ed299dd2..82e74098d248 100644 --- a/cpp/src/arrow/c/bridge.cc +++ b/cpp/src/arrow/c/bridge.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -603,7 +604,7 @@ struct ArrayExporter { }); if (need_variadic_buffer_sizes) { - auto variadic_buffers = util::span(data->buffers).subspan(2); + auto variadic_buffers = std::span(data->buffers).subspan(2); export_.variadic_buffer_sizes_.resize(variadic_buffers.size()); size_t i = 0; for (const auto& buf : variadic_buffers) { diff --git a/cpp/src/arrow/c/bridge_test.cc b/cpp/src/arrow/c/bridge_test.cc index c6a5e01e0380..cb204806f90c 100644 --- a/cpp/src/arrow/c/bridge_test.cc +++ b/cpp/src/arrow/c/bridge_test.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -592,8 +593,8 @@ struct ArrayExportChecker { ASSERT_EQ(c_export->buffers[i], expected_ptr); } if (has_variadic_buffer_sizes) { - auto variadic_buffers = util::span(expected_data.buffers).subspan(2); - auto variadic_buffer_sizes = util::span( + auto variadic_buffers = std::span(expected_data.buffers).subspan(2); + auto variadic_buffer_sizes = std::span( static_cast(c_export->buffers[c_export->n_buffers - 1]), variadic_buffers.size()); for (auto [buf, size] : Zip(variadic_buffers, variadic_buffer_sizes)) { diff --git a/cpp/src/arrow/chunk_resolver.cc b/cpp/src/arrow/chunk_resolver.cc index 7fc259f38c24..07ad807be689 100644 --- a/cpp/src/arrow/chunk_resolver.cc +++ b/cpp/src/arrow/chunk_resolver.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "arrow/array.h" @@ -28,8 +29,6 @@ namespace arrow { -using util::span; - namespace { template int64_t GetLength(const T& array) { @@ -44,7 +43,7 @@ int64_t GetLength>( } template -inline std::vector MakeChunksOffsets(span chunks) { +inline std::vector MakeChunksOffsets(std::span chunks) { std::vector offsets(chunks.size() + 1); int64_t offset = 0; std::transform(chunks.begin(), chunks.end(), offsets.begin(), @@ -114,13 +113,13 @@ void ResolveManyInline(uint32_t num_offsets, const int64_t* signed_offsets, } // namespace ChunkResolver::ChunkResolver(const ArrayVector& chunks) noexcept - : offsets_(MakeChunksOffsets(span(chunks))), cached_chunk_(0) {} + : offsets_(MakeChunksOffsets(std::span(chunks))), cached_chunk_(0) {} -ChunkResolver::ChunkResolver(span chunks) noexcept +ChunkResolver::ChunkResolver(std::span chunks) noexcept : offsets_(MakeChunksOffsets(chunks)), cached_chunk_(0) {} ChunkResolver::ChunkResolver(const RecordBatchVector& batches) noexcept - : offsets_(MakeChunksOffsets(span(batches))), cached_chunk_(0) {} + : offsets_(MakeChunksOffsets(std::span(batches))), cached_chunk_(0) {} ChunkResolver::ChunkResolver(ChunkResolver&& other) noexcept : offsets_(std::move(other.offsets_)), diff --git a/cpp/src/arrow/chunk_resolver.h b/cpp/src/arrow/chunk_resolver.h index 3d6458167fac..fbf84876e3dc 100644 --- a/cpp/src/arrow/chunk_resolver.h +++ b/cpp/src/arrow/chunk_resolver.h @@ -21,12 +21,12 @@ #include #include #include +#include #include #include #include "arrow/type_fwd.h" #include "arrow/util/macros.h" -#include "arrow/util/span.h" namespace arrow { @@ -77,7 +77,7 @@ class ARROW_EXPORT ChunkResolver { public: explicit ChunkResolver(const ArrayVector& chunks) noexcept; - explicit ChunkResolver(util::span chunks) noexcept; + explicit ChunkResolver(std::span chunks) noexcept; explicit ChunkResolver(const RecordBatchVector& batches) noexcept; /// \brief Construct a ChunkResolver from a vector of chunks.size() + 1 offsets. diff --git a/cpp/src/arrow/compute/kernels/aggregate_pivot.cc b/cpp/src/arrow/compute/kernels/aggregate_pivot.cc index 504c7cdd26d5..50047f5ef1ce 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_pivot.cc +++ b/cpp/src/arrow/compute/kernels/aggregate_pivot.cc @@ -29,7 +29,6 @@ namespace arrow::compute::internal { namespace { using arrow::internal::VisitSetBitRunsVoid; -using arrow::util::span; struct PivotImpl : public ScalarAggregator { Status Init(const PivotWiderOptions& options, const std::vector& in_types, diff --git a/cpp/src/arrow/compute/kernels/chunked_internal.cc b/cpp/src/arrow/compute/kernels/chunked_internal.cc index bf574fcafd0f..39495ea0e681 100644 --- a/cpp/src/arrow/compute/kernels/chunked_internal.cc +++ b/cpp/src/arrow/compute/kernels/chunked_internal.cc @@ -18,6 +18,7 @@ #include "arrow/compute/kernels/chunked_internal.h" #include +#include #include "arrow/record_batch.h" #include "arrow/util/logging_internal.h" @@ -32,7 +33,7 @@ std::vector GetArrayPointers(const ArrayVector& arrays) { } std::vector ChunkedIndexMapper::GetChunkLengths( - util::span chunks) { + std::span chunks) { std::vector chunk_lengths(chunks.size()); for (int64_t i = 0; i < static_cast(chunks.size()); ++i) { chunk_lengths[i] = chunks[i]->length(); diff --git a/cpp/src/arrow/compute/kernels/chunked_internal.h b/cpp/src/arrow/compute/kernels/chunked_internal.h index 330bd185f25a..2dcfa0047ea0 100644 --- a/cpp/src/arrow/compute/kernels/chunked_internal.h +++ b/cpp/src/arrow/compute/kernels/chunked_internal.h @@ -20,13 +20,13 @@ #include #include #include +#include #include #include #include "arrow/array.h" #include "arrow/chunk_resolver.h" #include "arrow/compute/kernels/codegen_internal.h" -#include "arrow/util/span.h" #include "arrow/util/visibility.h" namespace arrow::compute::internal { @@ -92,13 +92,13 @@ static_assert(sizeof(uint64_t) == sizeof(CompressedChunkLocation)); class ChunkedArrayResolver { private: ChunkResolver resolver_; - util::span chunks_; + std::span chunks_; std::vector owned_chunks_; public: explicit ChunkedArrayResolver(std::vector&& chunks) : resolver_(chunks), chunks_(chunks), owned_chunks_(std::move(chunks)) {} - explicit ChunkedArrayResolver(util::span chunks) + explicit ChunkedArrayResolver(std::span chunks) : resolver_(chunks), chunks_(chunks) {} ARROW_DEFAULT_MOVE_AND_ASSIGN(ChunkedArrayResolver); @@ -129,8 +129,8 @@ class ARROW_EXPORT ChunkedIndexMapper { public: ChunkedIndexMapper(const std::vector& chunks, uint64_t* indices_begin, uint64_t* indices_end) - : ChunkedIndexMapper(util::span(chunks), indices_begin, indices_end) {} - ChunkedIndexMapper(util::span chunks, uint64_t* indices_begin, + : ChunkedIndexMapper(std::span(chunks), indices_begin, indices_end) {} + ChunkedIndexMapper(std::span chunks, uint64_t* indices_begin, uint64_t* indices_end) : chunk_lengths_(GetChunkLengths(chunks)), indices_begin_(indices_begin), @@ -154,7 +154,7 @@ class ARROW_EXPORT ChunkedIndexMapper { Status PhysicalToLogical(); private: - static std::vector GetChunkLengths(util::span chunks); + static std::vector GetChunkLengths(std::span chunks); static std::vector GetChunkLengths(const RecordBatchVector& chunks); std::vector chunk_lengths_; diff --git a/cpp/src/arrow/compute/kernels/hash_aggregate.cc b/cpp/src/arrow/compute/kernels/hash_aggregate.cc index 3ab7ff065b28..73ae31f7ea46 100644 --- a/cpp/src/arrow/compute/kernels/hash_aggregate.cc +++ b/cpp/src/arrow/compute/kernels/hash_aggregate.cc @@ -45,14 +45,12 @@ #include "arrow/util/checked_cast.h" #include "arrow/util/int_util_overflow.h" #include "arrow/util/ree_util.h" -#include "arrow/util/span.h" #include "arrow/visit_type_inline.h" namespace arrow::compute::internal { using ::arrow::internal::checked_cast; using ::arrow::internal::FirstTimeBitmapWriter; -using ::arrow::util::span; namespace { diff --git a/cpp/src/arrow/compute/kernels/hash_aggregate_numeric.cc b/cpp/src/arrow/compute/kernels/hash_aggregate_numeric.cc index aa231430aa06..30504ba87b6b 100644 --- a/cpp/src/arrow/compute/kernels/hash_aggregate_numeric.cc +++ b/cpp/src/arrow/compute/kernels/hash_aggregate_numeric.cc @@ -32,7 +32,6 @@ #include "arrow/compute/row/grouper.h" #include "arrow/util/checked_cast.h" #include "arrow/util/int128_internal.h" -#include "arrow/util/span.h" #include "arrow/util/tdigest_internal.h" #include "arrow/visit_type_inline.h" diff --git a/cpp/src/arrow/compute/kernels/hash_aggregate_pivot.cc b/cpp/src/arrow/compute/kernels/hash_aggregate_pivot.cc index f60aa367ca25..030a862afc09 100644 --- a/cpp/src/arrow/compute/kernels/hash_aggregate_pivot.cc +++ b/cpp/src/arrow/compute/kernels/hash_aggregate_pivot.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -33,13 +34,10 @@ #include "arrow/util/bit_block_counter.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging_internal.h" -#include "arrow/util/span.h" #include "arrow/visit_type_inline.h" namespace arrow::compute::internal { -using ::arrow::util::span; - namespace { // ---------------------------------------------------------------------- @@ -57,7 +55,7 @@ struct GroupedPivotAccumulator { return Status::OK(); } - Status Consume(span groups, const std::shared_ptr& keys, + Status Consume(std::span groups, const std::shared_ptr& keys, const ArraySpan& values) { // To dispatch the values into the right (group, key) coordinates, // we first compute a vector of take indices for each output column. @@ -178,8 +176,8 @@ struct GroupedPivotAccumulator { return MergeColumns(std::move(new_columns)); } - Status Consume(span groups, std::optional maybe_key, - const ArraySpan& values) { + Status Consume(std::span groups, + std::optional maybe_key, const ArraySpan& values) { if (!maybe_key.has_value()) { // Nothing to update return Status::OK(); diff --git a/cpp/src/arrow/compute/kernels/pivot_internal.cc b/cpp/src/arrow/compute/kernels/pivot_internal.cc index e9065152d7ff..c9ccc7d60766 100644 --- a/cpp/src/arrow/compute/kernels/pivot_internal.cc +++ b/cpp/src/arrow/compute/kernels/pivot_internal.cc @@ -37,8 +37,6 @@ namespace arrow::compute::internal { -using ::arrow::util::span; - struct ConcretePivotWiderKeyMapper : public PivotWiderKeyMapper { Status Init(const DataType& key_type, const PivotWiderOptions* options, ExecContext* ctx) { diff --git a/cpp/src/arrow/compute/kernels/vector_rank.cc b/cpp/src/arrow/compute/kernels/vector_rank.cc index ef7419ea7c54..adac794902dc 100644 --- a/cpp/src/arrow/compute/kernels/vector_rank.cc +++ b/cpp/src/arrow/compute/kernels/vector_rank.cc @@ -27,8 +27,6 @@ namespace arrow::compute::internal { -using ::arrow::util::span; - namespace { // ---------------------------------------------------------------------- @@ -103,7 +101,8 @@ Result DoSortAndMarkDuplicate( physical_chunks, order, null_placement)); if (needs_duplicates) { const auto arrays = GetArrayPointers(physical_chunks); - auto value_selector = [resolver = ChunkedArrayResolver(span(arrays))](int64_t index) { + auto value_selector = [resolver = + ChunkedArrayResolver(std::span(arrays))](int64_t index) { return resolver.Resolve(index).Value(); }; MarkDuplicates(sorted, value_selector); diff --git a/cpp/src/arrow/compute/kernels/vector_sort.cc b/cpp/src/arrow/compute/kernels/vector_sort.cc index 41cb0a357a4c..3befb7a1441e 100644 --- a/cpp/src/arrow/compute/kernels/vector_sort.cc +++ b/cpp/src/arrow/compute/kernels/vector_sort.cc @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include "arrow/compute/function.h" @@ -26,7 +27,6 @@ namespace arrow { using internal::checked_cast; -using util::span; namespace compute { namespace internal { @@ -177,7 +177,8 @@ class ChunkedArraySorter : public TypeVisitor { template void MergeNonNulls(CompressedChunkLocation* range_begin, CompressedChunkLocation* range_middle, - CompressedChunkLocation* range_end, span arrays, + CompressedChunkLocation* range_end, + std::span arrays, CompressedChunkLocation* temp_indices) { using ArrowType = typename ArrayType::TypeClass; @@ -202,7 +203,8 @@ class ChunkedArraySorter : public TypeVisitor { } template - auto ChunkValue(span arrays, CompressedChunkLocation loc) const { + auto ChunkValue(std::span arrays, + CompressedChunkLocation loc) const { return ResolvedChunk(arrays[loc.chunk_index()], static_cast(loc.index_in_chunk())) .template Value(); diff --git a/cpp/src/arrow/compute/kernels/vector_sort_internal.h b/cpp/src/arrow/compute/kernels/vector_sort_internal.h index 49704ff80699..93ff9b19205f 100644 --- a/cpp/src/arrow/compute/kernels/vector_sort_internal.h +++ b/cpp/src/arrow/compute/kernels/vector_sort_internal.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "arrow/array.h" #include "arrow/compute/api_vector.h" @@ -272,7 +273,7 @@ NullPartitionResult PartitionNullsOnly(uint64_t* indices_begin, uint64_t* indice template ChunkedNullPartitionResult PartitionNullsOnly(CompressedChunkLocation* locations_begin, CompressedChunkLocation* locations_end, - util::span chunks, + std::span chunks, int64_t null_count, NullPlacement null_placement) { if (null_count == 0) { diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h index 7f8a4a7ef856..f6178c7f5832 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h @@ -31,6 +31,7 @@ #define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING namespace ODBC { +using arrow::flight::sql::odbc::WcsToUtf8; // Return the number of bytes required for the conversion. template @@ -99,6 +100,24 @@ inline std::string SqlWcharToString(SQLWCHAR* wchar_msg, SQLINTEGER msg_len = SQ return std::string(utf8_str.begin(), utf8_str.end()); } +/// \brief Convert vector of SqlWchar to standard string +/// \param[in] wchar_vec SqlWchar vector to convert +/// \return wchar_vec in std::string format +inline std::string SqlWcharToString(const std::vector& wchar_vec) { + if (wchar_vec.empty() || wchar_vec[0] == 0) { + return {}; + } + + auto end = std::find(wchar_vec.begin(), wchar_vec.end(), 0); + size_t actual_len = std::distance(wchar_vec.begin(), end); + + thread_local std::vector utf8_str; + + WcsToUtf8((void*)wchar_vec.data(), static_cast(actual_len), &utf8_str); + + return std::string(utf8_str.begin(), utf8_str.end()); +} + inline std::string SqlStringToString(const unsigned char* sql_str, int32_t sql_str_len = SQL_NTS) { std::string res; diff --git a/cpp/src/arrow/flight/sql/odbc/tests/columns_test.cc b/cpp/src/arrow/flight/sql/odbc/tests/columns_test.cc index 6ac3cde874b8..d88a569c8ffc 100644 --- a/cpp/src/arrow/flight/sql/odbc/tests/columns_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/tests/columns_test.cc @@ -127,12 +127,12 @@ void CheckRemoteSQLColumns( } void CheckSQLColAttribute(SQLHSTMT stmt, SQLUSMALLINT idx, - const std::wstring& expected_column_name, + const std::string& expected_column_name, SQLLEN expected_data_type, SQLLEN expected_concise_type, SQLLEN expected_display_size, SQLLEN expected_prec_scale, SQLLEN expected_length, - const std::wstring& expected_literal_prefix, - const std::wstring& expected_literal_suffix, + const std::string& expected_literal_prefix, + const std::string& expected_literal_suffix, SQLLEN expected_column_size, SQLLEN expected_column_scale, SQLLEN expected_column_nullability, SQLLEN expected_num_prec_radix, SQLLEN expected_octet_length, @@ -215,10 +215,10 @@ void CheckSQLColAttribute(SQLHSTMT stmt, SQLUSMALLINT idx, EXPECT_EQ(SQL_SUCCESS, SQLColAttribute(stmt, idx, SQL_DESC_UNSIGNED, 0, 0, nullptr, &unsigned_col)); - std::wstring name_str = ConvertToWString(name, name_len); - std::wstring base_column_name_str = ConvertToWString(base_column_name, column_name_len); - std::wstring label_str = ConvertToWString(label, label_len); - std::wstring prefixStr = ConvertToWString(prefix, prefix_len); + std::string name_str = ODBC::SqlWcharToString(name); + std::string base_column_name_str = ODBC::SqlWcharToString(base_column_name); + std::string label_str = ODBC::SqlWcharToString(label); + std::string prefix_str = ODBC::SqlWcharToString(prefix); // Assume column name, base column name, and label are equivalent in the result set EXPECT_EQ(expected_column_name, name_str); @@ -229,7 +229,7 @@ void CheckSQLColAttribute(SQLHSTMT stmt, SQLUSMALLINT idx, EXPECT_EQ(expected_display_size, display_size); EXPECT_EQ(expected_prec_scale, prec_scale); EXPECT_EQ(expected_length, length); - EXPECT_EQ(expected_literal_prefix, prefixStr); + EXPECT_EQ(expected_literal_prefix, prefix_str); EXPECT_EQ(expected_column_size, size); EXPECT_EQ(expected_column_scale, scale); EXPECT_EQ(expected_column_nullability, nullability); @@ -241,7 +241,7 @@ void CheckSQLColAttribute(SQLHSTMT stmt, SQLUSMALLINT idx, #ifndef __APPLE__ void CheckSQLColAttributes(SQLHSTMT stmt, SQLUSMALLINT idx, - const std::wstring& expected_column_name, + const std::string& expected_column_name, SQLLEN expected_data_type, SQLLEN expected_display_size, SQLLEN expected_prec_scale, SQLLEN expected_length, SQLLEN expected_column_size, SQLLEN expected_column_scale, @@ -295,8 +295,8 @@ void CheckSQLColAttributes(SQLHSTMT stmt, SQLUSMALLINT idx, EXPECT_EQ(SQL_SUCCESS, SQLColAttributes(stmt, idx, SQL_COLUMN_UNSIGNED, 0, 0, nullptr, &unsigned_col)); - std::wstring name_str = ConvertToWString(name, name_len); - std::wstring label_str = ConvertToWString(label, label_len); + std::string name_str = ODBC::SqlWcharToString(name); + std::string label_str = ODBC::SqlWcharToString(label); EXPECT_EQ(expected_column_name, name_str); EXPECT_EQ(expected_column_name, label_str); @@ -326,8 +326,9 @@ void GetSQLColAttributeString(SQLHSTMT stmt, const std::wstring& wsql, SQLUSMALL std::vector str_val(kOdbcBufferSize); SQLSMALLINT str_len = 0; - ASSERT_EQ(SQL_SUCCESS, SQLColAttribute(stmt, idx, field_identifier, &str_val[0], - (SQLSMALLINT)str_val.size(), &str_len, nullptr)); + ASSERT_EQ(SQL_SUCCESS, + SQLColAttribute(stmt, idx, field_identifier, &str_val[0], + static_cast(str_val.size()), &str_len, nullptr)); value = ConvertToWString(str_val, str_len); } @@ -347,9 +348,9 @@ void GetSQLColAttributesString(SQLHSTMT stmt, const std::wstring& wsql, SQLUSMAL std::vector str_val(kOdbcBufferSize); SQLSMALLINT str_len = 0; - ASSERT_EQ(SQL_SUCCESS, - SQLColAttributes(stmt, idx, field_identifier, &str_val[0], - (SQLSMALLINT)str_val.size(), &str_len, nullptr)); + ASSERT_EQ(SQL_SUCCESS, SQLColAttributes(stmt, idx, field_identifier, &str_val[0], + static_cast(str_val.size()), + &str_len, nullptr)); value = ConvertToWString(str_val, str_len); } @@ -1223,24 +1224,23 @@ TEST_F(ColumnsMockTest, TestSQLColumnsInvalidTablePattern) { } TYPED_TEST(ColumnsTest, SQLColAttributeTestInputData) { - std::wstring wsql = L"SELECT 1 as col1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1 as col1;"; + SQLSMALLINT wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))) + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)) << GetOdbcErrorMessage(SQL_HANDLE_DBC, conn); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); SQLUSMALLINT idx = 1; - std::vector character_attr(kOdbcBufferSize); + SQLWCHAR character_attr[kOdbcBufferSize]; SQLSMALLINT character_attr_len = 0; SQLLEN numeric_attr = 0; // All character values populated - EXPECT_EQ(SQL_SUCCESS, SQLColAttribute(stmt, idx, SQL_DESC_NAME, &character_attr[0], - (SQLSMALLINT)character_attr.size(), - &character_attr_len, nullptr)); + EXPECT_EQ(SQL_SUCCESS, + SQLColAttribute(stmt, idx, SQL_DESC_NAME, character_attr, + std::wcslen(character_attr), &character_attr_len, nullptr)); // All numeric values populated EXPECT_EQ(SQL_SUCCESS, @@ -1255,11 +1255,10 @@ TYPED_TEST(ColumnsTest, SQLColAttributeTestInputData) { } TYPED_TEST(ColumnsTest, SQLColAttributeGetCharacterLen) { - std::wstring wsql = L"SELECT 1 as col1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1 as col1;"; + SQLSMALLINT wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1272,44 +1271,41 @@ TYPED_TEST(ColumnsTest, SQLColAttributeGetCharacterLen) { } TYPED_TEST(ColumnsTest, SQLColAttributeInvalidFieldId) { - std::wstring wsql = L"SELECT 1 as col1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1 as col1;"; + SQLSMALLINT wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); SQLUSMALLINT invalid_field_id = -100; SQLUSMALLINT idx = 1; - std::vector character_attr(kOdbcBufferSize); + SQLWCHAR character_attr[kOdbcBufferSize]; SQLSMALLINT character_attr_len = 0; SQLLEN numeric_attr = 0; - ASSERT_EQ(SQL_ERROR, SQLColAttribute(stmt, idx, invalid_field_id, &character_attr[0], - (SQLSMALLINT)character_attr.size(), - &character_attr_len, nullptr)); + ASSERT_EQ(SQL_ERROR, + SQLColAttribute(stmt, idx, invalid_field_id, character_attr, + std::wcslen(character_attr), &character_attr_len, nullptr)); // Verify invalid descriptor field identifier error state is returned VerifyOdbcErrorState(SQL_HANDLE_STMT, stmt, kErrorStateHY091); } TYPED_TEST(ColumnsTest, SQLColAttributeInvalidColId) { - std::wstring wsql = L"SELECT 1 as col1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1 as col1;"; + SQLSMALLINT wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); SQLUSMALLINT invalid_col_id = 2; - std::vector character_attr(kOdbcBufferSize); + SQLWCHAR character_attr[kOdbcBufferSize]; SQLSMALLINT character_attr_len = 0; - ASSERT_EQ( - SQL_ERROR, - SQLColAttribute(stmt, invalid_col_id, SQL_DESC_BASE_COLUMN_NAME, &character_attr[0], - (SQLSMALLINT)character_attr.size(), &character_attr_len, nullptr)); + ASSERT_EQ(SQL_ERROR, SQLColAttribute(stmt, invalid_col_id, SQL_DESC_BASE_COLUMN_NAME, + character_attr, std::wcslen(character_attr), + &character_attr_len, nullptr)); // Verify invalid descriptor index error state is returned VerifyOdbcErrorState(SQL_HANDLE_STMT, stmt, kErrorState07009); } @@ -1317,81 +1313,80 @@ TYPED_TEST(ColumnsTest, SQLColAttributeInvalidColId) { TEST_F(ColumnsMockTest, TestSQLColAttributeAllTypes) { CreateAllDataTypeTable(); - std::wstring wsql = L"SELECT * from AllTypesTable;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT * from AllTypesTable;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); CheckSQLColAttribute(stmt, 1, - std::wstring(L"bigint_col"), // expected_column_name - SQL_BIGINT, // expected_data_type - SQL_BIGINT, // expected_concise_type - 20, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 10, // expected_num_prec_radix - 8, // expected_octet_length - SQL_PRED_NONE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "bigint_col", // expected_column_name + SQL_BIGINT, // expected_data_type + SQL_BIGINT, // expected_concise_type + 20, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 10, // expected_num_prec_radix + 8, // expected_octet_length + SQL_PRED_NONE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 2, - std::wstring(L"char_col"), // expected_column_name - SQL_WVARCHAR, // expected_data_type - SQL_WVARCHAR, // expected_concise_type - 0, // expected_display_size - SQL_FALSE, // expected_prec_scale - 0, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 0, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 0, // expected_octet_length - SQL_PRED_NONE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "char_col", // expected_column_name + SQL_WVARCHAR, // expected_data_type + SQL_WVARCHAR, // expected_concise_type + 0, // expected_display_size + SQL_FALSE, // expected_prec_scale + 0, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 0, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 0, // expected_octet_length + SQL_PRED_NONE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttribute(stmt, 3, - std::wstring(L"varbinary_col"), // expected_column_name - SQL_BINARY, // expected_data_type - SQL_BINARY, // expected_concise_type - 0, // expected_display_size - SQL_FALSE, // expected_prec_scale - 0, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 0, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 0, // expected_octet_length - SQL_PRED_NONE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "varbinary_col", // expected_column_name + SQL_BINARY, // expected_data_type + SQL_BINARY, // expected_concise_type + 0, // expected_display_size + SQL_FALSE, // expected_prec_scale + 0, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 0, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 0, // expected_octet_length + SQL_PRED_NONE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttribute(stmt, 4, - std::wstring(L"double_col"), // expected_column_name - SQL_DOUBLE, // expected_data_type - SQL_DOUBLE, // expected_concise_type - 24, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 2, // expected_num_prec_radix - 8, // expected_octet_length - SQL_PRED_NONE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "double_col", // expected_column_name + SQL_DOUBLE, // expected_data_type + SQL_DOUBLE, // expected_concise_type + 24, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 2, // expected_num_prec_radix + 8, // expected_octet_length + SQL_PRED_NONE, // expected_searchable + SQL_FALSE); // expected_unsigned_column DropAllDataTypeTable(); } @@ -1402,60 +1397,59 @@ TEST_F(ColumnsOdbcV2MockTest, TestSQLColAttributesAllTypes) { // Tests ODBC 2.0 API SQLColAttributes CreateAllDataTypeTable(); - std::wstring wsql = L"SELECT * from AllTypesTable;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT * from AllTypesTable;"; + SQLSMALLINT wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); CheckSQLColAttributes(stmt, 1, - std::wstring(L"bigint_col"), // expected_column_name - SQL_BIGINT, // expected_data_type - 20, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_PRED_NONE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "bigint_col", // expected_column_name + SQL_BIGINT, // expected_data_type + 20, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_PRED_NONE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttributes(stmt, 2, - std::wstring(L"char_col"), // expected_column_name - SQL_WVARCHAR, // expected_data_type - 0, // expected_display_size - SQL_FALSE, // expected_prec_scale - 0, // expected_length - 0, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_PRED_NONE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "char_col", // expected_column_name + SQL_WVARCHAR, // expected_data_type + 0, // expected_display_size + SQL_FALSE, // expected_prec_scale + 0, // expected_length + 0, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_PRED_NONE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttributes(stmt, 3, - std::wstring(L"varbinary_col"), // expected_column_name - SQL_BINARY, // expected_data_type - 0, // expected_display_size - SQL_FALSE, // expected_prec_scale - 0, // expected_length - 0, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_PRED_NONE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "varbinary_col", // expected_column_name + SQL_BINARY, // expected_data_type + 0, // expected_display_size + SQL_FALSE, // expected_prec_scale + 0, // expected_length + 0, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_PRED_NONE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttributes(stmt, 4, - std::wstring(L"double_col"), // expected_column_name - SQL_DOUBLE, // expected_data_type - 24, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_PRED_NONE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "double_col", // expected_column_name + SQL_DOUBLE, // expected_data_type + 24, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_PRED_NONE, // expected_searchable + SQL_FALSE); // expected_unsigned_column DropAllDataTypeTable(); } @@ -1464,453 +1458,450 @@ TEST_F(ColumnsOdbcV2MockTest, TestSQLColAttributesAllTypes) { TEST_F(ColumnsRemoteTest, TestSQLColAttributeAllTypes) { // Test assumes there is a table $scratch.ODBCTest in remote server - std::wstring wsql = L"SELECT * from $scratch.ODBCTest;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT * from $scratch.ODBCTest;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); CheckSQLColAttribute(stmt, 1, - std::wstring(L"sinteger_max"), // expected_column_name - SQL_INTEGER, // expected_data_type - SQL_INTEGER, // expected_concise_type - 11, // expected_display_size - SQL_FALSE, // expected_prec_scale - 4, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 4, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 10, // expected_num_prec_radix - 4, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "sinteger_max", // expected_column_name + SQL_INTEGER, // expected_data_type + SQL_INTEGER, // expected_concise_type + 11, // expected_display_size + SQL_FALSE, // expected_prec_scale + 4, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 4, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 10, // expected_num_prec_radix + 4, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 2, - std::wstring(L"sbigint_max"), // expected_column_name - SQL_BIGINT, // expected_data_type - SQL_BIGINT, // expected_concise_type - 20, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 10, // expected_num_prec_radix - 8, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "sbigint_max", // expected_column_name + SQL_BIGINT, // expected_data_type + SQL_BIGINT, // expected_concise_type + 20, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 10, // expected_num_prec_radix + 8, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 3, - std::wstring(L"decimal_positive"), // expected_column_name - SQL_DECIMAL, // expected_data_type - SQL_DECIMAL, // expected_concise_type - 40, // expected_display_size - SQL_FALSE, // expected_prec_scale - 19, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 19, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 10, // expected_num_prec_radix - 40, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "decimal_positive", // expected_column_name + SQL_DECIMAL, // expected_data_type + SQL_DECIMAL, // expected_concise_type + 40, // expected_display_size + SQL_FALSE, // expected_prec_scale + 19, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 19, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 10, // expected_num_prec_radix + 40, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 4, - std::wstring(L"float_max"), // expected_column_name - SQL_FLOAT, // expected_data_type - SQL_FLOAT, // expected_concise_type - 24, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 2, // expected_num_prec_radix - 8, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "float_max", // expected_column_name + SQL_FLOAT, // expected_data_type + SQL_FLOAT, // expected_concise_type + 24, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 2, // expected_num_prec_radix + 8, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 5, - std::wstring(L"double_max"), // expected_column_name - SQL_DOUBLE, // expected_data_type - SQL_DOUBLE, // expected_concise_type - 24, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 2, // expected_num_prec_radix - 8, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "double_max", // expected_column_name + SQL_DOUBLE, // expected_data_type + SQL_DOUBLE, // expected_concise_type + 24, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 2, // expected_num_prec_radix + 8, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 6, - std::wstring(L"bit_true"), // expected_column_name - SQL_BIT, // expected_data_type - SQL_BIT, // expected_concise_type - 1, // expected_display_size - SQL_FALSE, // expected_prec_scale - 1, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 1, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 1, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "bit_true", // expected_column_name + SQL_BIT, // expected_data_type + SQL_BIT, // expected_concise_type + 1, // expected_display_size + SQL_FALSE, // expected_prec_scale + 1, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 1, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 1, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttribute(stmt, 7, - std::wstring(L"date_max"), // expected_column_name - SQL_DATETIME, // expected_data_type - SQL_TYPE_DATE, // expected_concise_type - 10, // expected_display_size - SQL_FALSE, // expected_prec_scale - 10, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 10, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 6, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "date_max", // expected_column_name + SQL_DATETIME, // expected_data_type + SQL_TYPE_DATE, // expected_concise_type + 10, // expected_display_size + SQL_FALSE, // expected_prec_scale + 10, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 10, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 6, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttribute(stmt, 8, - std::wstring(L"time_max"), // expected_column_name - SQL_DATETIME, // expected_data_type - SQL_TYPE_TIME, // expected_concise_type - 12, // expected_display_size - SQL_FALSE, // expected_prec_scale - 12, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 12, // expected_column_size - 3, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 6, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "time_max", // expected_column_name + SQL_DATETIME, // expected_data_type + SQL_TYPE_TIME, // expected_concise_type + 12, // expected_display_size + SQL_FALSE, // expected_prec_scale + 12, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 12, // expected_column_size + 3, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 6, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttribute(stmt, 9, - std::wstring(L"timestamp_max"), // expected_column_name - SQL_DATETIME, // expected_data_type - SQL_TYPE_TIMESTAMP, // expected_concise_type - 23, // expected_display_size - SQL_FALSE, // expected_prec_scale - 23, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 23, // expected_column_size - 3, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 16, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "timestamp_max", // expected_column_name + SQL_DATETIME, // expected_data_type + SQL_TYPE_TIMESTAMP, // expected_concise_type + 23, // expected_display_size + SQL_FALSE, // expected_prec_scale + 23, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 23, // expected_column_size + 3, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 16, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column } // iODBC does not support SQLColAttribute in ODBC 2.0 mode. #ifndef __APPLE__ TEST_F(ColumnsOdbcV2RemoteTest, TestSQLColAttributeAllTypes) { // Test assumes there is a table $scratch.ODBCTest in remote server - std::wstring wsql = L"SELECT * from $scratch.ODBCTest;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT * from $scratch.ODBCTest;"; + SQLSMALLINT wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); CheckSQLColAttribute(stmt, 1, - std::wstring(L"sinteger_max"), // expected_column_name - SQL_INTEGER, // expected_data_type - SQL_INTEGER, // expected_concise_type - 11, // expected_display_size - SQL_FALSE, // expected_prec_scale - 4, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 4, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 10, // expected_num_prec_radix - 4, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "sinteger_max", // expected_column_name + SQL_INTEGER, // expected_data_type + SQL_INTEGER, // expected_concise_type + 11, // expected_display_size + SQL_FALSE, // expected_prec_scale + 4, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 4, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 10, // expected_num_prec_radix + 4, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 2, - std::wstring(L"sbigint_max"), // expected_column_name - SQL_BIGINT, // expected_data_type - SQL_BIGINT, // expected_concise_type - 20, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 10, // expected_num_prec_radix - 8, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "sbigint_max", // expected_column_name + SQL_BIGINT, // expected_data_type + SQL_BIGINT, // expected_concise_type + 20, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 10, // expected_num_prec_radix + 8, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 3, - std::wstring(L"decimal_positive"), // expected_column_name - SQL_DECIMAL, // expected_data_type - SQL_DECIMAL, // expected_concise_type - 40, // expected_display_size - SQL_FALSE, // expected_prec_scale - 19, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 19, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 10, // expected_num_prec_radix - 40, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "decimal_positive", // expected_column_name + SQL_DECIMAL, // expected_data_type + SQL_DECIMAL, // expected_concise_type + 40, // expected_display_size + SQL_FALSE, // expected_prec_scale + 19, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 19, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 10, // expected_num_prec_radix + 40, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 4, - std::wstring(L"float_max"), // expected_column_name - SQL_FLOAT, // expected_data_type - SQL_FLOAT, // expected_concise_type - 24, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 2, // expected_num_prec_radix - 8, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "float_max", // expected_column_name + SQL_FLOAT, // expected_data_type + SQL_FLOAT, // expected_concise_type + 24, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 2, // expected_num_prec_radix + 8, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 5, - std::wstring(L"double_max"), // expected_column_name - SQL_DOUBLE, // expected_data_type - SQL_DOUBLE, // expected_concise_type - 24, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 2, // expected_num_prec_radix - 8, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "double_max", // expected_column_name + SQL_DOUBLE, // expected_data_type + SQL_DOUBLE, // expected_concise_type + 24, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 2, // expected_num_prec_radix + 8, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttribute(stmt, 6, - std::wstring(L"bit_true"), // expected_column_name - SQL_BIT, // expected_data_type - SQL_BIT, // expected_concise_type - 1, // expected_display_size - SQL_FALSE, // expected_prec_scale - 1, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 1, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 1, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "bit_true", // expected_column_name + SQL_BIT, // expected_data_type + SQL_BIT, // expected_concise_type + 1, // expected_display_size + SQL_FALSE, // expected_prec_scale + 1, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 1, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 1, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttribute(stmt, 7, - std::wstring(L"date_max"), // expected_column_name - SQL_DATETIME, // expected_data_type - SQL_DATE, // expected_concise_type - 10, // expected_display_size - SQL_FALSE, // expected_prec_scale - 10, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 10, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 6, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "date_max", // expected_column_name + SQL_DATETIME, // expected_data_type + SQL_DATE, // expected_concise_type + 10, // expected_display_size + SQL_FALSE, // expected_prec_scale + 10, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 10, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 6, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttribute(stmt, 8, - std::wstring(L"time_max"), // expected_column_name - SQL_DATETIME, // expected_data_type - SQL_TIME, // expected_concise_type - 12, // expected_display_size - SQL_FALSE, // expected_prec_scale - 12, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 12, // expected_column_size - 3, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 6, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "time_max", // expected_column_name + SQL_DATETIME, // expected_data_type + SQL_TIME, // expected_concise_type + 12, // expected_display_size + SQL_FALSE, // expected_prec_scale + 12, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 12, // expected_column_size + 3, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 6, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttribute(stmt, 9, - std::wstring(L"timestamp_max"), // expected_column_name - SQL_DATETIME, // expected_data_type - SQL_TIMESTAMP, // expected_concise_type - 23, // expected_display_size - SQL_FALSE, // expected_prec_scale - 23, // expected_length - std::wstring(L""), // expected_literal_prefix - std::wstring(L""), // expected_literal_suffix - 23, // expected_column_size - 3, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - 0, // expected_num_prec_radix - 16, // expected_octet_length - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "timestamp_max", // expected_column_name + SQL_DATETIME, // expected_data_type + SQL_TIMESTAMP, // expected_concise_type + 23, // expected_display_size + SQL_FALSE, // expected_prec_scale + 23, // expected_length + "", // expected_literal_prefix + "", // expected_literal_suffix + 23, // expected_column_size + 3, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + 0, // expected_num_prec_radix + 16, // expected_octet_length + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column } // iODBC does not support SQLColAttributes for ODBC 3.0 attributes. TEST_F(ColumnsOdbcV2RemoteTest, TestSQLColAttributesAllTypes) { // Tests ODBC 2.0 API SQLColAttributes // Test assumes there is a table $scratch.ODBCTest in remote server - std::wstring wsql = L"SELECT * from $scratch.ODBCTest;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT * from $scratch.ODBCTest;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); CheckSQLColAttributes(stmt, 1, - std::wstring(L"sinteger_max"), // expected_column_name - SQL_INTEGER, // expected_data_type - 11, // expected_display_size - SQL_FALSE, // expected_prec_scale - 4, // expected_length - 4, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "sinteger_max", // expected_column_name + SQL_INTEGER, // expected_data_type + 11, // expected_display_size + SQL_FALSE, // expected_prec_scale + 4, // expected_length + 4, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttributes(stmt, 2, - std::wstring(L"sbigint_max"), // expected_column_name - SQL_BIGINT, // expected_data_type - 20, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "sbigint_max", // expected_column_name + SQL_BIGINT, // expected_data_type + 20, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttributes(stmt, 3, - std::wstring(L"decimal_positive"), // expected_column_name - SQL_DECIMAL, // expected_data_type - 40, // expected_display_size - SQL_FALSE, // expected_prec_scale - 19, // expected_length - 19, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "decimal_positive", // expected_column_name + SQL_DECIMAL, // expected_data_type + 40, // expected_display_size + SQL_FALSE, // expected_prec_scale + 19, // expected_length + 19, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttributes(stmt, 4, - std::wstring(L"float_max"), // expected_column_name - SQL_FLOAT, // expected_data_type - 24, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "float_max", // expected_column_name + SQL_FLOAT, // expected_data_type + 24, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttributes(stmt, 5, - std::wstring(L"double_max"), // expected_column_name - SQL_DOUBLE, // expected_data_type - 24, // expected_display_size - SQL_FALSE, // expected_prec_scale - 8, // expected_length - 8, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_SEARCHABLE, // expected_searchable - SQL_FALSE); // expected_unsigned_column + "double_max", // expected_column_name + SQL_DOUBLE, // expected_data_type + 24, // expected_display_size + SQL_FALSE, // expected_prec_scale + 8, // expected_length + 8, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_SEARCHABLE, // expected_searchable + SQL_FALSE); // expected_unsigned_column CheckSQLColAttributes(stmt, 6, - std::wstring(L"bit_true"), // expected_column_name - SQL_BIT, // expected_data_type - 1, // expected_display_size - SQL_FALSE, // expected_prec_scale - 1, // expected_length - 1, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "bit_true", // expected_column_name + SQL_BIT, // expected_data_type + 1, // expected_display_size + SQL_FALSE, // expected_prec_scale + 1, // expected_length + 1, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttributes(stmt, 7, - std::wstring(L"date_max"), // expected_column_name - SQL_DATE, // expected_data_type - 10, // expected_display_size - SQL_FALSE, // expected_prec_scale - 10, // expected_length - 10, // expected_column_size - 0, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "date_max", // expected_column_name + SQL_DATE, // expected_data_type + 10, // expected_display_size + SQL_FALSE, // expected_prec_scale + 10, // expected_length + 10, // expected_column_size + 0, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttributes(stmt, 8, - std::wstring(L"time_max"), // expected_column_name - SQL_TIME, // expected_data_type - 12, // expected_display_size - SQL_FALSE, // expected_prec_scale - 12, // expected_length - 12, // expected_column_size - 3, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "time_max", // expected_column_name + SQL_TIME, // expected_data_type + 12, // expected_display_size + SQL_FALSE, // expected_prec_scale + 12, // expected_length + 12, // expected_column_size + 3, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column CheckSQLColAttributes(stmt, 9, - std::wstring(L"timestamp_max"), // expected_column_name - SQL_TIMESTAMP, // expected_data_type - 23, // expected_display_size - SQL_FALSE, // expected_prec_scale - 23, // expected_length - 23, // expected_column_size - 3, // expected_column_scale - SQL_NULLABLE, // expected_column_nullability - SQL_SEARCHABLE, // expected_searchable - SQL_TRUE); // expected_unsigned_column + "timestamp_max", // expected_column_name + SQL_TIMESTAMP, // expected_data_type + 23, // expected_display_size + SQL_FALSE, // expected_prec_scale + 23, // expected_length + 23, // expected_column_size + 3, // expected_column_scale + SQL_NULLABLE, // expected_column_nullability + SQL_SEARCHABLE, // expected_searchable + SQL_TRUE); // expected_unsigned_column } #endif // __APPLE__ @@ -2254,7 +2245,7 @@ TEST_F(ColumnsMockTest, SQLDescribeColValidateInput) { SQLUSMALLINT bookmark_column = 0; SQLUSMALLINT out_of_range_column = 4; SQLUSMALLINT negative_column = -1; - SQLWCHAR column_name[1024]; + SQLWCHAR column_name[1024] = {0}; SQLSMALLINT buf_char_len = static_cast(sizeof(column_name) / GetSqlWCharSize()); SQLSMALLINT name_length = 0; diff --git a/cpp/src/arrow/flight/sql/odbc/tests/errors_test.cc b/cpp/src/arrow/flight/sql/odbc/tests/errors_test.cc index 307b4812517a..38f8d342a0bd 100644 --- a/cpp/src/arrow/flight/sql/odbc/tests/errors_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/tests/errors_test.cc @@ -401,11 +401,10 @@ TYPED_TEST(ErrorsTest, TestSQLErrorStmtError) { // When application passes buffer length greater than SQL_MAX_MESSAGE_LENGTH (512), // DM passes 512 as buffer length to SQLError. - std::wstring wsql = L"1"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"1"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_ERROR, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_ERROR, SQLExecDirect(stmt, wsql, wsql_len)); SQLWCHAR sql_state[6] = {0}; SQLINTEGER native_error = 0; @@ -428,11 +427,10 @@ TYPED_TEST(ErrorsTest, TestSQLErrorStmtError) { TYPED_TEST(ErrorsTest, TestSQLErrorStmtWarning) { // Test ODBC 2.0 API SQLError. - std::wstring wsql = L"SELECT 'VERY LONG STRING here' AS string_col;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 'VERY LONG STRING here' AS string_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -534,11 +532,10 @@ TYPED_TEST(ErrorsOdbcV2Test, TestSQLErrorStmtError) { // When application passes buffer length greater than SQL_MAX_MESSAGE_LENGTH (512), // DM passes 512 as buffer length to SQLError. - std::wstring wsql = L"SELECT * from non_existent_table;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT * from non_existent_table;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_ERROR, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_ERROR, SQLExecDirect(stmt, wsql, wsql_len)); SQLWCHAR sql_state[6] = {0}; SQLINTEGER native_error = 0; @@ -559,11 +556,10 @@ TYPED_TEST(ErrorsOdbcV2Test, TestSQLErrorStmtError) { TYPED_TEST(ErrorsOdbcV2Test, TestSQLErrorStmtWarning) { // Test ODBC 2.0 API SQLError. - std::wstring wsql = L"SELECT 'VERY LONG STRING here' AS string_col;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 'VERY LONG STRING here' AS string_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); diff --git a/cpp/src/arrow/flight/sql/odbc/tests/statement_attr_test.cc b/cpp/src/arrow/flight/sql/odbc/tests/statement_attr_test.cc index 18dca6e24ba6..1bbd3c1b74e0 100644 --- a/cpp/src/arrow/flight/sql/odbc/tests/statement_attr_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/tests/statement_attr_test.cc @@ -332,11 +332,10 @@ TYPED_TEST(StatementAttributeTest, TestSQLGetStmtAttrRowBindType) { } TYPED_TEST(StatementAttributeTest, TestSQLGetStmtAttrRowNumber) { - std::wstring wsql = L"SELECT 1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); diff --git a/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc b/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc index a69ba9dd3c17..dd33ce9ae973 100644 --- a/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc @@ -38,11 +38,10 @@ using TestTypes = ::testing::Types; TYPED_TEST_SUITE(StatementTest, TestTypes); TYPED_TEST(StatementTest, TestSQLExecDirectSimpleQuery) { - std::wstring wsql = L"SELECT 1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -66,21 +65,19 @@ TYPED_TEST(StatementTest, TestSQLExecDirectSimpleQuery) { } TYPED_TEST(StatementTest, TestSQLExecDirectInvalidQuery) { - std::wstring wsql = L"SELECT;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_ERROR, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_ERROR, SQLExecDirect(stmt, wsql, wsql_len)); // ODBC provides generic error code HY000 to all statement errors VerifyOdbcErrorState(SQL_HANDLE_STMT, stmt, kErrorStateHY000); } TYPED_TEST(StatementTest, TestSQLExecuteSimpleQuery) { - std::wstring wsql = L"SELECT 1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLPrepare(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLPrepare(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLExecute(stmt)); @@ -107,10 +104,10 @@ TYPED_TEST(StatementTest, TestSQLExecuteSimpleQuery) { } TYPED_TEST(StatementTest, TestSQLPrepareInvalidQuery) { - std::wstring wsql = L"SELECT;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_ERROR, SQLPrepare(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_ERROR, SQLPrepare(stmt, wsql, wsql_len)); // ODBC provides generic error code HY000 to all statement errors VerifyOdbcErrorState(SQL_HANDLE_STMT, stmt, kErrorStateHY000); @@ -349,15 +346,14 @@ TEST_F(StatementRemoteTest, TestSQLExecDirectTimeQuery) { // Mock server test is skipped due to limitation on the mock server. // Time type from mock server does not include the fraction - std::wstring wsql = + SQLWCHAR wsql[] = LR"( SELECT CAST(TIME '00:00:00' AS TIME) AS time_min, CAST(TIME '23:59:59' AS TIME) AS time_max; )"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLSMALLINT wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -382,11 +378,10 @@ TEST_F(StatementMockTest, TestSQLExecDirectVarbinaryQuery) { // Have binary test on mock test base as remote test servers tend to have different // formats for binary data - std::wstring wsql = L"SELECT X'ABCDEF' AS c_varbinary;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT X'ABCDEF' AS c_varbinary;"; + SQLSMALLINT wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -578,15 +573,14 @@ TEST_F(StatementRemoteTest, TestSQLExecDirectTimeQueryDefaultType) { // Mock server test is skipped due to limitation on the mock server. // Time type from mock server does not include the fraction - std::wstring wsql = + SQLWCHAR wsql[] = LR"( SELECT CAST(TIME '00:00:00' AS TIME) AS time_min, CAST(TIME '23:59:59' AS TIME) AS time_max; )"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -612,11 +606,10 @@ TEST_F(StatementRemoteTest, TestSQLExecDirectVarbinaryQueryDefaultType) { // Mock server has type `DENSE_UNION` for varbinary. // Note that not all remote servers support "from_hex" function - std::wstring wsql = L"SELECT from_hex('ABCDEF') AS c_varbinary;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT from_hex('ABCDEF') AS c_varbinary;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -634,11 +627,10 @@ TEST_F(StatementRemoteTest, TestSQLExecDirectVarbinaryQueryDefaultType) { // TODO(GH-48730): Enable this test when ARD/IRD descriptor support is fully implemented TYPED_TEST(StatementTest, DISABLED_TestGetDataPrecisionScaleUsesIRDAsDefault) { // Verify that SQLGetData uses IRD precision/scale as defaults when ARD values are unset - std::wstring wsql = L"SELECT CAST('123.45' AS NUMERIC) as decimal_col;"; - std::vector sql(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT CAST('123.45' AS NUMERIC) as decimal_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql[0], static_cast(sql.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -680,11 +672,10 @@ TYPED_TEST(StatementTest, DISABLED_TestGetDataPrecisionScaleUsesIRDAsDefault) { TYPED_TEST(StatementTest, DISABLED_TestGetDataPrecisionScaleUsesARDWhenSet) { // Verify that SQLGetData uses ARD precision/scale when set, for both SQL_ARD_TYPE and // SQL_C_DEFAULT - std::wstring wsql = L"SELECT CAST('123.45' AS NUMERIC) as decimal_col;"; - std::vector sql(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT CAST('123.45' AS NUMERIC) as decimal_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql[0], static_cast(sql.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -726,11 +717,10 @@ TYPED_TEST(StatementTest, DISABLED_TestGetDataPrecisionScaleUsesARDWhenSet) { TYPED_TEST(StatementTest, TestSQLExecDirectGuidQueryUnsupported) { // Query GUID as string as SQLite does not support GUID - std::wstring wsql = L"SELECT 'C77313CF-4E08-47CE-B6DF-94DD2FCF3541' AS guid;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 'C77313CF-4E08-47CE-B6DF-94DD2FCF3541' AS guid;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -743,7 +733,7 @@ TYPED_TEST(StatementTest, TestSQLExecDirectGuidQueryUnsupported) { } TYPED_TEST(StatementTest, TestSQLExecDirectRowFetching) { - std::wstring wsql = + SQLWCHAR wsql[] = LR"( SELECT 1 AS small_table UNION ALL @@ -751,10 +741,9 @@ TYPED_TEST(StatementTest, TestSQLExecDirectRowFetching) { UNION ALL SELECT 3; )"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); // Fetch row 1 ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -802,7 +791,7 @@ TYPED_TEST(StatementTest, TestSQLFetchScrollRowFetching) { SQLLEN rows_fetched; SQLSetStmtAttr(stmt, SQL_ATTR_ROWS_FETCHED_PTR, &rows_fetched, 0); - std::wstring wsql = + SQLWCHAR wsql[] = LR"( SELECT 1 AS small_table UNION ALL @@ -810,10 +799,9 @@ TYPED_TEST(StatementTest, TestSQLFetchScrollRowFetching) { UNION ALL SELECT 3; )"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); // Fetch row 1 ASSERT_EQ(SQL_SUCCESS, SQLFetchScroll(stmt, SQL_FETCH_NEXT, 0)); @@ -865,11 +853,10 @@ TYPED_TEST(StatementTest, TestSQLFetchScrollRowFetching) { TYPED_TEST(StatementTest, TestSQLFetchScrollUnsupportedOrientation) { // SQL_FETCH_NEXT is the only supported fetch orientation. - std::wstring wsql = L"SELECT 1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_ERROR, SQLFetchScroll(stmt, SQL_FETCH_PRIOR, 0)); @@ -903,11 +890,10 @@ TYPED_TEST(StatementTest, TestSQLFetchScrollUnsupportedOrientation) { } TYPED_TEST(StatementTest, TestSQLExecDirectVarcharTruncation) { - std::wstring wsql = L"SELECT 'VERY LONG STRING here' AS string_col;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 'VERY LONG STRING here' AS string_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -953,11 +939,10 @@ TYPED_TEST(StatementTest, TestSQLExecDirectVarcharTruncation) { } TYPED_TEST(StatementTest, TestSQLExecDirectWVarcharTruncation) { - std::wstring wsql = L"SELECT 'VERY LONG Unicode STRING 句子 here' AS wstring_col;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 'VERY LONG Unicode STRING 句子 here' AS wstring_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1007,11 +992,10 @@ TEST_F(StatementMockTest, TestSQLExecDirectVarbinaryTruncation) { // Have binary test on mock test base as remote test servers tend to have different // formats for binary data - std::wstring wsql = L"SELECT X'ABCDEFAB' AS c_varbinary;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT X'ABCDEFAB' AS c_varbinary;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1052,9 +1036,9 @@ TYPED_TEST(StatementTest, DISABLED_TestSQLExecDirectFloatTruncation) { // GH-46985: return warning message instead of error on float truncation case std::wstring wsql; if constexpr (std::is_same_v) { - wsql = std::wstring(L"SELECT CAST(1.234 AS REAL) AS float_val"); + wsql = L"SELECT CAST(1.234 AS REAL) AS float_val"; } else if constexpr (std::is_same_v) { - wsql = std::wstring(L"SELECT CAST(1.234 AS FLOAT) AS float_val"); + wsql = L"SELECT CAST(1.234 AS FLOAT) AS float_val"; } std::vector sql0(wsql.begin(), wsql.end()); @@ -1077,11 +1061,10 @@ TEST_F(StatementRemoteTest, TestSQLExecDirectNullQuery) { // Limitation on mock test server prevents null from working properly, so use remote // server instead. Mock server has type `DENSE_UNION` for null column data. - std::wstring wsql = L"SELECT null as null_col;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT null as null_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1099,17 +1082,16 @@ TEST_F(StatementMockTest, TestSQLExecDirectTruncationQueryNullIndicator) { // Have binary test on mock test base as remote test servers tend to have different // formats for binary data - std::wstring wsql = + SQLWCHAR wsql[] = LR"( SELECT 1, 'VERY LONG STRING here' AS string_col, 'VERY LONG Unicode STRING 句子 here' AS wstring_col, X'ABCDEFAB' AS c_varbinary; )"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1150,11 +1132,10 @@ TEST_F(StatementRemoteTest, TestSQLExecDirectNullQueryNullIndicator) { // Limitation on mock test server prevents null from working properly, so use remote // server instead. Mock server has type `DENSE_UNION` for null column data. - std::wstring wsql = L"SELECT null as null_col;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT null as null_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1651,15 +1632,14 @@ TEST_F(StatementRemoteTest, TestSQLBindColTimeQuery) { ASSERT_EQ(SQL_SUCCESS, SQLBindCol(stmt, 2, SQL_C_TYPE_TIME, &time_var_max, buf_len, &ind)); - std::wstring wsql = + SQLWCHAR wsql[] = LR"( SELECT CAST(TIME '00:00:00' AS TIME) AS time_min, CAST(TIME '23:59:59' AS TIME) AS time_max; )"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1685,11 +1665,10 @@ TEST_F(StatementMockTest, TestSQLBindColVarbinaryQuery) { ASSERT_EQ(SQL_SUCCESS, SQLBindCol(stmt, 1, SQL_C_BINARY, &varbinary_val[0], buf_len, &ind)); - std::wstring wsql = L"SELECT X'ABCDEF' AS c_varbinary;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT X'ABCDEF' AS c_varbinary;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1708,11 +1687,10 @@ TEST_F(StatementRemoteTest, TestSQLBindColNullQuery) { ASSERT_EQ(SQL_SUCCESS, SQLBindCol(stmt, 1, SQL_C_LONG, &val, 0, &ind)); - std::wstring wsql = L"SELECT null as null_col;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT null as null_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1728,11 +1706,10 @@ TEST_F(StatementRemoteTest, TestSQLBindColNullQueryNullIndicator) { ASSERT_EQ(SQL_SUCCESS, SQLBindCol(stmt, 1, SQL_C_LONG, &val, 0, 0)); - std::wstring wsql = L"SELECT null as null_col;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT null as null_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_ERROR, SQLFetch(stmt)); // Verify invalid null indicator is reported, as it is required @@ -1748,7 +1725,7 @@ TYPED_TEST(StatementTest, TestSQLBindColRowFetching) { // should be updated after every SQLFetch call. ASSERT_EQ(SQL_SUCCESS, SQLBindCol(stmt, 1, SQL_C_LONG, &val, buf_len, &ind)); - std::wstring wsql = + SQLWCHAR wsql[] = LR"( SELECT 1 AS small_table UNION ALL @@ -1756,10 +1733,9 @@ TYPED_TEST(StatementTest, TestSQLBindColRowFetching) { UNION ALL SELECT 3; )"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); // Fetch row 1 ASSERT_EQ(SQL_SUCCESS, SQLFetch(stmt)); @@ -1799,7 +1775,7 @@ TYPED_TEST(StatementTest, TestSQLBindColRowArraySize) { ASSERT_EQ(SQL_SUCCESS, SQLSetStmtAttr(stmt, SQL_ATTR_ROWS_FETCHED_PTR, &rows_fetched, 0)); - std::wstring wsql = + SQLWCHAR wsql[] = LR"( SELECT 1 AS small_table UNION ALL @@ -1807,10 +1783,9 @@ TYPED_TEST(StatementTest, TestSQLBindColRowArraySize) { UNION ALL SELECT 3; )"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLSetStmtAttr(stmt, SQL_ATTR_ROW_ARRAY_SIZE, reinterpret_cast(rows), 0)); @@ -1918,7 +1893,7 @@ TYPED_TEST(StatementTest, TestSQLExtendedFetchRowFetching) { ASSERT_EQ(SQL_SUCCESS, SQLSetStmtAttr(stmt, SQL_ROWSET_SIZE, reinterpret_cast(rows), 0)); - std::wstring wsql = + SQLWCHAR wsql[] = LR"( SELECT 1 AS small_table UNION ALL @@ -1926,10 +1901,9 @@ TYPED_TEST(StatementTest, TestSQLExtendedFetchRowFetching) { UNION ALL SELECT 3; )"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); // Fetch row 1-3. SQLULEN row_count; @@ -1965,11 +1939,10 @@ TEST_F(StatementRemoteTest, DISABLED_TestSQLExtendedFetchQueryNullIndicator) { ASSERT_EQ(SQL_SUCCESS, SQLBindCol(stmt, 1, SQL_C_LONG, &val, 0, nullptr)); - std::wstring wsql = L"SELECT null as null_col;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT null as null_col;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); SQLULEN row_count1; SQLUSMALLINT row_status1[1]; @@ -1982,11 +1955,11 @@ TEST_F(StatementRemoteTest, DISABLED_TestSQLExtendedFetchQueryNullIndicator) { TYPED_TEST(StatementTest, TestSQLMoreResultsNoData) { // Verify SQLMoreResults returns SQL_NO_DATA by default. - std::wstring wsql = L"SELECT 1;"; - std::vector sql0(wsql.begin(), wsql.end()); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + SQLWCHAR wsql[] = L"SELECT 1;"; + SQLINTEGER wsql_len = std::wcslen(wsql); + + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_NO_DATA, SQLMoreResults(stmt)); } @@ -2209,21 +2182,19 @@ TYPED_TEST(StatementTest, SQLRowCountFunctionSequenceErrorOnNoQuery) { } TYPED_TEST(StatementTest, TestSQLFreeStmtSQLClose) { - std::wstring wsql = L"SELECT 1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLFreeStmt(stmt, SQL_CLOSE)); } TYPED_TEST(StatementTest, TestSQLCloseCursor) { - std::wstring wsql = L"SELECT 1;"; - std::vector sql0(wsql.begin(), wsql.end()); + SQLWCHAR wsql[] = L"SELECT 1;"; + SQLINTEGER wsql_len = std::wcslen(wsql); - ASSERT_EQ(SQL_SUCCESS, - SQLExecDirect(stmt, &sql0[0], static_cast(sql0.size()))); + ASSERT_EQ(SQL_SUCCESS, SQLExecDirect(stmt, wsql, wsql_len)); ASSERT_EQ(SQL_SUCCESS, SQLCloseCursor(stmt)); } diff --git a/cpp/src/arrow/integration/json_internal.cc b/cpp/src/arrow/integration/json_internal.cc index 4b0fc693931c..2dc6b3611802 100644 --- a/cpp/src/arrow/integration/json_internal.cc +++ b/cpp/src/arrow/integration/json_internal.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -48,7 +49,6 @@ #include "arrow/util/key_value_metadata.h" #include "arrow/util/logging_internal.h" #include "arrow/util/range.h" -#include "arrow/util/span.h" #include "arrow/util/string.h" #include "arrow/util/value_parsing.h" #include "arrow/visit_array_inline.h" @@ -1461,11 +1461,10 @@ class ArrayReader { GetMemberArray(obj_, "VARIADIC_DATA_BUFFERS")); using internal::Zip; - using util::span; BufferVector buffers; buffers.resize(json_variadic_bufs.Size() + 2); - for (auto [json_buf, buf] : Zip(json_variadic_bufs, span{buffers}.subspan(2))) { + for (auto [json_buf, buf] : Zip(json_variadic_bufs, std::span{buffers}.subspan(2))) { ARROW_ASSIGN_OR_RAISE(auto hex_string, GetStringView(json_buf)); ARROW_ASSIGN_OR_RAISE( buf, AllocateBuffer(static_cast(hex_string.size()) / 2, pool_)); @@ -1482,8 +1481,8 @@ class ArrayReader { ARROW_ASSIGN_OR_RAISE( buffers[1], AllocateBuffer(length_ * sizeof(BinaryViewType::c_type), pool_)); - span views{buffers[1]->mutable_data_as(), - static_cast(length_)}; + std::span views{buffers[1]->mutable_data_as(), + static_cast(length_)}; int64_t null_count = 0; for (auto [json_view, out_view, is_valid] : Zip(json_views, views, is_valid_)) { diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index a41b63f07b3e..4352716ebd76 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -74,7 +74,6 @@ add_arrow_test(utility-test rows_to_batches_test.cc secure_string_test.cc small_vector_test.cc - span_test.cc stl_util_test.cc string_test.cc tdigest_test.cc diff --git a/cpp/src/arrow/util/align_util.h b/cpp/src/arrow/util/align_util.h index a03a92cc0a66..0b23c6d77bd9 100644 --- a/cpp/src/arrow/util/align_util.h +++ b/cpp/src/arrow/util/align_util.h @@ -46,7 +46,8 @@ inline BitmapWordAlignParams BitmapWordAlign(const uint8_t* data, int64_t bit_of int64_t length) { // TODO: We can remove this condition once CRAN upgrades its macOS // SDK from 11.3. -#if defined(__clang__) && !defined(__cpp_lib_bitops) && !defined(__EMSCRIPTEN__) + // __apple_build_version__ should be defined only on Apple clang +#if defined(__apple_build_version__) && !defined(__cpp_lib_bitops) static_assert((ALIGN_IN_BYTES != 0) && ((ALIGN_IN_BYTES & (ALIGN_IN_BYTES - 1)) == 0), "ALIGN_IN_BYTES should be a positive power of two"); #else diff --git a/cpp/src/arrow/util/binary_view_util.h b/cpp/src/arrow/util/binary_view_util.h index eb079e2c548a..3eae84bd0f2a 100644 --- a/cpp/src/arrow/util/binary_view_util.h +++ b/cpp/src/arrow/util/binary_view_util.h @@ -21,7 +21,6 @@ #include #include "arrow/type.h" -#include "arrow/util/span.h" namespace arrow::util { diff --git a/cpp/src/arrow/util/bit_util.h b/cpp/src/arrow/util/bit_util.h index a6eb540fa9be..bffc3c14a106 100644 --- a/cpp/src/arrow/util/bit_util.h +++ b/cpp/src/arrow/util/bit_util.h @@ -139,9 +139,10 @@ static inline uint64_t TrailingBits(uint64_t v, int num_bits) { static inline int Log2(uint64_t x) { // DCHECK_GT(x, 0); - // TODO: We can remove this condition once CRAN upgrades its macOS - // SDK from 11.3. -#if defined(__clang__) && !defined(__cpp_lib_bitops) && !defined(__EMSCRIPTEN__) +// TODO: We can remove this condition once CRAN upgrades its macOS +// SDK from 11.3. +// __apple_build_version__ should be defined only on Apple clang +#if defined(__apple_build_version__) && !defined(__cpp_lib_bitops) return std::log2p1(x - 1); #else return std::bit_width(x - 1); diff --git a/cpp/src/arrow/util/bitmap.h b/cpp/src/arrow/util/bitmap.h index 141d558c3a8c..49f319081fa4 100644 --- a/cpp/src/arrow/util/bitmap.h +++ b/cpp/src/arrow/util/bitmap.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,6 @@ #include "arrow/util/compare.h" #include "arrow/util/endian.h" #include "arrow/util/functional.h" -#include "arrow/util/span.h" #include "arrow/util/string_util.h" #include "arrow/util/visibility.h" @@ -155,7 +155,7 @@ class ARROW_EXPORT Bitmap : public util::ToStringOstreamable, Bitmap bitmaps[N]; int64_t offsets[N]; int64_t bit_length = BitLength(bitmaps_arg, N); - util::span words[N]; + std::span words[N]; for (size_t i = 0; i < N; ++i) { bitmaps[i] = bitmaps_arg[i]; offsets[i] = bitmaps[i].template word_offset(); @@ -384,7 +384,7 @@ class ARROW_EXPORT Bitmap : public util::ToStringOstreamable, int64_t length() const { return length_; } /// span of all bytes which contain any bit in this Bitmap - util::span bytes() const { + std::span bytes() const { auto byte_offset = offset_ / 8; auto byte_count = bit_util::CeilDiv(offset_ + length_, 8) - byte_offset; return {data_ + byte_offset, static_cast(byte_count)}; @@ -404,7 +404,7 @@ class ARROW_EXPORT Bitmap : public util::ToStringOstreamable, /// \warning The words may contain bytes which lie outside the buffer or are /// uninitialized. template - util::span words() const { + std::span words() const { auto bytes_addr = reinterpret_cast(bytes().data()); auto words_addr = bytes_addr - bytes_addr % sizeof(Word); auto word_byte_count = diff --git a/cpp/src/arrow/util/bitmap_builders.cc b/cpp/src/arrow/util/bitmap_builders.cc index 000dda718d0d..5fadddc6308b 100644 --- a/cpp/src/arrow/util/bitmap_builders.cc +++ b/cpp/src/arrow/util/bitmap_builders.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -33,7 +34,7 @@ namespace internal { namespace { -void FillBitsFromBytes(util::span bytes, uint8_t* bits) { +void FillBitsFromBytes(std::span bytes, uint8_t* bits) { for (size_t i = 0; i < bytes.size(); ++i) { if (bytes[i] > 0) { bit_util::SetBit(bits, i); @@ -43,7 +44,7 @@ void FillBitsFromBytes(util::span bytes, uint8_t* bits) { } // namespace -Result> BytesToBits(util::span bytes, +Result> BytesToBits(std::span bytes, MemoryPool* pool) { int64_t bit_length = bit_util::BytesForBits(bytes.size()); diff --git a/cpp/src/arrow/util/bitmap_builders.h b/cpp/src/arrow/util/bitmap_builders.h index 4bf2edfdcbd6..ae9ea35047e2 100644 --- a/cpp/src/arrow/util/bitmap_builders.h +++ b/cpp/src/arrow/util/bitmap_builders.h @@ -19,11 +19,11 @@ #include #include +#include #include #include "arrow/result.h" #include "arrow/type_fwd.h" -#include "arrow/util/span.h" #include "arrow/util/visibility.h" namespace arrow { @@ -37,7 +37,7 @@ Result> BitmapAllButOne(MemoryPool* pool, int64_t length /// \brief Convert vector of bytes to bitmap buffer ARROW_EXPORT -Result> BytesToBits(util::span bytes, +Result> BytesToBits(std::span bytes, MemoryPool* pool = default_memory_pool()); } // namespace internal diff --git a/cpp/src/arrow/util/bpacking_simd_kernel_internal.h b/cpp/src/arrow/util/bpacking_simd_kernel_internal.h index 70329dcb34c2..318f348b4a7b 100644 --- a/cpp/src/arrow/util/bpacking_simd_kernel_internal.h +++ b/cpp/src/arrow/util/bpacking_simd_kernel_internal.h @@ -153,14 +153,15 @@ template constexpr bool IsNeon = std::is_base_of_v; /// Wrapper around ``xsimd::bitwise_lshift`` with optimizations for non implemented sizes. -// -// We replace the variable left shift by a variable multiply with a power of two. -// -// This trick is borrowed from Daniel Lemire and Leonid Boytsov, Decoding billions of -// integers per second through vectorization, Software Practice & Experience 45 (1), 2015. -// http://arxiv.org/abs/1209.2137 -// +/// +/// We replace the variable left shift by a variable multiply with a power of two. +/// +/// This trick is borrowed from Daniel Lemire and Leonid Boytsov, Decoding billions of +/// integers per second through vectorization, Software Practice & Experience 45 (1), +/// 2015. http://arxiv.org/abs/1209.2137 +/// /// TODO(xsimd) Tracking in https://github.com/xtensor-stack/xsimd/pull/1220 +/// When migrating, be sure to use batch_constant overload, and not the batch one. template auto left_shift(const xsimd::batch& batch, xsimd::batch_constant shifts) @@ -200,14 +201,16 @@ auto left_shift(const xsimd::batch& batch, return xsimd::bitwise_cast(shifted0 | shifted1); } - // TODO(xsimd) bug fixed likely in xsimd>14.0.0 + // TODO(xsimd) bug fixed in xsimd 14.1.0 // https://github.com/xtensor-stack/xsimd/pull/1266 +#if XSIMD_VERSION_MAJOR < 14 || ((XSIMD_VERSION_MAJOR == 14) && XSIMD_VERSION_MINOR == 0) if constexpr (IsNeon) { using SInt = std::make_signed_t; constexpr auto signed_shifts = xsimd::batch_constant(kShifts)...>(); return xsimd::kernel::bitwise_lshift(batch, signed_shifts.as_batch(), Arch{}); } +#endif return batch << shifts; } @@ -237,7 +240,8 @@ auto right_shift_by_excess(const xsimd::batch& batch, constexpr auto IntSize = sizeof(Int); // Architecture for which there is no variable right shift but a larger fallback exists. - /// TODO(xsimd) Tracking for Avx2 in https://github.com/xtensor-stack/xsimd/pull/1220 + // TODO(xsimd) Tracking for Avx2 in https://github.com/xtensor-stack/xsimd/pull/1220 + // When migrating, be sure to use batch_constant overload, and not the batch one. if constexpr (kIsAvx2 && (IntSize == sizeof(uint8_t) || IntSize == sizeof(uint16_t))) { using twice_uint = SizedUint<2 * IntSize>; @@ -265,14 +269,16 @@ auto right_shift_by_excess(const xsimd::batch& batch, return xsimd::bitwise_rshift(left_shift(batch, kLShifts)); } - // TODO(xsimd) bug fixed likely in xsimd>14.0.0 + // TODO(xsimd) bug fixed in xsimd 14.1.0 // https://github.com/xtensor-stack/xsimd/pull/1266 +#if XSIMD_VERSION_MAJOR < 14 || ((XSIMD_VERSION_MAJOR == 14) && XSIMD_VERSION_MINOR == 0) if constexpr (IsNeon) { using SInt = std::make_signed_t; constexpr auto signed_shifts = xsimd::batch_constant(kShifts)...>(); return xsimd::kernel::bitwise_rshift(batch, signed_shifts.as_batch(), Arch{}); } +#endif return batch >> shifts; } diff --git a/cpp/src/arrow/util/float16_test.cc b/cpp/src/arrow/util/float16_test.cc index 5918381a2699..284bf718833f 100644 --- a/cpp/src/arrow/util/float16_test.cc +++ b/cpp/src/arrow/util/float16_test.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -24,7 +25,6 @@ #include "arrow/testing/gtest_util.h" #include "arrow/util/endian.h" #include "arrow/util/float16.h" -#include "arrow/util/span.h" #include "arrow/util/ubsan.h" namespace arrow::util { @@ -45,7 +45,7 @@ class Float16ConversionTest : public ::testing::Test { T output; }; - static void TestRoundTrip(span test_cases) { + static void TestRoundTrip(std::span test_cases) { for (size_t index = 0; index < test_cases.size(); ++index) { ARROW_SCOPED_TRACE("i=", index); const auto& tc = test_cases[index]; @@ -61,7 +61,7 @@ class Float16ConversionTest : public ::testing::Test { } } - static void TestRoundTripFromNaN(span test_cases) { + static void TestRoundTripFromNaN(std::span test_cases) { for (size_t i = 0; i < test_cases.size(); ++i) { ARROW_SCOPED_TRACE("i=", i); const auto input = test_cases[i]; @@ -143,7 +143,7 @@ void Float16ConversionTest::TestRoundTrip() { {Limits::lowest(), 0b1111110000000000u, -Limits::infinity()}, }; - TestRoundTrip(span(test_cases, std::size(test_cases))); + TestRoundTrip(std::span(test_cases, std::size(test_cases))); } template <> @@ -182,7 +182,7 @@ void Float16ConversionTest::TestRoundTrip() { {Limits::lowest(), 0b1111110000000000u, -Limits::infinity()}, }; - TestRoundTrip(span(test_cases, std::size(test_cases))); + TestRoundTrip(std::span(test_cases, std::size(test_cases))); } template <> @@ -190,7 +190,7 @@ void Float16ConversionTest::TestRoundTripFromNaN() { const float test_cases[] = { Limits::quiet_NaN(), F32(0x7f800001u), F32(0xff800001u), F32(0x7fc00000u), F32(0xffc00000u), F32(0x7fffffffu), F32(0xffffffffu)}; - TestRoundTripFromNaN(span(test_cases, std::size(test_cases))); + TestRoundTripFromNaN(std::span(test_cases, std::size(test_cases))); } template <> @@ -199,7 +199,7 @@ void Float16ConversionTest::TestRoundTripFromNaN() { F64(0xfff0000000000001u), F64(0x7ff8000000000000u), F64(0xfff8000000000000u), F64(0x7fffffffffffffffu), F64(0xffffffffffffffffu)}; - TestRoundTripFromNaN(span(test_cases, std::size(test_cases))); + TestRoundTripFromNaN(std::span(test_cases, std::size(test_cases))); } using NativeFloatTypes = ::testing::Types; diff --git a/cpp/src/arrow/util/meson.build b/cpp/src/arrow/util/meson.build index ccccd68d1f1d..e21537b0d43a 100644 --- a/cpp/src/arrow/util/meson.build +++ b/cpp/src/arrow/util/meson.build @@ -159,7 +159,6 @@ install_headers( 'rows_to_batches.h', 'simd.h', 'small_vector.h', - 'span.h', 'string_util.h', 'string.h', 'task_group.h', @@ -210,7 +209,6 @@ utility_test_srcs = [ 'reflection_test.cc', 'rows_to_batches_test.cc', 'small_vector_test.cc', - 'span_test.cc', 'stl_util_test.cc', 'string_test.cc', 'tdigest_test.cc', diff --git a/cpp/src/arrow/util/rle_encoding_test.cc b/cpp/src/arrow/util/rle_encoding_test.cc index 5cfa64563a09..788450c67c90 100644 --- a/cpp/src/arrow/util/rle_encoding_test.cc +++ b/cpp/src/arrow/util/rle_encoding_test.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -36,7 +37,6 @@ #include "arrow/util/bit_util.h" #include "arrow/util/io_util.h" #include "arrow/util/rle_encoding_internal.h" -#include "arrow/util/span.h" namespace arrow::util { @@ -463,7 +463,7 @@ void TestRleBitPackedParser(std::vector bytes, rle_size_t bit_width, EXPECT_EQ(decoded, expected); } -void TestRleBitPackedParserError(span bytes, rle_size_t bit_width) { +void TestRleBitPackedParserError(std::span bytes, rle_size_t bit_width) { auto parser = RleBitPackedParser(bytes.data(), static_cast(bytes.size()), bit_width); EXPECT_FALSE(parser.exhausted()); @@ -483,7 +483,7 @@ void TestRleBitPackedParserError(span bytes, rle_size_t bit_width void TestRleBitPackedParserError(const std::vector& bytes, rle_size_t bit_width) { - TestRleBitPackedParserError(span(bytes), bit_width); + TestRleBitPackedParserError(std::span(bytes), bit_width); } TEST(RleBitPacked, RleBitPackedParser) { @@ -632,7 +632,7 @@ TEST(RleBitPacked, RleBitPackedParserErrors) { // (we pass a span<> on invalid memory, but only the reachable part should be read) std::vector bytes = {0x81, 0x80, 0x80, 0x80, 0x02}; TestRleBitPackedParserError( - /* bytes= */ span(bytes.data(), 1ULL << 30), + /* bytes= */ std::span(bytes.data(), 1ULL << 30), /* bit_width= */ 1); } @@ -918,7 +918,8 @@ TEST(BitRle, Random) { } // TODO: We can remove this condition once CRAN upgrades its macOS // SDK from 11.3. -#if defined(__clang__) && !defined(__cpp_lib_bitops) && !defined(__EMSCRIPTEN__) + // __apple_build_version__ should be defined only on Apple clang +#if defined(__apple_build_version__) && !defined(__cpp_lib_bitops) if (!CheckRoundTrip(values, std::log2p1(values.size()))) { #else if (!CheckRoundTrip(values, std::bit_width(values.size()))) { diff --git a/cpp/src/arrow/util/secure_string.cc b/cpp/src/arrow/util/secure_string.cc index bd52c55f3120..b3c5abb58f11 100644 --- a/cpp/src/arrow/util/secure_string.cc +++ b/cpp/src/arrow/util/secure_string.cc @@ -31,9 +31,10 @@ # include #endif +#include + #include "arrow/util/logging.h" #include "arrow/util/secure_string.h" -#include "arrow/util/span.h" namespace arrow::util { @@ -181,11 +182,11 @@ std::size_t SecureString::length() const { return secret_.length(); } std::size_t SecureString::capacity() const { return secret_.capacity(); } -span SecureString::as_span() { +std::span SecureString::as_span() { return {reinterpret_cast(secret_.data()), secret_.size()}; } -span SecureString::as_span() const { +std::span SecureString::as_span() const { return {reinterpret_cast(secret_.data()), secret_.size()}; } diff --git a/cpp/src/arrow/util/secure_string.h b/cpp/src/arrow/util/secure_string.h index 30088c78d4c3..36602c6a92f4 100644 --- a/cpp/src/arrow/util/secure_string.h +++ b/cpp/src/arrow/util/secure_string.h @@ -18,9 +18,9 @@ #pragma once #include +#include #include -#include "arrow/util/span.h" #include "arrow/util/visibility.h" namespace arrow::util { @@ -56,8 +56,8 @@ class ARROW_EXPORT SecureString { [[nodiscard]] std::size_t length() const; [[nodiscard]] std::size_t capacity() const; - [[nodiscard]] span as_span(); - [[nodiscard]] span as_span() const; + [[nodiscard]] std::span as_span(); + [[nodiscard]] std::span as_span() const; [[nodiscard]] std::string_view as_view() const; void Dispose(); diff --git a/cpp/src/arrow/util/secure_string_test.cc b/cpp/src/arrow/util/secure_string_test.cc index 213a4b11f206..f2dfda5ca64a 100644 --- a/cpp/src/arrow/util/secure_string_test.cc +++ b/cpp/src/arrow/util/secure_string_test.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include "arrow/util/secure_string.h" @@ -475,16 +476,17 @@ TEST(TestSecureString, AsSpan) { auto mutable_span = secret.as_span(); std::string expected = "hello world"; - span expected_span = {reinterpret_cast(expected.data()), expected.size()}; - ASSERT_EQ(const_span, expected_span); - ASSERT_EQ(mutable_span, expected_span); + std::span expected_span = {reinterpret_cast(expected.data()), + expected.size()}; + ASSERT_TRUE(std::ranges::equal(const_span, expected_span)); + ASSERT_TRUE(std::ranges::equal(mutable_span, expected_span)); // modify secret through mutual span // the const span shares the same secret, so it is changed as well mutable_span[0] = 'H'; expected_span[0] = 'H'; - ASSERT_EQ(const_span, expected_span); - ASSERT_EQ(mutable_span, expected_span); + ASSERT_TRUE(std::ranges::equal(const_span, expected_span)); + ASSERT_TRUE(std::ranges::equal(mutable_span, expected_span)); } TEST(TestSecureString, AsView) { diff --git a/cpp/src/arrow/util/span.h b/cpp/src/arrow/util/span.h deleted file mode 100644 index abe8e61bebb9..000000000000 --- a/cpp/src/arrow/util/span.h +++ /dev/null @@ -1,132 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -namespace arrow::util { - -template -class span; - -/// std::span polyfill. -/// -/// Does not support static extents. -template -class span { - static_assert(sizeof(T), - R"( -std::span allows contiguous_iterators instead of just pointers, the enforcement -of which requires T to be a complete type. arrow::util::span does not support -contiguous_iterators, but T is still required to be a complete type to prevent -writing code which would break when it is replaced by std::span.)"); - - public: - using element_type = T; - using value_type = std::remove_cv_t; - using iterator = T*; - using const_iterator = const T*; - - span() = default; - span(const span&) = default; - span& operator=(const span&) = default; - - template >> - // NOLINTNEXTLINE runtime/explicit - constexpr span(span mut) : span{mut.data(), mut.size()} {} - - constexpr span(T* data, size_t count) : data_{data}, size_{count} {} - - constexpr span(T* begin, T* end) - : data_{begin}, size_{static_cast(end - begin)} {} - - template ())), - typename RS = decltype(std::size(std::declval())), - typename E = std::enable_if_t && - std::is_constructible_v>> - // NOLINTNEXTLINE runtime/explicit, non-const reference - constexpr span(R&& range) : data_{std::data(range)}, size_{std::size(range)} {} - - constexpr T* begin() const { return data_; } - constexpr T* end() const { return data_ + size_; } - constexpr T* data() const { return data_; } - - constexpr size_t size() const { return size_; } - constexpr size_t size_bytes() const { return size_ * sizeof(T); } - constexpr bool empty() const { return size_ == 0; } - - constexpr T& operator[](size_t i) { return data_[i]; } - constexpr const T& operator[](size_t i) const { return data_[i]; } - - constexpr span subspan(size_t offset) const { - if (offset > size_) return {data_, data_}; - return {data_ + offset, size_ - offset}; - } - - constexpr span subspan(size_t offset, size_t count) const { - auto out = subspan(offset); - if (count < out.size_) { - out.size_ = count; - } - return out; - } - - constexpr bool operator==(const span& other) const { - if (size_ != other.size_) return false; - - if constexpr (std::is_integral_v) { - if (size_ == 0) { - return true; // memcmp does not handle null pointers, even if size_ == 0 - } - return std::memcmp(data_, other.data_, size_bytes()) == 0; - } else { - T* ptr = data_; - for (const T& e : other) { - if (*ptr++ != e) return false; - } - return true; - } - } - constexpr bool operator!=(const span& other) const { return !(*this == other); } - - private: - T* data_{}; - size_t size_{}; -}; - -template -span(R& range) -> span>; - -template -span(T*, size_t) -> span; - -template -constexpr span as_bytes(span s) { - return {reinterpret_cast(s.data()), s.size_bytes()}; -} - -template -constexpr span as_writable_bytes(span s) { - return {reinterpret_cast(s.data()), s.size_bytes()}; -} - -} // namespace arrow::util diff --git a/cpp/src/arrow/util/span_test.cc b/cpp/src/arrow/util/span_test.cc deleted file mode 100644 index 01460c2bd037..000000000000 --- a/cpp/src/arrow/util/span_test.cc +++ /dev/null @@ -1,204 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include - -#include - -#include "arrow/testing/gtest_util.h" -#include "arrow/testing/matchers.h" -#include "arrow/util/span.h" - -using testing::ElementsAre; -using testing::ElementsAreArray; -using testing::PrintToString; - -namespace arrow::util { - -template -std::ostream& operator<<(std::ostream& os, const span& span) { - // Inefficient but good enough for testing - os << PrintToString(std::vector(span.begin(), span.end())); - return os; -} - -TEST(Span, Construction) { - // const spans may be constructed from mutable spans - static_assert(std::is_constructible_v, span>); - // ... but mutable spans may be constructed from const spans - static_assert(!std::is_constructible_v, span>); - - int arr[] = {1, 2, 3}; - constexpr int const_arr[] = {7, 8, 9}; - - static_assert(std::is_constructible_v, decltype(arr)&>); - static_assert(!std::is_constructible_v, decltype(const_arr)&>); - - static_assert(std::is_constructible_v, decltype(arr)&>); - static_assert(std::is_constructible_v, decltype(const_arr)&>); - - static_assert(std::is_constructible_v, std::vector&>); - static_assert(!std::is_constructible_v, const std::vector&>); - static_assert(!std::is_constructible_v, std::vector&&>); - - static_assert(std::is_constructible_v, std::vector&>); - static_assert(std::is_constructible_v, const std::vector&>); - // const spans may even be constructed from rvalue ranges - static_assert(std::is_constructible_v, std::vector&&>); - - EXPECT_THAT(span(const_arr), ElementsAreArray(const_arr)); - EXPECT_THAT(span(arr), ElementsAreArray(arr)); - - static_assert(!std::is_constructible_v, decltype(const_arr)&>); - static_assert(!std::is_constructible_v, decltype(const_arr)&>); -} - -TEST(Span, TemplateArgumentDeduction) { - int arr[3]; - const int const_arr[] = {1, 2, 3}; - std::vector vec; - const std::vector const_vec; - static_assert(std::is_same_v>); - static_assert(std::is_same_v>); - static_assert(std::is_same_v>); - static_assert(std::is_same_v>); -} - -TEST(Span, Size) { - int arr[] = {1, 2, 3}; - EXPECT_EQ(span(arr).size(), 3); - EXPECT_EQ(span(arr).size_bytes(), sizeof(int) * 3); - - std::vector vec; - EXPECT_TRUE(span(vec).empty()); - EXPECT_EQ(span(vec).size(), 0); - EXPECT_EQ(span(vec).size_bytes(), 0); - - vec.resize(999); - EXPECT_FALSE(span(vec).empty()); - EXPECT_EQ(span(vec).size(), 999); - EXPECT_EQ(span(vec).size_bytes(), sizeof(int) * 999); -} - -TEST(Span, Equality) { - auto check_eq = [](auto l, auto r) { - ARROW_SCOPED_TRACE("l = ", l, ", r = ", r); - EXPECT_TRUE(l == r); - EXPECT_FALSE(l != r); - }; - auto check_ne = [](auto l, auto r) { - ARROW_SCOPED_TRACE("l = ", l, ", r = ", r); - EXPECT_TRUE(l != r); - EXPECT_FALSE(l == r); - }; - - { - // exercise integral branch with memcmp - check_eq(span(), span()); - - int arr[] = {1, 2, 3}; - check_eq(span(arr), span(arr)); - check_eq(span(arr).subspan(1), span(arr).subspan(1)); - check_ne(span(arr).subspan(1), span(arr).subspan(2)); - - std::vector vec{1, 2, 3}; - check_eq(span(vec), span(arr)); - check_eq(span(vec).subspan(1), span(arr).subspan(1)); - - vec = {2, 3, 4}; - check_ne(span(vec), span(arr)); - check_eq(span(vec).subspan(0, 2), span(arr).subspan(1)); - - // 0-sized - vec = {}; - check_ne(span(vec), span(arr)); - check_eq(span(vec), span(arr).subspan(3)); - } - { - // exercise non-integral branch with for loop - check_eq(span(), span()); - - std::string arr[] = {"a", "b", "c"}; - check_eq(span(arr), span(arr)); - check_eq(span(arr).subspan(1), span(arr).subspan(1)); - - std::vector vec{"a", "b", "c"}; - check_eq(span(vec), span(arr)); - check_eq(span(vec).subspan(1), span(arr).subspan(1)); - - vec = {"b", "c", "d"}; - check_ne(span(vec), span(arr)); - check_eq(span(vec).subspan(0, 2), span(arr).subspan(1)); - - // 0-sized - vec = {}; - check_ne(span(vec), span(arr)); - check_eq(span(vec), span(arr).subspan(3)); - } -} - -TEST(Span, SubSpan) { - int arr[] = {1, 2, 3}; - span s(arr); - - auto ExpectIdentical = [](span l, span r) { - EXPECT_EQ(l.data(), r.data()); - EXPECT_EQ(l.size(), r.size()); - }; - - ExpectIdentical(s.subspan(0), s); - ExpectIdentical(s.subspan(0, s.size()), s); - - for (size_t offset = 0; offset < s.size(); ++offset) { - span expected(arr + offset, s.size() - offset); - ExpectIdentical(s.subspan(offset), expected); - ExpectIdentical(s.subspan(offset, s.size() * 3), expected); - } - EXPECT_TRUE(s.subspan(s.size()).empty()); - EXPECT_TRUE(s.subspan(s.size() * 3).empty()); - - for (size_t length = 0; length < s.size(); ++length) { - span expected(arr, length); - ExpectIdentical(s.subspan(0, length), expected); - } - - ExpectIdentical(s.subspan(1, 1), span(arr + 1, 1)); -} - -TEST(Span, Mutation) { - size_t arr[] = {9, 9, 9, 9, 9}; - - span s(arr); - for (size_t i = 0; i < s.size(); ++i) { - s[i] = i * i; - } - - EXPECT_THAT(arr, ElementsAre(0, 1, 4, 9, 16)); - - auto set = [](span lhs, size_t rhs) { - for (size_t& i : lhs) { - i = rhs; - } - }; - set(span(arr), 0); - set(span(arr).subspan(1), 1); - set(span(arr).subspan(2, 2), 23); - EXPECT_THAT(arr, ElementsAre(0, 1, 23, 23, 1)); -} - -} // namespace arrow::util diff --git a/cpp/src/parquet/chunker_internal.cc b/cpp/src/parquet/chunker_internal.cc index 1e1238e83504..794075b73379 100644 --- a/cpp/src/parquet/chunker_internal.cc +++ b/cpp/src/parquet/chunker_internal.cc @@ -88,7 +88,8 @@ uint64_t CalculateMask(int64_t min_chunk_size, int64_t max_chunk_size, int norm_ // by taking the floor(log2(target_size)) // TODO: We can remove this condition once CRAN upgrades its macOS // SDK from 11.3. -#if defined(__clang__) && !defined(__cpp_lib_bitops) && !defined(__EMSCRIPTEN__) + // __apple_build_version__ should be defined only on Apple clang +#if defined(__apple_build_version__) && !defined(__cpp_lib_bitops) auto target_bits = std::log2p1(static_cast(target_size)); #else auto target_bits = std::bit_width(static_cast(target_size)); diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 797d435e73e8..1f1197f95a0e 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -1791,7 +1792,7 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, } auto add_levels = [](std::vector& level_histogram, - ::arrow::util::span levels, int16_t max_level) { + std::span levels, int16_t max_level) { if (max_level == 0) { return; } diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index 75eccc96a442..2c70f63b6b8e 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -1169,7 +1169,8 @@ void DeltaBitPackEncoder::FlushBlock() { // See overflow comment above. // TODO: We can remove this condition once CRAN upgrades its macOS // SDK from 11.3. -#if defined(__clang__) && !defined(__cpp_lib_bitops) && !defined(__EMSCRIPTEN__) + // __apple_build_version__ should be defined only on Apple clang +#if defined(__apple_build_version__) && !defined(__cpp_lib_bitops) const auto bit_width = bit_width_data[i] = std::log2p1(static_cast(max_delta) - static_cast(min_delta)); #else diff --git a/cpp/src/parquet/encoding_test.cc b/cpp/src/parquet/encoding_test.cc index 5115516f9fcd..9c88eb468a49 100644 --- a/cpp/src/parquet/encoding_test.cc +++ b/cpp/src/parquet/encoding_test.cc @@ -41,7 +41,6 @@ #include "arrow/util/bitmap_writer.h" #include "arrow/util/checked_cast.h" #include "arrow/util/endian.h" -#include "arrow/util/span.h" #include "arrow/util/string.h" #include "parquet/encoding.h" #include "parquet/platform.h" @@ -52,7 +51,6 @@ using arrow::default_memory_pool; using arrow::MemoryPool; using arrow::internal::checked_cast; -using arrow::util::span; namespace bit_util = arrow::bit_util; @@ -1507,7 +1505,8 @@ class TestByteStreamSplitEncoding : public TestEncodingBase { USING_BASE_MEMBERS(); template - void CheckDecode(span encoded_data, span expected_decoded_data, + void CheckDecode(std::span encoded_data, + std::span expected_decoded_data, const ColumnDescriptor* descr = nullptr) { static_assert(sizeof(U) == sizeof(c_type)); static_assert(std::is_same_v == std::is_same_v); @@ -1525,8 +1524,9 @@ class TestByteStreamSplitEncoding : public TestEncodingBase { if constexpr (std::is_same_v) { auto type_length = descr->type_length(); for (int i = 0; i < num_elements; ++i) { - ASSERT_EQ(span(expected_decoded_data[i].ptr, type_length), - span(decoded_data[i].ptr, type_length)); + ASSERT_TRUE(std::ranges::equal( + std::span(expected_decoded_data[i].ptr, type_length), + std::span(decoded_data[i].ptr, type_length))); } } else { for (int i = 0; i < num_elements; ++i) { @@ -1537,7 +1537,8 @@ class TestByteStreamSplitEncoding : public TestEncodingBase { } template - void CheckEncode(span data, span expected_encoded_data, + void CheckEncode(std::span data, + std::span expected_encoded_data, const ColumnDescriptor* descr = nullptr) { static_assert(sizeof(U) == sizeof(c_type)); static_assert(std::is_same_v == std::is_same_v); @@ -1590,7 +1591,8 @@ void TestByteStreamSplitEncoding::CheckDecode() { const std::vector expected_output{ FLBA{&raw_expected_output[0]}, FLBA{&raw_expected_output[3]}, FLBA{&raw_expected_output[6]}, FLBA{&raw_expected_output[9]}}; - CheckDecode(span{data}, span{expected_output}, FLBAColumnDescriptor(3).get()); + CheckDecode(std::span{data}, std::span{expected_output}, + FLBAColumnDescriptor(3).get()); } // - type_length = 1 { @@ -1599,7 +1601,8 @@ void TestByteStreamSplitEncoding::CheckDecode() { const std::vector expected_output{FLBA{&raw_expected_output[0]}, FLBA{&raw_expected_output[1]}, FLBA{&raw_expected_output[2]}}; - CheckDecode(span{data}, span{expected_output}, FLBAColumnDescriptor(1).get()); + CheckDecode(std::span{data}, std::span{expected_output}, + FLBAColumnDescriptor(1).get()); } } else if constexpr (sizeof(c_type) == 4) { // INT32, FLOAT @@ -1607,14 +1610,14 @@ void TestByteStreamSplitEncoding::CheckDecode() { 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC}; const auto expected_output = ToLittleEndian({0xAA774411U, 0xBB885522U, 0xCC996633U}); - CheckDecode(span{data}, span{expected_output}); + CheckDecode(std::span{data}, std::span{expected_output}); } else { // INT64, DOUBLE const std::vector data{0xDE, 0xC0, 0x37, 0x13, 0x11, 0x22, 0x33, 0x44, 0xAA, 0xBB, 0xCC, 0xDD, 0x55, 0x66, 0x77, 0x88}; const auto expected_output = ToLittleEndian({0x7755CCAA331137DEULL, 0x8866DDBB442213C0ULL}); - CheckDecode(span{data}, span{expected_output}); + CheckDecode(std::span{data}, std::span{expected_output}); } } @@ -1630,7 +1633,8 @@ void TestByteStreamSplitEncoding::CheckEncode() { FLBA{&raw_data[6]}, FLBA{&raw_data[9]}}; const std::vector expected_output{0x11, 0x44, 0x77, 0xAA, 0x22, 0x55, 0x88, 0xBB, 0x33, 0x66, 0x99, 0xCC}; - CheckEncode(span{data}, span{expected_output}, FLBAColumnDescriptor(3).get()); + CheckEncode(std::span{data}, std::span{expected_output}, + FLBAColumnDescriptor(3).get()); } // - type_length = 1 { @@ -1638,14 +1642,15 @@ void TestByteStreamSplitEncoding::CheckEncode() { const std::vector data{FLBA{&raw_data[0]}, FLBA{&raw_data[1]}, FLBA{&raw_data[2]}}; const std::vector expected_output{0x11, 0x22, 0x33}; - CheckEncode(span{data}, span{expected_output}, FLBAColumnDescriptor(1).get()); + CheckEncode(std::span{data}, std::span{expected_output}, + FLBAColumnDescriptor(1).get()); } } else if constexpr (sizeof(c_type) == 4) { // INT32, FLOAT const auto data = ToLittleEndian({0xaabbccddUL, 0x11223344UL}); const std::vector expected_output{0xdd, 0x44, 0xcc, 0x33, 0xbb, 0x22, 0xaa, 0x11}; - CheckEncode(span{data}, span{expected_output}); + CheckEncode(std::span{data}, std::span{expected_output}); } else { // INT64, DOUBLE const auto data = ToLittleEndian( @@ -1654,7 +1659,7 @@ void TestByteStreamSplitEncoding::CheckEncode() { 0x48, 0x08, 0xb8, 0x47, 0x07, 0xb7, 0x46, 0x06, 0xb6, 0x45, 0x05, 0xb5, 0x44, 0x04, 0xb4, 0x43, 0x03, 0xb3, 0x42, 0x02, 0xb2, 0x41, 0x01, 0xb1, }; - CheckEncode(span{data}, span{expected_output}); + CheckEncode(std::span{data}, std::span{expected_output}); } } @@ -2173,7 +2178,7 @@ std::shared_ptr DeltaEncode(std::vector lengths) { return encoder->FlushValues(); } -std::shared_ptr DeltaEncode(::arrow::util::span lengths) { +std::shared_ptr DeltaEncode(std::span lengths) { auto encoder = MakeTypedEncoder(Encoding::DELTA_BINARY_PACKED); encoder->Put(lengths.data(), static_cast(lengths.size())); return encoder->FlushValues(); @@ -2181,8 +2186,8 @@ std::shared_ptr DeltaEncode(::arrow::util::span lengths) std::shared_ptr DeltaEncode(std::shared_ptr<::arrow::Array>& lengths) { auto data = ::arrow::internal::checked_pointer_cast(lengths); - auto span = ::arrow::util::span{data->raw_values(), - static_cast(lengths->length())}; + auto span = std::span{data->raw_values(), + static_cast(lengths->length())}; return DeltaEncode(span); } diff --git a/cpp/src/parquet/encryption/encryption.h b/cpp/src/parquet/encryption/encryption.h index b4634b704735..023a536fd91f 100644 --- a/cpp/src/parquet/encryption/encryption.h +++ b/cpp/src/parquet/encryption/encryption.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -95,7 +96,7 @@ class PARQUET_EXPORT KeyAccessDeniedException : public ParquetException { : ParquetException(columnPath.c_str()) {} }; -inline ::arrow::util::span str2span(const std::string& str) { +inline std::span str2span(const std::string& str) { if (str.empty()) { return {}; } diff --git a/cpp/src/parquet/encryption/encryption_internal.cc b/cpp/src/parquet/encryption/encryption_internal.cc index 9400fae0adf7..1eaebb96cac8 100644 --- a/cpp/src/parquet/encryption/encryption_internal.cc +++ b/cpp/src/parquet/encryption/encryption_internal.cc @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -33,7 +34,6 @@ #include "parquet/encryption/openssl_internal.h" #include "parquet/exception.h" -using ::arrow::util::span; using parquet::ParquetException; namespace parquet::encryption { @@ -109,12 +109,13 @@ class AesEncryptor::AesEncryptorImpl : public AesCryptoContext { explicit AesEncryptorImpl(ParquetCipher::type alg_id, int32_t key_len, bool metadata, bool write_length); - int32_t Encrypt(span plaintext, span key, - span aad, span ciphertext); + int32_t Encrypt(std::span plaintext, std::span key, + std::span aad, std::span ciphertext); - int32_t SignedFooterEncrypt(span footer, span key, - span aad, span nonce, - span encrypted_footer); + int32_t SignedFooterEncrypt(std::span footer, + std::span key, std::span aad, + std::span nonce, + std::span encrypted_footer); [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const { if (plaintext_len < 0) { @@ -135,12 +136,12 @@ class AesEncryptor::AesEncryptorImpl : public AesCryptoContext { private: [[nodiscard]] CipherContext MakeCipherContext() const; - int32_t GcmEncrypt(span plaintext, span key, - span nonce, span aad, - span ciphertext); + int32_t GcmEncrypt(std::span plaintext, std::span key, + std::span nonce, std::span aad, + std::span ciphertext); - int32_t CtrEncrypt(span plaintext, span key, - span nonce, span ciphertext); + int32_t CtrEncrypt(std::span plaintext, std::span key, + std::span nonce, std::span ciphertext); }; AesEncryptor::AesEncryptorImpl::AesEncryptorImpl(ParquetCipher::type alg_id, @@ -174,8 +175,9 @@ AesCryptoContext::CipherContext AesEncryptor::AesEncryptorImpl::MakeCipherContex } int32_t AesEncryptor::AesEncryptorImpl::SignedFooterEncrypt( - span footer, span key, span aad, - span nonce, span encrypted_footer) { + std::span footer, std::span key, + std::span aad, std::span nonce, + std::span encrypted_footer) { if (static_cast(key_length_) != key.size()) { std::stringstream ss; ss << "Wrong key length " << key.size() << ". Should be " << key_length_; @@ -196,10 +198,10 @@ int32_t AesEncryptor::AesEncryptorImpl::SignedFooterEncrypt( return GcmEncrypt(footer, key, nonce, aad, encrypted_footer); } -int32_t AesEncryptor::AesEncryptorImpl::Encrypt(span plaintext, - span key, - span aad, - span ciphertext) { +int32_t AesEncryptor::AesEncryptorImpl::Encrypt(std::span plaintext, + std::span key, + std::span aad, + std::span ciphertext) { if (static_cast(key_length_) != key.size()) { std::stringstream ss; ss << "Wrong key length " << key.size() << ". Should be " << key_length_; @@ -225,11 +227,11 @@ int32_t AesEncryptor::AesEncryptorImpl::Encrypt(span plaintext, return CtrEncrypt(plaintext, key, nonce, ciphertext); } -int32_t AesEncryptor::AesEncryptorImpl::GcmEncrypt(span plaintext, - span key, - span nonce, - span aad, - span ciphertext) { +int32_t AesEncryptor::AesEncryptorImpl::GcmEncrypt(std::span plaintext, + std::span key, + std::span nonce, + std::span aad, + std::span ciphertext) { int len; int32_t ciphertext_len; @@ -304,10 +306,10 @@ int32_t AesEncryptor::AesEncryptorImpl::GcmEncrypt(span plaintext return length_buffer_length_ + buffer_size; } -int32_t AesEncryptor::AesEncryptorImpl::CtrEncrypt(span plaintext, - span key, - span nonce, - span ciphertext) { +int32_t AesEncryptor::AesEncryptorImpl::CtrEncrypt(std::span plaintext, + std::span key, + std::span nonce, + std::span ciphertext) { int len; int32_t ciphertext_len; @@ -371,11 +373,11 @@ int32_t AesEncryptor::AesEncryptorImpl::CtrEncrypt(span plaintext AesEncryptor::~AesEncryptor() = default; -int32_t AesEncryptor::SignedFooterEncrypt(span footer, - span key, - span aad, - span nonce, - span encrypted_footer) { +int32_t AesEncryptor::SignedFooterEncrypt(std::span footer, + std::span key, + std::span aad, + std::span nonce, + std::span encrypted_footer) { return impl_->SignedFooterEncrypt(footer, key, aad, nonce, encrypted_footer); } @@ -383,8 +385,9 @@ int32_t AesEncryptor::CiphertextLength(int64_t plaintext_len) const { return impl_->CiphertextLength(plaintext_len); } -int32_t AesEncryptor::Encrypt(span plaintext, span key, - span aad, span ciphertext) { +int32_t AesEncryptor::Encrypt(std::span plaintext, + std::span key, std::span aad, + std::span ciphertext) { return impl_->Encrypt(plaintext, key, aad, ciphertext); } @@ -398,8 +401,8 @@ class AesDecryptor::AesDecryptorImpl : AesCryptoContext { explicit AesDecryptorImpl(ParquetCipher::type alg_id, int32_t key_len, bool metadata, bool contains_length); - int32_t Decrypt(span ciphertext, span key, - span aad, span plaintext); + int32_t Decrypt(std::span ciphertext, std::span key, + std::span aad, std::span plaintext); [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const { if (ciphertext_len < ciphertext_size_delta_) { @@ -431,17 +434,18 @@ class AesDecryptor::AesDecryptorImpl : AesCryptoContext { /// Get the actual ciphertext length, inclusive of the length buffer length, /// and validate that the provided buffer size is large enough. - [[nodiscard]] int32_t GetCiphertextLength(span ciphertext) const; + [[nodiscard]] int32_t GetCiphertextLength(std::span ciphertext) const; - int32_t GcmDecrypt(span ciphertext, span key, - span aad, span plaintext); + int32_t GcmDecrypt(std::span ciphertext, std::span key, + std::span aad, std::span plaintext); - int32_t CtrDecrypt(span ciphertext, span key, - span plaintext); + int32_t CtrDecrypt(std::span ciphertext, std::span key, + std::span plaintext); }; -int32_t AesDecryptor::Decrypt(span ciphertext, span key, - span aad, span plaintext) { +int32_t AesDecryptor::Decrypt(std::span ciphertext, + std::span key, std::span aad, + std::span plaintext) { return impl_->Decrypt(ciphertext, key, aad, plaintext); } @@ -502,7 +506,7 @@ int32_t AesDecryptor::CiphertextLength(int32_t plaintext_len) const { } int32_t AesDecryptor::AesDecryptorImpl::GetCiphertextLength( - span ciphertext) const { + std::span ciphertext) const { if (length_buffer_length_ > 0) { // Note: length_buffer_length_ must be either 0 or kBufferSizeLength if (ciphertext.size() < static_cast(kBufferSizeLength)) { @@ -547,10 +551,10 @@ int32_t AesDecryptor::AesDecryptorImpl::GetCiphertextLength( } } -int32_t AesDecryptor::AesDecryptorImpl::GcmDecrypt(span ciphertext, - span key, - span aad, - span plaintext) { +int32_t AesDecryptor::AesDecryptorImpl::GcmDecrypt(std::span ciphertext, + std::span key, + std::span aad, + std::span plaintext) { int len; int32_t plaintext_len; @@ -622,9 +626,9 @@ int32_t AesDecryptor::AesDecryptorImpl::GcmDecrypt(span ciphertex return plaintext_len; } -int32_t AesDecryptor::AesDecryptorImpl::CtrDecrypt(span ciphertext, - span key, - span plaintext) { +int32_t AesDecryptor::AesDecryptorImpl::CtrDecrypt(std::span ciphertext, + std::span key, + std::span plaintext) { int len; int32_t plaintext_len; @@ -681,10 +685,10 @@ int32_t AesDecryptor::AesDecryptorImpl::CtrDecrypt(span ciphertex return plaintext_len; } -int32_t AesDecryptor::AesDecryptorImpl::Decrypt(span ciphertext, - span key, - span aad, - span plaintext) { +int32_t AesDecryptor::AesDecryptorImpl::Decrypt(std::span ciphertext, + std::span key, + std::span aad, + std::span plaintext) { if (static_cast(key_length_) != key.size()) { std::stringstream ss; ss << "Wrong key length " << key.size() << ". Should be " << key_length_; diff --git a/cpp/src/parquet/encryption/encryption_internal.h b/cpp/src/parquet/encryption/encryption_internal.h index 062527495659..de256809fa96 100644 --- a/cpp/src/parquet/encryption/encryption_internal.h +++ b/cpp/src/parquet/encryption/encryption_internal.h @@ -18,10 +18,10 @@ #pragma once #include +#include #include #include -#include "arrow/util/span.h" #include "parquet/properties.h" #include "parquet/types.h" @@ -62,17 +62,14 @@ class PARQUET_EXPORT AesEncryptor { /// Encrypts plaintext with the key and aad. Key length is passed only for validation. /// If different from value in constructor, exception will be thrown. - int32_t Encrypt(::arrow::util::span plaintext, - ::arrow::util::span key, - ::arrow::util::span aad, - ::arrow::util::span ciphertext); + int32_t Encrypt(std::span plaintext, std::span key, + std::span aad, std::span ciphertext); /// Encrypts plaintext footer, in order to compute footer signature (tag). - int32_t SignedFooterEncrypt(::arrow::util::span footer, - ::arrow::util::span key, - ::arrow::util::span aad, - ::arrow::util::span nonce, - ::arrow::util::span encrypted_footer); + int32_t SignedFooterEncrypt(std::span footer, + std::span key, std::span aad, + std::span nonce, + std::span encrypted_footer); private: // PIMPL Idiom @@ -107,10 +104,8 @@ class PARQUET_EXPORT AesDecryptor { /// validation. If different from value in constructor, exception will be thrown. /// The caller is responsible for ensuring that the plaintext buffer is at least as /// large as PlaintextLength(ciphertext_len). - int32_t Decrypt(::arrow::util::span ciphertext, - ::arrow::util::span key, - ::arrow::util::span aad, - ::arrow::util::span plaintext); + int32_t Decrypt(std::span ciphertext, std::span key, + std::span aad, std::span plaintext); private: // PIMPL Idiom diff --git a/cpp/src/parquet/encryption/encryption_internal_nossl.cc b/cpp/src/parquet/encryption/encryption_internal_nossl.cc index 2450f8654d6f..74de130cc9a5 100644 --- a/cpp/src/parquet/encryption/encryption_internal_nossl.cc +++ b/cpp/src/parquet/encryption/encryption_internal_nossl.cc @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include + #include "parquet/encryption/encryption_internal.h" #include "parquet/exception.h" @@ -33,11 +35,11 @@ class AesEncryptor::AesEncryptorImpl {}; AesEncryptor::~AesEncryptor() {} -int32_t AesEncryptor::SignedFooterEncrypt(::arrow::util::span footer, - ::arrow::util::span key, - ::arrow::util::span aad, - ::arrow::util::span nonce, - ::arrow::util::span encrypted_footer) { +int32_t AesEncryptor::SignedFooterEncrypt(std::span footer, + std::span key, + std::span aad, + std::span nonce, + std::span encrypted_footer) { ThrowOpenSSLRequiredException(); return -1; } @@ -47,10 +49,9 @@ int32_t AesEncryptor::CiphertextLength(int64_t plaintext_len) const { return -1; } -int32_t AesEncryptor::Encrypt(::arrow::util::span plaintext, - ::arrow::util::span key, - ::arrow::util::span aad, - ::arrow::util::span ciphertext) { +int32_t AesEncryptor::Encrypt(std::span plaintext, + std::span key, std::span aad, + std::span ciphertext) { ThrowOpenSSLRequiredException(); return -1; } @@ -62,10 +63,9 @@ AesEncryptor::AesEncryptor(ParquetCipher::type alg_id, int32_t key_len, bool met class AesDecryptor::AesDecryptorImpl {}; -int32_t AesDecryptor::Decrypt(::arrow::util::span ciphertext, - ::arrow::util::span key, - ::arrow::util::span aad, - ::arrow::util::span plaintext) { +int32_t AesDecryptor::Decrypt(std::span ciphertext, + std::span key, std::span aad, + std::span plaintext) { ThrowOpenSSLRequiredException(); return -1; } diff --git a/cpp/src/parquet/encryption/encryption_internal_test.cc b/cpp/src/parquet/encryption/encryption_internal_test.cc index bf6607e32877..006554a5cb75 100644 --- a/cpp/src/parquet/encryption/encryption_internal_test.cc +++ b/cpp/src/parquet/encryption/encryption_internal_test.cc @@ -16,6 +16,7 @@ // under the License. #include +#include #include "parquet/encryption/encryption_internal.h" @@ -97,8 +98,7 @@ class TestAesEncryption : public ::testing::Test { int32_t expected_plaintext_length = decryptor.PlaintextLength(ciphertext_length); std::vector decrypted_text(expected_plaintext_length, '\0'); - ::arrow::util::span truncated_ciphertext(ciphertext.data(), - ciphertext_length - 1); + std::span truncated_ciphertext(ciphertext.data(), ciphertext_length - 1); EXPECT_THROW(decryptor.Decrypt(truncated_ciphertext, str2span(key_), str2span(aad_), decrypted_text), ParquetException); diff --git a/cpp/src/parquet/encryption/internal_file_decryptor.cc b/cpp/src/parquet/encryption/internal_file_decryptor.cc index b90d31585597..11e3b4e54bb6 100644 --- a/cpp/src/parquet/encryption/internal_file_decryptor.cc +++ b/cpp/src/parquet/encryption/internal_file_decryptor.cc @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include + #include "parquet/encryption/internal_file_decryptor.h" #include "arrow/util/logging.h" @@ -47,8 +49,8 @@ int32_t Decryptor::CiphertextLength(int32_t plaintext_len) const { return aes_decryptor_->CiphertextLength(plaintext_len); } -int32_t Decryptor::Decrypt(::arrow::util::span ciphertext, - ::arrow::util::span plaintext) { +int32_t Decryptor::Decrypt(std::span ciphertext, + std::span plaintext) { return aes_decryptor_->Decrypt(ciphertext, key_.as_span(), str2span(aad_), plaintext); } diff --git a/cpp/src/parquet/encryption/internal_file_decryptor.h b/cpp/src/parquet/encryption/internal_file_decryptor.h index a365b4df4bf9..1343769ef364 100644 --- a/cpp/src/parquet/encryption/internal_file_decryptor.h +++ b/cpp/src/parquet/encryption/internal_file_decryptor.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -52,8 +53,7 @@ class PARQUET_EXPORT Decryptor { [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const; [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const; - int32_t Decrypt(::arrow::util::span ciphertext, - ::arrow::util::span plaintext); + int32_t Decrypt(std::span ciphertext, std::span plaintext); private: std::unique_ptr aes_decryptor_; diff --git a/cpp/src/parquet/encryption/internal_file_encryptor.cc b/cpp/src/parquet/encryption/internal_file_encryptor.cc index f6fc08499847..23dbfe34d393 100644 --- a/cpp/src/parquet/encryption/internal_file_encryptor.cc +++ b/cpp/src/parquet/encryption/internal_file_encryptor.cc @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "parquet/encryption/internal_file_encryptor.h" +#include + #include "arrow/util/secure_string.h" #include "parquet/encryption/encryption.h" #include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/internal_file_encryptor.h" using arrow::util::SecureString; @@ -37,8 +39,8 @@ int32_t Encryptor::CiphertextLength(int64_t plaintext_len) const { return aes_encryptor_->CiphertextLength(plaintext_len); } -int32_t Encryptor::Encrypt(::arrow::util::span plaintext, - ::arrow::util::span ciphertext) { +int32_t Encryptor::Encrypt(std::span plaintext, + std::span ciphertext) { return aes_encryptor_->Encrypt(plaintext, key_.as_span(), str2span(aad_), ciphertext); } diff --git a/cpp/src/parquet/encryption/internal_file_encryptor.h b/cpp/src/parquet/encryption/internal_file_encryptor.h index ee15fe32de96..c444140a05a6 100644 --- a/cpp/src/parquet/encryption/internal_file_encryptor.h +++ b/cpp/src/parquet/encryption/internal_file_encryptor.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -44,8 +45,7 @@ class PARQUET_EXPORT Encryptor { [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const; - int32_t Encrypt(::arrow::util::span plaintext, - ::arrow::util::span ciphertext); + int32_t Encrypt(std::span plaintext, std::span ciphertext); bool EncryptColumnMetaData( bool encrypted_footer, diff --git a/cpp/src/parquet/encryption/key_toolkit_internal.cc b/cpp/src/parquet/encryption/key_toolkit_internal.cc index 60a8a52206c3..d304041e3ea7 100644 --- a/cpp/src/parquet/encryption/key_toolkit_internal.cc +++ b/cpp/src/parquet/encryption/key_toolkit_internal.cc @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include + #include "arrow/util/base64.h" #include "arrow/util/secure_string.h" @@ -38,8 +40,8 @@ std::string EncryptKeyLocally(const SecureString& key_bytes, int32_t encrypted_key_len = key_encryptor.CiphertextLength(static_cast(key_bytes.size())); std::string encrypted_key(encrypted_key_len, '\0'); - ::arrow::util::span encrypted_key_span( - reinterpret_cast(&encrypted_key[0]), encrypted_key_len); + std::span encrypted_key_span(reinterpret_cast(&encrypted_key[0]), + encrypted_key_len); encrypted_key_len = key_encryptor.Encrypt(key_bytes.as_span(), master_key.as_span(), str2span(aad), encrypted_key_span); diff --git a/cpp/src/parquet/geospatial/util_internal.cc b/cpp/src/parquet/geospatial/util_internal.cc index d5c8d6628849..59551ae87035 100644 --- a/cpp/src/parquet/geospatial/util_internal.cc +++ b/cpp/src/parquet/geospatial/util_internal.cc @@ -17,6 +17,7 @@ #include "parquet/geospatial/util_internal.h" +#include #include #include "arrow/util/endian.h" @@ -147,11 +148,11 @@ std::vector WKBGeometryBounder::GeometryTypes() const { } void WKBGeometryBounder::MergeGeometry(std::string_view bytes_wkb) { - MergeGeometry(::arrow::util::span(reinterpret_cast(bytes_wkb.data()), - bytes_wkb.size())); + MergeGeometry( + std::span(reinterpret_cast(bytes_wkb.data()), bytes_wkb.size())); } -void WKBGeometryBounder::MergeGeometry(::arrow::util::span bytes_wkb) { +void WKBGeometryBounder::MergeGeometry(std::span bytes_wkb) { WKBBuffer src{bytes_wkb.data(), static_cast(bytes_wkb.size())}; MergeGeometryInternal(&src, /*record_wkb_type=*/true); if (src.size() != 0) { @@ -160,7 +161,12 @@ void WKBGeometryBounder::MergeGeometry(::arrow::util::span bytes_ } } -void WKBGeometryBounder::MergeGeometryInternal(WKBBuffer* src, bool record_wkb_type) { +void WKBGeometryBounder::MergeGeometryInternal(WKBBuffer* src, bool record_wkb_type, + int depth) { + if (depth > 128) { + throw ParquetException("WKB geometry has too many levels of nesting"); + } + uint8_t endian = src->ReadUInt8(); #if ARROW_LITTLE_ENDIAN bool swap = endian != 0x01; @@ -208,7 +214,7 @@ void WKBGeometryBounder::MergeGeometryInternal(WKBBuffer* src, bool record_wkb_t case GeometryType::kGeometryCollection: { uint32_t n_parts = src->ReadUInt32(swap); for (uint32_t i = 0; i < n_parts; i++) { - MergeGeometryInternal(src, /*record_wkb_type*/ false); + MergeGeometryInternal(src, /*record_wkb_type*/ false, depth + 1); } break; } diff --git a/cpp/src/parquet/geospatial/util_internal.h b/cpp/src/parquet/geospatial/util_internal.h index 2de9e1d076f6..b25e1e16366f 100644 --- a/cpp/src/parquet/geospatial/util_internal.h +++ b/cpp/src/parquet/geospatial/util_internal.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -94,19 +95,19 @@ struct PARQUET_EXPORT BoundingBox { BoundingBox& operator=(const BoundingBox&) = default; /// \brief Update the X and Y bounds to ensure these bounds contain coord - void UpdateXY(::arrow::util::span coord) { + void UpdateXY(std::span coord) { DCHECK_EQ(coord.size(), 2); UpdateInternal(coord); } /// \brief Update the X, Y, and Z bounds to ensure these bounds contain coord - void UpdateXYZ(::arrow::util::span coord) { + void UpdateXYZ(std::span coord) { DCHECK_EQ(coord.size(), 3); UpdateInternal(coord); } /// \brief Update the X, Y, and M bounds to ensure these bounds contain coord - void UpdateXYM(::arrow::util::span coord) { + void UpdateXYM(std::span coord) { DCHECK_EQ(coord.size(), 3); min[0] = std::min(min[0], coord[0]); min[1] = std::min(min[1], coord[1]); @@ -117,7 +118,7 @@ struct PARQUET_EXPORT BoundingBox { } /// \brief Update the X, Y, Z, and M bounds to ensure these bounds contain coord - void UpdateXYZM(::arrow::util::span coord) { + void UpdateXYZM(std::span coord) { DCHECK_EQ(coord.size(), 4); UpdateInternal(coord); } @@ -190,13 +191,13 @@ class PARQUET_EXPORT WKBGeometryBounder { /// the geometry is added to the internal geometry type list. void MergeGeometry(std::string_view bytes_wkb); - void MergeGeometry(::arrow::util::span bytes_wkb); + void MergeGeometry(std::span bytes_wkb); /// \brief Accumulate the bounds of a previously-calculated BoundingBox void MergeBox(const BoundingBox& box) { box_.Merge(box); } /// \brief Accumulate a previously-calculated list of geometry types - void MergeGeometryTypes(::arrow::util::span geospatial_types) { + void MergeGeometryTypes(std::span geospatial_types) { geospatial_types_.insert(geospatial_types.begin(), geospatial_types.end()); } @@ -216,7 +217,7 @@ class PARQUET_EXPORT WKBGeometryBounder { BoundingBox box_; std::unordered_set geospatial_types_; - void MergeGeometryInternal(WKBBuffer* src, bool record_wkb_type); + void MergeGeometryInternal(WKBBuffer* src, bool record_wkb_type, int depth = 0); void MergeSequence(WKBBuffer* src, Dimensions dimensions, uint32_t n_coords, bool swap); }; diff --git a/cpp/src/parquet/geospatial/util_internal_test.cc b/cpp/src/parquet/geospatial/util_internal_test.cc index 96f90b64b594..527bcc505051 100644 --- a/cpp/src/parquet/geospatial/util_internal_test.cc +++ b/cpp/src/parquet/geospatial/util_internal_test.cc @@ -545,4 +545,47 @@ INSTANTIATE_TEST_SUITE_P( MakeWKBPointTestCase{{30, 10, 40, 300}, false, true}, MakeWKBPointTestCase{{30, 10, 40, 300}, true, true})); +TEST(TestGeometryUtil, TestWKBBounderErrorForDeepNesting) { + // Construct a nested GeometryCollection with 200 levels + std::vector nested_wkb; + int num_levels = 200; + + for (int i = 0; i < num_levels; i++) { + nested_wkb.push_back(0x01); // little endian + nested_wkb.push_back(0x07); // geometry collection + nested_wkb.push_back(0x00); + nested_wkb.push_back(0x00); + nested_wkb.push_back(0x00); + + nested_wkb.push_back(0x01); // 1 part + nested_wkb.push_back(0x00); + nested_wkb.push_back(0x00); + nested_wkb.push_back(0x00); + } + + // Final part is an empty geometry collection + nested_wkb.push_back(0x01); // little endian + nested_wkb.push_back(0x07); // geometry collection + nested_wkb.push_back(0x00); + nested_wkb.push_back(0x00); + nested_wkb.push_back(0x00); + + nested_wkb.push_back(0x00); // 0 parts + nested_wkb.push_back(0x00); + nested_wkb.push_back(0x00); + nested_wkb.push_back(0x00); + + WKBGeometryBounder bounder; + EXPECT_THROW( + { + try { + bounder.MergeGeometry(nested_wkb); + } catch (const ParquetException& e) { + EXPECT_THAT(e.what(), ::testing::HasSubstr("too many levels of nesting")); + throw; + } + }, + ParquetException); +} + } // namespace parquet::geospatial diff --git a/cpp/src/parquet/metadata.cc b/cpp/src/parquet/metadata.cc index 505ace275b11..46cd1e442461 100644 --- a/cpp/src/parquet/metadata.cc +++ b/cpp/src/parquet/metadata.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -809,12 +810,11 @@ class FileMetaData::FileMetaDataImpl { uint32_t serialized_len = metadata_len_; ThriftSerializer serializer; serializer.SerializeToBuffer(metadata_.get(), &serialized_len, &serialized_data); - ::arrow::util::span serialized_data_span(serialized_data, - serialized_len); + std::span serialized_data_span(serialized_data, serialized_len); // encrypt with nonce - ::arrow::util::span nonce(reinterpret_cast(signature), - encryption::kNonceLength); + std::span nonce(reinterpret_cast(signature), + encryption::kNonceLength); auto tag = reinterpret_cast(signature) + encryption::kNonceLength; const SecureString& key = file_decryptor_->GetFooterKey(); @@ -867,8 +867,7 @@ class FileMetaData::FileMetaDataImpl { uint8_t* serialized_data; uint32_t serialized_len; serializer.SerializeToBuffer(metadata_.get(), &serialized_len, &serialized_data); - ::arrow::util::span serialized_data_span(serialized_data, - serialized_len); + std::span serialized_data_span(serialized_data, serialized_len); // encrypt the footer key std::vector encrypted_data(encryptor->CiphertextLength(serialized_len)); @@ -1773,8 +1772,7 @@ class ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilderImpl { serializer.SerializeToBuffer(&column_chunk_->meta_data, &serialized_len, &serialized_data); - ::arrow::util::span serialized_data_span(serialized_data, - serialized_len); + std::span serialized_data_span(serialized_data, serialized_len); std::vector encrypted_data(encryptor->CiphertextLength(serialized_len)); int32_t encrypted_len = encryptor->Encrypt(serialized_data_span, encrypted_data); diff --git a/cpp/src/parquet/size_statistics.cc b/cpp/src/parquet/size_statistics.cc index 0b30d7c8bbd7..2ef783ebc9e2 100644 --- a/cpp/src/parquet/size_statistics.cc +++ b/cpp/src/parquet/size_statistics.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -32,8 +33,7 @@ namespace parquet { namespace { -void MergeLevelHistogram(::arrow::util::span histogram, - ::arrow::util::span other) { +void MergeLevelHistogram(std::span histogram, std::span other) { ARROW_DCHECK_EQ(histogram.size(), other.size()); std::transform(histogram.begin(), histogram.end(), other.begin(), histogram.begin(), std::plus<>()); @@ -143,8 +143,7 @@ std::ostream& operator<<(std::ostream& os, const SizeStatistics& size_stats) { return os; } -void UpdateLevelHistogram(::arrow::util::span levels, - ::arrow::util::span histogram) { +void UpdateLevelHistogram(std::span levels, std::span histogram) { const int64_t num_levels = static_cast(levels.size()); DCHECK_GE(histogram.size(), 1); const int16_t max_level = static_cast(histogram.size() - 1); diff --git a/cpp/src/parquet/size_statistics.h b/cpp/src/parquet/size_statistics.h index ec79b8c4f8b8..5b41b0596b21 100644 --- a/cpp/src/parquet/size_statistics.h +++ b/cpp/src/parquet/size_statistics.h @@ -20,9 +20,9 @@ #include #include #include +#include #include -#include "arrow/util/span.h" #include "parquet/platform.h" #include "parquet/type_fwd.h" @@ -96,7 +96,6 @@ PARQUET_EXPORT std::ostream& operator<<(std::ostream&, const SizeStatistics&); PARQUET_EXPORT -void UpdateLevelHistogram(::arrow::util::span levels, - ::arrow::util::span histogram); +void UpdateLevelHistogram(std::span levels, std::span histogram); } // namespace parquet diff --git a/cpp/src/parquet/size_statistics_test.cc b/cpp/src/parquet/size_statistics_test.cc index 6e8cec9a1301..8c36d6b680d5 100644 --- a/cpp/src/parquet/size_statistics_test.cc +++ b/cpp/src/parquet/size_statistics_test.cc @@ -25,7 +25,6 @@ #include "arrow/buffer.h" #include "arrow/table.h" #include "arrow/testing/gtest_util.h" -#include "arrow/util/span.h" #include "parquet/arrow/reader.h" #include "parquet/arrow/schema.h" #include "parquet/arrow/writer.h" diff --git a/cpp/src/parquet/stream_writer.h b/cpp/src/parquet/stream_writer.h index 762651402221..c4a3ccfe2faf 100644 --- a/cpp/src/parquet/stream_writer.h +++ b/cpp/src/parquet/stream_writer.h @@ -22,12 +22,11 @@ #include #include #include +#include #include #include #include -#include "arrow/util/span.h" - #include "parquet/column_writer.h" #include "parquet/file_writer.h" @@ -154,7 +153,7 @@ class PARQUET_EXPORT StreamWriter { StreamWriter& operator<<(::std::string_view v); /// \brief Helper class to write variable length raw data. - using RawDataView = ::arrow::util::span; + using RawDataView = std::span; /// \brief Output operators for variable length raw data. StreamWriter& operator<<(RawDataView v); diff --git a/cpp/src/parquet/thrift_internal.h b/cpp/src/parquet/thrift_internal.h index 1ffe99eb3c96..5d14ae4e289c 100644 --- a/cpp/src/parquet/thrift_internal.h +++ b/cpp/src/parquet/thrift_internal.h @@ -21,8 +21,8 @@ #include #include - #include +#include #include #include #include @@ -580,7 +580,7 @@ class ThriftDeserializer { // decrypt auto decrypted_buffer = AllocateBuffer( decryptor->pool(), decryptor->PlaintextLength(static_cast(clen))); - ::arrow::util::span cipher_buf(buf, clen); + std::span cipher_buf(buf, clen); uint32_t decrypted_buffer_len = decryptor->Decrypt(cipher_buf, decrypted_buffer->mutable_span_as()); if (decrypted_buffer_len <= 0) { @@ -690,7 +690,7 @@ class ThriftSerializer { uint32_t out_length, Encryptor* encryptor) { auto cipher_buffer = AllocateBuffer(encryptor->pool(), encryptor->CiphertextLength(out_length)); - ::arrow::util::span out_span(out_buffer, out_length); + std::span out_span(out_buffer, out_length); int32_t cipher_buffer_len = encryptor->Encrypt(out_span, cipher_buffer->mutable_span_as()); diff --git a/dev/archery/setup.py b/dev/archery/setup.py index 13ba39659b96..23bb1096d760 100755 --- a/dev/archery/setup.py +++ b/dev/archery/setup.py @@ -37,9 +37,9 @@ 'integration': ['cffi', 'numpy'], 'integration-java': ['jpype1'], 'numpydoc': ['numpydoc==1.1.0'], - 'release': ['pygithub', jinja_req, 'semver', 'gitpython'], + 'release': ['pygithub<2.9.0', jinja_req, 'semver', 'gitpython'], } -extras['bot'] = extras['crossbow'] + ['pygithub'] +extras['bot'] = extras['crossbow'] + ['pygithub<2.9.0'] extras['all'] = list(set(functools.reduce(operator.add, extras.values()))) setup( diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index f91b8de474c2..9e7deb237251 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -538,7 +538,7 @@ test_python() { show_header "Build and test Python libraries" # Build and test Python - maybe_setup_virtualenv + maybe_setup_virtualenv -r python/requirements-build.txt maybe_setup_conda --file ci/conda_env_python.txt if [ "${USE_CONDA}" -gt 0 ]; then @@ -570,7 +570,9 @@ test_python() { pushd python # Build pyarrow - python -m pip install -e . + python -m pip install --no-build-isolation . + + popd # Check mandatory and optional imports python -c " @@ -601,12 +603,10 @@ import pyarrow.parquet # Install test dependencies - pip install -r requirements-test.txt + pip install -r python/requirements-test.txt # Execute pyarrow unittests - pytest pyarrow -v - - popd + pytest --pyargs pyarrow -v } test_glib() { diff --git a/dev/tasks/r/github.linux.cran.yml b/dev/tasks/r/github.linux.cran.yml index 8e073db775f6..0bad0e2b66e1 100644 --- a/dev/tasks/r/github.linux.cran.yml +++ b/dev/tasks/r/github.linux.cran.yml @@ -33,13 +33,12 @@ jobs: - { r_image: "ubuntu-clang" } # ~ r-devel-linux-x86_64-debian-clang - { r_image: "ubuntu-next" } # ~ r-patched-linux-x86_64 - { r_image: "ubuntu-release" } # ~ r-release-linux-x86_64 - - { r_image: "clang20", skip_vignettes: true, r_update_clang: true } # ~ r-devel-linux-x86_64-fedora-clang + - { r_image: "clang22", skip_vignettes: true } # ~ r-devel-linux-x86_64-fedora-clang env: R_ORG: "rhub" R_IMAGE: {{ '${{ matrix.config.r_image }}' }} R_TAG: "latest" ARROW_R_DEV: "FALSE" - R_UPDATE_CLANG: {{ '${{ matrix.config.r_update_clang }}' }} steps: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_install_archery()|indent }} diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index b488476cd591..35be2acbd9bf 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -374,10 +374,10 @@ jobs: arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - name: Install R package system dependencies (Linux) if: matrix.platform.name == 'Linux' - run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev + run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev - name: Install R package system dependencies (macOS) if: matrix.platform.name == 'macOS' - run: brew install sccache openssl curl + run: brew install sccache openssl curl libuv - name: Remove arrow/ run: | rm -rf arrow/ diff --git a/dev/tasks/verify-rc/github.macos.yml b/dev/tasks/verify-rc/github.macos.yml index d20d78307b0f..207fef29181c 100644 --- a/dev/tasks/verify-rc/github.macos.yml +++ b/dev/tasks/verify-rc/github.macos.yml @@ -50,6 +50,10 @@ jobs: brew uninstall pkg-config@0.29.2 || : {% endif %} + # Workaround for https://github.com/grpc/grpc/issues/41755 + # Remove once the runner ships a newer Homebrew. + brew update + brew bundle --file=arrow/cpp/Brewfile brew bundle --file=arrow/c_glib/Brewfile diff --git a/docs/source/cpp/api/array.rst b/docs/source/cpp/api/array.rst index 91aa5da67358..6b0355a10c18 100644 --- a/docs/source/cpp/api/array.rst +++ b/docs/source/cpp/api/array.rst @@ -92,8 +92,8 @@ Extension arrays .. doxygenclass:: arrow::ExtensionArray :members: -Run-End Encoded Array ---------------------- +Run-end encoded +--------------- .. doxygenclass:: arrow::RunEndEncodedArray :members: @@ -116,6 +116,18 @@ Chunked Arrays :project: arrow_cpp :members: +Non-owning data class +===================== + +.. warning:: + As this class doesn't keep alive the objects and data it points to, their + lifetime must be ensured separately. Use :class:`arrow::ArraySpan` only + when you can guarantee that lifetime and need a lightweight non-owning + view; otherwise, prefer :class:`arrow::ArrayData`. + +.. doxygenclass:: arrow::ArraySpan + :members: + Utilities ========= diff --git a/docs/source/cpp/api/builder.rst b/docs/source/cpp/api/builder.rst index 1342ba2655f5..d6532b03a2b9 100644 --- a/docs/source/cpp/api/builder.rst +++ b/docs/source/cpp/api/builder.rst @@ -15,10 +15,14 @@ .. specific language governing permissions and limitations .. under the License. +.. _cpp-api-array-builders: + ============== Array Builders ============== +.. seealso:: :ref:`cpp-api-buffer-builders` for direct construction of array buffers + .. doxygenclass:: arrow::ArrayBuilder :members: diff --git a/docs/source/cpp/api/memory.rst b/docs/source/cpp/api/memory.rst index 9d12e4bdf0d9..49d30f683f6f 100644 --- a/docs/source/cpp/api/memory.rst +++ b/docs/source/cpp/api/memory.rst @@ -55,6 +55,16 @@ Buffers .. doxygenclass:: arrow::ResizableBuffer :members: +Non-owning Buffer +----------------- + +.. warning:: + This class is exposed solely as a building block for :class:`arrow::ArraySpan`. + For any other purpose, please use :class:`arrow::Buffer`. + +.. doxygenclass:: arrow::BufferSpan + :members: + Memory Pools ------------ @@ -91,6 +101,8 @@ Slicing .. doxygengroup:: buffer-slicing-functions :content-only: +.. _cpp-api-buffer-builders: + Buffer Builders --------------- diff --git a/docs/source/cpp/conventions.rst b/docs/source/cpp/conventions.rst index 8ea625c0b882..e82fd3ecb1b8 100644 --- a/docs/source/cpp/conventions.rst +++ b/docs/source/cpp/conventions.rst @@ -20,6 +20,8 @@ .. cpp:namespace:: arrow +.. _cpp-conventions: + Conventions =========== @@ -43,6 +45,10 @@ Safe pointers Arrow objects are usually passed and stored using safe pointers -- most of the time :class:`std::shared_ptr` but sometimes also :class:`std::unique_ptr`. +Non-owning alternatives exist for the rare situations where the overhead of +a safe pointer is considered unacceptable: :class:`ArraySpan` and :class:`BufferSpan`. +Their usage in third-party code is not recommended. + Immutability ------------ @@ -104,4 +110,3 @@ For example:: .. seealso:: :doc:`API reference for error reporting ` - diff --git a/docs/source/cpp/csv.rst b/docs/source/cpp/csv.rst index bcb17bdc587b..74ee0bb4fbbf 100644 --- a/docs/source/cpp/csv.rst +++ b/docs/source/cpp/csv.rst @@ -30,6 +30,8 @@ to create Arrow Tables or a stream of Arrow RecordBatches. .. seealso:: :ref:`CSV reader/writer API reference `. +.. _cpp-csv-reading: + Reading CSV files ================= diff --git a/docs/source/cpp/ipc.rst b/docs/source/cpp/ipc.rst index ce4175bca0e5..14ae060e5ed5 100644 --- a/docs/source/cpp/ipc.rst +++ b/docs/source/cpp/ipc.rst @@ -33,6 +33,8 @@ lower level input/output, handled through the :doc:`IO interfaces `. For reading, there is also an event-driven API that enables feeding arbitrary data into the IPC decoding layer asynchronously. +.. _cpp-ipc-reading: + Reading IPC streams and files ============================= diff --git a/docs/source/cpp/parquet.rst b/docs/source/cpp/parquet.rst index 8c55ec5d53d0..045f7f80f634 100644 --- a/docs/source/cpp/parquet.rst +++ b/docs/source/cpp/parquet.rst @@ -32,6 +32,8 @@ is a space-efficient columnar storage format for complex data. The Parquet C++ implementation is part of the Apache Arrow project and benefits from tight integration with the Arrow C++ classes and facilities. +.. _cpp-parquet-reading: + Reading Parquet files ===================== diff --git a/docs/source/cpp/security.rst b/docs/source/cpp/security.rst new file mode 100644 index 000000000000..591207448770 --- /dev/null +++ b/docs/source/cpp/security.rst @@ -0,0 +1,142 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +.. default-domain:: cpp + +.. _cpp-security: + +======================= +Security Considerations +======================= + +.. important:: + This document describes the security model for using the Arrow C++ APIs. + For better understanding of this document, we recommend that you first read + the :ref:`overall security model ` for the Arrow project. + +Parameter mismatch +================== + +Many Arrow C++ APIs report errors using the :class:`arrow::Status` and +:class:`arrow::Result` types. Such APIs can be assumed to detect common errors +in the provided arguments. However, there are also often implicit +pre-conditions that have to be upheld; these can usually be deduced from the +semantics of an API as described by its documentation. + +.. seealso:: Arrow C++ :ref:`cpp-conventions` + +Pointer validity +---------------- + +Pointers are always assumed to be valid and point to memory of the size required +by the API. In particular, it is *forbidden to pass a null pointer* except where +the API documentation explicitly says otherwise. + +Type restrictions +----------------- + +Some APIs are specified to operate on specific Arrow data types and may not +verify that their arguments conform to the expected data types. Passing the +wrong kind of data as input may lead to undefined behavior. + +.. _cpp-valid-data: + +Data validity +============= + +Arrow data, for example passed as :class:`arrow::Array` or :class:`arrow::Table`, +is always assumed to be :ref:`valid `. If your program may +encounter invalid data, it must explicitly check its validity by calling one of +the following validation APIs. + +Structural validity +------------------- + +The ``Validate`` methods exposed on various Arrow C++ classes perform relatively +inexpensive validity checks that the data is structurally valid. This implies +checking the number of buffers, child arrays, and other similar conditions. + +* :func:`arrow::Array::Validate` +* :func:`arrow::RecordBatch::Validate` +* :func:`arrow::ChunkedArray::Validate` +* :func:`arrow::Table::Validate` +* :func:`arrow::Scalar::Validate` + +These checks typically are constant-time against the number of rows in the data, +but linear in the number of descendant fields. They can be good enough to detect +potential bugs in your own code. However, they are not enough to detect all classes of +invalid data, and they won't protect against all kinds of malicious payloads. + +Full validity +------------- + +The ``ValidateFull`` methods exposed by the same classes perform the same validity +checks as the ``Validate`` methods, but they also check the data extensively for +any non-conformance to the Arrow spec. In particular, they check all the offsets +of variable-length data types, which is of fundamental importance when ingesting +untrusted data from sources such as the IPC format (otherwise the variable-length +offsets could point outside of the corresponding data buffer). They also check +for invalid values, such as invalid UTF-8 strings or decimal values out of range +for the advertised precision. + +* :func:`arrow::Array::ValidateFull` +* :func:`arrow::RecordBatch::ValidateFull` +* :func:`arrow::ChunkedArray::ValidateFull` +* :func:`arrow::Table::ValidateFull` +* :func:`arrow::Scalar::ValidateFull` + +"Safe" and "unsafe" APIs +======================== + +Some APIs are exposed in both "safe" and "unsafe" variants. The naming convention +for such pairs varies: sometimes the former has a ``Safe`` suffix (for example +``SliceSafe`` vs. ``Slice``), sometimes the latter has an ``Unsafe`` prefix or +suffix (for example ``Append`` vs. ``UnsafeAppend``). + +In all cases, the "unsafe" API is intended as a more efficient API that +eschews some of the checks that the "safe" API performs. It is then up to the +caller to ensure that the preconditions are met, otherwise undefined behavior +may ensue. + +The API documentation usually spells out the differences between "safe" and "unsafe" +variants, but these typically fall into two categories: + +* structural checks, such as passing the right Arrow data type or numbers of buffers; +* allocation size checks, such as having preallocated enough data for the given input + arguments (this is typical of the :ref:`array builders ` + and :ref:`buffer builders `). + +Ingesting untrusted data +======================== + +As an exception to the above (see :ref:`cpp-valid-data`), some APIs support ingesting +untrusted, potentially malicious data. These are: + +* the :ref:`IPC reader ` APIs +* the :ref:`Parquet reader ` APIs +* the :ref:`CSV reader ` APIs + +You must not assume that they will always return valid Arrow data. The reason +for not validating data automatically is that validation can be expensive but +unnecessary when reading from trusted data sources. + +Instead, when using these APIs with potentially invalid data (such as data coming +from an untrusted source), you **must** follow these steps: + +1. Check any error returned by the API, as with any other API +2. If the API returned successfully, validate the returned Arrow data in full + (see "Full validity" above) diff --git a/docs/source/cpp/user_guide.rst b/docs/source/cpp/user_guide.rst index 094859f9c570..722e9e50afa8 100644 --- a/docs/source/cpp/user_guide.rst +++ b/docs/source/cpp/user_guide.rst @@ -39,6 +39,7 @@ User Guide json dataset flight + security gdb threading opentelemetry diff --git a/docs/source/developers/python/building.rst b/docs/source/developers/python/building.rst index 39d72b7c7d3c..1d40d78f0abb 100644 --- a/docs/source/developers/python/building.rst +++ b/docs/source/developers/python/building.rst @@ -434,6 +434,11 @@ To build PyArrow run: Note that bundled Arrow C++ libraries will not be automatically updated when rebuilding Arrow C++. +This creates an *editable install*, meaning changes to the Python source code +will be reflected immediately without needing to reinstall the package. +The ``--no-build-isolation`` flag ensures that the build uses your current +environment's dependencies instead of creating an isolated one. + To set the number of threads used to compile PyArrow's C++/Cython components, set the ``CMAKE_BUILD_PARALLEL_LEVEL`` environment variable. @@ -450,7 +455,7 @@ A ``relwithdebinfo`` build can be created similarly. Self-Contained Wheel ^^^^^^^^^^^^^^^^^^^^ -If you're preparing a PyArrow wheel for distribution (e.g., for PyPI), you’ll +If you're preparing a PyArrow wheel for distribution (e.g., for PyPI), you'll need to build a self-contained wheel (including the Arrow and Parquet C++ libraries). This ensures that all necessary native libraries are bundled inside the wheel, so users can install it without needing to have Arrow or Parquet @@ -467,22 +472,6 @@ To do this, set the ``PYARROW_BUNDLE_ARROW_CPP`` environment variable before bui This option is typically only needed for releases or distribution scenarios, not for local development. -Editable install -^^^^^^^^^^^^^^^^ - -To install an editable PyArrow build, run the following command from the -``arrow/python`` directory: - -.. code-block:: - - pip install -e . --no-build-isolation - -This creates an *editable install*, meaning changes to the Python source code -will be reflected immediately without needing to reinstall the package. -The ``--no-build-isolation`` flag ensures that the build uses your current -environment's dependencies instead of creating an isolated one. This is -especially useful during development and debugging. - .. _stale_artifacts: Deleting stale build artifacts diff --git a/docs/source/developers/python/development.rst b/docs/source/developers/python/development.rst index 5757b761875a..e70fb4430757 100644 --- a/docs/source/developers/python/development.rst +++ b/docs/source/developers/python/development.rst @@ -195,6 +195,17 @@ for ``.py`` files or for ``.pyx`` and ``.pxi`` files. In this case you will also need to install the `pytest-cython `_ plugin. +.. note:: + Cython ``.pxi`` files are included in ``.pyx`` files at compile time, + so ``--doctest-cython`` cannot be run directly on ``.pxi`` files. + In PyArrow, all ``.pxi`` files are included into ``lib.pyx``, so run + doctests on that file:: + + $ python -m pytest --doctest-cython path/to/lib.pyx + + Any doctest errors originating from ``.pxi`` files will appear under + ``lib.pyx``, not the original ``.pxi`` filename. + Testing Documentation Examples ------------------------------- diff --git a/docs/source/format/Intro.rst b/docs/source/format/Intro.rst index ce35402bff51..e98bd40e0107 100644 --- a/docs/source/format/Intro.rst +++ b/docs/source/format/Intro.rst @@ -296,6 +296,13 @@ key is the field name and the child array its values. The field (key) is saved in the schema and the values of a specific field (key) are saved in the child array. +Since child arrays are independent, Arrow does not enforce physical +consistency between the struct's validity bitmap and those of it's children. +Logically, a struct row is only valid if both the parent and the child +bitmaps have a value of 1 for that slot (a logical AND operation). +This allows for "hidden" data to exist in child arrays at null struct +positions (see ``alice`` below). + .. figure:: ./images/struct-diagram.svg :alt: Diagram is showing the difference between the struct data type presented in a Table and the data actually stored in computer diff --git a/docs/source/format/Security.rst b/docs/source/format/Security.rst index e14f07143ce6..8e630ea9a55f 100644 --- a/docs/source/format/Security.rst +++ b/docs/source/format/Security.rst @@ -26,10 +26,15 @@ data from untrusted sources. It focuses specifically on data passed in a standardized serialized form (such as a IPC stream), as opposed to an implementation-specific native representation (such as ``arrow::Array`` in C++). -.. note:: +.. important:: Implementation-specific concerns, such as bad API usage, are out of scope for this document. Please refer to the implementation's own documentation. +.. seealso:: + + Arrow C++ :ref:`cpp-security` + Security model for Arrow C++ APIs + Who should read this ==================== @@ -49,6 +54,8 @@ You should read this document if you belong to either of these two categories: Columnar Format =============== +.. _format-invalid-data: + Invalid data ------------ @@ -89,8 +96,6 @@ explicitly validates any Arrow data it receives under serialized form from untrusted sources. Many Arrow implementations provide explicit APIs to perform such validation. -.. TODO: link to some validation APIs for the main implementations here? - Advice for implementors ''''''''''''''''''''''' diff --git a/docs/source/format/images/struct-diagram.svg b/docs/source/format/images/struct-diagram.svg index 81ff29689a85..17053414e057 100644 --- a/docs/source/format/images/struct-diagram.svg +++ b/docs/source/format/images/struct-diagram.svg @@ -1,6 +1,6 @@ - + - + - structPhysical layout - Struct Layoutcolumn 4{"name": "joe", "id": 1}{"name": null, "id": 2}null{"name": "jane", "id": null}{"name": "mark", "id": 4}column 4Values bufferValidity bitmap bufferOffset bufferValidity bitmap buffer0001110100011011children arraysValues bufferjoemarkjaneVariable-size Binary child arrayFixed-size Primitive child arrayValidity bitmap buffer000010110 3 3 3 7 111 2 _ 4 _ \ No newline at end of file + structPhysical layout - Struct Layoutcolumn 4{"name": "joe", "id": 1}{"name": null, "id": 2}null{"name": "jane", "id": null}{"name": "mark", "id": 4}column 4Values bufferValidity bitmap bufferOffset bufferValidity bitmap buffer0001110100011011children arraysValues bufferjoealicemarkjaneVariable-size Binary child arrayFixed-size Primitive child arrayValidity bitmap buffer000010110 3 3 8 12 161 2 _ 4 _ diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 6395b3e1e7a0..bfbcd131416e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -293,6 +293,15 @@ message(STATUS "Found Cython version: ${CYTHON_VERSION}") include(GNUInstallDirs) find_package(Arrow REQUIRED) +# When not bundling Arrow C++ libraries on macOS, add the Arrow library +# directory to the RPATH so that the extensions can find libarrow at runtime. +if(APPLE + AND NOT PYARROW_BUNDLE_ARROW_CPP + AND ARROW_SHARED_LIB) + get_filename_component(_arrow_lib_dir "${ARROW_SHARED_LIB}" DIRECTORY) + list(APPEND CMAKE_INSTALL_RPATH "${_arrow_lib_dir}") +endif() + macro(define_option name description arrow_option) set("PYARROW_${name}" "AUTO" @@ -968,6 +977,9 @@ foreach(module ${CYTHON_EXTENSIONS}) continue() endif() endif() + if(output MATCHES "\\.h$") + continue() + endif() install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${output} DESTINATION ".") endforeach() endforeach() diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index ed9d20beb6b3..79985530af17 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -332,6 +332,22 @@ cdef class ReadOptions(_Weakrefable): except TypeError: return False + def _repr_base(self): + return (f""" + use_threads={self.use_threads}, + block_size={self.block_size}, + skip_rows={self.skip_rows}, + skip_rows_after_names={self.skip_rows_after_names}, + column_names={self.column_names}, + autogenerate_column_names={self.autogenerate_column_names}, + encoding={self.encoding!r}""") + + def __repr__(self): + return (f"({self._repr_base()})") + + def __str__(self): + return (f"ReadOptions({self._repr_base()})") + cdef class ParseOptions(_Weakrefable): """ @@ -585,6 +601,23 @@ cdef class ParseOptions(_Weakrefable): except TypeError: return False + def _repr_base(self): + return (f""" + delimiter={self.delimiter!r}, + quote_char={self.quote_char!r}, + double_quote={self.double_quote}, + escape_char={self.escape_char!r}, + newlines_in_values={self.newlines_in_values}, + ignore_empty_lines={self.ignore_empty_lines}, + invalid_row_handler={getattr(self.invalid_row_handler, '__name__', + self.invalid_row_handler)}""") + + def __repr__(self): + return (f"({self._repr_base()})") + + def __str__(self): + return (f"ParseOptions({self._repr_base()})") + cdef class _ISO8601(_Weakrefable): """ @@ -1108,6 +1141,28 @@ cdef class ConvertOptions(_Weakrefable): except TypeError: return False + def _repr_base(self): + return (f""" + check_utf8={self.check_utf8}, + column_types={self.column_types}, + null_values={self.null_values}, + true_values={self.true_values}, + false_values={self.false_values}, + decimal_point={self.decimal_point!r}, + strings_can_be_null={self.strings_can_be_null}, + quoted_strings_can_be_null={self.quoted_strings_can_be_null}, + include_columns={self.include_columns}, + include_missing_columns={self.include_missing_columns}, + auto_dict_encode={self.auto_dict_encode}, + auto_dict_max_cardinality={self.auto_dict_max_cardinality}, + timestamp_parsers={[str(i) for i in self.timestamp_parsers]}""") + + def __repr__(self): + return (f"({self._repr_base()})") + + def __str__(self): + return (f"ConvertOptions({self._repr_base()})") + cdef _get_reader(input_file, ReadOptions read_options, shared_ptr[CInputStream]* out): @@ -1459,6 +1514,19 @@ cdef class WriteOptions(_Weakrefable): def validate(self): check_status(self.options.get().Validate()) + def _repr_base(self): + return (f""" + include_header={self.include_header}, + batch_size={self.batch_size}, + delimiter={self.delimiter!r}, + quoting_style={self.quoting_style!r}""") + + def __repr__(self): + return (f"({self._repr_base()})") + + def __str__(self): + return (f"WriteOptions({self._repr_base()})") + cdef _get_write_options(WriteOptions write_options, CCSVWriteOptions* out): if write_options is None: diff --git a/python/pyarrow/_json.pyx b/python/pyarrow/_json.pyx index 07e615dd5e48..f8373feeabe5 100644 --- a/python/pyarrow/_json.pyx +++ b/python/pyarrow/_json.pyx @@ -105,6 +105,16 @@ cdef class ReadOptions(_Weakrefable): except TypeError: return False + def __repr__(self): + return (f"""( + use_threads={self.use_threads}, + block_size={self.block_size})""") + + def __str__(self): + return (f"""ReadOptions( + use_threads={self.use_threads}, + block_size={self.block_size})""") + @staticmethod cdef ReadOptions wrap(CJSONReadOptions options): out = ReadOptions() @@ -244,6 +254,18 @@ cdef class ParseOptions(_Weakrefable): except TypeError: return False + def _repr_base(self): + return (f""" + explicit_schema={self.explicit_schema}, + newlines_in_values={self.newlines_in_values}, + unexpected_field_behavior={self.unexpected_field_behavior!r}""") + + def __repr__(self): + return (f"({self._repr_base()})") + + def __str__(self): + return (f"ParseOptions({self._repr_base()})") + @staticmethod cdef ParseOptions wrap(CJSONParseOptions options): out = ParseOptions() diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index 6185d5f2392c..db6a6b56ac4c 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -297,7 +297,10 @@ cdef class KmsConnectionConfig(_Weakrefable): @staticmethod cdef wrap(const CKmsConnectionConfig& config): result = KmsConnectionConfig() - result.configuration = make_shared[CKmsConnectionConfig](move(config)) + # We require a copy of the config because the input is + # a const reference owned by C++. + cdef CKmsConnectionConfig config_copy = config + result.configuration = make_shared[CKmsConnectionConfig](move(config_copy)) return result diff --git a/python/pyarrow/tests/test_cpp_internals.py b/python/pyarrow/tests/test_cpp_internals.py index 7508d8f0b981..fbbf95e81b6c 100644 --- a/python/pyarrow/tests/test_cpp_internals.py +++ b/python/pyarrow/tests/test_cpp_internals.py @@ -49,9 +49,8 @@ def test_pyarrow_include(): # created. Either with PyArrow C++ header files or with # Arrow C++ and PyArrow C++ header files together - source = os.path.dirname(os.path.abspath(__file__)) - pyarrow_dir = pjoin(source, '..') - pyarrow_include = pjoin(pyarrow_dir, 'include') + import pyarrow + pyarrow_include = pyarrow.get_include() pyarrow_cpp_include = pjoin(pyarrow_include, 'arrow', 'python') assert os.path.exists(pyarrow_include) diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index dce605c7156d..d608d2bee5eb 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -213,6 +213,18 @@ def test_read_options(pickle_module): opts.column_names = ('a', 'b') opts.validate() + expected_repr_inner = """ + use_threads=True, + block_size=1048576, + skip_rows=0, + skip_rows_after_names=0, + column_names=['a', 'b'], + autogenerate_column_names=True, + encoding='utf8'""" + + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ReadOptions({expected_repr_inner})" + def test_parse_options(pickle_module): cls = ParseOptions @@ -273,6 +285,18 @@ def test_parse_options(pickle_module): opts.escape_char = "\r" opts.validate() + expected_repr_inner = r""" + delimiter=',', + quote_char='"', + double_quote=True, + escape_char='\r', + newlines_in_values=False, + ignore_empty_lines=True, + invalid_row_handler=None""" + + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ParseOptions({expected_repr_inner})" + def test_convert_options(pickle_module): cls = ConvertOptions @@ -354,6 +378,23 @@ def test_convert_options(pickle_module): assert opts.auto_dict_max_cardinality == 999 assert opts.timestamp_parsers == [ISO8601, '%Y-%m-%d'] + expected_repr_inner = (""" + check_utf8=True, + column_types={'a': DataType(null)}, + null_values=['N', 'nn'], + true_values=['T', 'tt'], + false_values=['F', 'ff'], + decimal_point='.', + strings_can_be_null=False, + quoted_strings_can_be_null=True, + include_columns=[], + include_missing_columns=False, + auto_dict_encode=False, + auto_dict_max_cardinality=999, + timestamp_parsers=['ISO8601', '%Y-%m-%d']""") + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ConvertOptions({expected_repr_inner})" + def test_write_options(): cls = WriteOptions @@ -378,6 +419,15 @@ def test_write_options(): opts.batch_size = 0 opts.validate() + expected_repr_inner = """ + include_header=True, + batch_size=0, + delimiter=',', + quoting_style='needed'""" + + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"WriteOptions({expected_repr_inner})" + class BaseTestCSV(abc.ABC): """Common tests which are shared by streaming and non streaming readers""" diff --git a/python/pyarrow/tests/test_json.py b/python/pyarrow/tests/test_json.py index c3f9fe333bd0..8d5e6f43db06 100644 --- a/python/pyarrow/tests/test_json.py +++ b/python/pyarrow/tests/test_json.py @@ -80,6 +80,13 @@ def test_read_options(pickle_module): assert opts.block_size == 1234 assert opts.use_threads is False + expected_repr_inner = """ + use_threads=False, + block_size=1234""" + + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ReadOptions({expected_repr_inner})" + check_options_class_pickling(cls, pickler=pickle_module, block_size=1234, use_threads=False) @@ -94,6 +101,14 @@ def test_parse_options(pickle_module): opts.newlines_in_values = True assert opts.newlines_in_values is True + expected_repr_inner = """ + explicit_schema=None, + newlines_in_values=True, + unexpected_field_behavior='infer'""" + + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ParseOptions({expected_repr_inner})" + schema = pa.schema([pa.field('foo', pa.int32())]) opts.explicit_schema = schema assert opts.explicit_schema == schema diff --git a/r/NEWS.md b/r/NEWS.md index 12d4047d8fca..2308c18d2912 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -18,6 +18,17 @@ --> # arrow 23.0.1.9000 +# arrow 23.0.1.2 + +## Minor improvements and fixes + +- Update use of internal function for non-API call CRAN requirements (#49530) + +# arrow 23.0.1.1 + +## Minor improvements and fixes + +- Refine checks for ensuring building with macOS libtool instead of GNU libtool (#49370) # arrow 23.0.1 diff --git a/r/src/arrow_cpp11.h b/r/src/arrow_cpp11.h index 6ea11ea008da..d28ad0feded2 100644 --- a/r/src/arrow_cpp11.h +++ b/r/src/arrow_cpp11.h @@ -38,17 +38,6 @@ #define ARROW_R_DCHECK(EXPR) #endif -#if (R_VERSION < R_Version(3, 5, 0)) -#define LOGICAL_RO(x) ((const int*)LOGICAL(x)) -#define INTEGER_RO(x) ((const int*)INTEGER(x)) -#define REAL_RO(x) ((const double*)REAL(x)) -#define COMPLEX_RO(x) ((const Rcomplex*)COMPLEX(x)) -#define STRING_PTR_RO(x) ((const SEXP*)STRING_PTR(x)) -#define RAW_RO(x) ((const Rbyte*)RAW(x)) -#define DATAPTR_RO(x) ((const void*)STRING_PTR(x)) -#define DATAPTR(x) (void*)STRING_PTR(x) -#endif - // R_altrep_class_name and R_altrep_class_package don't exist before R 4.6 #if R_VERSION < R_Version(4, 6, 0) inline SEXP R_altrep_class_name(SEXP x) { @@ -219,8 +208,12 @@ Pointer r6_to_pointer(SEXP self) { cpp11::stop("Invalid R object for %s, must be an ArrowObject", type_name.c_str()); } +#if R_VERSION >= R_Version(4, 5, 0) + SEXP xp = R_getVarEx(arrow::r::symbols::xp, self, FALSE, R_UnboundValue); +#else SEXP xp = Rf_findVarInFrame(self, arrow::r::symbols::xp); - if (xp == R_NilValue) { +#endif + if (xp == R_UnboundValue || xp == R_NilValue) { cpp11::stop("Invalid: self$`.:xp:.` is NULL"); } @@ -234,7 +227,11 @@ Pointer r6_to_pointer(SEXP self) { template void r6_reset_pointer(SEXP r6) { +#if R_VERSION >= R_Version(4, 5, 0) + SEXP xp = R_getVarEx(arrow::r::symbols::xp, r6, FALSE, R_UnboundValue); +#else SEXP xp = Rf_findVarInFrame(r6, arrow::r::symbols::xp); +#endif void* p = R_ExternalPtrAddr(xp); if (p != nullptr) { delete reinterpret_cast*>(p); diff --git a/r/tests/testthat/test-dplyr-mutate.R b/r/tests/testthat/test-dplyr-mutate.R index e9a52c60e876..3e116a1012ca 100644 --- a/r/tests/testthat/test-dplyr-mutate.R +++ b/r/tests/testthat/test-dplyr-mutate.R @@ -152,6 +152,9 @@ test_that("transmute() with unsupported arguments", { }) test_that("transmute() defuses dots arguments (ARROW-13262)", { + # There is a sanitizer issue when stringi compiles with bundled ICU + # see https://github.com/gagolews/stringi/issues/525 + skip_on_linux_devel() expect_snapshot( tbl |> Table$create() |> diff --git a/r/vignettes/developers/setup.Rmd b/r/vignettes/developers/setup.Rmd index e61436df31db..d13fc53db1ee 100644 --- a/r/vignettes/developers/setup.Rmd +++ b/r/vignettes/developers/setup.Rmd @@ -350,7 +350,7 @@ sudo apt update sudo apt install -y -V libarrow-dev ``` -```{bash, save=run & !sys_install & macos} +```{bash, save=run & !sys_install} MAKEFLAGS="LDFLAGS=" R CMD INSTALL . ``` diff --git a/ruby/Rakefile b/ruby/Rakefile index 7f26773403a6..6ff20915bcfb 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -35,9 +35,11 @@ end packages.each do |package| namespace package do + package_dir = File.join(base_dir, package) + desc "Run test for #{package}" task :test do - cd(File.join(base_dir, package)) do + cd(package_dir) do if ENV["USE_BUNDLER"] sh("bundle", "exec", "rake", "test") else @@ -46,9 +48,22 @@ packages.each do |package| end end + desc "Run benchmark for #{package}" + task :benchmark do + cd(package_dir) do + if File.directory?("benchmark") + if ENV["USE_BUNDLER"] + sh("bundle", "exec", "rake", "benchmark") + else + ruby("-S", "rake", "benchmark") + end + end + end + end + desc "Install #{package}" task :install do - cd(File.join(base_dir, package)) do + cd(package_dir) do if ENV["USE_BUNDLER"] sh("bundle", "exec", "rake", "install") else @@ -70,6 +85,9 @@ end desc "Run test for all packages" task test: sorted_packages.collect {|package| "#{package}:test"} +desc "Run benchmark for all packages" +task benchmark: sorted_packages.collect {|package| "#{package}:benchmark"} + desc "Install all packages" task install: sorted_packages.collect {|package| "#{package}:install"} diff --git a/ruby/red-arrow-format/Gemfile b/ruby/red-arrow-format/Gemfile index 2307252d9e60..34c981237c59 100644 --- a/ruby/red-arrow-format/Gemfile +++ b/ruby/red-arrow-format/Gemfile @@ -21,6 +21,11 @@ source "https://rubygems.org/" gemspec -gem "rake" gem "red-arrow", path: "../red-arrow" -gem "test-unit" + +group :development do + gem "benchmark-driver" + gem "rake" + gem "stringio" + gem "test-unit" +end diff --git a/ruby/red-arrow-format/Rakefile b/ruby/red-arrow-format/Rakefile index f50f18f3b828..3671f35d6e32 100644 --- a/ruby/red-arrow-format/Rakefile +++ b/ruby/red-arrow-format/Rakefile @@ -39,6 +39,28 @@ task :test do end end +benchmark_tasks = [] +namespace :benchmark do + Dir.glob("benchmark/*.yaml").sort.each do |yaml| + name = File.basename(yaml, ".*") + command_line = [ + RbConfig.ruby, "-v", "-S", "benchmark-driver", File.expand_path(yaml), + ] + + desc "Run #{name} benchmark" + task name do + puts("```console") + puts("$ #{command_line.join(" ")}") + sh(*command_line, verbose: false) + puts("```") + end + benchmark_tasks << "benchmark:#{name}" + end +end + +desc "Run all benchmarks" +task :benchmark => benchmark_tasks + namespace :flat_buffers do desc "Generate FlatBuffers code" task :generate do diff --git a/ruby/red-arrow-format/benchmark/file-reader.yaml b/ruby/red-arrow-format/benchmark/file-reader.yaml new file mode 100644 index 000000000000..25e8c73ef1c4 --- /dev/null +++ b/ruby/red-arrow-format/benchmark/file-reader.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +prelude: | + Warning[:experimental] = false + + require "arrow" + require "arrow-format" + + seed = 29 + random = Random.new(seed) + + n_columns = 100 + n_rows = 10000 + max_uint32 = 2 ** 32 - 1 + arrays = n_columns.times.collect do |i| + if i.even? + Arrow::UInt32Array.new(n_rows.times.collect {random.rand(max_uint32)}) + else + Arrow::BinaryArray.new(n_rows.times.collect {random.bytes(random.rand(10))}) + end + end + table = Arrow::Table.new(arrays.collect.with_index {|array, i| [i, array]}) + buffer = Arrow::ResizableBuffer.new(4096) + table.save(buffer, format: :arrow_file) + + GC.start + GC.disable +benchmark: + "Arrow::Table.load": | + Arrow::Table.load(buffer, format: :arrow_file) + "Arrow::RecordBatchFileReader": | + Arrow::BufferInputStream.open(buffer) do |input| + Arrow::RecordBatchFileReader.new(input).each do + end + end + "ArrowFormat::FileReader": | + ArrowFormat::FileReader.new(buffer.data.to_s).each do + end diff --git a/ruby/red-arrow-format/benchmark/streaming-reader.yaml b/ruby/red-arrow-format/benchmark/streaming-reader.yaml new file mode 100644 index 000000000000..f1b383395b6e --- /dev/null +++ b/ruby/red-arrow-format/benchmark/streaming-reader.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +prelude: | + Warning[:experimental] = false + + require "arrow" + require "arrow-format" + + seed = 29 + random = Random.new(seed) + + n_columns = 100 + n_rows = 10000 + max_uint32 = 2 ** 32 - 1 + arrays = n_columns.times.collect do |i| + if i.even? + Arrow::UInt32Array.new(n_rows.times.collect {random.rand(max_uint32)}) + else + Arrow::BinaryArray.new(n_rows.times.collect {random.bytes(random.rand(10))}) + end + end + table = Arrow::Table.new(arrays.collect.with_index {|array, i| [i, array]}) + buffer = Arrow::ResizableBuffer.new(4096) + table.save(buffer, format: :arrow_streaming) + + GC.start + GC.disable +benchmark: + "Arrow::Table.load": | + Arrow::Table.load(buffer, format: :arrow_streaming) + "Arrow::RecordBatchStreamReader": | + Arrow::BufferInputStream.open(buffer) do |input| + Arrow::RecordBatchStreamReader.new(input).each do + end + end + "ArrowFormat::StreamingReader": | + ArrowFormat::StreamingReader.new(buffer.data.to_s).each do + end diff --git a/ruby/red-arrow-format/lib/arrow-format/file-reader.rb b/ruby/red-arrow-format/lib/arrow-format/file-reader.rb index 7c749e5fbf8e..cec371109622 100644 --- a/ruby/red-arrow-format/lib/arrow-format/file-reader.rb +++ b/ruby/red-arrow-format/lib/arrow-format/file-reader.rb @@ -35,6 +35,7 @@ class FileReader FOOTER_SIZE_SIZE = IO::Buffer.size_of(FOOTER_SIZE_FORMAT) attr_reader :schema + attr_reader :metadata def initialize(input) case input when IO @@ -47,6 +48,7 @@ def initialize(input) validate @footer = read_footer + @metadata = read_custom_metadata(@footer.custom_metadata) @record_batch_blocks = @footer.record_batches || [] @schema = read_schema(@footer.schema) @dictionaries = read_dictionaries diff --git a/ruby/red-arrow-format/lib/arrow-format/file-writer.rb b/ruby/red-arrow-format/lib/arrow-format/file-writer.rb index 27b6b55bbf9a..2ac469518082 100644 --- a/ruby/red-arrow-format/lib/arrow-format/file-writer.rb +++ b/ruby/red-arrow-format/lib/arrow-format/file-writer.rb @@ -29,26 +29,33 @@ def start(schema) super end - def finish - super - write_footer + def finish(metadata=nil) + super() + write_footer(metadata) write_data(MAGIC) @output end private - def build_footer + def build_footer(metadata) fb_footer = FB::Footer::Data.new fb_footer.version = FB::MetadataVersion::V5 fb_footer.schema = @fb_schema fb_footer.dictionaries = @fb_dictionary_blocks fb_footer.record_batches = @fb_record_batch_blocks - # fb_footer.custom_metadata = ... # TODO + if metadata + fb_footer.custom_metadata = metadata.collect do |key, value| + fb_key_value = FB::KeyValue::Data.new + fb_key_value.key = key + fb_key_value.value = value + fb_key_value + end + end FB::Footer.serialize(fb_footer) end - def write_footer - footer = build_footer + def write_footer(metadata) + footer = build_footer(metadata) write_data(footer) write_data([footer.bytesize].pack("l<")) end diff --git a/ruby/red-arrow-format/lib/arrow-format/streaming-reader.rb b/ruby/red-arrow-format/lib/arrow-format/streaming-reader.rb index f81cfe8913a4..1a9f71ac9cb4 100644 --- a/ruby/red-arrow-format/lib/arrow-format/streaming-reader.rb +++ b/ruby/red-arrow-format/lib/arrow-format/streaming-reader.rb @@ -22,7 +22,17 @@ class StreamingReader include Enumerable def initialize(input) - @input = input + case input + when File + @input = IO::Buffer.map(input, nil, 0, IO::Buffer::READONLY) + @offset = 0 + when String + @input = IO::Buffer.for(input) + @offset = 0 + else + @input = input + end + @on_read = nil @pull_reader = StreamingPullReader.new do |record_batch| @on_read.call(record_batch) if @on_read @@ -53,11 +63,18 @@ def consume next_size = @pull_reader.next_required_size return false if next_size.zero? - next_chunk = @input.read(next_size, @buffer) - return false if next_chunk.nil? + if @input.is_a?(IO::Buffer) + next_chunk = @input.slice(@offset, next_size) + @offset += next_size + @pull_reader.consume(next_chunk) + true + else + next_chunk = @input.read(next_size, @buffer) + return false if next_chunk.nil? - @pull_reader.consume(IO::Buffer.for(next_chunk)) - true + @pull_reader.consume(IO::Buffer.for(next_chunk)) + true + end end def ensure_schema diff --git a/ruby/red-arrow-format/test/helper.rb b/ruby/red-arrow-format/test/helper.rb index 394d92d0dd4c..29fbfaec4c5f 100644 --- a/ruby/red-arrow-format/test/helper.rb +++ b/ruby/red-arrow-format/test/helper.rb @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +require "stringio" require "tmpdir" require "test-unit" diff --git a/ruby/red-arrow-format/test/test-reader.rb b/ruby/red-arrow-format/test/test-reader.rb index d59a93ce184b..1d3202f4f720 100644 --- a/ruby/red-arrow-format/test/test-reader.rb +++ b/ruby/red-arrow-format/test/test-reader.rb @@ -26,9 +26,7 @@ def roundtrip(data) else table = data end - path = File.join(tmp_dir, "data.#{file_extension}") - table.save(path) - File.open(path, "rb") do |input| + open_input(table, tmp_dir) do |input| reader = reader_class.new(input) case data when Arrow::Array @@ -677,8 +675,61 @@ def test_dictionary end end -class TestFileReader < Test::Unit::TestCase +module FileReaderTests + def test_custom_metadata_footer + Dir.mktmpdir do |tmp_dir| + table = Arrow::Table.new(value: Arrow::Int8Array.new([1, 2, 3])) + metadata = { + "key1" => "value1", + "key2" => "value2", + } + open_input(table, tmp_dir, metadata: metadata) do |input| + reader = reader_class.new(input) + assert_equal(metadata, reader.metadata) + end + ensure + GC.start + end + end +end + +module FileInput + def open_input(table, tmp_dir, **options, &block) + path = File.join(tmp_dir, "data.#{file_extension}") + table.save(path, **options) + File.open(path, "rb", &block) + end +end + +module PipeInput + def open_input(table, tmp_dir, **options) + buffer = Arrow::ResizableBuffer.new(4096) + table.save(buffer, format: format, **options) + IO.pipe do |input, output| + write_thread = Thread.new do + output.write(buffer.data.to_s) + end + begin + yield(input) + ensure + write_thread.join + end + end + end +end + +module StringInput + def open_input(table, tmp_dir, **options) + buffer = Arrow::ResizableBuffer.new(4096) + table.save(buffer, format: format, **options) + yield(buffer.data.to_s) + end +end + +class TestFileReaderFileInput < Test::Unit::TestCase include ReaderTests + include FileReaderTests + include FileInput def file_extension "arrow" @@ -689,8 +740,23 @@ def reader_class end end -class TestStreamingReader < Test::Unit::TestCase +class TestFileReaderStringInput < Test::Unit::TestCase + include ReaderTests + include FileReaderTests + include StringInput + + def format + :arrow_file + end + + def reader_class + ArrowFormat::FileReader + end +end + +class TestStreamingReaderFileInupt < Test::Unit::TestCase include ReaderTests + include FileInput def file_extension "arrows" @@ -700,3 +766,29 @@ def reader_class ArrowFormat::StreamingReader end end + +class TestStreamingReaderPipeInupt < Test::Unit::TestCase + include ReaderTests + include PipeInput + + def format + :arrow_streaming + end + + def reader_class + ArrowFormat::StreamingReader + end +end + +class TestStreamingReaderStringInupt < Test::Unit::TestCase + include ReaderTests + include StringInput + + def format + :arrow_streaming + end + + def reader_class + ArrowFormat::StreamingReader + end +end diff --git a/ruby/red-arrow-format/test/test-writer.rb b/ruby/red-arrow-format/test/test-writer.rb index 72776f01ab8e..55b3c22b7a96 100644 --- a/ruby/red-arrow-format/test/test-writer.rb +++ b/ruby/red-arrow-format/test/test-writer.rb @@ -924,6 +924,26 @@ def test_dictionary end end +module FileWriterTests + def test_custom_metadata_footer + output = StringIO.new(+"".b) + writer = writer_class.new(output) + field = ArrowFormat::Field.new("value", ArrowFormat::BooleanType.new) + schema = ArrowFormat::Schema.new([field]) + writer.start(schema) + metadata = { + "key1" => "value1", + "key2" => "value2", + } + writer.finish(metadata) + buffer = Arrow::Buffer.new(output.string) + Arrow::BufferInputStream.open(buffer) do |input| + reader = Arrow::RecordBatchFileReader.new(input) + assert_equal(metadata, reader.metadata) + end + end +end + module WriterDictionaryDeltaTests def build_schema(value_type) index_type = ArrowFormat::Int32Type.singleton @@ -1513,6 +1533,7 @@ def read(path) sub_test_case("Basic") do include WriterTests + include FileWriterTests end sub_test_case("Dictionary: delta") do diff --git a/ruby/red-arrow/lib/arrow/table-saver.rb b/ruby/red-arrow/lib/arrow/table-saver.rb index c33e64143873..d456f235e5c9 100644 --- a/ruby/red-arrow/lib/arrow/table-saver.rb +++ b/ruby/red-arrow/lib/arrow/table-saver.rb @@ -130,9 +130,9 @@ def open_output_stream(&block) end end - def save_raw(writer_class) + def save_raw(writer_class, *args) open_output_stream do |output| - writer_class.open(output, @table.schema) do |writer| + writer_class.open(output, @table.schema, *args) do |writer| writer.write_table(@table) end end @@ -144,7 +144,7 @@ def save_as_arrow # @since 1.0.0 def save_as_arrow_file - save_raw(RecordBatchFileWriter) + save_raw(RecordBatchFileWriter, nil, @options[:metadata]) end # @deprecated Use `format: :arrow_batch` instead.