Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
27fdd35
MINOR: [Docs] Clarify struct validity masking with 'hidden data' exa…
pucke-dev Mar 20, 2026
8a8e574
GH-49503: [Docs][Python] Documenting .pxi doctests are tested via lib…
bhavanaeh Mar 20, 2026
10eaafd
GH-49544: [Ruby] Add benchmark for readers (#49545)
kou Mar 21, 2026
c24bc29
GH-49576: [Ruby] Add support for custom metadata in Footer (#49577)
kou Mar 22, 2026
1ffcb4e
GH-49559: [C++][Parquet] Fix uncontrolled recursion in WKBGeometryBou…
wgtmac Mar 23, 2026
ba273ca
GH-49569: [CI][Python][C++] Add check targetting Apple clang on decid…
raulcd Mar 23, 2026
d28aea9
GH-49380: [R] Remove hidden CI test chunks from setup.Rmd to fix r-de…
vanshaj2023 Mar 23, 2026
999a662
GH-49579: [C++] Fix xsimd 14.1.0 build failure (#49580)
AntoinePrv Mar 23, 2026
af184f5
GH-49561: [C++][FlightRPC][ODBC] Use SQLWCHAR array for wide string l…
justing-bq Mar 24, 2026
df9dbbc
GH-48588 [C++] Migrate to stdlib span (#49492)
Anakin100100 Mar 24, 2026
ee7e210
GH-49529: [R] CI job shows NOTE due to "non-API call" Rf_findVarInFra…
thisisnic Mar 24, 2026
a942ed6
GH-49578: [CI][R] gcc sanitizer failure (#49581)
jonkeane Mar 24, 2026
ae5137e
MINOR: [R]: Update NEWS.md for 23.0.1.X-r releases (#49588)
thisisnic Mar 24, 2026
f9315d4
GH-49565: [Python] Copy CKmsConnectionConfig instead of trying to mov…
raulcd Mar 25, 2026
618d5fb
GH-47389: [Python] CSV and JSON options lack a nice repr/str (#47397)
thisisnic Mar 25, 2026
c9cc3b8
GH-49596: [CI][Dev] Pin PyGithub to < 2.9 to fix broken archery (#49597)
raulcd Mar 25, 2026
2c2a61c
GH-49566: [Python] Skip header files when installing compiled Cython …
AlenkaF Mar 25, 2026
1fb0e25
GH-49572 : [Python][Docs] Remove editable section and consolidate the…
AlenkaF Mar 25, 2026
4feb9bd
GH-49593: [R][CI] Add libuv-dev to CI jobs due to update to fs packag…
thisisnic Mar 25, 2026
bbfb242
GH-49563: [C++][CMake] Remove clang/infer tools detection (#49575)
alinaliBQ Mar 25, 2026
3843f52
GH-49274: [Doc][C++] Document security model for Arrow C++
pitrou Mar 10, 2026
dc9edf1
GH-49274: [Doc][C++] Address review follow-ups
zanmato1984 Mar 26, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .env
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ R_ORG=rhub
R_TAG=latest

# Env vars for R builds
R_UPDATE_CLANG=false
R_CUSTOM_CCACHE=false
ARROW_R_DEV=TRUE
R_PRUNE_DEPS=FALSE
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ruby.yml
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ jobs:
run: archery docker push ubuntu-ruby

macos:
name: ARM64 macOS 14 GLib & Ruby
name: ARM64 macOS GLib & Ruby
runs-on: macos-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 60
Expand Down
29 changes: 29 additions & 0 deletions c_glib/arrow-glib/reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -668,6 +668,35 @@ garrow_record_batch_file_reader_read_record_batch(GArrowRecordBatchFileReader *r
}
}

/**
* garrow_record_batch_file_reader_get_metadata:
* @reader: A #GArrowRecordBatchFileReader.
*
* Returns: (nullable) (element-type utf8 utf8) (transfer full):
* The metadata in the footer.
*
* Since: 24.0.0
*/
GHashTable *
garrow_record_batch_file_reader_get_metadata(GArrowRecordBatchFileReader *reader)
{
auto arrow_reader = garrow_record_batch_file_reader_get_raw(reader);
auto arrow_metadata = arrow_reader->metadata();

if (!arrow_metadata) {
return nullptr;
}

auto metadata = g_hash_table_new(g_str_hash, g_str_equal);
const auto n = arrow_metadata->size();
for (int64_t i = 0; i < n; ++i) {
g_hash_table_insert(metadata,
const_cast<gchar *>(arrow_metadata->key(i).c_str()),
const_cast<gchar *>(arrow_metadata->value(i).c_str()));
}
return metadata;
}

struct GArrowFeatherFileReaderPrivate
{
std::shared_ptr<arrow::ipc::feather::Reader> feather_reader;
Expand Down
4 changes: 4 additions & 0 deletions c_glib/arrow-glib/reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ garrow_record_batch_file_reader_read_record_batch(GArrowRecordBatchFileReader *r
guint i,
GError **error);

GARROW_AVAILABLE_IN_24_0
GHashTable *
garrow_record_batch_file_reader_get_metadata(GArrowRecordBatchFileReader *reader);

#define GARROW_TYPE_FEATHER_FILE_READER (garrow_feather_file_reader_get_type())
GARROW_AVAILABLE_IN_ALL
G_DECLARE_DERIVABLE_TYPE(GArrowFeatherFileReader,
Expand Down
42 changes: 39 additions & 3 deletions c_glib/arrow-glib/writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
#include <arrow-glib/array.hpp>
#include <arrow-glib/enums.h>
#include <arrow-glib/error.hpp>
#include <arrow-glib/internal-hash-table.hpp>
#include <arrow-glib/ipc-options.hpp>
#include <arrow-glib/record-batch.hpp>
#include <arrow-glib/schema.hpp>
#include <arrow-glib/table.hpp>
Expand Down Expand Up @@ -288,16 +290,50 @@ GArrowRecordBatchFileWriter *
garrow_record_batch_file_writer_new(GArrowOutputStream *sink,
GArrowSchema *schema,
GError **error)
{
return garrow_record_batch_file_writer_new_full(sink, schema, nullptr, nullptr, error);
}

/**
* garrow_record_batch_file_writer_new_full:
* @sink: The output of the writer.
* @schema: The schema of the writer.
* @options: (nullable): The options for serialization.
* @metadata: (nullable) (element-type utf8 utf8): The custom metadata in
* the footer.
* @error: (nullable): Return location for a #GError or %NULL.
*
* Returns: (nullable): A newly created #GArrowRecordBatchFileWriter
* or %NULL on error.
*
* Since: 24.0.0
*/
GArrowRecordBatchFileWriter *
garrow_record_batch_file_writer_new_full(GArrowOutputStream *sink,
GArrowSchema *schema,
GArrowWriteOptions *options,
GHashTable *metadata,
GError **error)
{
auto arrow_sink = garrow_output_stream_get_raw(sink);
auto arrow_schema = garrow_schema_get_raw(schema);
arrow::ipc::IpcWriteOptions arrow_options = arrow::ipc::IpcWriteOptions::Defaults();
if (options) {
arrow_options = *garrow_write_options_get_raw(options);
}
std::shared_ptr<arrow::KeyValueMetadata> arrow_metadata;
if (metadata) {
arrow_metadata = garrow_internal_hash_table_to_metadata(metadata);
}

std::shared_ptr<arrow::ipc::RecordBatchWriter> arrow_writer;
auto arrow_writer_result = arrow::ipc::MakeFileWriter(arrow_sink, arrow_schema);
if (garrow::check(error, arrow_writer_result, "[record-batch-file-writer][open]")) {
auto arrow_writer_result =
arrow::ipc::MakeFileWriter(arrow_sink, arrow_schema, arrow_options, arrow_metadata);
if (garrow::check(error, arrow_writer_result, "[record-batch-file-writer][new]")) {
auto arrow_writer = *arrow_writer_result;
return garrow_record_batch_file_writer_new_raw(&arrow_writer);
} else {
return NULL;
return nullptr;
}
}

Expand Down
9 changes: 9 additions & 0 deletions c_glib/arrow-glib/writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#pragma once

#include <arrow-glib/array.h>
#include <arrow-glib/ipc-options.h>
#include <arrow-glib/record-batch.h>
#include <arrow-glib/schema.h>

Expand Down Expand Up @@ -94,6 +95,14 @@ garrow_record_batch_file_writer_new(GArrowOutputStream *sink,
GArrowSchema *schema,
GError **error);

GARROW_AVAILABLE_IN_24_0
GArrowRecordBatchFileWriter *
garrow_record_batch_file_writer_new_full(GArrowOutputStream *sink,
GArrowSchema *schema,
GArrowWriteOptions *options,
GHashTable *metadata,
GError **error);

/**
* GArrowCSVQuotingStyle:
* @GARROW_CSV_QUOTING_STYLE_NEEDED: Only enclose values in quotes which need them.
Expand Down
32 changes: 32 additions & 0 deletions c_glib/test/test-file-writer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -88,4 +88,36 @@ def test_write_table
input.close
end
end

def test_footer_custom_metadata
tempfile = Tempfile.open("arrow-ipc-file-writer")
output = Arrow::FileOutputStream.new(tempfile.path, false)

array = build_boolean_array([true, false, true])
field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
schema = Arrow::Schema.new([field])

options = Arrow::WriteOptions.new
metadata = {"key1" => "value1", "key2" => "value2"}
begin
file_writer = Arrow::RecordBatchFileWriter.new(output,
schema,
options,
metadata)
file_writer.close
assert do
file_writer.closed?
end
ensure
output.close
end

input = Arrow::MemoryMappedInputStream.new(tempfile.path)
begin
file_reader = Arrow::RecordBatchFileReader.new(input)
assert_equal(metadata, file_reader.metadata)
ensure
input.close
end
end
end
3 changes: 0 additions & 3 deletions ci/docker/linux-r.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,6 @@ ENV R_PRUNE_DEPS=${r_prune_deps}
ARG r_custom_ccache=false
ENV R_CUSTOM_CCACHE=${r_custom_ccache}

ARG r_update_clang=false
ENV R_UPDATE_CLANG=${r_update_clang}

ARG tz="UTC"
ENV TZ=${tz}

Expand Down
11 changes: 0 additions & 11 deletions ci/scripts/r_docker_configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,17 +85,6 @@ else
$PACKAGE_MANAGER install -y rsync cmake curl
fi

# Update clang version to latest available.
# This is only for rhub/clang20. If we change the base image from rhub/clang20,
# we need to update this part too.
if [ "$R_UPDATE_CLANG" = true ]; then
apt update -y --allow-releaseinfo-change # flag needed for when debian version changes
apt install -y gnupg
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/llvm.gpg
echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-20 main" > /etc/apt/sources.list.d/llvm20.list
apt update -y --allow-releaseinfo-change # flag needed for when debian version changes
apt install -y clang-20 lld-20
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}'
12 changes: 7 additions & 5 deletions ci/scripts/r_install_system_dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,19 @@ else
apt-get update
fi

# Install curl and OpenSSL (technically, only needed for S3/GCS support, but
# installing the R curl package fails without it)
# Install curl, OpenSSL, and libuv
# - curl/OpenSSL: technically only needed for S3/GCS support, but
# installing the R curl package fails without it
# - libuv: required by the fs R package (no longer bundles libuv by default)
case "$PACKAGE_MANAGER" in
apt-get)
apt-get install -y libcurl4-openssl-dev libssl-dev
apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev
;;
apk)
$PACKAGE_MANAGER add curl-dev openssl-dev
$PACKAGE_MANAGER add curl-dev openssl-dev libuv-dev
;;
*)
$PACKAGE_MANAGER install -y libcurl-devel openssl-devel
$PACKAGE_MANAGER install -y libcurl-devel openssl-devel libuv-devel
;;
esac

Expand Down
1 change: 1 addition & 0 deletions ci/scripts/r_revdepcheck.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ apt install -y \
libbz2-dev \
libc-ares-dev \
libcurl4-openssl-dev \
libuv1-dev \
libgflags-dev \
libgoogle-glog-dev \
liblz4-dev \
Expand Down
1 change: 0 additions & 1 deletion compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1667,7 +1667,6 @@ services:
tz: ${TZ}
r_prune_deps: ${R_PRUNE_DEPS}
r_custom_ccache: ${R_CUSTOM_CCACHE}
r_update_clang: ${R_UPDATE_CLANG}
shm_size: *shm-size
environment:
<<: [*common, *sccache]
Expand Down
27 changes: 0 additions & 27 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -211,17 +211,6 @@ else()
set(MSVC_TOOLCHAIN FALSE)
endif()

find_package(ClangTools)
find_package(InferTools)
if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1"
OR CLANG_TIDY_FOUND
OR INFER_FOUND)
# Generate a Clang compile_commands.json "compilation database" file for use
# with various development tools, such as Vim's YouCompleteMe plugin.
# See http://clang.llvm.org/docs/JSONCompilationDatabase.html
set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
endif()

# Needed for Gandiva.
# Use the first Python installation on PATH, not the newest one
set(Python3_FIND_STRATEGY "LOCATION")
Expand Down Expand Up @@ -649,22 +638,6 @@ if(UNIX)
VERBATIM)
endif(UNIX)

#
# "make infer" target
#

if(${INFER_FOUND})
# runs infer capture
add_custom_target(infer ${BUILD_SUPPORT_DIR}/run-infer.sh ${INFER_BIN}
${CMAKE_BINARY_DIR}/compile_commands.json 1)
# runs infer analyze
add_custom_target(infer-analyze ${BUILD_SUPPORT_DIR}/run-infer.sh ${INFER_BIN}
${CMAKE_BINARY_DIR}/compile_commands.json 2)
# runs infer report
add_custom_target(infer-report ${BUILD_SUPPORT_DIR}/run-infer.sh ${INFER_BIN}
${CMAKE_BINARY_DIR}/compile_commands.json 3)
endif()

#
# Link targets
#
Expand Down
Loading
Loading