From 27103d92a5738922d724c6685d44056a9b604431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a?= Date: Fri, 17 Oct 2025 22:08:51 +0000 Subject: [PATCH 1/3] GH-47848: [C++] Add TimeUnit::PICO This enum value is unused for now, but I plan to introduce a timestamp128 type in the future which supports this. --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/array/diff.cc | 9 ++ cpp/src/arrow/c/bridge.cc | 15 ++- cpp/src/arrow/compute/api_scalar.cc | 4 +- .../arrow/compute/kernels/codegen_internal.cc | 3 + .../compute/kernels/scalar_cast_temporal.cc | 2 + .../arrow/compute/kernels/scalar_cast_test.cc | 11 +- .../compute/kernels/scalar_if_else_test.cc | 5 +- .../compute/kernels/scalar_temporal_test.cc | 27 ++-- cpp/src/arrow/type.cc | 10 +- cpp/src/arrow/type_fwd.h | 2 +- cpp/src/arrow/util/int128_internal.cc | 120 ++++++++++++++++++ cpp/src/arrow/util/int128_internal.h | 10 ++ cpp/src/arrow/vendored/datetime/date.h | 4 + cpp/src/parquet/arrow/reader_internal.cc | 2 + cpp/src/parquet/arrow/schema.cc | 3 + cpp/src/parquet/column_writer.cc | 2 + 17 files changed, 209 insertions(+), 21 deletions(-) create mode 100644 cpp/src/arrow/util/int128_internal.cc diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 25e5749335a0..e91fa9dee278 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -509,6 +509,7 @@ set(ARROW_UTIL_SRCS util/future.cc util/hashing.cc util/int_util.cc + util/int128_internal.cc util/io_util.cc util/list_util.cc util/logger.cc diff --git a/cpp/src/arrow/array/diff.cc b/cpp/src/arrow/array/diff.cc index fd907e3c7b29..8b816d426e90 100644 --- a/cpp/src/arrow/array/diff.cc +++ b/cpp/src/arrow/array/diff.cc @@ -44,6 +44,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/checked_cast.h" #include "arrow/util/float16.h" +#include "arrow/util/int128_internal.h" #include "arrow/util/logging_internal.h" #include "arrow/util/range.h" #include "arrow/util/ree_util.h" @@ -57,6 +58,7 @@ namespace arrow { using internal::checked_cast; using internal::checked_pointer_cast; using internal::MakeLazyRange; +using internal::int128_t; namespace { @@ -855,6 +857,7 @@ class MakeFormatterImpl { // Using unqualified `format` directly would produce ambiguous // lookup because of `std::format` (ARROW-15520). namespace avd = arrow_vendored::date; + using picoseconds = std::chrono::duration; using std::chrono::nanoseconds; using std::chrono::microseconds; using std::chrono::milliseconds; @@ -863,6 +866,9 @@ class MakeFormatterImpl { static avd::sys_days epoch{avd::jan / 1 / 1970}; switch (unit) { + case TimeUnit::PICO: + *os << avd::format(fmt, static_cast(value) + epoch); + break; case TimeUnit::NANO: *os << avd::format(fmt, static_cast(value) + epoch); break; @@ -879,6 +885,9 @@ class MakeFormatterImpl { return; } switch (unit) { + case TimeUnit::PICO: + *os << avd::format(fmt, static_cast(value)); + break; case TimeUnit::NANO: *os << avd::format(fmt, static_cast(value)); break; diff --git a/cpp/src/arrow/c/bridge.cc b/cpp/src/arrow/c/bridge.cc index dd25ed299dd2..6ebd3d4ae4b5 100644 --- a/cpp/src/arrow/c/bridge.cc +++ b/cpp/src/arrow/c/bridge.cc @@ -1221,21 +1221,28 @@ struct SchemaImporter { ARROW_ASSIGN_OR_RAISE(auto unit, f_parser_.ParseTimeUnit()); if (unit == TimeUnit::SECOND || unit == TimeUnit::MILLI) { return ProcessPrimitive(time32(unit)); - } else { + } else if (unit != TimeUnit::PICO) { return ProcessPrimitive(time64(unit)); } + return f_parser_.Invalid(); } Status ProcessDuration() { ARROW_ASSIGN_OR_RAISE(auto unit, f_parser_.ParseTimeUnit()); - return ProcessPrimitive(duration(unit)); + if (unit != TimeUnit::PICO) { + return ProcessPrimitive(duration(unit)); + } + return f_parser_.Invalid(); } Status ProcessTimestamp() { ARROW_ASSIGN_OR_RAISE(auto unit, f_parser_.ParseTimeUnit()); RETURN_NOT_OK(f_parser_.CheckNext(':')); - type_ = timestamp(unit, std::string(f_parser_.Rest())); - return Status::OK(); + if (unit != TimeUnit::PICO) { + type_ = timestamp(unit, std::string(f_parser_.Rest())); + return Status::OK(); + } + return f_parser_.Invalid(); } Status ProcessFixedSizeBinary() { diff --git a/cpp/src/arrow/compute/api_scalar.cc b/cpp/src/arrow/compute/api_scalar.cc index b43eca542f36..6d3c896a8fd9 100644 --- a/cpp/src/arrow/compute/api_scalar.cc +++ b/cpp/src/arrow/compute/api_scalar.cc @@ -58,7 +58,7 @@ struct EnumTraits template <> struct EnumTraits : BasicEnumTraits { + TimeUnit::type::MICRO, TimeUnit::type::NANO, TimeUnit::type::PICO> { static std::string name() { return "TimeUnit::type"; } static std::string value_name(TimeUnit::type value) { switch (value) { @@ -70,6 +70,8 @@ struct EnumTraits return "MICRO"; case TimeUnit::type::NANO: return "NANO"; + case TimeUnit::type::PICO: + return "PICO"; } return ""; } diff --git a/cpp/src/arrow/compute/kernels/codegen_internal.cc b/cpp/src/arrow/compute/kernels/codegen_internal.cc index 10ed9344d976..51ffca23a5c2 100644 --- a/cpp/src/arrow/compute/kernels/codegen_internal.cc +++ b/cpp/src/arrow/compute/kernels/codegen_internal.cc @@ -342,6 +342,9 @@ TypeHolder CommonTemporal(const TypeHolder* begin, size_t count) { case TimeUnit::MICRO: case TimeUnit::NANO: return time64(finest_unit); + case TimeUnit::PICO: + // Type not implemented. + return TypeHolder(nullptr); } } return TypeHolder(nullptr); diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc b/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc index d076186e5635..9aba29a60aed 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc @@ -140,6 +140,8 @@ Status ExtractTemporal(KernelContext* ctx, const ExecSpan& batch, ExecResult* ou case TimeUnit::NANO: return TemporalComponentExtract::Exec(ctx, batch, out, args...); + case TimeUnit::PICO: + return Status::Invalid("Picoseconds not yet supported: ", ty); } return Status::Invalid("Unknown timestamp unit: ", ty); } diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc index 44b50b31f758..737a95c72e5d 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc @@ -2396,7 +2396,8 @@ TEST(Cast, TimestampToDate) { CheckCast(timestamps, date_64); CheckCast(timestamps_extreme, date_32_extreme); CheckCast(timestamps_extreme, date_64_extreme); - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u); CheckCast(ArrayFromJSON(unit, kTimestampSecondsJson), date_32); CheckCast(ArrayFromJSON(unit, kTimestampSecondsJson), date_64); @@ -2438,7 +2439,8 @@ TEST_F(CastTimezone, ZonedTimestampToDate) { 1230422400000, 1230508800000, 1325376000000, null ])"); - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto timestamps = ArrayFromJSON(timestamp(u, "Australia/Broken_Hill"), kTimestampSecondsJson); CheckCast(timestamps, date_32); @@ -2446,7 +2448,7 @@ TEST_F(CastTimezone, ZonedTimestampToDate) { } // Invalid timezone - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto timestamps = ArrayFromJSON(timestamp(u, "Mars/Mariner_Valley"), kTimestampSecondsJson); CheckCastFails(timestamps, CastOptions::Unsafe(date32())); @@ -2583,7 +2585,8 @@ TEST(Cast, TimestampToTime) { CheckCast(timestamps_s, times_ms); // Invalid timezone - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto timestamps = ArrayFromJSON(timestamp(u, "Mars/Mariner_Valley"), kTimestampSecondsJson); if (u == TimeUnit::SECOND || u == TimeUnit::MILLI) { diff --git a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc index 85d152aa8cd2..1f17318ca1e0 100644 --- a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc @@ -421,7 +421,10 @@ TEST_F(TestIfElseKernel, IfElseMultiType) { } TEST_F(TestIfElseKernel, TimestampTypes) { - for (const auto unit : TimeUnit::values()) { + for (const auto unit : { + TimeUnit::SECOND, + TimeUnit::NANO, + }) { auto ty = timestamp(unit); CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), ArrayFromJSON(ty, "[1, 2, 3, 4]"), diff --git a/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc b/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc index 3350fb805c47..a8c9765f1c82 100644 --- a/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc @@ -608,7 +608,8 @@ TEST_F(ScalarTemporalTest, TestTemporalComponentExtractionAllTemporalTypes) { } TEST_F(ScalarTemporalTest, TestTemporalComponentExtractionWithDifferentUnits) { - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u); CheckScalarUnary("year", unit, times_seconds_precision, int64(), year); CheckScalarUnary("is_leap_year", unit, times_seconds_precision, boolean(), @@ -814,7 +815,8 @@ TEST_F(ScalarTemporalTest, TestZoned1) { } TEST_F(ScalarTemporalTest, TestZoned2) { - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u, "Australia/Broken_Hill"); auto month = "[1, 3, 1, 5, 1, 12, 12, 12, 1, 1, 1, 1, 12, 12, 12, 1, null]"; auto day = "[1, 1, 1, 18, 1, 31, 30, 31, 1, 3, 4, 1, 31, 28, 29, 1, null]"; @@ -906,8 +908,9 @@ TEST_F(ScalarTemporalTest, TestNonexistentTimezone) { auto nonexistent_timezones = { "Mars/Mariner_Valley", "+25:00", "-25:00", "15:00", "5:00", "500", "+05:00:00", "+050000"}; + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; for (auto timezone : nonexistent_timezones) { - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto ts_type = timestamp(u, timezone); auto timestamp_array = std::make_shared>( ts_type, 2, data_buffer, null_buffer, 0); @@ -1026,7 +1029,8 @@ TEST_F(ScalarTemporalTest, DayOfWeek) { } TEST_F(ScalarTemporalTest, TestTemporalDifference) { - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u); auto arr1 = ArrayFromJSON(unit, times_seconds_precision); auto arr2 = ArrayFromJSON(unit, times_seconds_precision2); @@ -1879,8 +1883,9 @@ TEST_F(ScalarTemporalTest, TestLocalTimestamp) { "2009-12-30 18:50:20", "2009-12-31 19:55:25", "2010-01-02 21:00:30", "2010-01-03 22:05:35", "2005-12-31 23:10:40", "2005-12-31 00:15:45", "2008-12-27 14:30:00", "2008-12-28 14:30:00", "2011-12-31 15:32:03", null])"; + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; - for (auto u : TimeUnit::values()) { + for (auto u : units) { CheckScalarUnary("local_timestamp", timestamp(u), times_seconds_precision, timestamp(u), times_seconds_precision); CheckScalarUnary("local_timestamp", timestamp(u, "UTC"), times_seconds_precision, @@ -1912,8 +1917,9 @@ TEST_F(ScalarTemporalTest, TestAssumeTimezone) { auto options_tbilisi = AssumeTimezoneOptions(timezone_tbilisi); auto options_tbilisi_offset = AssumeTimezoneOptions(timezone_tbilisi_offset); auto options_invalid = AssumeTimezoneOptions("Europe/Brusselsss"); + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto unit = timestamp(u); auto unit_utc = timestamp(u, timezone_utc); auto unit_kolkata = timestamp(u, timezone_kolkata); @@ -1956,8 +1962,9 @@ TEST_F(ScalarTemporalTest, TestAssumeTimezoneAmbiguous) { AssumeTimezoneOptions(timezone, AssumeTimezoneOptions::AMBIGUOUS_LATEST); auto options_raise = AssumeTimezoneOptions(timezone, AssumeTimezoneOptions::AMBIGUOUS_RAISE); + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto unit = timestamp(u); auto unit_local = timestamp(u, timezone); ASSERT_RAISES(Invalid, AssumeTimezone(ArrayFromJSON(unit, times), options_raise)); @@ -1989,8 +1996,9 @@ TEST_F(ScalarTemporalTest, TestAssumeTimezoneNonexistent) { auto options_earliest = AssumeTimezoneOptions(timezone, AssumeTimezoneOptions::AMBIGUOUS_RAISE, AssumeTimezoneOptions::NONEXISTENT_EARLIEST); + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto unit = timestamp(u); auto unit_local = timestamp(u, timezone); ASSERT_RAISES(Invalid, AssumeTimezone(ArrayFromJSON(unit, times), options_raise)); @@ -2245,7 +2253,8 @@ TEST_F(ScalarTemporalTest, StrftimeInvalidLocale) { } TEST_F(ScalarTemporalTest, TestTemporalDifferenceZoned) { - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u, "Pacific/Marquesas"); auto arr1 = ArrayFromJSON(unit, times_seconds_precision); auto arr2 = ArrayFromJSON(unit, times_seconds_precision2); diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index 2e9d860a8d6f..67277e83ff9c 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -249,6 +249,8 @@ std::string ToString(TimeUnit::type unit) { return "us"; case TimeUnit::NANO: return "ns"; + case TimeUnit::PICO: + return "ps"; default: DCHECK(false); return ""; @@ -1190,6 +1192,9 @@ std::ostream& operator<<(std::ostream& os, TimeUnit::type unit) { case TimeUnit::NANO: os << "ns"; break; + case TimeUnit::PICO: + os << "ps"; + break; } return os; } @@ -2767,6 +2772,8 @@ static char TimeUnitFingerprint(TimeUnit::type unit) { return 'u'; case TimeUnit::NANO: return 'n'; + case TimeUnit::PICO: + return 'p'; default: DCHECK(false) << "Unexpected TimeUnit"; return '\0'; @@ -3542,7 +3549,8 @@ const std::vector>& PrimitiveTypes() { const std::vector& TimeUnit::values() { static std::vector units = {TimeUnit::SECOND, TimeUnit::MILLI, - TimeUnit::MICRO, TimeUnit::NANO}; + TimeUnit::MICRO, TimeUnit::NANO, + TimeUnit::PICO}; return units; } diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h index be26c40dc1f4..8d2988e2e54b 100644 --- a/cpp/src/arrow/type_fwd.h +++ b/cpp/src/arrow/type_fwd.h @@ -269,7 +269,7 @@ struct Date64Scalar; struct ARROW_EXPORT TimeUnit { /// The unit for a time or timestamp DataType - enum type { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 }; + enum type { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3, PICO = 4 }; /// Iterate over all valid time units static const std::vector& values(); diff --git a/cpp/src/arrow/util/int128_internal.cc b/cpp/src/arrow/util/int128_internal.cc new file mode 100644 index 000000000000..d62ac7f4dde5 --- /dev/null +++ b/cpp/src/arrow/util/int128_internal.cc @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/util/int128_internal.h" + +// From StackOverlow user Mark Adler: https://stackoverflow.com/a/72651639/101923 +static void out128(std::ostream& out, arrow::internal::uint128_t val, int neg) { + // Note if the number is zero. (No hex or octal prefix in this case.) + auto zero = val == 0; + + // Note if upper-case letters requested. + auto state = out.flags(); + auto upper = (state & std::ios_base::uppercase) != 0; + + // Set base for digits. + unsigned base = state & std::ios_base::hex ? 16 : + state & std::ios_base::oct ? 8 : + 10; + + // Space for digits and prefix. Generate digits starting at the end of the + // string, going backwards. num will be the digit string. Terminate it. + char str[47]; + auto end = str + sizeof(str), num = end; + *--num = 0; + + // Compute and place digits in base base. + do { + char dig = val % base; + val /= base; + dig += dig < 10 ? '0' : (upper ? 'A' : 'a') - 10; + *--num = dig; + } while (val); + + // Prepend octal number with a zero if requested. + if (state & std::ios_base::showbase && base == 8 && !zero) + *--num = '0'; + + // pre will be the prefix string. Terminate it. + auto pre = num; + *--pre = 0; + + // Put a plus or minus sign in the prefix as appropriate. + if (base == 10) { + if (neg) + *--pre = '-'; + else if (state & std::ios_base::showpos) + *--pre = '+'; + } + + // Prefix a hexadecimal number if requested. + else if (state & std::ios_base::showbase && base == 16 && !zero) { + *--pre = upper ? 'X' : 'x'; + *--pre = '0'; + } + + // Compute the number of pad characters and get the fill character. + auto len = (num - pre) + (end - num) - 2; + auto pad = out.width(); + out.width(0); + pad = pad > len ? pad - len : 0; + char fill = out.fill(); + + // Put the padding before prefix if neither left nor internal requested. + if (!(state & (std::ios_base::internal | std::ios_base::left))) + while (pad) { + out << fill; + pad--; + } + + // Write prefix. + out << pre; + + // Put the padding between the prefix and the digits if requested. + if (state & std::ios_base::internal) + while (pad) { + out << fill; + pad--; + } + + // Write digits. + out << num; + + // Put number to the left of padding, if requested. + if (state & std::ios_base::left) + while (pad) { + out << fill; + pad--; + } +} + +// Overload << for an unsigned 128-bit integer. +std::ostream& operator<<(std::ostream& out, arrow::internal::uint128_t val) { + out128(out, val, 0); + return out; +} + +// Overload << for a signed 128-bit integer. Negation of the most negative +// signed value gives the correct unsigned absolute value. +std::ostream& operator<<(std::ostream& out, arrow::internal::int128_t val) { + auto state = out.flags(); + if (val < 0 && !(state & (std::ios_base::hex | std::ios_base::oct))) + out128(out, -(arrow::internal::uint128_t)val, 1); + else + out128(out, (arrow::internal::uint128_t)val, 0); + return out; +} diff --git a/cpp/src/arrow/util/int128_internal.h b/cpp/src/arrow/util/int128_internal.h index 201e4a134919..f1990f40bc64 100644 --- a/cpp/src/arrow/util/int128_internal.h +++ b/cpp/src/arrow/util/int128_internal.h @@ -16,6 +16,9 @@ // under the License. #pragma once +#include +#include + #include "arrow/util/config.h" #include "arrow/util/macros.h" @@ -43,3 +46,10 @@ using boost::multiprecision::uint128_t; } // namespace internal } // namespace arrow + +// Write the 128-bit integer val to out, with a minus sign if decimal and neg +// is true. Obey all of the ostream settings of out for integer display: octal +// or hexadecimal, upper case letters, plus sign, fill character and width, and +// fill placement. +std::ostream& operator<<(std::ostream& out, arrow::internal::uint128_t val); +std::ostream& operator<<(std::ostream& out, arrow::internal::int128_t val); diff --git a/cpp/src/arrow/vendored/datetime/date.h b/cpp/src/arrow/vendored/datetime/date.h index 1b06182a6daa..213925c1ab00 100644 --- a/cpp/src/arrow/vendored/datetime/date.h +++ b/cpp/src/arrow/vendored/datetime/date.h @@ -67,6 +67,8 @@ #include #include +#include "arrow/util/int128_internal.h" + #ifdef __GNUC__ # pragma GCC diagnostic push # if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 7) @@ -167,6 +169,8 @@ template using ratio_divide = decltype(std::ratio_divide{}); } // namespace detail +using ::operator<<; + //-----------+ // Interface | //-----------+ diff --git a/cpp/src/parquet/arrow/reader_internal.cc b/cpp/src/parquet/arrow/reader_internal.cc index 12f36fe39cf8..f28a78940f8f 100644 --- a/cpp/src/parquet/arrow/reader_internal.cc +++ b/cpp/src/parquet/arrow/reader_internal.cc @@ -516,6 +516,8 @@ Status TransferInt96(RecordReader* reader, MemoryPool* pool, *data_ptr++ = 0; } else { switch (int96_arrow_time_unit) { + case ::arrow::TimeUnit::PICO: + return Status::Invalid("Picoseconds not supported."); case ::arrow::TimeUnit::NANO: *data_ptr++ = Int96GetNanoSeconds(values[i]); break; diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc index 293ae94b94dd..58feacf34a5d 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc @@ -195,6 +195,7 @@ static std::shared_ptr TimestampLogicalTypeFromArrowTimestamp return LogicalType::Timestamp(utc, LogicalType::TimeUnit::NANOS, /*is_from_converted_type=*/false, /*force_set_converted_type=*/false); + case ::arrow::TimeUnit::PICO: case ::arrow::TimeUnit::SECOND: // No equivalent parquet logical type. break; @@ -231,6 +232,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type, case ::arrow::TimeUnit::MILLI: case ::arrow::TimeUnit::MICRO: break; + case ::arrow::TimeUnit::PICO: case ::arrow::TimeUnit::NANO: case ::arrow::TimeUnit::SECOND: return Status::NotImplemented("For Parquet version ", @@ -244,6 +246,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type, case ::arrow::TimeUnit::MICRO: case ::arrow::TimeUnit::NANO: break; + case ::arrow::TimeUnit::PICO: case ::arrow::TimeUnit::SECOND: return Status::NotImplemented("For Parquet version ", ::parquet::ParquetVersionToString(version), diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 94b67dfa807e..e98835d7461f 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -2239,6 +2239,8 @@ struct SerializeFunctor { const int64_t* input = array.raw_values(); const auto& type = static_cast(*array.type()); switch (type.unit()) { + case ::arrow::TimeUnit::PICO: + return Status::Invalid("Picoseconds not supported."); case ::arrow::TimeUnit::NANO: INT96_CONVERT_LOOP(internal::NanosecondsToImpalaTimestamp); break; From 035c0c52c8832e13eb4aa91e2e066f286cb7ce6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a?= Date: Fri, 17 Oct 2025 22:08:51 +0000 Subject: [PATCH 2/3] GH-47848: [C++] Add TimeUnit::PICO This enum value is unused for now, but I plan to introduce a timestamp128 type in the future which supports this. --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/array/diff.cc | 9 ++ cpp/src/arrow/c/bridge.cc | 15 ++- cpp/src/arrow/compute/api_scalar.cc | 4 +- .../arrow/compute/kernels/codegen_internal.cc | 3 + .../compute/kernels/scalar_cast_temporal.cc | 2 + .../arrow/compute/kernels/scalar_cast_test.cc | 11 +- .../compute/kernels/scalar_if_else_test.cc | 5 +- .../compute/kernels/scalar_temporal_test.cc | 27 ++-- cpp/src/arrow/type.cc | 10 +- cpp/src/arrow/type_fwd.h | 2 +- cpp/src/arrow/util/int128_internal.cc | 120 ++++++++++++++++++ cpp/src/arrow/util/int128_internal.h | 10 ++ cpp/src/arrow/vendored/datetime/date.h | 4 + cpp/src/parquet/arrow/reader_internal.cc | 2 + cpp/src/parquet/arrow/schema.cc | 3 + cpp/src/parquet/column_writer.cc | 2 + 17 files changed, 209 insertions(+), 21 deletions(-) create mode 100644 cpp/src/arrow/util/int128_internal.cc diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 25e5749335a0..e91fa9dee278 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -509,6 +509,7 @@ set(ARROW_UTIL_SRCS util/future.cc util/hashing.cc util/int_util.cc + util/int128_internal.cc util/io_util.cc util/list_util.cc util/logger.cc diff --git a/cpp/src/arrow/array/diff.cc b/cpp/src/arrow/array/diff.cc index fd907e3c7b29..8b816d426e90 100644 --- a/cpp/src/arrow/array/diff.cc +++ b/cpp/src/arrow/array/diff.cc @@ -44,6 +44,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/checked_cast.h" #include "arrow/util/float16.h" +#include "arrow/util/int128_internal.h" #include "arrow/util/logging_internal.h" #include "arrow/util/range.h" #include "arrow/util/ree_util.h" @@ -57,6 +58,7 @@ namespace arrow { using internal::checked_cast; using internal::checked_pointer_cast; using internal::MakeLazyRange; +using internal::int128_t; namespace { @@ -855,6 +857,7 @@ class MakeFormatterImpl { // Using unqualified `format` directly would produce ambiguous // lookup because of `std::format` (ARROW-15520). namespace avd = arrow_vendored::date; + using picoseconds = std::chrono::duration; using std::chrono::nanoseconds; using std::chrono::microseconds; using std::chrono::milliseconds; @@ -863,6 +866,9 @@ class MakeFormatterImpl { static avd::sys_days epoch{avd::jan / 1 / 1970}; switch (unit) { + case TimeUnit::PICO: + *os << avd::format(fmt, static_cast(value) + epoch); + break; case TimeUnit::NANO: *os << avd::format(fmt, static_cast(value) + epoch); break; @@ -879,6 +885,9 @@ class MakeFormatterImpl { return; } switch (unit) { + case TimeUnit::PICO: + *os << avd::format(fmt, static_cast(value)); + break; case TimeUnit::NANO: *os << avd::format(fmt, static_cast(value)); break; diff --git a/cpp/src/arrow/c/bridge.cc b/cpp/src/arrow/c/bridge.cc index dd25ed299dd2..6ebd3d4ae4b5 100644 --- a/cpp/src/arrow/c/bridge.cc +++ b/cpp/src/arrow/c/bridge.cc @@ -1221,21 +1221,28 @@ struct SchemaImporter { ARROW_ASSIGN_OR_RAISE(auto unit, f_parser_.ParseTimeUnit()); if (unit == TimeUnit::SECOND || unit == TimeUnit::MILLI) { return ProcessPrimitive(time32(unit)); - } else { + } else if (unit != TimeUnit::PICO) { return ProcessPrimitive(time64(unit)); } + return f_parser_.Invalid(); } Status ProcessDuration() { ARROW_ASSIGN_OR_RAISE(auto unit, f_parser_.ParseTimeUnit()); - return ProcessPrimitive(duration(unit)); + if (unit != TimeUnit::PICO) { + return ProcessPrimitive(duration(unit)); + } + return f_parser_.Invalid(); } Status ProcessTimestamp() { ARROW_ASSIGN_OR_RAISE(auto unit, f_parser_.ParseTimeUnit()); RETURN_NOT_OK(f_parser_.CheckNext(':')); - type_ = timestamp(unit, std::string(f_parser_.Rest())); - return Status::OK(); + if (unit != TimeUnit::PICO) { + type_ = timestamp(unit, std::string(f_parser_.Rest())); + return Status::OK(); + } + return f_parser_.Invalid(); } Status ProcessFixedSizeBinary() { diff --git a/cpp/src/arrow/compute/api_scalar.cc b/cpp/src/arrow/compute/api_scalar.cc index b43eca542f36..6d3c896a8fd9 100644 --- a/cpp/src/arrow/compute/api_scalar.cc +++ b/cpp/src/arrow/compute/api_scalar.cc @@ -58,7 +58,7 @@ struct EnumTraits template <> struct EnumTraits : BasicEnumTraits { + TimeUnit::type::MICRO, TimeUnit::type::NANO, TimeUnit::type::PICO> { static std::string name() { return "TimeUnit::type"; } static std::string value_name(TimeUnit::type value) { switch (value) { @@ -70,6 +70,8 @@ struct EnumTraits return "MICRO"; case TimeUnit::type::NANO: return "NANO"; + case TimeUnit::type::PICO: + return "PICO"; } return ""; } diff --git a/cpp/src/arrow/compute/kernels/codegen_internal.cc b/cpp/src/arrow/compute/kernels/codegen_internal.cc index 10ed9344d976..51ffca23a5c2 100644 --- a/cpp/src/arrow/compute/kernels/codegen_internal.cc +++ b/cpp/src/arrow/compute/kernels/codegen_internal.cc @@ -342,6 +342,9 @@ TypeHolder CommonTemporal(const TypeHolder* begin, size_t count) { case TimeUnit::MICRO: case TimeUnit::NANO: return time64(finest_unit); + case TimeUnit::PICO: + // Type not implemented. + return TypeHolder(nullptr); } } return TypeHolder(nullptr); diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc b/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc index d076186e5635..9aba29a60aed 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc @@ -140,6 +140,8 @@ Status ExtractTemporal(KernelContext* ctx, const ExecSpan& batch, ExecResult* ou case TimeUnit::NANO: return TemporalComponentExtract::Exec(ctx, batch, out, args...); + case TimeUnit::PICO: + return Status::Invalid("Picoseconds not yet supported: ", ty); } return Status::Invalid("Unknown timestamp unit: ", ty); } diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc index 44b50b31f758..737a95c72e5d 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc @@ -2396,7 +2396,8 @@ TEST(Cast, TimestampToDate) { CheckCast(timestamps, date_64); CheckCast(timestamps_extreme, date_32_extreme); CheckCast(timestamps_extreme, date_64_extreme); - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u); CheckCast(ArrayFromJSON(unit, kTimestampSecondsJson), date_32); CheckCast(ArrayFromJSON(unit, kTimestampSecondsJson), date_64); @@ -2438,7 +2439,8 @@ TEST_F(CastTimezone, ZonedTimestampToDate) { 1230422400000, 1230508800000, 1325376000000, null ])"); - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto timestamps = ArrayFromJSON(timestamp(u, "Australia/Broken_Hill"), kTimestampSecondsJson); CheckCast(timestamps, date_32); @@ -2446,7 +2448,7 @@ TEST_F(CastTimezone, ZonedTimestampToDate) { } // Invalid timezone - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto timestamps = ArrayFromJSON(timestamp(u, "Mars/Mariner_Valley"), kTimestampSecondsJson); CheckCastFails(timestamps, CastOptions::Unsafe(date32())); @@ -2583,7 +2585,8 @@ TEST(Cast, TimestampToTime) { CheckCast(timestamps_s, times_ms); // Invalid timezone - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto timestamps = ArrayFromJSON(timestamp(u, "Mars/Mariner_Valley"), kTimestampSecondsJson); if (u == TimeUnit::SECOND || u == TimeUnit::MILLI) { diff --git a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc index 85d152aa8cd2..1f17318ca1e0 100644 --- a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc @@ -421,7 +421,10 @@ TEST_F(TestIfElseKernel, IfElseMultiType) { } TEST_F(TestIfElseKernel, TimestampTypes) { - for (const auto unit : TimeUnit::values()) { + for (const auto unit : { + TimeUnit::SECOND, + TimeUnit::NANO, + }) { auto ty = timestamp(unit); CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), ArrayFromJSON(ty, "[1, 2, 3, 4]"), diff --git a/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc b/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc index 3350fb805c47..a8c9765f1c82 100644 --- a/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc @@ -608,7 +608,8 @@ TEST_F(ScalarTemporalTest, TestTemporalComponentExtractionAllTemporalTypes) { } TEST_F(ScalarTemporalTest, TestTemporalComponentExtractionWithDifferentUnits) { - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u); CheckScalarUnary("year", unit, times_seconds_precision, int64(), year); CheckScalarUnary("is_leap_year", unit, times_seconds_precision, boolean(), @@ -814,7 +815,8 @@ TEST_F(ScalarTemporalTest, TestZoned1) { } TEST_F(ScalarTemporalTest, TestZoned2) { - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u, "Australia/Broken_Hill"); auto month = "[1, 3, 1, 5, 1, 12, 12, 12, 1, 1, 1, 1, 12, 12, 12, 1, null]"; auto day = "[1, 1, 1, 18, 1, 31, 30, 31, 1, 3, 4, 1, 31, 28, 29, 1, null]"; @@ -906,8 +908,9 @@ TEST_F(ScalarTemporalTest, TestNonexistentTimezone) { auto nonexistent_timezones = { "Mars/Mariner_Valley", "+25:00", "-25:00", "15:00", "5:00", "500", "+05:00:00", "+050000"}; + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; for (auto timezone : nonexistent_timezones) { - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto ts_type = timestamp(u, timezone); auto timestamp_array = std::make_shared>( ts_type, 2, data_buffer, null_buffer, 0); @@ -1026,7 +1029,8 @@ TEST_F(ScalarTemporalTest, DayOfWeek) { } TEST_F(ScalarTemporalTest, TestTemporalDifference) { - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u); auto arr1 = ArrayFromJSON(unit, times_seconds_precision); auto arr2 = ArrayFromJSON(unit, times_seconds_precision2); @@ -1879,8 +1883,9 @@ TEST_F(ScalarTemporalTest, TestLocalTimestamp) { "2009-12-30 18:50:20", "2009-12-31 19:55:25", "2010-01-02 21:00:30", "2010-01-03 22:05:35", "2005-12-31 23:10:40", "2005-12-31 00:15:45", "2008-12-27 14:30:00", "2008-12-28 14:30:00", "2011-12-31 15:32:03", null])"; + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; - for (auto u : TimeUnit::values()) { + for (auto u : units) { CheckScalarUnary("local_timestamp", timestamp(u), times_seconds_precision, timestamp(u), times_seconds_precision); CheckScalarUnary("local_timestamp", timestamp(u, "UTC"), times_seconds_precision, @@ -1912,8 +1917,9 @@ TEST_F(ScalarTemporalTest, TestAssumeTimezone) { auto options_tbilisi = AssumeTimezoneOptions(timezone_tbilisi); auto options_tbilisi_offset = AssumeTimezoneOptions(timezone_tbilisi_offset); auto options_invalid = AssumeTimezoneOptions("Europe/Brusselsss"); + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto unit = timestamp(u); auto unit_utc = timestamp(u, timezone_utc); auto unit_kolkata = timestamp(u, timezone_kolkata); @@ -1956,8 +1962,9 @@ TEST_F(ScalarTemporalTest, TestAssumeTimezoneAmbiguous) { AssumeTimezoneOptions(timezone, AssumeTimezoneOptions::AMBIGUOUS_LATEST); auto options_raise = AssumeTimezoneOptions(timezone, AssumeTimezoneOptions::AMBIGUOUS_RAISE); + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto unit = timestamp(u); auto unit_local = timestamp(u, timezone); ASSERT_RAISES(Invalid, AssumeTimezone(ArrayFromJSON(unit, times), options_raise)); @@ -1989,8 +1996,9 @@ TEST_F(ScalarTemporalTest, TestAssumeTimezoneNonexistent) { auto options_earliest = AssumeTimezoneOptions(timezone, AssumeTimezoneOptions::AMBIGUOUS_RAISE, AssumeTimezoneOptions::NONEXISTENT_EARLIEST); + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; - for (auto u : TimeUnit::values()) { + for (auto u : units) { auto unit = timestamp(u); auto unit_local = timestamp(u, timezone); ASSERT_RAISES(Invalid, AssumeTimezone(ArrayFromJSON(unit, times), options_raise)); @@ -2245,7 +2253,8 @@ TEST_F(ScalarTemporalTest, StrftimeInvalidLocale) { } TEST_F(ScalarTemporalTest, TestTemporalDifferenceZoned) { - for (auto u : TimeUnit::values()) { + auto units = {TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO}; + for (auto u : units) { auto unit = timestamp(u, "Pacific/Marquesas"); auto arr1 = ArrayFromJSON(unit, times_seconds_precision); auto arr2 = ArrayFromJSON(unit, times_seconds_precision2); diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index 2e9d860a8d6f..67277e83ff9c 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -249,6 +249,8 @@ std::string ToString(TimeUnit::type unit) { return "us"; case TimeUnit::NANO: return "ns"; + case TimeUnit::PICO: + return "ps"; default: DCHECK(false); return ""; @@ -1190,6 +1192,9 @@ std::ostream& operator<<(std::ostream& os, TimeUnit::type unit) { case TimeUnit::NANO: os << "ns"; break; + case TimeUnit::PICO: + os << "ps"; + break; } return os; } @@ -2767,6 +2772,8 @@ static char TimeUnitFingerprint(TimeUnit::type unit) { return 'u'; case TimeUnit::NANO: return 'n'; + case TimeUnit::PICO: + return 'p'; default: DCHECK(false) << "Unexpected TimeUnit"; return '\0'; @@ -3542,7 +3549,8 @@ const std::vector>& PrimitiveTypes() { const std::vector& TimeUnit::values() { static std::vector units = {TimeUnit::SECOND, TimeUnit::MILLI, - TimeUnit::MICRO, TimeUnit::NANO}; + TimeUnit::MICRO, TimeUnit::NANO, + TimeUnit::PICO}; return units; } diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h index be26c40dc1f4..8d2988e2e54b 100644 --- a/cpp/src/arrow/type_fwd.h +++ b/cpp/src/arrow/type_fwd.h @@ -269,7 +269,7 @@ struct Date64Scalar; struct ARROW_EXPORT TimeUnit { /// The unit for a time or timestamp DataType - enum type { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 }; + enum type { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3, PICO = 4 }; /// Iterate over all valid time units static const std::vector& values(); diff --git a/cpp/src/arrow/util/int128_internal.cc b/cpp/src/arrow/util/int128_internal.cc new file mode 100644 index 000000000000..d62ac7f4dde5 --- /dev/null +++ b/cpp/src/arrow/util/int128_internal.cc @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/util/int128_internal.h" + +// From StackOverlow user Mark Adler: https://stackoverflow.com/a/72651639/101923 +static void out128(std::ostream& out, arrow::internal::uint128_t val, int neg) { + // Note if the number is zero. (No hex or octal prefix in this case.) + auto zero = val == 0; + + // Note if upper-case letters requested. + auto state = out.flags(); + auto upper = (state & std::ios_base::uppercase) != 0; + + // Set base for digits. + unsigned base = state & std::ios_base::hex ? 16 : + state & std::ios_base::oct ? 8 : + 10; + + // Space for digits and prefix. Generate digits starting at the end of the + // string, going backwards. num will be the digit string. Terminate it. + char str[47]; + auto end = str + sizeof(str), num = end; + *--num = 0; + + // Compute and place digits in base base. + do { + char dig = val % base; + val /= base; + dig += dig < 10 ? '0' : (upper ? 'A' : 'a') - 10; + *--num = dig; + } while (val); + + // Prepend octal number with a zero if requested. + if (state & std::ios_base::showbase && base == 8 && !zero) + *--num = '0'; + + // pre will be the prefix string. Terminate it. + auto pre = num; + *--pre = 0; + + // Put a plus or minus sign in the prefix as appropriate. + if (base == 10) { + if (neg) + *--pre = '-'; + else if (state & std::ios_base::showpos) + *--pre = '+'; + } + + // Prefix a hexadecimal number if requested. + else if (state & std::ios_base::showbase && base == 16 && !zero) { + *--pre = upper ? 'X' : 'x'; + *--pre = '0'; + } + + // Compute the number of pad characters and get the fill character. + auto len = (num - pre) + (end - num) - 2; + auto pad = out.width(); + out.width(0); + pad = pad > len ? pad - len : 0; + char fill = out.fill(); + + // Put the padding before prefix if neither left nor internal requested. + if (!(state & (std::ios_base::internal | std::ios_base::left))) + while (pad) { + out << fill; + pad--; + } + + // Write prefix. + out << pre; + + // Put the padding between the prefix and the digits if requested. + if (state & std::ios_base::internal) + while (pad) { + out << fill; + pad--; + } + + // Write digits. + out << num; + + // Put number to the left of padding, if requested. + if (state & std::ios_base::left) + while (pad) { + out << fill; + pad--; + } +} + +// Overload << for an unsigned 128-bit integer. +std::ostream& operator<<(std::ostream& out, arrow::internal::uint128_t val) { + out128(out, val, 0); + return out; +} + +// Overload << for a signed 128-bit integer. Negation of the most negative +// signed value gives the correct unsigned absolute value. +std::ostream& operator<<(std::ostream& out, arrow::internal::int128_t val) { + auto state = out.flags(); + if (val < 0 && !(state & (std::ios_base::hex | std::ios_base::oct))) + out128(out, -(arrow::internal::uint128_t)val, 1); + else + out128(out, (arrow::internal::uint128_t)val, 0); + return out; +} diff --git a/cpp/src/arrow/util/int128_internal.h b/cpp/src/arrow/util/int128_internal.h index 201e4a134919..f1990f40bc64 100644 --- a/cpp/src/arrow/util/int128_internal.h +++ b/cpp/src/arrow/util/int128_internal.h @@ -16,6 +16,9 @@ // under the License. #pragma once +#include +#include + #include "arrow/util/config.h" #include "arrow/util/macros.h" @@ -43,3 +46,10 @@ using boost::multiprecision::uint128_t; } // namespace internal } // namespace arrow + +// Write the 128-bit integer val to out, with a minus sign if decimal and neg +// is true. Obey all of the ostream settings of out for integer display: octal +// or hexadecimal, upper case letters, plus sign, fill character and width, and +// fill placement. +std::ostream& operator<<(std::ostream& out, arrow::internal::uint128_t val); +std::ostream& operator<<(std::ostream& out, arrow::internal::int128_t val); diff --git a/cpp/src/arrow/vendored/datetime/date.h b/cpp/src/arrow/vendored/datetime/date.h index 1b06182a6daa..213925c1ab00 100644 --- a/cpp/src/arrow/vendored/datetime/date.h +++ b/cpp/src/arrow/vendored/datetime/date.h @@ -67,6 +67,8 @@ #include #include +#include "arrow/util/int128_internal.h" + #ifdef __GNUC__ # pragma GCC diagnostic push # if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 7) @@ -167,6 +169,8 @@ template using ratio_divide = decltype(std::ratio_divide{}); } // namespace detail +using ::operator<<; + //-----------+ // Interface | //-----------+ diff --git a/cpp/src/parquet/arrow/reader_internal.cc b/cpp/src/parquet/arrow/reader_internal.cc index 12f36fe39cf8..f28a78940f8f 100644 --- a/cpp/src/parquet/arrow/reader_internal.cc +++ b/cpp/src/parquet/arrow/reader_internal.cc @@ -516,6 +516,8 @@ Status TransferInt96(RecordReader* reader, MemoryPool* pool, *data_ptr++ = 0; } else { switch (int96_arrow_time_unit) { + case ::arrow::TimeUnit::PICO: + return Status::Invalid("Picoseconds not supported."); case ::arrow::TimeUnit::NANO: *data_ptr++ = Int96GetNanoSeconds(values[i]); break; diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc index 293ae94b94dd..58feacf34a5d 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc @@ -195,6 +195,7 @@ static std::shared_ptr TimestampLogicalTypeFromArrowTimestamp return LogicalType::Timestamp(utc, LogicalType::TimeUnit::NANOS, /*is_from_converted_type=*/false, /*force_set_converted_type=*/false); + case ::arrow::TimeUnit::PICO: case ::arrow::TimeUnit::SECOND: // No equivalent parquet logical type. break; @@ -231,6 +232,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type, case ::arrow::TimeUnit::MILLI: case ::arrow::TimeUnit::MICRO: break; + case ::arrow::TimeUnit::PICO: case ::arrow::TimeUnit::NANO: case ::arrow::TimeUnit::SECOND: return Status::NotImplemented("For Parquet version ", @@ -244,6 +246,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type, case ::arrow::TimeUnit::MICRO: case ::arrow::TimeUnit::NANO: break; + case ::arrow::TimeUnit::PICO: case ::arrow::TimeUnit::SECOND: return Status::NotImplemented("For Parquet version ", ::parquet::ParquetVersionToString(version), diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 94b67dfa807e..e98835d7461f 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -2239,6 +2239,8 @@ struct SerializeFunctor { const int64_t* input = array.raw_values(); const auto& type = static_cast(*array.type()); switch (type.unit()) { + case ::arrow::TimeUnit::PICO: + return Status::Invalid("Picoseconds not supported."); case ::arrow::TimeUnit::NANO: INT96_CONVERT_LOOP(internal::NanosecondsToImpalaTimestamp); break; From 02d01260c6f1db56ec82b7480ce098c144f9b5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a?= Date: Fri, 31 Oct 2025 21:33:54 +0000 Subject: [PATCH 3/3] format with pre-commit --- cpp/src/arrow/array/diff.cc | 2 +- .../compute/kernels/scalar_if_else_test.cc | 6 +- cpp/src/arrow/type.cc | 5 +- cpp/src/arrow/util/int128_internal.cc | 168 +++++++++--------- cpp/src/arrow/util/int128_internal.h | 2 +- testing | 2 +- 6 files changed, 91 insertions(+), 94 deletions(-) diff --git a/cpp/src/arrow/array/diff.cc b/cpp/src/arrow/array/diff.cc index 8b816d426e90..f43c967a11ec 100644 --- a/cpp/src/arrow/array/diff.cc +++ b/cpp/src/arrow/array/diff.cc @@ -57,8 +57,8 @@ namespace arrow { using internal::checked_cast; using internal::checked_pointer_cast; -using internal::MakeLazyRange; using internal::int128_t; +using internal::MakeLazyRange; namespace { diff --git a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc index 1f17318ca1e0..c592ef2ddc79 100644 --- a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc @@ -422,9 +422,9 @@ TEST_F(TestIfElseKernel, IfElseMultiType) { TEST_F(TestIfElseKernel, TimestampTypes) { for (const auto unit : { - TimeUnit::SECOND, - TimeUnit::NANO, - }) { + TimeUnit::SECOND, + TimeUnit::NANO, + }) { auto ty = timestamp(unit); CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), ArrayFromJSON(ty, "[1, 2, 3, 4]"), diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index 67277e83ff9c..8cdf245ac6c2 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -3548,9 +3548,8 @@ const std::vector>& PrimitiveTypes() { } const std::vector& TimeUnit::values() { - static std::vector units = {TimeUnit::SECOND, TimeUnit::MILLI, - TimeUnit::MICRO, TimeUnit::NANO, - TimeUnit::PICO}; + static std::vector units = { + TimeUnit::SECOND, TimeUnit::MILLI, TimeUnit::MICRO, TimeUnit::NANO, TimeUnit::PICO}; return units; } diff --git a/cpp/src/arrow/util/int128_internal.cc b/cpp/src/arrow/util/int128_internal.cc index d62ac7f4dde5..87265d0e24c6 100644 --- a/cpp/src/arrow/util/int128_internal.cc +++ b/cpp/src/arrow/util/int128_internal.cc @@ -19,102 +19,100 @@ // From StackOverlow user Mark Adler: https://stackoverflow.com/a/72651639/101923 static void out128(std::ostream& out, arrow::internal::uint128_t val, int neg) { - // Note if the number is zero. (No hex or octal prefix in this case.) - auto zero = val == 0; - - // Note if upper-case letters requested. - auto state = out.flags(); - auto upper = (state & std::ios_base::uppercase) != 0; - - // Set base for digits. - unsigned base = state & std::ios_base::hex ? 16 : - state & std::ios_base::oct ? 8 : - 10; - - // Space for digits and prefix. Generate digits starting at the end of the - // string, going backwards. num will be the digit string. Terminate it. - char str[47]; - auto end = str + sizeof(str), num = end; - *--num = 0; - - // Compute and place digits in base base. - do { - char dig = val % base; - val /= base; - dig += dig < 10 ? '0' : (upper ? 'A' : 'a') - 10; - *--num = dig; - } while (val); - - // Prepend octal number with a zero if requested. - if (state & std::ios_base::showbase && base == 8 && !zero) - *--num = '0'; - - // pre will be the prefix string. Terminate it. - auto pre = num; - *--pre = 0; - - // Put a plus or minus sign in the prefix as appropriate. - if (base == 10) { - if (neg) - *--pre = '-'; - else if (state & std::ios_base::showpos) - *--pre = '+'; + // Note if the number is zero. (No hex or octal prefix in this case.) + auto zero = val == 0; + + // Note if upper-case letters requested. + auto state = out.flags(); + auto upper = (state & std::ios_base::uppercase) != 0; + + // Set base for digits. + unsigned base = state & std::ios_base::hex ? 16 : state & std::ios_base::oct ? 8 : 10; + + // Space for digits and prefix. Generate digits starting at the end of the + // string, going backwards. num will be the digit string. Terminate it. + char str[47]; + auto end = str + sizeof(str), num = end; + *--num = 0; + + // Compute and place digits in base base. + do { + char dig = val % base; + val /= base; + dig += dig < 10 ? '0' : (upper ? 'A' : 'a') - 10; + *--num = dig; + } while (val); + + // Prepend octal number with a zero if requested. + if (state & std::ios_base::showbase && base == 8 && !zero) *--num = '0'; + + // pre will be the prefix string. Terminate it. + auto pre = num; + *--pre = 0; + + // Put a plus or minus sign in the prefix as appropriate. + if (base == 10) { + if (neg) + *--pre = '-'; + else if (state & std::ios_base::showpos) + *--pre = '+'; + } else if (state & std::ios_base::showbase && base == 16 && !zero) { + // Prefix a hexadecimal number if requested. + *--pre = upper ? 'X' : 'x'; + *--pre = '0'; + } + + // Compute the number of pad characters and get the fill character. + auto len = (num - pre) + (end - num) - 2; + auto pad = out.width(); + out.width(0); + pad = pad > len ? pad - len : 0; + char fill = out.fill(); + + // Put the padding before prefix if neither left nor internal requested. + if (!(state & (std::ios_base::internal | std::ios_base::left))) { + while (pad) { + out << fill; + pad--; } + } - // Prefix a hexadecimal number if requested. - else if (state & std::ios_base::showbase && base == 16 && !zero) { - *--pre = upper ? 'X' : 'x'; - *--pre = '0'; + // Write prefix. + out << pre; + + // Put the padding between the prefix and the digits if requested. + if (state & std::ios_base::internal) { + while (pad) { + out << fill; + pad--; } + } - // Compute the number of pad characters and get the fill character. - auto len = (num - pre) + (end - num) - 2; - auto pad = out.width(); - out.width(0); - pad = pad > len ? pad - len : 0; - char fill = out.fill(); - - // Put the padding before prefix if neither left nor internal requested. - if (!(state & (std::ios_base::internal | std::ios_base::left))) - while (pad) { - out << fill; - pad--; - } - - // Write prefix. - out << pre; - - // Put the padding between the prefix and the digits if requested. - if (state & std::ios_base::internal) - while (pad) { - out << fill; - pad--; - } - - // Write digits. - out << num; - - // Put number to the left of padding, if requested. - if (state & std::ios_base::left) - while (pad) { - out << fill; - pad--; - } + // Write digits. + out << num; + + // Put number to the left of padding, if requested. + if (state & std::ios_base::left) { + while (pad) { + out << fill; + pad--; + } + } } // Overload << for an unsigned 128-bit integer. std::ostream& operator<<(std::ostream& out, arrow::internal::uint128_t val) { - out128(out, val, 0); - return out; + out128(out, val, 0); + return out; } // Overload << for a signed 128-bit integer. Negation of the most negative // signed value gives the correct unsigned absolute value. std::ostream& operator<<(std::ostream& out, arrow::internal::int128_t val) { - auto state = out.flags(); - if (val < 0 && !(state & (std::ios_base::hex | std::ios_base::oct))) - out128(out, -(arrow::internal::uint128_t)val, 1); - else - out128(out, (arrow::internal::uint128_t)val, 0); - return out; + auto state = out.flags(); + if (val < 0 && !(state & (std::ios_base::hex | std::ios_base::oct))) + out128(out, -(arrow::internal::uint128_t)val, 1); + else + out128(out, (arrow::internal::uint128_t)val, 0); + return out; } diff --git a/cpp/src/arrow/util/int128_internal.h b/cpp/src/arrow/util/int128_internal.h index f1990f40bc64..f7c26c8c9fcc 100644 --- a/cpp/src/arrow/util/int128_internal.h +++ b/cpp/src/arrow/util/int128_internal.h @@ -16,8 +16,8 @@ // under the License. #pragma once -#include #include +#include #include "arrow/util/config.h" #include "arrow/util/macros.h" diff --git a/testing b/testing index 8da05fdd62a7..9a02925d1ba8 160000 --- a/testing +++ b/testing @@ -1 +1 @@ -Subproject commit 8da05fdd62a7243ef77aa9757acb62e0586a4d0c +Subproject commit 9a02925d1ba80bd493b6d4da6e8a777588d57ac4