GH-50339: [R] read_ipc_stream fails to unify nested uint64 fields inside a Struct array across record batches (#50374)

### Rationale for this change

Attempting downcasting on UINT64  Fields when converting them from Arrow to R works fine on individual arrays, but fails on list arrays, because we only use the individual row to work out whether to downcast.

I considered a few different solutions but I concluded that this one is the least complex while presenting a good solution -  having UINT64  always be converted to doubles and never attempt downcasting.  

The impact here is that people who are expecting integers via downcasting will see a change in behaviour, but I think there's a strong argument here that if you are using UINT64  to store values that can be downcast, you probably need to be using UINT32 anyway.

Other options considered:   fix this in `Converter_List`  and check values to see if they can be downcast:  significantly more complex and only fixes one level of nesting.

I checked duckdb and they always convert UINT64 to doubles.

### What changes are included in this PR?

Never try to downcast UNIT64 values when converting to R

### Are these changes tested?

Yes

### Are there any user-facing changes?

Breaking change as mentioned in rationale above.

* GitHub Issue: #50339

Lead-authored-by: Nic Crane <thisisnic@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Nic Crane <thisisnic@gmail.com>

Author: thisisnic

r/NEWS.md

@@ -19,7 +19,10 @@
 
 # arrow 24.0.0.9000
 
-# arrow 24.0.0
+- Arrow `uint64` types are now always converted to R `double` (numeric) vectors,
+  regardless of the values. Previously, small `uint64` values were converted to
+  R `integer`, which could cause inconsistent types within list columns when
+  different list elements had different value ranges (#50339).
 
 # arrow 24.0.0
 

r/R/type.R

@@ -356,13 +356,14 @@ NestedType <- R6Class("NestedType", inherit = DataType)
 #' `date32()` creates a datetime type with a "day" unit, like the R `Date`
 #' class. `date64()` has a "ms" unit.
 #'
-#' `uint32` (32 bit unsigned integer), `uint64` (64 bit unsigned integer), and
-#' `int64` (64-bit signed integer) types may contain values that exceed the
-#' range of R's `integer` type (32-bit signed integer). When these arrow objects
-#' are translated to R objects, `uint32` and `uint64` are converted to `double`
-#' ("numeric") and `int64` is converted to `bit64::integer64`. For `int64`
-#' types, this conversion can be disabled (so that `int64` always yields a
-#' `bit64::integer64` object) by setting `options(arrow.int64_downcast =
+#' `uint64` (64 bit unsigned integer) is always converted to `double`
+#' ("numeric") in R. Note that doubles cannot exactly represent all uint64
+#' values; precision may be lost for values above 2^53. `uint32` (32 bit unsigned integer) and `int64` (64-bit
+#' signed integer) types may contain values that exceed the range of R's
+#' `integer` type (32-bit signed integer). When they do, `uint32` is converted
+#' to `double` ("numeric") and `int64` is converted to `bit64::integer64`. For
+#' `int64` types, this conversion can be disabled (so that `int64` always yields
+#' a `bit64::integer64` object) by setting `options(arrow.int64_downcast =
 #' FALSE)`.
 #'
 #' `decimal128()` creates a `Decimal128Type`. Arrow decimals are fixed-point

r/man/data-type.Rd

@@ -1296,13 +1296,8 @@ std::shared_ptr<Converter> Converter::Make(
       }
 
     case Type::UINT64:
-      if (ArraysCanFitInteger(chunked_array->chunks())) {
-        return std::make_shared<arrow::r::Converter_Int<arrow::UInt64Type>>(
-            chunked_array);
-      } else {
-        return std::make_shared<arrow::r::Converter_Double<arrow::UInt64Type>>(
-            chunked_array);
-      }
+      return std::make_shared<arrow::r::Converter_Double<arrow::UInt64Type>>(
+          chunked_array);
 
     case Type::HALF_FLOAT:
       return std::make_shared<arrow::r::Converter_Double<arrow::HalfFloatType>>(

r/src/array_to_vector.cpp

@@ -521,23 +521,25 @@ test_that("Array<int8>$as_vector() converts to integer (ARROW-3794)", {
   expect_as_vector(a, u8)
 })
 
-test_that("Arrays of {,u}int{32,64} convert to integer if they can fit", {
+test_that("Arrays of uint32 and int64 convert to integer if they can fit", {
   u32 <- arrow_array(1L)$cast(uint32())
   expect_identical(as.vector(u32), 1L)
 
-  u64 <- arrow_array(1L)$cast(uint64())
-  expect_identical(as.vector(u64), 1L)
-
   i64 <- arrow_array(bit64::as.integer64(1:10))
   expect_identical(as.vector(i64), 1:10)
 })
 
-test_that("Arrays of uint{32,64} convert to numeric if they can't fit integer", {
+test_that("Arrays of uint32 convert to numeric if they can't fit integer", {
   u32 <- arrow_array(bit64::as.integer64(1) + MAX_INT)$cast(uint32())
   expect_identical(as.vector(u32), 1 + MAX_INT)
+})
 
-  u64 <- arrow_array(bit64::as.integer64(1) + MAX_INT)$cast(uint64())
-  expect_identical(as.vector(u64), 1 + MAX_INT)
+test_that("Arrays of uint64 always convert to numeric (double)", {
+  u64_small <- arrow_array(1L)$cast(uint64())
+  expect_identical(as.vector(u64_small), 1)
+
+  u64_large <- arrow_array(bit64::as.integer64(1) + MAX_INT)$cast(uint64())
+  expect_identical(as.vector(u64_large), 1 + MAX_INT)
 })
 
 test_that("arrow_array() recognise arrow::Array (ARROW-3815)", {
@@ -1453,3 +1455,14 @@ test_that("Array handles negative fractional dates correctly (GH-46873)", {
   arr <- arrow_array(d)
   expect_equal(as.vector(arr), as.Date("1969-12-31", origin = "1970-01-01"))
 })
+
+test_that("uint64 inside list columns always converts to double (GH-50339)", {
+  list_arr <- arrow_array(
+    list(1, 9999999999),
+    type = list_of(uint64())
+  )
+
+  result <- as.vector(list_arr)
+  expect_type(result[[1]], "double")
+  expect_type(result[[2]], "double")
+})

r/src/arrowExports.cpp

@@ -97,12 +97,15 @@ If the value in R does not fall within the permissible range for the correspondi
 chunked_array(c(10L, 3L, 200L), type = int8())
 ```
 
-When translating from Arrow to R, integer types alway translate to R integers unless one of the following exceptions applies:
+When translating from Arrow to R, integer types always translate to R integers unless one of the following exceptions applies:
 
-- If the value of an Arrow uint32 or uint64 falls outside the range allowed for R integers, the result will be a numeric vector in R 
+- If the value of an Arrow uint32 falls outside the range allowed for R integers, the result will be a numeric vector in R
 - If the value of an Arrow int64 variable falls outside the range allowed for R integers, the result will be a `bit64::integer64` vector in R
 - If the user sets `options(arrow.int64_downcast = FALSE)`, the Arrow int64 type always yields a `bit64::integer64` vector in R
  regardless of the value
+ - Arrow uint64 types are always converted to numeric (double) vectors in R.
+  Note that doubles cannot exactly represent all uint64 values; precision may
+   be lost for values above 2^53.
 
 ## Floating point numeric types
 
@@ -351,7 +354,7 @@ to Arrow list type (which is a "list of" some type).
 | uint8               | integer                      |
 | uint16              | integer                      |
 | uint32              | integer ^1^                  |
-| uint64              | integer ^1^                  |
+| uint64              | double                       |
 | float16             | - ^2^                        |
 | float32             | double                       |
 | float64             | double                       |
@@ -376,9 +379,9 @@ to Arrow list type (which is a "list of" some type).
 | map                 | arrow_list ^5^               |
 | union               | - ^2^                       |
 
-^1^: These integer types may contain values that exceed the range of R's 
-`integer` type (32 bit signed integer). When they do, `uint32` and `uint64` are 
-converted to `double` ("numeric") and `int64` is converted to 
+^1^: These integer types may contain values that exceed the range of R's
+`integer` type (32 bit signed integer). When they do, `uint32` is
+converted to `double` ("numeric") and `int64` is converted to
 `bit64::integer64`. This conversion can be disabled (so that `int64` always
 yields a `bit64::integer64` vector) by setting `options(arrow.int64_downcast = FALSE)`.