fix(arrow/array): honor field nullability in comparison and JSON marshaling

arrow/array.go

@@ -128,6 +128,21 @@ type Array interface {
 	Release()
 }
 
+// NullableMarshaler is an optional interface an Array may implement to control
+// whether a value is rendered as JSON null based on the field-local nullability
+// from the schema, rather than solely on the array's own validity bitmap.
+//
+// It exists so containers (struct/union/dictionary/run-end-encoded/extension)
+// and RecordToJSON can propagate each field's Nullable flag down to its values
+// without a breaking change to the Array interface. Arrays that do not implement
+// it fall back to GetOneForMarshal, preserving the previous validity-based behavior.
+type NullableMarshaler interface {
+	// GetOneForMarshalNullable returns the value at i for json.Marshal. When
+	// nullable is false, a value is returned even if the array's validity bitmap
+	// marks it null (a non-nullable field must not serialize as null).
+	GetOneForMarshalNullable(i int, nullable bool) interface{}
+}
+
 // ValueType is a generic constraint for valid Arrow primitive types
 type ValueType interface {
 	bool | FixedWidthType | string | []byte

arrow/array/binary.go

@@ -152,13 +152,17 @@ func (a *Binary) setData(data *Data) {
 	}
 }
 
-func (a *Binary) GetOneForMarshal(i int) interface{} {
-	if a.IsNull(i) {
+func (a *Binary) GetOneForMarshalNullable(i int, nullable bool) interface{} {
+	if nullable && a.IsNull(i) {
 		return nil
 	}
 	return a.Value(i)
 }
 
+func (a *Binary) GetOneForMarshal(i int) interface{} {
+	return a.GetOneForMarshalNullable(i, true)
+}
+
 func (a *Binary) MarshalJSON() ([]byte, error) {
 	vals := make([]interface{}, a.Len())
 	for i := 0; i < a.Len(); i++ {
@@ -223,9 +227,9 @@ func (a *Binary) ValidateFull() error {
 	return nil
 }
 
-func arrayEqualBinary(left, right *Binary) bool {
+func arrayEqualBinary(left, right *Binary, opt equalOption) bool {
 	for i := 0; i < left.Len(); i++ {
-		if left.IsNull(i) {
+		if opt.nullable && left.IsNull(i) {
 			continue
 		}
 		if !bytes.Equal(left.Value(i), right.Value(i)) {
@@ -346,13 +350,17 @@ func (a *LargeBinary) setData(data *Data) {
 	}
 }
 
-func (a *LargeBinary) GetOneForMarshal(i int) interface{} {
-	if a.IsNull(i) {
+func (a *LargeBinary) GetOneForMarshalNullable(i int, nullable bool) interface{} {
+	if nullable && a.IsNull(i) {
 		return nil
 	}
 	return a.Value(i)
 }
 
+func (a *LargeBinary) GetOneForMarshal(i int) interface{} {
+	return a.GetOneForMarshalNullable(i, true)
+}
+
 func (a *LargeBinary) MarshalJSON() ([]byte, error) {
 	vals := make([]interface{}, a.Len())
 	for i := 0; i < a.Len(); i++ {
@@ -417,9 +425,9 @@ func (a *LargeBinary) ValidateFull() error {
 	return nil
 }
 
-func arrayEqualLargeBinary(left, right *LargeBinary) bool {
+func arrayEqualLargeBinary(left, right *LargeBinary, opt equalOption) bool {
 	for i := 0; i < left.Len(); i++ {
-		if left.IsNull(i) {
+		if opt.nullable && left.IsNull(i) {
 			continue
 		}
 		if !bytes.Equal(left.Value(i), right.Value(i)) {
@@ -522,13 +530,17 @@ func (a *BinaryView) ValueStr(i int) string {
 	return base64.StdEncoding.EncodeToString(a.Value(i))
 }
 
-func (a *BinaryView) GetOneForMarshal(i int) interface{} {
-	if a.IsNull(i) {
+func (a *BinaryView) GetOneForMarshalNullable(i int, nullable bool) interface{} {
+	if nullable && a.IsNull(i) {
 		return nil
 	}
 	return a.Value(i)
 }
 
+func (a *BinaryView) GetOneForMarshal(i int) interface{} {
+	return a.GetOneForMarshalNullable(i, true)
+}
+
 func (a *BinaryView) MarshalJSON() ([]byte, error) {
 	vals := make([]interface{}, a.Len())
 	for i := 0; i < a.Len(); i++ {
@@ -539,10 +551,10 @@ func (a *BinaryView) MarshalJSON() ([]byte, error) {
 	return json.Marshal(vals)
 }
 
-func arrayEqualBinaryView(left, right *BinaryView) bool {
+func arrayEqualBinaryView(left, right *BinaryView, opt equalOption) bool {
 	leftBufs, rightBufs := left.dataBuffers, right.dataBuffers
 	for i := 0; i < left.Len(); i++ {
-		if left.IsNull(i) {
+		if opt.nullable && left.IsNull(i) {
 			continue
 		}
 		if !left.ValueHeader(i).Equals(leftBufs, right.ValueHeader(i), rightBufs) {

arrow/array/boolean.go

@@ -90,13 +90,17 @@ func (a *Boolean) setData(data *Data) {
 	}
 }
 
-func (a *Boolean) GetOneForMarshal(i int) interface{} {
-	if a.IsValid(i) {
+func (a *Boolean) GetOneForMarshalNullable(i int, nullable bool) interface{} {
+	if !nullable || a.IsValid(i) {
 		return a.Value(i)
 	}
 	return nil
 }
 
+func (a *Boolean) GetOneForMarshal(i int) interface{} {
+	return a.GetOneForMarshalNullable(i, true)
+}
+
 func (a *Boolean) MarshalJSON() ([]byte, error) {
 	vals := make([]interface{}, a.Len())
 	for i := 0; i < a.Len(); i++ {
@@ -109,9 +113,9 @@ func (a *Boolean) MarshalJSON() ([]byte, error) {
 	return json.Marshal(vals)
 }
 
-func arrayEqualBoolean(left, right *Boolean) bool {
+func arrayEqualBoolean(left, right *Boolean, opt equalOption) bool {
 	for i := 0; i < left.Len(); i++ {
-		if left.IsNull(i) {
+		if opt.nullable && left.IsNull(i) {
 			continue
 		}
 		if left.Value(i) != right.Value(i) {