From 9a0489ac3fa97d542257ef56eda0b861811287e0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 1 Aug 2025 02:48:41 +0000 Subject: [PATCH 1/6] Initial plan From 441223b0f32bf92a10cca420a48c99adc6fd44c8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 1 Aug 2025 02:56:43 +0000 Subject: [PATCH 2/6] Fix slice negative length validation to check nulls first Co-authored-by: yinqingh <22501635+yinqingh@users.noreply.github.com> --- .../spark/sql/rapids/collectionOperations.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 9253982f328..be80d2e396d 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -216,6 +216,11 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) } else { val list = listCol.getBase val start = startCol.getBase + withResource(GpuDivModLike.mergeNulls(list, start)) { mergedNulls => + if (mergedNulls.getRowCount == mergedNulls.getNullCount) { + return GpuColumnVector.columnVectorFromNull(listCol.getRowCount.toInt, dataType) + } + } val length = lengthS.getValue.asInstanceOf[Int] withResource(Scalar.fromInt(0)) { zero => if (start.contains(zero)) { @@ -237,6 +242,14 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) val list = listCol.getBase val start = startCol.getBase val length = lengthCol.getBase + val mergedNullsList = withResource(GpuDivModLike.mergeNulls(list, start)) { mergedNulls => + GpuDivModLike.mergeNulls(mergedNulls, length) + } + withResource(mergedNullsList) { _ => + if (mergedNullsList.getRowCount == mergedNullsList.getNullCount) { + return GpuColumnVector.columnVectorFromNull(listCol.getRowCount.toInt, dataType) + } + } withResource(Scalar.fromInt(0)) { zero => if (start.contains(zero)) { throw RapidsErrorUtils.unexpectedValueForStartInFunctionError(prettyName) From 45316190a5ed0bd94e24967c60ac9550879b4b5d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 1 Aug 2025 03:57:27 +0000 Subject: [PATCH 3/6] Fix GPU slice validation to match CPU behavior with null inputs Added null checks in GpuSlice implementation before performing validation: - In doColumnar(listCol, startCol, lengthS): Check if all rows are null in list or start columns - In doColumnar(listCol, startCol, lengthCol): Check if all rows are null in list, start, or length columns This ensures CPU and GPU have consistent behavior when handling null inputs with negative length parameters. Improved comments to clarify the null handling behavior. Fixes #13164 Co-authored-by: pxLi <8086184+pxLi@users.noreply.github.com> --- .../org/apache/spark/sql/rapids/collectionOperations.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index be80d2e396d..01a5f64498e 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -169,7 +169,7 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) override def doColumnar(listCol: GpuColumnVector, startS: GpuScalar, lengthS: GpuScalar): ColumnVector = { - // When either start or length is null, return all nulls like the CPU does. + // When input column is all nulls or either start or length is null, return all nulls like the CPU does. if (listCol.getRowCount == listCol.numNulls() || !startS.isValid || !lengthS.isValid) { GpuColumnVector.columnVectorFromNull(listCol.getRowCount.toInt, dataType) } else { From 71251d2435fad52642c9099ddf1ab9ce45696f58 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 1 Aug 2025 03:58:19 +0000 Subject: [PATCH 4/6] Fix GPU slice validation to match CPU behavior with null inputs Added null checks in GpuSlice implementation before performing validation: - In doColumnar(listCol, startCol, lengthS): Check if all rows are null in list or start columns - In doColumnar(listCol, startCol, lengthCol): Check if all rows are null in list, start, or length columns This ensures CPU and GPU have consistent behavior when handling null inputs with negative length parameters. Improved comments to clarify the null handling behavior. Fixes #13164 Signed-off-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: pxLi <8086184+pxLi@users.noreply.github.com> --- .../scala/org/apache/spark/sql/rapids/collectionOperations.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 01a5f64498e..1307ca40aac 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -170,6 +170,7 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) override def doColumnar(listCol: GpuColumnVector, startS: GpuScalar, lengthS: GpuScalar): ColumnVector = { // When input column is all nulls or either start or length is null, return all nulls like the CPU does. + // This matches CPU behavior for slice function with null inputs. if (listCol.getRowCount == listCol.numNulls() || !startS.isValid || !lengthS.isValid) { GpuColumnVector.columnVectorFromNull(listCol.getRowCount.toInt, dataType) } else { From 97e5b4845a6742f5407a1a9201ba065db8e51467 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 1 Aug 2025 04:19:17 +0000 Subject: [PATCH 5/6] Fix method call for checking null count in GpuSlice implementation Co-authored-by: pxLi <8086184+pxLi@users.noreply.github.com> --- .../spark/sql/rapids/collectionOperations.scala | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 1307ca40aac..a379a28dc5a 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -162,8 +162,13 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) override def doColumnar(listS: GpuScalar, startCol: GpuColumnVector, lengthCol: GpuColumnVector): ColumnVector = { val numRows = startCol.getRowCount.toInt - withResource(GpuColumnVector.from(listS, numRows, dataType)) { listCol => - doColumnar(listCol, startCol, lengthCol) + // When list is null, return all nulls like the CPU does. + if (!listS.isValid) { + GpuColumnVector.columnVectorFromNull(numRows, dataType) + } else { + withResource(GpuColumnVector.from(listS, numRows, dataType)) { listCol => + doColumnar(listCol, startCol, lengthCol) + } } } @@ -171,7 +176,7 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) lengthS: GpuScalar): ColumnVector = { // When input column is all nulls or either start or length is null, return all nulls like the CPU does. // This matches CPU behavior for slice function with null inputs. - if (listCol.getRowCount == listCol.numNulls() || !startS.isValid || !lengthS.isValid) { + if (listCol.getRowCount == listCol.getBase.getNullCount || !startS.isValid || !lengthS.isValid) { GpuColumnVector.columnVectorFromNull(listCol.getRowCount.toInt, dataType) } else { val list = listCol.getBase @@ -190,7 +195,7 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) override def doColumnar(listCol: GpuColumnVector, startS: GpuScalar, lengthCol: GpuColumnVector): ColumnVector = { // When start is null, return all nulls like the CPU does. - if (listCol.getRowCount == listCol.numNulls() || !startS.isValid) { + if (listCol.getRowCount == listCol.getBase.getNullCount || !startS.isValid) { GpuColumnVector.columnVectorFromNull(listCol.getRowCount.toInt, dataType) } else { val list = listCol.getBase @@ -212,7 +217,7 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) override def doColumnar(listCol: GpuColumnVector, startCol: GpuColumnVector, lengthS: GpuScalar): ColumnVector = { // When length is null, return all nulls like the CPU does. - if (listCol.getRowCount == listCol.numNulls() || !lengthS.isValid) { + if (listCol.getRowCount == listCol.getBase.getNullCount || !lengthS.isValid) { GpuColumnVector.columnVectorFromNull(listCol.getRowCount.toInt, dataType) } else { val list = listCol.getBase @@ -237,7 +242,7 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) override def doColumnar(listCol: GpuColumnVector, startCol: GpuColumnVector, lengthCol: GpuColumnVector): ColumnVector = { - if (listCol.getRowCount == listCol.numNulls()) { + if (listCol.getRowCount == listCol.getBase.getNullCount) { GpuColumnVector.columnVectorFromNull(listCol.getRowCount.toInt, dataType) } else { val list = listCol.getBase From 4e60e36fec3ea778511998f2d1b4d7ccfa6a5bb0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 1 Aug 2025 04:24:28 +0000 Subject: [PATCH 6/6] Fix line length issue in GpuSlice comment and condition Co-authored-by: pxLi <8086184+pxLi@users.noreply.github.com> --- .../apache/spark/sql/rapids/collectionOperations.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index a379a28dc5a..f378c1f2c14 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -174,9 +174,11 @@ case class GpuSlice(x: Expression, start: Expression, length: Expression) override def doColumnar(listCol: GpuColumnVector, startS: GpuScalar, lengthS: GpuScalar): ColumnVector = { - // When input column is all nulls or either start or length is null, return all nulls like the CPU does. - // This matches CPU behavior for slice function with null inputs. - if (listCol.getRowCount == listCol.getBase.getNullCount || !startS.isValid || !lengthS.isValid) { + // When input column is all nulls or either start or length is null, + // return all nulls like the CPU does. This matches CPU behavior for + // slice function with null inputs. + if (listCol.getRowCount == listCol.getBase.getNullCount || + !startS.isValid || !lengthS.isValid) { GpuColumnVector.columnVectorFromNull(listCol.getRowCount.toInt, dataType) } else { val list = listCol.getBase