diff --git a/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileEndMsg.java b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileEndMsg.java new file mode 100644 index 00000000000..c19651fa101 --- /dev/null +++ b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileEndMsg.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids; + +import java.util.Objects; + +public class ProfileEndMsg implements ProfileMsg { + private static final long serialVersionUID = 1L; + + private final String executorId; + private final String path; + + public ProfileEndMsg(String executorId, String path) { + this.executorId = executorId; + this.path = path; + } + + public String executorId() { + return executorId; + } + + public String path() { + return path; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ProfileEndMsg)) { + return false; + } + ProfileEndMsg that = (ProfileEndMsg) other; + return Objects.equals(executorId, that.executorId) && + Objects.equals(path, that.path); + } + + @Override + public int hashCode() { + return Objects.hash(executorId, path); + } + + @Override + public String toString() { + return "ProfileEndMsg(" + executorId + "," + path + ")"; + } +} diff --git a/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileErrorMsg.java b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileErrorMsg.java new file mode 100644 index 00000000000..7aeeffc5c9d --- /dev/null +++ b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileErrorMsg.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids; + +import java.util.Objects; + +public class ProfileErrorMsg implements ProfileMsg { + private static final long serialVersionUID = 1L; + + private final String executorId; + private final String msg; + + public ProfileErrorMsg(String executorId, String msg) { + this.executorId = executorId; + this.msg = msg; + } + + public String executorId() { + return executorId; + } + + public String msg() { + return msg; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ProfileErrorMsg)) { + return false; + } + ProfileErrorMsg that = (ProfileErrorMsg) other; + return Objects.equals(executorId, that.executorId) && + Objects.equals(msg, that.msg); + } + + @Override + public int hashCode() { + return Objects.hash(executorId, msg); + } + + @Override + public String toString() { + return "ProfileErrorMsg(" + executorId + "," + msg + ")"; + } +} diff --git a/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileInitMsg.java b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileInitMsg.java new file mode 100644 index 00000000000..55a3b7627a7 --- /dev/null +++ b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileInitMsg.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids; + +import java.util.Objects; + +public class ProfileInitMsg implements ProfileMsg { + private static final long serialVersionUID = 1L; + + private final String executorId; + private final String path; + + public ProfileInitMsg(String executorId, String path) { + this.executorId = executorId; + this.path = path; + } + + public String executorId() { + return executorId; + } + + public String path() { + return path; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ProfileInitMsg)) { + return false; + } + ProfileInitMsg that = (ProfileInitMsg) other; + return Objects.equals(executorId, that.executorId) && + Objects.equals(path, that.path); + } + + @Override + public int hashCode() { + return Objects.hash(executorId, path); + } + + @Override + public String toString() { + return "ProfileInitMsg(" + executorId + "," + path + ")"; + } +} diff --git a/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileJobStageQueryMsg.java b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileJobStageQueryMsg.java new file mode 100644 index 00000000000..dc2a8666a31 --- /dev/null +++ b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileJobStageQueryMsg.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids; + +import java.util.Objects; + +public class ProfileJobStageQueryMsg implements ProfileMsg { + private static final long serialVersionUID = 1L; + + private final int[] activeJobs; + private final int[] activeStages; + + public ProfileJobStageQueryMsg(int[] activeJobs, int[] activeStages) { + this.activeJobs = activeJobs; + this.activeStages = activeStages; + } + + public int[] activeJobs() { + return activeJobs; + } + + public int[] activeStages() { + return activeStages; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ProfileJobStageQueryMsg)) { + return false; + } + ProfileJobStageQueryMsg that = (ProfileJobStageQueryMsg) other; + return Objects.equals(activeJobs, that.activeJobs) && + Objects.equals(activeStages, that.activeStages); + } + + @Override + public int hashCode() { + return Objects.hash(activeJobs, activeStages); + } + + @Override + public String toString() { + return "ProfileJobStageQueryMsg(" + activeJobs + "," + activeStages + ")"; + } +} diff --git a/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileMsg.java b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileMsg.java new file mode 100644 index 00000000000..1cbeababfbb --- /dev/null +++ b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileMsg.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids; + +import java.io.Serializable; + +public interface ProfileMsg extends Serializable { +} diff --git a/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileStatusMsg.java b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileStatusMsg.java new file mode 100644 index 00000000000..b5db1431c36 --- /dev/null +++ b/sql-plugin-columnar/src/main/java/com/nvidia/spark/rapids/ProfileStatusMsg.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids; + +import java.util.Objects; + +public class ProfileStatusMsg implements ProfileMsg { + private static final long serialVersionUID = 1L; + + private final String executorId; + private final String msg; + + public ProfileStatusMsg(String executorId, String msg) { + this.executorId = executorId; + this.msg = msg; + } + + public String executorId() { + return executorId; + } + + public String msg() { + return msg; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ProfileStatusMsg)) { + return false; + } + ProfileStatusMsg that = (ProfileStatusMsg) other; + return Objects.equals(executorId, that.executorId) && + Objects.equals(msg, that.msg); + } + + @Override + public int hashCode() { + return Objects.hash(executorId, msg); + } + + @Override + public String toString() { + return "ProfileStatusMsg(" + executorId + "," + msg + ")"; + } +} diff --git a/sql-plugin-columnar/src/main/java/org/apache/spark/sql/rapids/BasicColumnarWriteTaskStats.java b/sql-plugin-columnar/src/main/java/org/apache/spark/sql/rapids/BasicColumnarWriteTaskStats.java new file mode 100644 index 00000000000..7579ea33958 --- /dev/null +++ b/sql-plugin-columnar/src/main/java/org/apache/spark/sql/rapids/BasicColumnarWriteTaskStats.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.rapids; + +import java.util.Objects; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.execution.datasources.WriteTaskStats; + +import scala.collection.Seq; + +/** + * Simple metrics collected during an instance of GpuFileFormatDataWriter. + * These were first introduced in https://github.com/apache/spark/pull/18159 (SPARK-20703). + */ +public final class BasicColumnarWriteTaskStats implements WriteTaskStats { + private static final long serialVersionUID = 0L; + + private final Seq partitions; + private final int numFiles; + private final int numWriters; + private final long numBytes; + private final long numRows; + + public BasicColumnarWriteTaskStats( + Seq partitions, + int numFiles, + int numWriters, + long numBytes, + long numRows) { + this.partitions = partitions; + this.numFiles = numFiles; + this.numWriters = numWriters; + this.numBytes = numBytes; + this.numRows = numRows; + } + + public Seq partitions() { + return partitions; + } + + public int numFiles() { + return numFiles; + } + + public int numWriters() { + return numWriters; + } + + public long numBytes() { + return numBytes; + } + + public long numRows() { + return numRows; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof BasicColumnarWriteTaskStats)) { + return false; + } + BasicColumnarWriteTaskStats that = (BasicColumnarWriteTaskStats) other; + return numFiles == that.numFiles + && numWriters == that.numWriters + && numBytes == that.numBytes + && numRows == that.numRows + && Objects.equals(partitions, that.partitions); + } + + @Override + public int hashCode() { + return Objects.hash(partitions, numFiles, numWriters, numBytes, numRows); + } + + @Override + public String toString() { + return "BasicColumnarWriteTaskStats(" + partitions + "," + numFiles + "," + + numWriters + "," + numBytes + "," + numRows + ")"; + } +} diff --git a/sql-plugin-columnar/src/main/java/org/apache/spark/sql/rapids/NanoTime.java b/sql-plugin-columnar/src/main/java/org/apache/spark/sql/rapids/NanoTime.java new file mode 100644 index 00000000000..d862ae90372 --- /dev/null +++ b/sql-plugin-columnar/src/main/java/org/apache/spark/sql/rapids/NanoTime.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.rapids; + +import java.io.Serializable; +import java.util.Locale; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +public final class NanoTime implements Serializable { + private static final long serialVersionUID = 1L; + + private final Long value; + + public NanoTime(Long value) { + this.value = value; + } + + public Long value() { + return value; + } + + @Override + public String toString() { + long hours = TimeUnit.NANOSECONDS.toHours(value); + long remaining = value - TimeUnit.HOURS.toNanos(hours); + long minutes = TimeUnit.NANOSECONDS.toMinutes(remaining); + remaining -= TimeUnit.MINUTES.toNanos(minutes); + double seconds = ((double) remaining) / TimeUnit.SECONDS.toNanos(1); + return String.format(Locale.US, "%02d:%02d:%06.3f", hours, minutes, seconds); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof NanoTime)) { + return false; + } + NanoTime other = (NanoTime) obj; + return Objects.equals(value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(value); + } +} diff --git a/sql-plugin-columnar/src/main/java/org/apache/spark/sql/rapids/SizeInBytes.java b/sql-plugin-columnar/src/main/java/org/apache/spark/sql/rapids/SizeInBytes.java new file mode 100644 index 00000000000..7fb6c83f5e8 --- /dev/null +++ b/sql-plugin-columnar/src/main/java/org/apache/spark/sql/rapids/SizeInBytes.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.rapids; + +import java.io.Serializable; +import java.util.Objects; + +public final class SizeInBytes implements Serializable { + private static final long serialVersionUID = 1L; + + private static final String[] SIZE_UNIT_NAMES = {"B", "KB", "MB", "GB", "TB", "PB", "EB"}; + + private final Long value; + + public SizeInBytes(Long value) { + this.value = value; + } + + public Long value() { + return value; + } + + @Override + public String toString() { + long unitVal = value; + long remainVal = 0; + int unitIndex = 0; + while (unitIndex < SIZE_UNIT_NAMES.length && unitVal >= 1024) { + long nextUnitVal = unitVal >> 10; + remainVal = unitVal - (nextUnitVal << 10); + unitVal = nextUnitVal; + unitIndex += 1; + } + String finalVal = String.format("%.2f", unitVal + (remainVal / 1024.0)); + return finalVal + SIZE_UNIT_NAMES[unitIndex] + " (" + value + " bytes)"; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof SizeInBytes)) { + return false; + } + SizeInBytes other = (SizeInBytes) obj; + return Objects.equals(value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(value); + } +}