From dbe4e9925abeb4bd96e42ba089c98b41dd628c57 Mon Sep 17 00:00:00 2001 From: Sayed Bilal Bari Date: Thu, 30 Jan 2025 14:20:38 -0800 Subject: [PATCH 1/4] Changed g5 -> g6 instance Signed-off-by: sbari --- .../com/nvidia/spark/rapids/tool/Platform.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala index c42cb67b2..4b650ebf7 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala @@ -77,12 +77,12 @@ object InstanceInfo { // format (numGpus, numCores) -> InstanceInfo about that CSP node instance type object PlatformInstanceTypes { - val AWS_BY_GPUS_CORES = Map((1, 4) -> InstanceInfo(4, 16 * 1024, "g5.xlarge", 1), - (1, 8) -> InstanceInfo(8, 32 * 1024, "g5.2xlarge", 1), - (1, 16) -> InstanceInfo(16, 64 * 1024, "g5.4xlarge", 1), - (1, 32) -> InstanceInfo(32, 128 * 1024, "g5.8xlarge", 1), - (4, 48) -> InstanceInfo(48, 192 * 1024, "g5.12xlarge", 1), - (1, 64) -> InstanceInfo(64, 256 * 1024, "g5.16xlarge", 1) + val AWS_BY_GPUS_CORES = Map((1, 4) -> InstanceInfo(4, 16 * 1024, "g6.xlarge", 1), + (1, 8) -> InstanceInfo(8, 32 * 1024, "g6.2xlarge", 1), + (1, 16) -> InstanceInfo(16, 64 * 1024, "g6.4xlarge", 1), + (1, 32) -> InstanceInfo(32, 128 * 1024, "g6.8xlarge", 1), + (4, 48) -> InstanceInfo(48, 192 * 1024, "g6.12xlarge", 1), + (1, 64) -> InstanceInfo(64, 256 * 1024, "g6.16xlarge", 1) ) // Standard_NC4as_T4_v3 - only recommending nodes with T4's for now, add more later From ce0d774b746f9f81db92ced7431d18bea940606a Mon Sep 17 00:00:00 2001 From: sbari Date: Mon, 3 Feb 2025 11:06:03 -0800 Subject: [PATCH 2/4] Updates instance types for EMR from g5 to g6 Signed-off-by: sbari --- .../com/nvidia/spark/rapids/tool/Platform.scala | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala index 4b650ebf7..111c83599 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala @@ -77,7 +77,8 @@ object InstanceInfo { // format (numGpus, numCores) -> InstanceInfo about that CSP node instance type object PlatformInstanceTypes { - val AWS_BY_GPUS_CORES = Map((1, 4) -> InstanceInfo(4, 16 * 1024, "g6.xlarge", 1), + // Using G6 instances for EMR and AWS + val EMR_BY_GPUS_CORES = Map((1, 4) -> InstanceInfo(4, 16 * 1024, "g6.xlarge", 1), (1, 8) -> InstanceInfo(8, 32 * 1024, "g6.2xlarge", 1), (1, 16) -> InstanceInfo(16, 64 * 1024, "g6.4xlarge", 1), (1, 32) -> InstanceInfo(32, 128 * 1024, "g6.8xlarge", 1), @@ -85,6 +86,16 @@ object PlatformInstanceTypes { (1, 64) -> InstanceInfo(64, 256 * 1024, "g6.16xlarge", 1) ) + // Using G5 instances. To be updated once G6 availability on Databricks + // is consistent + val DATABRICKS_AWS_BY_GPUS_CORES = Map((1, 4) -> InstanceInfo(4, 16 * 1024, "g5.xlarge", 1), + (1, 8) -> InstanceInfo(8, 32 * 1024, "g5.2xlarge", 1), + (1, 16) -> InstanceInfo(16, 64 * 1024, "g5.4xlarge", 1), + (1, 32) -> InstanceInfo(32, 128 * 1024, "g5.8xlarge", 1), + (4, 48) -> InstanceInfo(48, 192 * 1024, "g5.12xlarge", 1), + (1, 64) -> InstanceInfo(64, 256 * 1024, "g5.16xlarge", 1) + ) + // Standard_NC4as_T4_v3 - only recommending nodes with T4's for now, add more later val AZURE_NCAS_T4_V3_BY_GPUS_CORES = Map( (1, 4) -> InstanceInfo(4, 28 * 1024, "Standard_NC4as_T4_v3", 1), // 1 GPU @@ -559,7 +570,7 @@ class DatabricksAwsPlatform(gpuDevice: Option[GpuDevice], override val defaultGpuDevice: GpuDevice = A10GGpu override def getInstanceByResourcesMap: Map[(Int, Int), InstanceInfo] = { - PlatformInstanceTypes.AWS_BY_GPUS_CORES + PlatformInstanceTypes.DATABRICKS_AWS_BY_GPUS_CORES } } @@ -629,7 +640,7 @@ class EmrPlatform(gpuDevice: Option[GpuDevice], } override def getInstanceByResourcesMap: Map[(Int, Int), InstanceInfo] = { - PlatformInstanceTypes.AWS_BY_GPUS_CORES + PlatformInstanceTypes.EMR_BY_GPUS_CORES } } From de499a245f8da778a476a05059b48e5a4d35db37 Mon Sep 17 00:00:00 2001 From: Sayed Bilal Bari Date: Mon, 3 Feb 2025 12:10:34 -0800 Subject: [PATCH 3/4] Update user tool tests - g6 recommendation for EMR Signed-off-by: sbari --- .../spark_rapids_tools_e2e/features/installation_checks.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_tools/tests/spark_rapids_tools_e2e/features/installation_checks.feature b/user_tools/tests/spark_rapids_tools_e2e/features/installation_checks.feature index d4b0c4f63..719e5b7ae 100644 --- a/user_tools/tests/spark_rapids_tools_e2e/features/installation_checks.feature +++ b/user_tools/tests/spark_rapids_tools_e2e/features/installation_checks.feature @@ -29,7 +29,7 @@ Feature: Tool Installation Checks Examples: | platform | cli | expected_stdout | | dataproc | gcloud | 2 x n1-standard-16 (1 T4 each) | - | emr | aws | 2 x g5.4xlarge | + | emr | aws | 2 x g6.4xlarge | | databricks-aws | aws | 2 x g5.4xlarge | | databricks-azure | az | 2 x Standard_NC16as_T4_v3 | From e000950658b6903fbbc6bb1568121a62cf021516 Mon Sep 17 00:00:00 2001 From: sbari Date: Mon, 3 Feb 2025 13:45:15 -0800 Subject: [PATCH 4/4] Review change - updated change Signed-off-by: sbari --- core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala index 111c83599..9e285bf5a 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala @@ -77,7 +77,7 @@ object InstanceInfo { // format (numGpus, numCores) -> InstanceInfo about that CSP node instance type object PlatformInstanceTypes { - // Using G6 instances for EMR and AWS + // Using G6 instances for EMR val EMR_BY_GPUS_CORES = Map((1, 4) -> InstanceInfo(4, 16 * 1024, "g6.xlarge", 1), (1, 8) -> InstanceInfo(8, 32 * 1024, "g6.2xlarge", 1), (1, 16) -> InstanceInfo(16, 64 * 1024, "g6.4xlarge", 1),