#686 - Set spark.app.name to jobName for all jobs (#696)

#686 - Set spark.app.name to jobName for all jobs

Author: jozefbakus

src/main/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/spark/SparkEmrClusterServiceImpl.scala

@@ -138,7 +138,7 @@ class SparkEmrClusterServiceImpl @Inject() (
   private def getSparkArgs(id: String, jobName: String, jobParameters: SparkInstanceParameters) = {
     val config = sparkConfig.emr
     val sparkSubmitConfs = Map("--deploy-mode" -> "cluster")
-    val confs = Map("spark.yarn.tags" -> id) ++
+    val confs = Map("spark.app.name" -> jobName, "spark.yarn.tags" -> id) ++
       config.additionalConfs ++
       jobParameters.additionalSparkConfig ++
       mergeAdditionalSparkConfig(config.additionalConfs, jobParameters.additionalSparkConfig)

src/main/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/spark/SparkYarnClusterServiceImpl.scala

@@ -86,6 +86,7 @@ class SparkYarnClusterServiceImpl @Inject() (implicit
       .setMainClass(jobParameters.mainClass)
       .setAppResource(jobParameters.jobJar)
       .setAppName(jobName)
+      .setConf("spark.app.name", jobName)
       .setConf("spark.yarn.tags", id)
       .addAppArgs(jobParameters.appArguments.toSeq.map(fix_json_for_yarn): _*)
       .addSparkArg("--verbose")

src/test/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/spark/SparkEmrClusterServiceTest.scala

@@ -129,6 +129,8 @@ class SparkEmrClusterServiceTest
       "--conf",
       "spark.executor.extraJavaOptions=-DGlobalExecutorOpt -DLocalExecutorOpt",
       "--conf",
+      s"spark.app.name=${jobInstance.jobName}",
+      "--conf",
       "spark.driver.memory=2g",
       "--conf",
       "spark.driver.extraJavaOptions=-DGlobalDriverOpt -DLocalDriverOpt",