Feature/405 persist yarn (#416)

kevinwallimann · web-flow · commit d1e02d7cd41f · 2021-04-06T13:52:32.000+02:00
* Persist application id to JobInstance

* Frontend impl

* Display N/A if there are no logs

* Satisfy sonarcloud

* Update db_latest sql
diff --git a/src/main/resources/db_scripts/db_script_latest.sql b/src/main/resources/db_scripts/db_script_latest.sql
@@ -31,6 +31,7 @@ create table "job_instance" (
   "key_value_pairs" VARCHAR NOT NULL,
   "job_status" VARCHAR NOT NULL,
   "executor_job_id" VARCHAR,
+  "application_id" VARCHAR,
   "created" TIMESTAMP NOT NULL,
   "updated" TIMESTAMP,
   "order" INTEGER NOT NULL,
diff --git a/src/main/resources/db_scripts/liquibase/db.changelog.yml b/src/main/resources/db_scripts/liquibase/db.changelog.yml
@@ -47,4 +47,8 @@ databaseChangeLog:
   - include:
       relativeToChangelogFile: true
       file: v0.3.4.add-scheduler-instances.yml
+  - include:
+      relativeToChangelogFile: true
+      file: v0.4.0.add-application-id.yml
+
 
diff --git a/src/main/resources/db_scripts/liquibase/v0.4.0.add-application-id.sql b/src/main/resources/db_scripts/liquibase/v0.4.0.add-application-id.sql
@@ -0,0 +1,16 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+alter table "job_instance" add column "application_id" VARCHAR;
diff --git a/src/main/resources/db_scripts/liquibase/v0.4.0.add-application-id.yml b/src/main/resources/db_scripts/liquibase/v0.4.0.add-application-id.yml
@@ -0,0 +1,25 @@
+#
+# Copyright 2018 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+databaseChangeLog:
+  - changeSet:
+      id: v0.4.0.add-application-id
+      logicalFilePath: v0.4.0.add-application-id
+      author: HyperdriveDevTeam@absa.africa
+      context: default
+      changes:
+        - sqlFile:
+            relativeToChangelogFile: true
+            path: v0.4.0.add-application-id.sql
diff --git a/src/main/scala/za/co/absa/hyperdrive/trigger/api/rest/controllers/AppInfoController.scala b/src/main/scala/za/co/absa/hyperdrive/trigger/api/rest/controllers/AppInfoController.scala
@@ -25,12 +25,15 @@ class AppInfoController {
   val environment: String = ""
   @Value("${version:Unknown}")
   val version: String = ""
+  @Value("${sparkYarnSink.hadoopResourceManagerUrlBase:Unknown}")
+  val resourceManagerUrl: String = ""
 
   @GetMapping(path = Array("/app/info"))
   def appInfo(): AppInfo = {
     AppInfo(
       environment = environment,
-      version = version
+      version = version,
+      resourceManagerUrl = resourceManagerUrl
     )
   }
 
diff --git a/src/main/scala/za/co/absa/hyperdrive/trigger/api/rest/services/DagInstanceService.scala b/src/main/scala/za/co/absa/hyperdrive/trigger/api/rest/services/DagInstanceService.scala
@@ -60,6 +60,7 @@ class DagInstanceServiceImpl(override val jobTemplateService: JobTemplateService
         jobParameters = resolvedJobDefinition.jobParameters,
         jobStatus = initialJobStatus,
         executorJobId = None,
+        applicationId = None,
         created = now,
         updated = finished,
         order = resolvedJobDefinition.order,
diff --git a/src/main/scala/za/co/absa/hyperdrive/trigger/models/AppInfo.scala b/src/main/scala/za/co/absa/hyperdrive/trigger/models/AppInfo.scala
@@ -15,4 +15,8 @@
 
 package za.co.absa.hyperdrive.trigger.models
 
-case class AppInfo(environment: String, version: String)
+case class AppInfo(
+  environment: String,
+  version: String,
+  resourceManagerUrl: String
+)
diff --git a/src/main/scala/za/co/absa/hyperdrive/trigger/models/JobInstance.scala b/src/main/scala/za/co/absa/hyperdrive/trigger/models/JobInstance.scala
@@ -26,6 +26,7 @@ case class JobInstance(
   jobParameters: JobParameters,
   jobStatus: JobStatus,
   executorJobId: Option[String],
+  applicationId: Option[String],
   created: LocalDateTime,
   updated: Option[LocalDateTime],
   order: Int,
@@ -41,6 +42,7 @@ case class JobInstanceJoined(
   jobParameters: JobParameters,
   jobStatus: JobStatus,
   executorJobId: Option[String],
+  applicationId: Option[String],
   created: LocalDateTime,
   updated: Option[LocalDateTime],
   order: Int,
diff --git a/src/main/scala/za/co/absa/hyperdrive/trigger/models/tables/JobInstanceTable.scala b/src/main/scala/za/co/absa/hyperdrive/trigger/models/tables/JobInstanceTable.scala
@@ -37,6 +37,7 @@ trait JobInstanceTable {
     def keyValuePairs: Rep[Map[String, SortedMap[String, String]]] = column[Map[String, SortedMap[String, String]]]("key_value_pairs")
     def jobStatus: Rep[JobStatus] = column[JobStatus]("job_status")
     def executorJobId: Rep[Option[String]] = column[Option[String]]("executor_job_id")
+    def applicationId: Rep[Option[String]] = column[Option[String]]("application_id")
     def created: Rep[LocalDateTime] = column[LocalDateTime]("created")
     def updated: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("updated")
     def order: Rep[Int] = column[Int]("order")
@@ -54,6 +55,7 @@ trait JobInstanceTable {
       keyValuePairs,
       jobStatus,
       executorJobId,
+      applicationId,
       created,
       updated,
       order,
@@ -71,11 +73,12 @@ trait JobInstanceTable {
           ),
           jobStatus = jobInstanceTuple._6,
           executorJobId = jobInstanceTuple._7,
-          created = jobInstanceTuple._8,
-          updated = jobInstanceTuple._9,
-          order = jobInstanceTuple._10,
-          dagInstanceId = jobInstanceTuple._11,
-          id = jobInstanceTuple._12
+          applicationId = jobInstanceTuple._8,
+          created = jobInstanceTuple._9,
+          updated = jobInstanceTuple._10,
+          order = jobInstanceTuple._11,
+          dagInstanceId = jobInstanceTuple._12,
+          id = jobInstanceTuple._13
         ),
       (jobInstance: JobInstance) =>
         Option(
@@ -86,6 +89,7 @@ trait JobInstanceTable {
           jobInstance.jobParameters.keyValuePairs,
           jobInstance.jobStatus,
           jobInstance.executorJobId,
+          jobInstance.applicationId,
           jobInstance.created,
           jobInstance.updated,
           jobInstance.order,
diff --git a/src/main/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/spark/SparkExecutor.scala b/src/main/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/spark/SparkExecutor.scala
@@ -64,7 +64,9 @@ object SparkExecutor extends Executor {
         case Some(asd) => asd.apps.app
         case None => Seq.empty
       }) match {
-        case Seq(first) => updateJob(jobInstance.copy(jobStatus = getStatus(first.finalStatus)))
+        case Seq(first) => updateJob(jobInstance.copy(
+          applicationId = Some(first.id),
+          jobStatus = getStatus(first.finalStatus)))
         case _ if jobInstance.jobStatus == Submitting => updateJob(jobInstance.copy(jobStatus = SubmissionTimeout))
         case _ => updateJob(jobInstance.copy(jobStatus = Lost))
       }
diff --git a/src/test/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/shell/ShellExecutorTest.scala b/src/test/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/shell/ShellExecutorTest.scala
@@ -42,6 +42,7 @@ class ShellExecutorTest extends FlatSpec with Matchers with BeforeAndAfterAll wi
     jobParameters = JobParameters(variables = Map.empty[String, String], maps = Map.empty[String, List[String]]),
     jobStatus = InQueue,
     executorJobId = None,
+    applicationId = None,
     created = LocalDateTime.now(),
     updated = None,
     order = 0,
diff --git a/ui/src/app/app.component.spec.ts b/ui/src/app/app.component.spec.ts
@@ -21,6 +21,7 @@ import { AppComponent } from './app.component';
 import * as fromApp from './stores/app.reducers';
 import { selectAuthState } from './stores/app.reducers';
 import { Router } from '@angular/router';
+import { AppInfoModelFactory } from './models/appInfo.model';
 
 describe('AppComponent', () => {
   let underTest: AppComponent;
@@ -37,10 +38,7 @@ describe('AppComponent', () => {
 
   const initialApplicationState = {
     loading: false,
-    appInfo: {
-      environment: 'Undefined',
-      version: 'Undefined',
-    },
+    appInfo: AppInfoModelFactory.create('Undefined', 'Undefined', 'Undefined'),
   };
 
   const initialAppState = {
diff --git a/ui/src/app/components/auth/login/login.component.spec.ts b/ui/src/app/components/auth/login/login.component.spec.ts
@@ -19,6 +19,7 @@ import { ClarityModule } from '@clr/angular';
 import { FormsModule } from '@angular/forms';
 import { LoginComponent } from './login.component';
 import * as fromApp from '../../../stores/app.reducers';
+import { AppInfoModelFactory } from '../../../models/appInfo.model';
 
 describe('LoginComponent', () => {
   let component: LoginComponent;
@@ -33,10 +34,7 @@ describe('LoginComponent', () => {
 
   const initialApplicationState = {
     loading: true,
-    appInfo: {
-      environment: 'Undefined',
-      version: 'Undefined',
-    },
+    appInfo: AppInfoModelFactory.create('Undefined', 'Undefined', 'Undefined'),
   };
 
   const initialAppState = {
diff --git a/ui/src/app/components/runs/run-detail/run-detail.component.html b/ui/src/app/components/runs/run-detail/run-detail.component.html
@@ -19,6 +19,7 @@
     <clr-dg-column>Created</clr-dg-column>
     <clr-dg-column>Updated</clr-dg-column>
     <clr-dg-column>Status</clr-dg-column>
+    <clr-dg-column>Logs</clr-dg-column>
 
     <clr-dg-placeholder>We couldn't find any jobs run!</clr-dg-placeholder>
 
@@ -38,5 +39,15 @@
         <clr-icon *ngSwitchDefault shape="help" class="is-solid" style="color: darkgoldenrod"></clr-icon>
         {{jobInstance.jobStatus.name}}
       </clr-dg-cell>
+      <clr-dg-cell>
+        <ng-template [ngIf]="jobInstance.applicationId" [ngIfElse]="noApplicationId">
+          <a href="{{ getApplicationIdUrl(appInfo.resourceManagerUrl, jobInstance.applicationId) }}" target="_blank"
+             style="margin-right: 2px" rel=noopener>Logs</a>
+          <clr-icon shape="pop-out"></clr-icon>
+        </ng-template>
+        <ng-template #noApplicationId>
+          N/A
+        </ng-template>
+      </clr-dg-cell>
     </clr-dg-row>
   </clr-datagrid>
diff --git a/ui/src/app/components/runs/run-detail/run-detail.component.spec.ts b/ui/src/app/components/runs/run-detail/run-detail.component.spec.ts
@@ -22,13 +22,17 @@ import { AppState } from '../../../stores/app.reducers';
 import { Store } from '@ngrx/store';
 import { Subject } from 'rxjs';
 import { GetDagRunDetail } from '../../../stores/runs/runs.actions';
+import { AppInfoModelFactory } from '../../../models/appInfo.model';
 
 describe('RunDetailComponent', () => {
   let underTest: RunDetailComponent;
   let fixture: ComponentFixture<RunDetailComponent>;
   let store: Store<AppState>;
 
   const initialAppState = {
+    application: {
+      appInfo: AppInfoModelFactory.create('Undefined', 'Undefined', 'Undefined'),
+    },
     runs: {
       detail: {
         loading: true,
@@ -78,4 +82,14 @@ describe('RunDetailComponent', () => {
       expect(storeSpy).toHaveBeenCalledWith(new GetDagRunDetail(underTest.dagRunId));
     });
   }));
+
+  it('getApplicationId() should return a valid url ', async(() => {
+    const expectedUrl = 'http://localhost:8088/cluster/app/applicationId_1234';
+    const url1 = underTest.getApplicationIdUrl('http://localhost:8088', 'applicationId_1234');
+    expect(url1).toBe(expectedUrl);
+    const url2 = underTest.getApplicationIdUrl('http://localhost:8088/', 'applicationId_1234');
+    expect(url2).toBe(expectedUrl);
+    const url3 = underTest.getApplicationIdUrl('http://localhost:8088', '/applicationId_1234');
+    expect(url3).toBe(expectedUrl);
+  }));
 });
diff --git a/ui/src/app/components/runs/run-detail/run-detail.component.ts b/ui/src/app/components/runs/run-detail/run-detail.component.ts
@@ -17,9 +17,10 @@ import { Component, Input, OnDestroy, OnInit } from '@angular/core';
 import { jobStatuses } from '../../../models/enums/jobStatuses.constants';
 import { JobInstanceModel } from '../../../models/jobInstance.model';
 import { Store } from '@ngrx/store';
-import { AppState, selectRunState } from '../../../stores/app.reducers';
+import { AppState, selectApplicationState, selectRunState } from '../../../stores/app.reducers';
 import { GetDagRunDetail } from '../../../stores/runs/runs.actions';
 import { Subject, Subscription } from 'rxjs';
+import { AppInfoModel } from '../../../models/appInfo.model';
 
 @Component({
   selector: 'app-run-detail',
@@ -31,8 +32,11 @@ export class RunDetailComponent implements OnInit, OnDestroy {
   @Input() refreshSubject: Subject<boolean> = new Subject<boolean>();
   runDetailSubscription: Subscription;
   refreshSubscription: Subscription;
+  applicationStateSubscription: Subscription;
 
   jobInstances: JobInstanceModel[];
+  appInfo: AppInfoModel;
+
   loading = true;
 
   jobStatuses = jobStatuses;
@@ -52,6 +56,10 @@ export class RunDetailComponent implements OnInit, OnDestroy {
         this.onRefresh();
       }
     });
+
+    this.applicationStateSubscription = this.store.select(selectApplicationState).subscribe((state) => {
+      this.appInfo = state.appInfo;
+    });
   }
 
   onRefresh() {
@@ -61,5 +69,19 @@ export class RunDetailComponent implements OnInit, OnDestroy {
   ngOnDestroy(): void {
     !!this.runDetailSubscription && this.runDetailSubscription.unsubscribe();
     !!this.refreshSubscription && this.refreshSubscription.unsubscribe();
+    !!this.applicationStateSubscription && this.applicationStateSubscription.unsubscribe();
+  }
+
+  getApplicationIdUrl(resourceManagerUrl: string, applicationId: string): string {
+    let url = resourceManagerUrl;
+    if (!url.endsWith('/')) {
+      url += '/';
+    }
+    url += 'cluster/app';
+    if (!applicationId.startsWith('/')) {
+      url += '/';
+    }
+    url += applicationId;
+    return url;
   }
 }
diff --git a/ui/src/app/models/appInfo.model.ts b/ui/src/app/models/appInfo.model.ts
@@ -16,10 +16,11 @@
 export type AppInfoModel = {
   environment: string;
   version: string;
+  resourceManagerUrl: string;
 };
 
 export class AppInfoModelFactory {
-  static create(environment: string, version: string): AppInfoModel {
-    return { environment: environment, version: version };
+  static create(environment: string, version: string, resourceManagerUrl: string): AppInfoModel {
+    return { environment: environment, version: version, resourceManagerUrl: resourceManagerUrl };
   }
 }
diff --git a/ui/src/app/models/jobInstance.model.ts b/ui/src/app/models/jobInstance.model.ts
@@ -19,6 +19,7 @@ export type JobInstanceModel = {
   id: number;
   jobName: string;
   jobType: JobType;
+  applicationId: string;
   created: Date;
   updated: Date;
   jobStatus: JobStatus;
@@ -34,12 +35,22 @@ export class JobInstanceModelFactory {
     id: number,
     jobName: string,
     jobType: JobType,
+    applicationId: string,
     created: Date,
     updated: Date,
     jobStatus: JobStatus,
     order: number,
   ): JobInstanceModel {
-    return { id: id, jobName: jobName, jobType: jobType, created: created, updated: updated, jobStatus: jobStatus, order: order };
+    return {
+      id: id,
+      jobName: jobName,
+      jobType: jobType,
+      applicationId: applicationId,
+      created: created,
+      updated: updated,
+      jobStatus: jobStatus,
+      order: order,
+    };
   }
 }
 
diff --git a/ui/src/app/services/appInfo/appInfo.service.spec.ts b/ui/src/app/services/appInfo/appInfo.service.spec.ts
@@ -42,7 +42,7 @@ describe('AppInfoService', () => {
   });
 
   it('getAppInfo() should return app info', () => {
-    const appInfo = AppInfoModelFactory.create('environment', 'version');
+    const appInfo = AppInfoModelFactory.create('environment', 'version', 'localhost:8088');
 
     underTest.getAppInfo().subscribe(
       (data) => expect(data).toEqual(appInfo),
diff --git a/ui/src/app/stores/application/application.effects.spec.ts b/ui/src/app/stores/application/application.effects.spec.ts
@@ -33,10 +33,7 @@ describe('ApplicationEffects', () => {
 
   const initialAppState = {
     loading: true,
-    appInfo: {
-      environment: 'Undefined',
-      version: 'Undefined',
-    },
+    appInfo: AppInfoModelFactory.create('Undefined', 'Undefined', 'Undefined'),
   };
 
   beforeEach(() => {
@@ -56,7 +53,7 @@ describe('ApplicationEffects', () => {
 
   describe('appInfoLoad', () => {
     it('should return app info when service returns success', () => {
-      const response = AppInfoModelFactory.create('environment', 'version');
+      const response = AppInfoModelFactory.create('environment', 'version', 'localhost:8088');
 
       const action = new LoadAppInfo();
       mockActions = cold('-a', { a: action });
diff --git a/ui/src/app/stores/application/application.reducers.spec.ts b/ui/src/app/stores/application/application.reducers.spec.ts
diff --git a/ui/src/app/stores/application/application.reducers.ts b/ui/src/app/stores/application/application.reducers.ts
diff --git a/ui/src/app/stores/runs/runs.effects.spec.ts b/ui/src/app/stores/runs/runs.effects.spec.ts
diff --git a/ui/src/app/stores/runs/runs.reducers.spec.ts b/ui/src/app/stores/runs/runs.reducers.spec.ts
