Skip to content

Commit 53735a1

Browse files
raghuAtWingsraghuAtWings
authored
[fix]: [DEL-4236]: Delegate returns a RC of 403 in logs, but we need more details (#34013)
* [fix]: [DEL-4236]: Delegate returns a RC of 403 in logs, but we need more details * [fix]: [DEL-4236]: Delegate returns a RC of 403 in logs, but we need more details * [fix]: [DEL-4286]: Delegate Logs for Kryo exceptions not adding the MDC in the logs making hard to troubleshoot
1 parent bae3f57 commit 53735a1

File tree

1 file changed

+68
-58
lines changed

1 file changed

+68
-58
lines changed

‎260-delegate/src/main/java/io/harness/delegate/service/DelegateAgentServiceImpl.java‎

Lines changed: 68 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1899,72 +1899,74 @@ private void dispatchDelegateTaskAsync(DelegateTaskEvent delegateTaskEvent) {
18991899
}
19001900

19011901
private void dispatchDelegateTask(DelegateTaskEvent delegateTaskEvent) {
1902-
log.info("DelegateTaskEvent received - {}", delegateTaskEvent);
1903-
String delegateTaskId = delegateTaskEvent.getDelegateTaskId();
1902+
try (TaskLogContext ignore = new TaskLogContext(delegateTaskEvent.getDelegateTaskId(), OVERRIDE_ERROR)) {
1903+
log.info("DelegateTaskEvent received - {}", delegateTaskEvent);
1904+
String delegateTaskId = delegateTaskEvent.getDelegateTaskId();
19041905

1905-
try {
1906-
if (frozen.get()) {
1907-
log.info(
1908-
"Delegate process with detected time out of sync or with revoked token is running. Won't acquire tasks.");
1909-
return;
1910-
}
1906+
try {
1907+
if (frozen.get()) {
1908+
log.info(
1909+
"Delegate process with detected time out of sync or with revoked token is running. Won't acquire tasks.");
1910+
return;
1911+
}
19111912

1912-
if (!acquireTasks.get()) {
1913-
log.info("[Old] Upgraded process is running. Won't acquire task while completing other tasks");
1914-
return;
1915-
}
1913+
if (!acquireTasks.get()) {
1914+
log.info("[Old] Upgraded process is running. Won't acquire task while completing other tasks");
1915+
return;
1916+
}
19161917

1917-
if (upgradePending.get() && !delegateTaskEvent.isSync()) {
1918-
log.info("[Old] Upgrade pending, won't acquire async task");
1919-
return;
1920-
}
1918+
if (upgradePending.get() && !delegateTaskEvent.isSync()) {
1919+
log.info("[Old] Upgrade pending, won't acquire async task");
1920+
return;
1921+
}
19211922

1922-
if (currentlyAcquiringTasks.contains(delegateTaskId)) {
1923-
log.info("Task [DelegateTaskEvent: {}] currently acquiring. Don't acquire again", delegateTaskEvent);
1924-
return;
1925-
}
1923+
if (currentlyAcquiringTasks.contains(delegateTaskId)) {
1924+
log.info("Task [DelegateTaskEvent: {}] currently acquiring. Don't acquire again", delegateTaskEvent);
1925+
return;
1926+
}
19261927

1927-
if (currentlyValidatingTasks.containsKey(delegateTaskId)) {
1928-
log.info("Task [DelegateTaskEvent: {}] already validating. Don't validate again", delegateTaskEvent);
1929-
return;
1930-
}
1928+
if (currentlyValidatingTasks.containsKey(delegateTaskId)) {
1929+
log.info("Task [DelegateTaskEvent: {}] already validating. Don't validate again", delegateTaskEvent);
1930+
return;
1931+
}
19311932

1932-
currentlyAcquiringTasks.add(delegateTaskId);
1933+
currentlyAcquiringTasks.add(delegateTaskId);
19331934

1934-
log.debug("Try to acquire DelegateTask - accountId: {}", accountId);
1935+
log.debug("Try to acquire DelegateTask - accountId: {}", accountId);
19351936

1936-
DelegateTaskPackage delegateTaskPackage = executeRestCall(
1937-
delegateAgentManagerClient.acquireTask(delegateId, delegateTaskId, accountId, delegateInstanceId));
1938-
if (delegateTaskPackage == null || delegateTaskPackage.getData() == null) {
1939-
log.warn("Delegate task data not available for task: {} - accountId: {}", delegateTaskId,
1940-
delegateTaskEvent.getAccountId());
1941-
return;
1942-
} else {
1943-
log.info("received task package {} for delegateInstance {}", delegateTaskPackage, delegateInstanceId);
1944-
}
1937+
DelegateTaskPackage delegateTaskPackage = executeRestCall(
1938+
delegateAgentManagerClient.acquireTask(delegateId, delegateTaskId, accountId, delegateInstanceId));
1939+
if (delegateTaskPackage == null || delegateTaskPackage.getData() == null) {
1940+
log.warn("Delegate task data not available for task: {} - accountId: {}", delegateTaskId,
1941+
delegateTaskEvent.getAccountId());
1942+
return;
1943+
} else {
1944+
log.info("received task package {} for delegateInstance {}", delegateTaskPackage, delegateInstanceId);
1945+
}
19451946

1946-
if (isEmpty(delegateTaskPackage.getDelegateInstanceId())) {
1947-
// Not whitelisted. Perform validation.
1948-
// TODO: Remove this once TaskValidation does not use secrets
1947+
if (isEmpty(delegateTaskPackage.getDelegateInstanceId())) {
1948+
// Not whitelisted. Perform validation.
1949+
// TODO: Remove this once TaskValidation does not use secrets
1950+
1951+
// applyDelegateSecretFunctor(delegatePackage);
1952+
DelegateValidateTask delegateValidateTask = getDelegateValidateTask(delegateTaskEvent, delegateTaskPackage);
1953+
injector.injectMembers(delegateValidateTask);
1954+
currentlyValidatingTasks.put(delegateTaskPackage.getDelegateTaskId(), delegateTaskPackage);
1955+
updateCounterIfLessThanCurrent(maxValidatingTasksCount, currentlyValidatingTasks.size());
1956+
delegateValidateTask.validationResults();
1957+
} else if (delegateInstanceId.equals(delegateTaskPackage.getDelegateInstanceId())) {
1958+
applyDelegateSecretFunctor(delegateTaskPackage);
1959+
// Whitelisted. Proceed immediately.
1960+
log.info("Delegate {} whitelisted for task and accountId: {}", delegateId, accountId);
1961+
executeTask(delegateTaskPackage);
1962+
}
19491963

1950-
// applyDelegateSecretFunctor(delegatePackage);
1951-
DelegateValidateTask delegateValidateTask = getDelegateValidateTask(delegateTaskEvent, delegateTaskPackage);
1952-
injector.injectMembers(delegateValidateTask);
1953-
currentlyValidatingTasks.put(delegateTaskPackage.getDelegateTaskId(), delegateTaskPackage);
1954-
updateCounterIfLessThanCurrent(maxValidatingTasksCount, currentlyValidatingTasks.size());
1955-
delegateValidateTask.validationResults();
1956-
} else if (delegateInstanceId.equals(delegateTaskPackage.getDelegateInstanceId())) {
1957-
applyDelegateSecretFunctor(delegateTaskPackage);
1958-
// Whitelisted. Proceed immediately.
1959-
log.info("Delegate {} whitelisted for task and accountId: {}", delegateId, accountId);
1960-
executeTask(delegateTaskPackage);
1964+
} catch (IOException e) {
1965+
log.error("Unable to get task for validation", e);
1966+
} finally {
1967+
currentlyAcquiringTasks.remove(delegateTaskId);
1968+
currentlyExecutingFutures.remove(delegateTaskId);
19611969
}
1962-
1963-
} catch (IOException e) {
1964-
log.error("Unable to get task for validation", e);
1965-
} finally {
1966-
currentlyAcquiringTasks.remove(delegateTaskId);
1967-
currentlyExecutingFutures.remove(delegateTaskId);
19681970
}
19691971
}
19701972

@@ -2280,7 +2282,12 @@ private Consumer<DelegateTaskResponse> getPostExecutionFunction(String taskId, L
22802282
log.info("Task {} response sent to manager", taskId);
22812283
break;
22822284
}
2283-
log.warn("Failed to send response for task {}: {}. {}", taskId, response == null ? "null" : response.code(),
2285+
log.warn("Failed to send response for task {}: {}. error: {}. requested url: {} {}", taskId,
2286+
response == null ? "null" : response.code(),
2287+
response == null || response.errorBody() == null ? "null" : response.errorBody().string(),
2288+
response == null || response.raw() == null || response.raw().request() == null
2289+
? "null"
2290+
: response.raw().request().url(),
22842291
attempt < (retries - 1) ? "Retrying." : "Giving up.");
22852292
if (attempt < retries - 1) {
22862293
// Do not sleep for last loop round, as we are going to fail.
@@ -2656,8 +2663,11 @@ private void sendErrorResponse(DelegateTaskPackage delegateTaskPackage, Exceptio
26562663
log.info("Task {} response sent to manager", taskId);
26572664
return;
26582665
}
2659-
log.warn(
2660-
"Failed to send response for task {}: {}. {}", taskId, resp == null ? "null" : resp.code(), "Retrying.");
2666+
log.warn("Failed to send response for task {}: {}. error: {}. requested url: {} {}", taskId,
2667+
resp == null ? "null" : resp.code(),
2668+
resp == null || resp.errorBody() == null ? "null" : resp.errorBody().string(),
2669+
resp == null || resp.raw() == null || resp.raw().request() == null ? "null" : resp.raw().request().url(),
2670+
"Retrying.");
26612671
sleep(ofSeconds(FibonacciBackOff.getFibonacciElement(attempt)));
26622672
}
26632673
} catch (Exception e) {

0 commit comments

Comments
 (0)