Arc 1192 update backfilling status

src/config/feature-flags.ts

@@ -26,6 +26,7 @@ export enum BooleanFlags {
 	ENABLE_SUBSCRIPTION_DEFERRED_INSTALL = "enable-subscription-deferred-install",
 	EARLY_EXIT_ON_VALIDATION_FAILED = "early-exit-on-validation-failed",
 	ENABLE_CONNECTED_REPOS_VIEW="enable-connected-repos-view",
+	ENABLE_BACKFILLING_STATUS_POLLING="enable-backfilling-status-polling",
 	USE_REST_API_FOR_DISCOVERY = "use-rest-api-for-discovery-again",
 	ENABLE_GENERIC_CONTAINERS = "enable-generic-containers",
 	ENABLE_GITHUB_SECURITY_IN_JIRA = "enable-github-security-in-jira",

src/routes/jira/jira-get-connections-backfill-status.test.ts

@@ -0,0 +1,191 @@
+import { getFrontendApp } from "~/src/app";
+import { Installation } from "models/installation";
+import { encodeSymmetric } from "atlassian-jwt";
+import { getLogger } from "config/logger";
+import { Subscription } from "models/subscription";
+import { DatabaseStateCreator } from "test/utils/database-state-creator";
+import supertest from "supertest";
+import { booleanFlag, BooleanFlags } from "config/feature-flags";
+import { when } from "jest-when";
+import { RepoSyncState } from "models/reposyncstate";
+
+jest.mock("config/feature-flags");
+
+describe("jira-get-connections-backfillStatus.test", () => {
+	let app;
+	let installation: Installation;
+	let subscription: Subscription;
+	let repoSyncState: RepoSyncState;
+	const generateJwt = async () => {
+		return encodeSymmetric(
+			{
+				qsh: "context-qsh",
+				iss: installation.plainClientKey,
+				sub: "myAccountId"
+			},
+			await installation.decrypt("encryptedSharedSecret", getLogger("test"))
+		);
+	};
+
+	beforeEach(async () => {
+		app = getFrontendApp();
+		const result = await new DatabaseStateCreator()
+			.withActiveRepoSyncState()
+			.create();
+		installation = result.installation;
+		subscription = result.subscription;
+		repoSyncState = result.repoSyncState!;
+		when(booleanFlag)
+			.calledWith(BooleanFlags.JIRA_ADMIN_CHECK)
+			.mockResolvedValue(true);
+	});
+
+	it("should return 401 when no JWT was provided", async () => {
+		const resp = await supertest(app).get(
+			`/jira/subscriptions/backfill-status/?subscriptionIds=${subscription.id}`
+		);
+		expect(resp.status).toStrictEqual(401);
+		expect(resp.text).toBe("Unauthorised");
+	});
+
+	it("should return 403 when not an admin", async () => {
+		const resp = await supertest(app)
+			.get(
+				`/jira/subscriptions/backfill-status?subscriptionIds=${subscription.id}`
+			)
+			.set(
+				"authorization",
+				`JWT ${await generateJwt()}`
+			);
+		expect(resp.status).toStrictEqual(403);
+	});
+
+	describe("admin and JWT are OK", () => {
+		beforeEach(() => {
+			const payload = {
+				accountId: "myAccountId",
+				globalPermissions: ["ADMINISTER"]
+			};
+			jiraNock
+				.post("/rest/api/latest/permissions/check", payload)
+				.reply(200, { globalPermissions: ["ADMINISTER"] });
+		});
+
+		it("should return 400 when no subscriptions were found", async () => {
+			const resp = await supertest(app)
+				.get(
+					`/jira/subscriptions/backfill-status?subscriptionIds=${
+						subscription.id + 1
+					}`
+				)
+				.set(
+					"authorization",
+					`JWT ${await generateJwt()}`
+				);
+			expect(resp.status).toStrictEqual(400);
+		});
+
+		it("should return 400 when no Missing Subscription IDs were found in query", async () => {
+			const resp = await supertest(app)
+				.get(`/jira/subscriptions/backfill-status`)
+				.set("authorization", `JWT ${await generateJwt()}`);
+			expect(resp.status).toStrictEqual(400);
+			expect(resp.text).toBe("Missing Subscription IDs");
+		});
+
+		it("should return 403 if the subscription belongs to a different user", async () => {
+			const result = await new DatabaseStateCreator()
+				.forJiraHost("https://another-one.atlassian.net")
+				.create();
+			const resp = await supertest(app)
+				.get(
+					`/jira/subscriptions/backfill-status?subscriptionIds=${result.subscription.id}`
+				)
+				.set(
+					"authorization",
+					`JWT ${await generateJwt()}`
+				);
+
+			expect(resp.status).toStrictEqual(403);
+		});
+
+		it("should return 200 if the subscription belongs to the same user", async () => {
+			const resp = await supertest(app)
+				.get(
+					`/jira/subscriptions/backfill-status?subscriptionIds=${subscription.id}`
+				)
+				.set(
+					"authorization",
+					`JWT ${await generateJwt()}`
+				);
+			expect(resp.status).toStrictEqual(200);
+		});
+
+		describe("happy paths", () => {
+			beforeEach(async () => {
+				const newRepoSyncStatesData: any[] = [];
+				for (let newRepoStateNo = 1; newRepoStateNo < 50; newRepoStateNo++) {
+					const newRepoSyncState = { ...repoSyncState.dataValues };
+					delete newRepoSyncState["id"];
+					delete newRepoSyncState["commitStatus"];
+					delete newRepoSyncState["branchStatus"];
+					newRepoSyncState["repoId"] = repoSyncState.repoId + newRepoStateNo;
+					newRepoSyncState["repoName"] =
+						repoSyncState.repoName + newRepoStateNo.toString();
+					newRepoSyncState["repoFullName"] =
+						repoSyncState.repoFullName +
+						String(newRepoStateNo).padStart(3, "0");
+					if (newRepoStateNo === 1) {
+						newRepoSyncState["commitStatus"] = "pending";
+						newRepoSyncState["branchStatus"] = "complete";
+						newRepoSyncState["pullStatus"] = "complete";
+						newRepoSyncState["buildStatus"] = "complete";
+						newRepoSyncState["deploymentStatus"] = "pending";
+					} else if (newRepoStateNo % 3 == 1) {
+						newRepoSyncState["commitStatus"] = "complete";
+						newRepoSyncState["branchStatus"] = "complete";
+						newRepoSyncState["pullStatus"] = "complete";
+						newRepoSyncState["buildStatus"] = "complete";
+						newRepoSyncState["deploymentStatus"] = "complete";
+					} else if (newRepoStateNo % 3 == 2) {
+						newRepoSyncState["commitStatus"] = "failed";
+						newRepoSyncState["branchStatus"] = "complete";
+						newRepoSyncState["pullStatus"] = "complete";
+						newRepoSyncState["buildStatus"] = "complete";
+						newRepoSyncState["deploymentStatus"] = "failed";
+					}
+					newRepoSyncStatesData.push(newRepoSyncState);
+				}
+				await RepoSyncState.bulkCreate(newRepoSyncStatesData);
+			});
+
+			it("should return 200 if the subscription belongs to the same user", async () => {
+				const resp = await supertest(app)
+					.get(
+						`/jira/subscriptions/backfill-status?subscriptionIds=${subscription.id}`
+					)
+					.set(
+						"authorization",
+						`JWT ${await generateJwt()}`
+					);
+				expect(resp.status).toStrictEqual(200);
+
+				expect(resp.body).toMatchObject({
+					data: {
+						subscriptions: {
+							[subscription.id]: {
+								isSyncComplete: false,
+								syncStatus: "IN PROGRESS",
+								totalRepos: 33,
+								syncedRepos: 17,
+								backfillSince: null
+							}
+						},
+						isBackfillComplete: false,
+						subscriptionIds: [subscription.id]
+					}
+				});
+			});
+		});
+	});
+});

src/routes/jira/jira-get-connections-backfill-status.ts

@@ -0,0 +1,130 @@
+import { NextFunction, Request, Response } from "express";
+import { groupBy } from "lodash";
+import { RepoSyncState } from "~/src/models/reposyncstate";
+import { Subscription, SyncStatus } from "~/src/models/subscription";
+import {
+	mapSyncStatus,
+	ConnectionSyncStatus,
+	getRetryableFailedSyncErrors
+} from "~/src/util/github-installations-helper";
+
+type SubscriptionBackfillState = {
+	totalRepos?: number;
+	syncedRepos?: number;
+	syncStatus: ConnectionSyncStatus;
+	isSyncComplete: boolean;
+	backfillSince?: string;
+	failedSyncErrors?: Record<string, number>;
+	syncWarning?: string;
+};
+
+type ErrorType = {
+	subscriptionId: string;
+	error: string;
+};
+type BackFillType = {
+	[key: string]: SubscriptionBackfillState;
+};
+
+export const JiraGetConnectionsBackfillStatus = async (
+	req: Request,
+	res: Response,
+	next: NextFunction
+): Promise<void> => {
+	try {
+		const { jiraHost: localJiraHost } = res.locals;
+		const subscriptionIds = String(req.query?.subscriptionIds)
+			.split(",")
+			.map(Number)
+			.filter(Boolean);
+
+		if (subscriptionIds.length === 0) {
+			req.log.warn("Missing Subscription IDs");
+			res.status(400).send("Missing Subscription IDs");
+			return;
+		}
+
+		const subscriptions = await Subscription.findAllForSubscriptionIds(
+			subscriptionIds
+		);
+
+		const resultSubscriptionIds = subscriptions.map(
+			(subscription) => subscription.id
+		);
+
+		if (!subscriptions || subscriptions.length === 0) {
+			req.log.error("Missing Subscription");
+			res.status(400).send("Missing Subscription");
+			return;
+		}
+
+		const jiraHostsMatched = subscriptions.every(
+			(subscription) => subscription?.jiraHost === localJiraHost
+		);
+
+		if (!jiraHostsMatched) {
+			req.log.error("mismatched Jira Host");
+			res.status(403).send("mismatched Jira Host");
+			return;
+		}
+		const subscriptionsById = groupBy(subscriptions, "id");
+		const { backfillStatus, errors } = await getBackfillStatus(
+			subscriptionsById
+		);
+		const isBackfillComplete = getBackfillCompletionStatus(backfillStatus);
+		res.status(200).send({
+			data: {
+				subscriptions: backfillStatus,
+				isBackfillComplete,
+				subscriptionIds: resultSubscriptionIds,
+				errors
+			}
+		});
+	} catch (error) {
+		req.log.error(
+			{ error },
+			"Failed to poll repo backfill status for provided subscription ID"
+		);
+		return next(
+			new Error(
+				`Failed to poll repo backfill status for provided subscription ID`
+			)
+		);
+	}
+};
+
+const getBackfillCompletionStatus = (backfillStatus: BackFillType): boolean =>
+	Object.values(backfillStatus).every(
+		(backFill: SubscriptionBackfillState): boolean => backFill?.isSyncComplete
+	);
+
+const getBackfillStatus = async (
+	subscriptionsById
+): Promise<{ backfillStatus: BackFillType; errors?: ErrorType[] }> => {
+	const backfillStatus: BackFillType = {};
+	const errors: ErrorType[] = [];
+	for (const subscriptionId in subscriptionsById) {
+		try {
+			const subscription = subscriptionsById[subscriptionId][0];
+			const isSyncComplete =
+				subscription?.syncStatus === SyncStatus.COMPLETE ||
+				subscription?.syncStatus === SyncStatus.FAILED;
+			const failedSyncErrors = await getRetryableFailedSyncErrors(subscription);
+
+			backfillStatus[subscriptionId] = {
+				isSyncComplete,
+				syncStatus: mapSyncStatus(subscription?.syncStatus),
+				totalRepos: subscription?.totalNumberOfRepos,
+				syncedRepos: await RepoSyncState.countFullySyncedReposForSubscription(
+					subscription
+				),
+				failedSyncErrors,
+				backfillSince: subscription?.backfillSince || null,
+				syncWarning: subscription.syncWarning
+			};
+		} catch (error) {
+			errors.push({ subscriptionId, error });
+		}
+	}
+	return { backfillStatus, errors };
+};

src/routes/jira/jira-get.ts

@@ -74,6 +74,7 @@ const renderJiraCloudAndEnterpriseServer = async (res: Response, req: Request):
 			useNewSPAExperience,
 			APP_URL: process.env.APP_URL,
 			enableRepoConnectedPage: await booleanFlag(BooleanFlags.ENABLE_CONNECTED_REPOS_VIEW, jiraHost),
+			enableBackfillingStatusPolling: await booleanFlag(BooleanFlags.ENABLE_BACKFILLING_STATUS_POLLING, jiraHost),
 			csrfToken: req.csrfToken(),
 			nonce
 		});

src/routes/jira/jira-router.ts

@@ -10,6 +10,7 @@ import { JiraConnectRouter } from "routes/jira/connect/jira-connect-router";
 import { body } from "express-validator";
 import { returnOnValidationError } from "routes/api/api-utils";
 import { jiraSymmetricJwtMiddleware } from "~/src/middleware/jira-symmetric-jwt-middleware";
+import { JiraGetConnectionsBackfillStatus } from "~/src/routes/jira/jira-get-connections-backfill-status";
 import { JiraConnectedReposGet } from "routes/jira/jira-connected-repos-get";
 import { jiraAdminPermissionsMiddleware } from "middleware/jira-admin-permission-middleware";
 import { JiraWorkspacesRouter } from "routes/jira/workspaces/jira-workspaces-router";
@@ -38,6 +39,7 @@ JiraRouter.use("/security", jiraSymmetricJwtMiddleware, JiraSecurityWorkspacesRo
 
 JiraRouter.get("/", csrfMiddleware, jiraSymmetricJwtMiddleware, jiraAdminPermissionsMiddleware, JiraGet);
 
+JiraRouter.get("/subscriptions/backfill-status", jiraSymmetricJwtMiddleware, jiraAdminPermissionsMiddleware, JiraGetConnectionsBackfillStatus);
 JiraRouter.get("/subscription/:subscriptionId/repos", csrfMiddleware, jiraSymmetricJwtMiddleware, jiraAdminPermissionsMiddleware, JiraConnectedReposGet);
 
 /********************************************************************************************************************

src/util/github-installations-helper.ts

@@ -31,7 +31,7 @@ interface GitHubCloudObj {
 }
 
 export type ConnectionSyncStatus = "IN PROGRESS" | "FINISHED" | "PENDING" | "FAILED" | undefined;
-const mapSyncStatus = (syncStatus: SyncStatus = SyncStatus.PENDING): ConnectionSyncStatus => {
+export const mapSyncStatus = (syncStatus: SyncStatus = SyncStatus.PENDING): ConnectionSyncStatus => {
 	switch (syncStatus) {
 		case "ACTIVE":
 			return "IN PROGRESS";