From 658a185e820e0f7a85848d2ed94ec07f89502151 Mon Sep 17 00:00:00 2001 From: benmartin-coforma Date: Thu, 9 Nov 2023 08:02:28 -0700 Subject: [PATCH 1/2] Add a backfill script for previousRevisions, submissionCount --- .../handlers/etl/addRevisionsToMcpar.test.ts | 77 +++++++++++++++++++ .../handlers/etl/addRevisionsToMcpar.ts | 45 +++++++++++ 2 files changed, 122 insertions(+) create mode 100644 services/app-api/handlers/etl/addRevisionsToMcpar.test.ts create mode 100644 services/app-api/handlers/etl/addRevisionsToMcpar.ts diff --git a/services/app-api/handlers/etl/addRevisionsToMcpar.test.ts b/services/app-api/handlers/etl/addRevisionsToMcpar.test.ts new file mode 100644 index 000000000..31e0e641e --- /dev/null +++ b/services/app-api/handlers/etl/addRevisionsToMcpar.test.ts @@ -0,0 +1,77 @@ +import { addRevisionsHandler } from "./addRevisionsToMcpar"; +import dynamodbLib from "../../utils/dynamo/dynamodb-lib"; +import { ReportStatus } from "../../utils/types"; + +jest.mock("../../utils/constants/constants", () => ({ + ...jest.requireActual("../../utils/constants/constants"), + reportTables: { + MCPAR: "local-mcpar-reports", + }, +})); + +jest.mock("../../utils/dynamo/dynamodb-lib", () => ({ + scanIterator: jest.fn(), + put: jest.fn(), +})); + +describe("addRevisionsHandler", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test("should add previousRevisions to reports without them", async () => { + (dynamodbLib.scanIterator as jest.Mock).mockReturnValue([ + { + reportId: "mock-id-1", + status: ReportStatus.SUBMITTED, + }, + { + reportId: "mock-id-2", + status: ReportStatus.IN_PROGRESS, + }, + { + reportId: "mock-id-3", + status: ReportStatus.NOT_STARTED, + previousRevisions: [], + submissionCount: 0, + locked: false, + }, + ]); + + const result = await addRevisionsHandler(); + + expect(result.statusCode).toBe(200); + const mockedPut = dynamodbLib.put as jest.Mock; + expect(mockedPut).toBeCalledTimes(2); + expect(mockedPut.mock.calls[0][0]).toEqual({ + TableName: "local-mcpar-reports", + Item: { + reportId: "mock-id-1", + status: ReportStatus.SUBMITTED, + previousRevisions: [], + submissionCount: 1, + locked: true, + }, + }); + expect(mockedPut.mock.calls[1][0]).toEqual({ + TableName: "local-mcpar-reports", + Item: { + reportId: "mock-id-2", + status: ReportStatus.IN_PROGRESS, + previousRevisions: [], + submissionCount: 0, + locked: false, + }, + }); + }); + + test("should return 500 if dynamo errors", async () => { + (dynamodbLib.scanIterator as jest.Mock).mockRejectedValue( + new Error("no DB for you") + ); + + const result = await addRevisionsHandler(); + + expect(result.statusCode).toBe(500); + }); +}); diff --git a/services/app-api/handlers/etl/addRevisionsToMcpar.ts b/services/app-api/handlers/etl/addRevisionsToMcpar.ts new file mode 100644 index 000000000..47f9be4e4 --- /dev/null +++ b/services/app-api/handlers/etl/addRevisionsToMcpar.ts @@ -0,0 +1,45 @@ +/* eslint-disable no-console */ +import { APIGatewayProxyResult } from "aws-lambda"; +import { ReportStatus, ReportType } from "../../utils/types"; +import { reportTables } from "../../utils/constants/constants"; +import dynamodbLib from "../../utils/dynamo/dynamodb-lib"; + +export const addRevisionsHandler = async (): Promise => { + try { + await addRevisionsToExistingMcparReports(); + + return { + statusCode: 200, + body: JSON.stringify({ + message: "Update complete", + }), + }; + } catch (err: any) { + console.error(err); + return { + statusCode: 500, + body: JSON.stringify({ + message: err.message, + }), + }; + } +}; + +const addRevisionsToExistingMcparReports = async () => { + const TableName = reportTables[ReportType.MCPAR]; + for await (const reportMetadata of dynamodbLib.scanIterator({ TableName })) { + if (reportMetadata.previousRevisions === undefined) { + const wasSubmitted = reportMetadata.status === ReportStatus.SUBMITTED; + + // TODO should we change lastAltered? + reportMetadata.previousRevisions = []; + reportMetadata.submissionCount = wasSubmitted ? 1 : 0; + reportMetadata.locked = wasSubmitted ? true : false; + + await dynamodbLib.put({ + TableName, + Item: reportMetadata, + }); + } + } +}; From cda77782869de0b178635b068f3f0fe5ef9b6376 Mon Sep 17 00:00:00 2001 From: benmartin-coforma Date: Mon, 13 Nov 2023 09:22:07 -0700 Subject: [PATCH 2/2] Wire the MCPAR ETL into serverless --- .../handlers/etl/addRevisionsToMcpar.test.ts | 8 ++++--- .../handlers/etl/addRevisionsToMcpar.ts | 23 ++++++++++--------- services/app-api/serverless.yml | 7 ++++++ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/services/app-api/handlers/etl/addRevisionsToMcpar.test.ts b/services/app-api/handlers/etl/addRevisionsToMcpar.test.ts index 31e0e641e..c3c71630f 100644 --- a/services/app-api/handlers/etl/addRevisionsToMcpar.test.ts +++ b/services/app-api/handlers/etl/addRevisionsToMcpar.test.ts @@ -66,9 +66,11 @@ describe("addRevisionsHandler", () => { }); test("should return 500 if dynamo errors", async () => { - (dynamodbLib.scanIterator as jest.Mock).mockRejectedValue( - new Error("no DB for you") - ); + jest.spyOn(console, "error").mockImplementationOnce(() => undefined); + + (dynamodbLib.scanIterator as jest.Mock).mockImplementationOnce(() => { + throw new Error("no DB for you"); + }); const result = await addRevisionsHandler(); diff --git a/services/app-api/handlers/etl/addRevisionsToMcpar.ts b/services/app-api/handlers/etl/addRevisionsToMcpar.ts index 47f9be4e4..726dd4667 100644 --- a/services/app-api/handlers/etl/addRevisionsToMcpar.ts +++ b/services/app-api/handlers/etl/addRevisionsToMcpar.ts @@ -28,18 +28,19 @@ export const addRevisionsHandler = async (): Promise => { const addRevisionsToExistingMcparReports = async () => { const TableName = reportTables[ReportType.MCPAR]; for await (const reportMetadata of dynamodbLib.scanIterator({ TableName })) { - if (reportMetadata.previousRevisions === undefined) { - const wasSubmitted = reportMetadata.status === ReportStatus.SUBMITTED; + if (reportMetadata.previousRevisions !== undefined) { + continue; + } - // TODO should we change lastAltered? - reportMetadata.previousRevisions = []; - reportMetadata.submissionCount = wasSubmitted ? 1 : 0; - reportMetadata.locked = wasSubmitted ? true : false; + const wasSubmitted = reportMetadata.status === ReportStatus.SUBMITTED; - await dynamodbLib.put({ - TableName, - Item: reportMetadata, - }); - } + reportMetadata.previousRevisions = []; + reportMetadata.submissionCount = wasSubmitted ? 1 : 0; + reportMetadata.locked = wasSubmitted ? true : false; + + await dynamodbLib.put({ + TableName, + Item: reportMetadata, + }); } }; diff --git a/services/app-api/serverless.yml b/services/app-api/serverless.yml index 2576409c7..d3977cfb8 100644 --- a/services/app-api/serverless.yml +++ b/services/app-api/serverless.yml @@ -160,6 +160,13 @@ provider: PRINCE_API_PATH: ${self:custom.princeApiPath} functions: + addRevisionsToMcpar: + handler: handlers/etl/addRevisionsToMcpar.addRevisionsHandler + timeout: 300 + warmup: + default: + enabled: false + maximumRetryAttempts: 0 numberFieldDataToFile: handler: handlers/etl/numberFieldDataToFile.exportNumericData timeout: 300