Adding unit tests for the non-hexagonal serverless application

simalexan · simalexan · commit b303713e5aa2 · 2020-06-24T15:25:06.000+02:00
diff --git a/README.md b/README.md
@@ -20,12 +20,15 @@ This project has the following folder structure:
 ├── package.json
 ├── samconfig.toml # AWS SAM config file, generated by SAM
 ├── src # Source code for all functions
-│   └── parse-covid-csv # Fuction source code
-│       ├── events # Events for local testing
-│       │   ├── context.ts
-│       │   └── event.json
-│       └── lambda.ts # Function source code
-├── template.yaml # Main CloudFormation file 
+│   ├── parse-covid-csv # Fuction source code
+│   │   ├── events # Events for local testing
+│   │   │   ├── context.ts
+│   │   │   └── event.json
+│   │   └── lambda.ts # Function source code
+│   └── tests
+│       └── parse-covid-csv
+│           └── lambda.test.ts
+├── template.yaml # Main CloudFormation file
 ├── tsconfig.json
 ├── webpack.config.js # Webpack config
 └── yarn.lock
@@ -79,8 +82,16 @@ sam deploy
 
 ### Testing
 
-This section will be added tomorrow.
+To run the application tests, run the following command:
+
+```bash
+npm test
+```
+
+This will run the jest tests for the sample serverless component, particularly the [`/src/tests/parse-covid-csv/lambda.test.ts`](./src/tests/parse-covid-csv/lambda.test.ts).
+
+Note. If you notice error logs during the test run, please do not get concerned, as the tests are also evaluating the error cases and it is recommended to trace the particular error logs.
 
 ## License
 
-MIT, see [LICENSE](LICENSE).
+MIT, see [LICENSE](LICENSE).
diff --git a/package-lock.json b/package-lock.json
diff --git a/src/tests/parse-covid-csv/lambda.test.ts b/src/tests/parse-covid-csv/lambda.test.ts
@@ -0,0 +1,230 @@
+import { Readable, Writable } from 'stream'
+import { PutItemInput } from 'aws-sdk/clients/dynamodb'
+import { GetObjectRequest } from 'aws-sdk/clients/s3'
+import s3Event from '../../parse-covid-csv/events/event.json'
+
+interface IMockGetObject {
+  createReadStream: jest.Mock
+}
+
+// This file represents common unit tests we see in projects.
+// These are not the best testing examples, you'll see the better approach
+// using hexagonal architecture in the following commits.
+describe('Parse Covid CSV', () => {
+  // We need to be able to check if these mocks are invoked in multiple tests,
+  // so we define them outside of the beforeEach block
+  const mockedS3GetObject = jest.fn()
+  const mockedDynamoDBPut = jest.fn()
+  const lambdaPath = '../../parse-covid-csv/lambda'
+
+  // Before each test, we'll create mocks of required libraries and modules.
+  // We can do this before all tests, but as we are using Node streams,
+  // we can't write to the stream that is already closed,
+  // so we'll re-create our mocks before each run.
+  // This is not an optimal solution, but it works.
+  beforeEach(function () {
+    // We can try to use the aws-sdk-mock library, but we need to mock fs too.
+    // It's easier to mock everything using the built-in Jest functionality.
+    // We are mocking the "clients/s3" path of the "aws-sdk" module,
+    // because our app imports this path
+    jest.mock('aws-sdk/clients/s3', () => {
+      // Instead of a real class, we'll return a mock class here
+      return class S3 {
+        // We only use the "getObject" method, so we mock that one only.
+        // We do not care about other methods.
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        getObject(_params: GetObjectRequest, _cb: () => {}): IMockGetObject {
+          return {
+            // Instead of a built-in "createReadStream" method, we'll return a simple value in our mock
+            createReadStream: mockedS3GetObject.mockReturnValue(Readable.from(['hello,name\n1,2\n'])),
+          }
+        }
+      }
+    })
+
+    // Similar to S3, we also need to mock DynamoDB DocumentClient
+    jest.mock('aws-sdk/clients/dynamodb', () => {
+      return { DocumentClient: class DocumentClient {
+        // We use only the "put" method of the DynamoDB DocumentClient
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        put(params: PutItemInput, cb: () => {}): jest.Mock {
+          // The DynamoDB put method returns a promise function.
+          // We'll simply return resolved promise from our mock,
+          // as that's enough for our current unit tests
+          return (mockedDynamoDBPut.mockReturnValue({
+            promise: () => Promise.resolve(),
+          }))(params, cb) // We also need to pass the arguments to our mock function
+        }
+      }}
+    })
+
+    // We also need to mock the uuid module, because it returns random values, which makes it hard to test.
+    // Other option would be to expect any string as an ID generated by the uuid module.
+    jest.mock('uuid', () => {
+      return {
+        v4: jest.fn().mockReturnValue('123'),
+      }
+    })
+  })
+
+  // After each test, we reset all mocks and reset the cache for all required node modules
+  afterEach(() => {
+    jest.resetAllMocks()
+    jest.resetModules()
+  })
+
+  test('should parse a single CSV file and store the row data into a database', async () => {
+    // As we are using streams, we need to mock the fs module too.
+    // We cand do this in the beforeEach section, but that creates some problems with writing to the finished writable stream.
+    // Also, we need to modify the fs.createReadStream function for almost all tests, and it's easier to do that in the test itself.
+    jest.mock('fs', () => {
+      return {
+        // The fs.createReadStream function returns a read stream with the test CSV data
+        createReadStream: jest.fn().mockImplementation(() => Readable.from(['hello,name\n1,2\n'])),
+        // The fs.createWriteStream function returns an empty writabe stream, so we can write to it
+        createWriteStream: jest.fn().mockImplementation(() => new Writable({ write: (_chunk, _encoding, done): void => done() })),
+      }
+    })
+    // We require and invoke our hadnler function.
+    // Moving this function to the top of this file would break the mocks from the beforeEach section.
+    const handler = require(lambdaPath).handler
+    await handler(s3Event)
+    // Then we test that all of our important mocks have been called.s
+    expect(mockedS3GetObject).toHaveBeenCalled()
+    expect(mockedDynamoDBPut).toHaveBeenCalled()
+    expect(mockedDynamoDBPut).toHaveBeenCalledTimes(1)
+    expect(mockedDynamoDBPut).toHaveBeenCalledWith({ Item: { hello: '1', name: '2', id: '123' }, TableName: ''}, undefined)
+  })
+
+  test('should parse a single CSV file with multiple rows and store the rows data into a database', async () => {
+    jest.mock('fs', () => {
+      return {
+        // We simply return a different CSV file content here
+        createReadStream: jest.fn().mockImplementation(() => Readable.from(['hello,name\n1,1\n2,2\n'])),
+        createWriteStream: jest.fn().mockImplementation(() => new Writable({ write: (_chunk, _encoding, done): void => done() })),
+      }
+    })
+    const handler = require(lambdaPath).handler
+    await handler(s3Event)
+    expect(mockedS3GetObject).toHaveBeenCalled()
+    expect(mockedDynamoDBPut).toHaveBeenCalled()
+    // And we test that both rows were saved to the DynamoDB table
+    expect(mockedDynamoDBPut).toHaveBeenCalledTimes(2)
+    // We test the arguments for the first invocation of our mockedDynamoDBPut
+    expect(mockedDynamoDBPut).toHaveBeenCalledWith({ Item: { hello: '1', name: '1', id: '123' }, TableName: '' }, undefined)
+    // Then we do the same for the next one
+    expect(mockedDynamoDBPut).toHaveBeenCalledWith({ Item: { hello: '2', name: '2', id: '123' }, TableName: '' }, undefined)
+  })
+
+  test('should parse a single empty CSV file, without storing the CSV data into the database', async () => {
+    jest.mock('fs', () => {
+      return {
+        // We send the empty CSV file in our fs.createReadStream mock
+        createReadStream: jest.fn().mockImplementation(() => Readable.from([''])),
+        createWriteStream: jest.fn().mockImplementation(() => new Writable({ write: (_chunk, _encoding, done): void => done() })),
+      }
+    })
+    const handler = require(lambdaPath).handler
+    await handler(s3Event)
+    expect(mockedS3GetObject).toHaveBeenCalled()
+    expect(mockedDynamoDBPut).not.toHaveBeenCalled()
+  })
+
+  test('should parse a single CSV file, with less row values than headers and store the ones properly parsed into the database', async () => {
+    jest.mock('fs', () => {
+      return {
+        // We send less row values than headers in a CSV file in our fs.createReadStream mock
+        createReadStream: jest.fn().mockImplementation(() => Readable.from(['hello,name,ignored,headers\n1,2\n'])),
+        createWriteStream: jest.fn().mockImplementation(() => new Writable({ write: (_chunk, _encoding, done): void => done() })),
+      }
+    })
+    const handler = require(lambdaPath).handler
+    await handler(s3Event)
+    expect(mockedS3GetObject).toHaveBeenCalled()
+    expect(mockedDynamoDBPut).toHaveBeenCalled()
+    expect(mockedDynamoDBPut).toHaveBeenCalledTimes(1)
+    expect(mockedDynamoDBPut).toHaveBeenCalledWith({ Item: { hello: '1', name: '2', id: '123' }, TableName: '' }, undefined)
+  })
+
+  test('should parse multiple CSV files and store a JSON string', async () => {
+    jest.mock('fs', () => {
+      return {
+        // We prepare a single row response from our
+        createReadStream: jest.fn().mockImplementation(() => Readable.from(['hello,name,\n1,2\n'])),
+        createWriteStream: jest.fn().mockImplementation(() => new Writable({ write: (_chunk, _encoding, done): void => done() })),
+      }
+    })
+    // Copy the S3 event, because we don't want to to reference and edit the existing one
+    const multipleFilesS3Event = Object.assign({}, s3Event)
+    // Copy the single S3 event record, and then push multiple values of it
+    // Don't worry, it will store them differently, because we are generating a different id for each CSV row
+    const recordCopy = Object.assign({}, s3Event.Records[0])
+    multipleFilesS3Event.Records.push(recordCopy)
+    multipleFilesS3Event.Records.push(recordCopy)
+    const handler = require(lambdaPath).handler
+    await handler(multipleFilesS3Event)
+    // Since there are three S3 files, the S3 getObject method needs to be called three times, to retrieve three documents
+    expect(mockedS3GetObject).toHaveBeenCalledTimes(3)
+    // Since there are three S3 files with a single row each, the DynamoDb put method needs to be called three times, to store the three rows
+    expect(mockedDynamoDBPut).toHaveBeenCalledTimes(3)
+    expect(mockedDynamoDBPut).toHaveBeenCalledWith({ Item: { hello: '1', name: '2', id: '123' }, TableName: '' }, undefined)
+  })
+
+  test('should throw an error when the CSV file is unavailable from the bucket', async () => {
+    // We define the error message for the S3 getObject operation, NoSuchKey Eception
+    const fileUnavailableError = 'An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist.'
+    jest.mock('fs', () => {
+      return {
+        createReadStream: jest.fn().mockImplementation(() => Readable.from(['hello,name\n1,2\n'])),
+        createWriteStream: jest.fn().mockImplementation(() => new Writable({ write: (_chunk, _encoding, done): void => done() })),
+      }
+    })
+    jest.mock('aws-sdk/clients/s3', () => {
+      // Instead of a real class, we'll return a mock class here
+      return class S3 {
+        // We need to setup the "getObject" method to throw the defined Error when trying to stream
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        getObject(_params: GetObjectRequest, _cb: () => {}): IMockGetObject {
+          return {
+            createReadStream: mockedS3GetObject.mockImplementation(() => { throw new Error(fileUnavailableError) }),
+          }
+        }
+      }
+    })
+
+    const handler = require(lambdaPath).handler
+    await expect(handler(s3Event)).rejects.toEqual(new Error(fileUnavailableError))
+    expect(mockedS3GetObject).toHaveBeenCalled()
+    expect(mockedDynamoDBPut).not.toHaveBeenCalled()
+  })
+
+  test('should throw an error if the parsed CSV data could not be stored into the database', async () => {
+    // We define the error message for the DynamoDB operation, Access Denied Exception is a standard one
+    // eslint-disable-next-line max-len
+    const accessDeniedException = 'AccessDeniedException: User: arn:aws:sts::111111111:assumed-role/lambdaRole/username is not authorized to perform: dynamodb:PutItem on resource: arn:aws:dynamodb:us-east-1:111111111:table/covid19-cases/index/type-created_at-index'
+    jest.mock('fs', () => {
+      return {
+        createReadStream: jest.fn().mockImplementation(() => Readable.from(['hello,name\n1,2\n'])),
+        createWriteStream: jest.fn().mockImplementation(() => new Writable({ write: (_chunk, _encoding, done): void => done() })),
+      }
+    })
+    // We also need to mock DynamoDB DocumentClient to properly throw this error
+    jest.mock('aws-sdk/clients/dynamodb', () => {
+      return { DocumentClient: class DocumentClient {
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        put(params: PutItemInput, cb: () => {}): jest.Mock {
+          // The DynamoDB put method returns a promise function.
+          // We'll return a rejected promise from our mock, with an error message
+          return (mockedDynamoDBPut.mockReturnValue({
+            promise: () => Promise.reject(new Error(accessDeniedException)),
+          }))(params, cb)
+        }
+      }}
+    })
+    const handler = require(lambdaPath).handler
+
+    await expect(handler(s3Event)).rejects.toEqual(new Error(accessDeniedException))
+    expect(mockedS3GetObject).toHaveBeenCalled()
+    expect(mockedDynamoDBPut).toHaveBeenCalled()
+  })
+})
