[L2B-4977] Refactor logger to allow for custom backends and formatters

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
       - run: yarn --frozen-lockfile
       - run: yarn build
@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
       - run: yarn --frozen-lockfile
       - run: yarn build
@@ -36,7 +36,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
       - run: yarn --frozen-lockfile
       - run: yarn build
@@ -48,7 +48,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
       - run: yarn --frozen-lockfile
       - run: yarn build

.github/workflows/release-canary.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
 
       - run: yarn --frozen-lockfile

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
 
       - run: yarn --frozen-lockfile

packages/backend-tools/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @l2beat/backend-tools
 
+## 0.6.0
+
+### Minor Changes
+
+- Refactor logger to allow for multiple backends and formatters
+
 ## 0.5.2
 
 ### Patch Changes

packages/backend-tools/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@l2beat/backend-tools",
   "description": "Common utilities for L2BEAT projects.",
-  "version": "0.5.2",
+  "version": "0.6.0",
   "license": "MIT",
   "repository": "https://github.com/l2beat/tools",
   "bugs": {
@@ -30,12 +30,16 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@elastic/elasticsearch": "^8.13.1",
     "chalk": "^4.1.2",
     "dotenv": "^16.3.1",
-    "error-stack-parser": "^2.1.4"
+    "error-stack-parser": "^2.1.4",
+    "uuid": "^9.0.1"
   },
   "devDependencies": {
     "@sinonjs/fake-timers": "^11.1.0",
-    "@types/sinonjs__fake-timers": "^8.1.2"
+    "@types/elasticsearch": "^5.0.43",
+    "@types/sinonjs__fake-timers": "^8.1.2",
+    "@types/uuid": "^9.0.8"
   }
 }

packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts

@@ -0,0 +1,91 @@
+import { expect, mockFn, MockObject, mockObject } from 'earl'
+
+import {
+  ElasticSearchBackend,
+  ElasticSearchBackendOptions,
+  UuidProvider,
+} from './ElasticSearchBackend'
+import { ElasticSearchClient } from './ElasticSearchClient'
+
+const flushInterval = 10
+const id = 'some-id'
+const indexPrefix = 'logs-'
+const indexName = createIndexName()
+const log = {
+  '@timestamp': '2024-04-24T21:02:30.916Z',
+  log: {
+    level: 'INFO',
+  },
+  message: 'Update started',
+}
+
+describe(ElasticSearchBackend.name, () => {
+  it("creates index if doesn't exist", async () => {
+    const clientMock = createClienMock(false)
+    const backendMock = createBackendMock(clientMock)
+
+    backendMock.log(JSON.stringify(log))
+
+    // wait for log flus
+    await delay(flushInterval + 10)
+
+    expect(clientMock.indexExist).toHaveBeenOnlyCalledWith(indexName)
+    expect(clientMock.indexCreate).toHaveBeenOnlyCalledWith(indexName)
+  })
+
+  it('does nothing if buffer is empty', async () => {
+    const clientMock = createClienMock(false)
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const backendMock = createBackendMock(clientMock)
+
+    // wait for log flush
+    await delay(flushInterval + 10)
+
+    expect(clientMock.bulk).not.toHaveBeenCalled()
+  })
+
+  it('pushes logs to ES if there is something in the buffer', async () => {
+    const clientMock = createClienMock(false)
+    const backendMock = createBackendMock(clientMock)
+
+    backendMock.log(JSON.stringify(log))
+
+    // wait for log flush
+    await delay(flushInterval + 10)
+
+    expect(clientMock.bulk).toHaveBeenOnlyCalledWith(
+      [{ id, ...log }],
+      indexName,
+    )
+  })
+})
+
+function createClienMock(indextExist = true) {
+  return mockObject<ElasticSearchClient>({
+    indexExist: mockFn(async (_: string): Promise<boolean> => indextExist),
+    indexCreate: mockFn(async (_: string): Promise<void> => {}),
+    bulk: mockFn(async (_: object[]): Promise<boolean> => true),
+  })
+}
+
+function createBackendMock(clientMock: MockObject<ElasticSearchClient>) {
+  const uuidProviderMock: UuidProvider = () => id
+
+  const options: ElasticSearchBackendOptions = {
+    node: 'node',
+    apiKey: 'apiKey',
+    indexPrefix,
+    flushInterval,
+  }
+
+  return new ElasticSearchBackend(options, clientMock, uuidProviderMock)
+}
+
+function createIndexName() {
+  const now = new Date()
+  return `${indexPrefix}-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}`
+}
+
+async function delay(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}

packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts

@@ -0,0 +1,102 @@
+import { v4 as uuidv4 } from 'uuid'
+
+import { LoggerBackend } from '../logger/interfaces'
+import {
+  ElasticSearchClient,
+  ElasticSearchClientOptions,
+} from './ElasticSearchClient'
+
+export interface ElasticSearchBackendOptions
+  extends ElasticSearchClientOptions {
+  flushInterval?: number
+  indexPrefix?: string
+}
+
+export type UuidProvider = () => string
+
+export class ElasticSearchBackend implements LoggerBackend {
+  private readonly buffer: string[]
+
+  constructor(
+    private readonly options: ElasticSearchBackendOptions,
+    private readonly client: ElasticSearchClient = new ElasticSearchClient(
+      options,
+    ),
+    private readonly uuidProvider: UuidProvider = uuidv4,
+  ) {
+    this.buffer = []
+    this.start()
+  }
+
+  public debug(message: string): void {
+    this.buffer.push(message)
+  }
+
+  public log(message: string): void {
+    this.buffer.push(message)
+  }
+
+  public warn(message: string): void {
+    this.buffer.push(message)
+  }
+
+  public error(message: string): void {
+    this.buffer.push(message)
+  }
+
+  private start(): void {
+    // eslint-disable-next-line @typescript-eslint/no-misused-promises
+    const interval = setInterval(async () => {
+      await this.flushLogs()
+    }, this.options.flushInterval ?? 10000)
+
+    // object will not require the Node.js event loop to remain active
+    // nodejs.org/api/timers.html#timers_timeout_unref
+    interval.unref()
+  }
+
+  private async flushLogs(): Promise<void> {
+    if (!this.buffer.length) {
+      return
+    }
+
+    try {
+      const index = await this.createIndex()
+
+      // copy buffer contents as it may change during async operations below
+      const batch = [...this.buffer]
+
+      //clear buffer
+      this.buffer.splice(0)
+
+      const documents = batch.map(
+        (log) =>
+          ({
+            id: this.uuidProvider(),
+            ...JSON.parse(log),
+          } as object),
+      )
+
+      const success = await this.client.bulk(documents, index)
+
+      if (!success) {
+        throw new Error('Failed to push liogs to Elastic Search node')
+      }
+    } catch (error) {
+      console.log(error)
+    }
+  }
+
+  private async createIndex(): Promise<string> {
+    const now = new Date()
+    const indexName = `${
+      this.options.indexPrefix ?? 'logs-'
+    }-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}`
+
+    const exist = await this.client.indexExist(indexName)
+    if (!exist) {
+      await this.client.indexCreate(indexName)
+    }
+    return indexName
+  }
+}

packages/backend-tools/src/elastic-search/ElasticSearchClient.ts

@@ -0,0 +1,40 @@
+import { Client } from '@elastic/elasticsearch'
+
+export interface ElasticSearchClientOptions {
+  node: string
+  apiKey: string
+}
+
+// hides complexity of ElastiSearch client API
+export class ElasticSearchClient {
+  private readonly client: Client
+
+  constructor(private readonly options: ElasticSearchClientOptions) {
+    this.client = new Client({
+      node: options.node,
+      auth: {
+        apiKey: options.apiKey,
+      },
+    })
+  }
+
+  public async bulk(documents: object[], index: string): Promise<boolean> {
+    const operations = documents.flatMap((doc: object) => [
+      { index: { _index: index } },
+      doc,
+    ])
+
+    const bulkResponse = await this.client.bulk({ refresh: true, operations })
+    return bulkResponse.errors
+  }
+
+  public async indexExist(index: string): Promise<boolean> {
+    return await this.client.indices.exists({ index })
+  }
+
+  public async indexCreate(index: string): Promise<void> {
+    await this.client.indices.create({
+      index,
+    })
+  }
+}

packages/backend-tools/src/index.ts

@@ -1,4 +1,9 @@
+export * from './elastic-search/ElasticSearchBackend'
 export * from './env'
+export * from './logger/interfaces'
+export * from './logger/LogFormatterEcs'
+export * from './logger/LogFormatterJson'
+export * from './logger/LogFormatterPretty'
 export * from './logger/Logger'
 export * from './rate-limit/RateLimiter'
 export * from './utils/assert'

packages/backend-tools/src/logger/LogFormatterEcs.ts

@@ -0,0 +1,31 @@
+import { LogEntry, LogFormatter } from './interfaces'
+import { toJSON } from './toJSON'
+
+// https://www.elastic.co/guide/en/ecs/8.11/ecs-reference.html
+export class LogFormatterEcs implements LogFormatter {
+  public format(entry: LogEntry): string {
+    const core = {
+      '@timestamp': entry.time.toISOString(),
+      log: {
+        level: entry.level,
+      },
+      service: {
+        name: entry.service,
+      },
+      message: entry.message,
+      error: entry.resolvedError
+        ? {
+            message: entry.resolvedError.error,
+            type: entry.resolvedError.name,
+            stack_trace: entry.resolvedError.stack,
+          }
+        : undefined,
+    }
+
+    try {
+      return toJSON({ ...core, parameters: entry.parameters })
+    } catch {
+      return toJSON({ ...core })
+    }
+  }
+}

packages/backend-tools/src/logger/LogFormatterJson.ts

@@ -0,0 +1,20 @@
+import { LogEntry, LogFormatter } from './interfaces'
+import { toJSON } from './toJSON'
+
+export class LogFormatterJson implements LogFormatter {
+  public format(entry: LogEntry): string {
+    const core = {
+      time: entry.time.toISOString(),
+      level: entry.level,
+      service: entry.service,
+      message: entry.message,
+      error: entry.resolvedError,
+    }
+
+    try {
+      return toJSON({ ...core, parameters: entry.parameters })
+    } catch {
+      return toJSON({ ...core })
+    }
+  }
+}