From e932b93d300291cd89f1aac022179a199f54b6df Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Wed, 24 Apr 2024 14:08:17 +0200 Subject: [PATCH 01/12] [L2B-4977] Refactor logger to allow for custom backends and formatters --- packages/backend-tools/package.json | 8 +- packages/backend-tools/src/index.ts | 5 + .../src/logger/ElasticSearchBackend.ts | 97 +++++++++ .../src/logger/LogFormatterEcs.ts | 30 +++ .../src/logger/LogFormatterJson.ts | 21 ++ .../src/logger/LogFormatterPretty.ts | 144 +++++++++++++ .../backend-tools/src/logger/Logger.test.ts | 44 ++-- packages/backend-tools/src/logger/Logger.ts | 192 +++++++----------- .../src/logger/formatLevelPretty.ts | 22 -- .../src/logger/formatParametersPretty.ts | 51 ----- .../src/logger/formatServicePretty.ts | 13 -- .../src/logger/formatTimePretty.ts | 27 --- .../backend-tools/src/logger/interfaces.ts | 39 ++++ packages/backend-tools/src/logger/toJSON.ts | 5 + yarn.lock | 64 +++++- 15 files changed, 507 insertions(+), 255 deletions(-) create mode 100644 packages/backend-tools/src/logger/ElasticSearchBackend.ts create mode 100644 packages/backend-tools/src/logger/LogFormatterEcs.ts create mode 100644 packages/backend-tools/src/logger/LogFormatterJson.ts create mode 100644 packages/backend-tools/src/logger/LogFormatterPretty.ts delete mode 100644 packages/backend-tools/src/logger/formatLevelPretty.ts delete mode 100644 packages/backend-tools/src/logger/formatParametersPretty.ts delete mode 100644 packages/backend-tools/src/logger/formatServicePretty.ts delete mode 100644 packages/backend-tools/src/logger/formatTimePretty.ts create mode 100644 packages/backend-tools/src/logger/interfaces.ts create mode 100644 packages/backend-tools/src/logger/toJSON.ts diff --git a/packages/backend-tools/package.json b/packages/backend-tools/package.json index 9d9f588e..9f2fab8c 100644 --- a/packages/backend-tools/package.json +++ b/packages/backend-tools/package.json @@ -30,12 +30,16 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@elastic/elasticsearch": "^8.13.1", "chalk": "^4.1.2", "dotenv": "^16.3.1", - "error-stack-parser": "^2.1.4" + "error-stack-parser": "^2.1.4", + "uuid": "^9.0.1" }, "devDependencies": { "@sinonjs/fake-timers": "^11.1.0", - "@types/sinonjs__fake-timers": "^8.1.2" + "@types/elasticsearch": "^5.0.43", + "@types/sinonjs__fake-timers": "^8.1.2", + "@types/uuid": "^9.0.8" } } diff --git a/packages/backend-tools/src/index.ts b/packages/backend-tools/src/index.ts index e2bc4377..37333619 100644 --- a/packages/backend-tools/src/index.ts +++ b/packages/backend-tools/src/index.ts @@ -1,4 +1,9 @@ export * from './env' +export * from './logger/ElasticSearchBackend' +export * from './logger/interfaces' +export * from './logger/LogFormatterEcs' +export * from './logger/LogFormatterJson' +export * from './logger/LogFormatterPretty' export * from './logger/Logger' export * from './rate-limit/RateLimiter' export * from './utils/assert' diff --git a/packages/backend-tools/src/logger/ElasticSearchBackend.ts b/packages/backend-tools/src/logger/ElasticSearchBackend.ts new file mode 100644 index 00000000..dd33c59d --- /dev/null +++ b/packages/backend-tools/src/logger/ElasticSearchBackend.ts @@ -0,0 +1,97 @@ +import { Client } from '@elastic/elasticsearch' +import { v4 as uuidv4 } from 'uuid'; + +import { LoggerBackend } from './interfaces'; + +export interface ElasticSearchBackendOptions { + node: string + apiKey: string + flushInterval?: number + indexPrefix?: string +} + +export class ElasticSearchBackend implements LoggerBackend { + + private readonly options: Required + private readonly buffer: string[] + private readonly client: Client; + + constructor(options: ElasticSearchBackendOptions) { + this.options = { + ...options, + flushInterval: options.flushInterval ?? 10000, + indexPrefix: options.indexPrefix ?? 'logs' + } + + this.client = new Client({ + node: options.node, + auth: { + apiKey: options.apiKey + } + }); + + this.buffer = [] + this.start() + } + + public debug(message: string): void { + this.buffer.push(message); + } + + public log(message: string): void { + this.buffer.push(message); + } + + public warn(message: string): void { + this.buffer.push(message); + } + + public error(message: string): void { + this.buffer.push(message); + } + + private start(): void { + // eslint-disable-next-line @typescript-eslint/no-misused-promises + setInterval(async () => { + await this.flushLogs() + }, this.options.flushInterval) + } + + private async flushLogs(): Promise { + try { + const index = await this.createIndex(); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const documents = this.buffer.map((message) => ({ + id: uuidv4(), + ...JSON.parse(message) + })) + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const operations = documents.flatMap(doc => [{ index: { _index: index } }, doc]) + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const bulkResponse = await this.client.bulk({ refresh: true, operations }) + + if (bulkResponse.errors) { + throw new Error('Failed to push liogs to Elastic Search node') + } + + } catch (error) { + console.log(error); + } + } + + private async createIndex(): Promise { + const now = new Date(); + const indexName = `${this.options.indexPrefix}-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` + + const exist = await this.client.indices.exists({ index: indexName }) + if (!exist) { + await this.client.indices.create({ + index: indexName + }) + } + return indexName + } +} diff --git a/packages/backend-tools/src/logger/LogFormatterEcs.ts b/packages/backend-tools/src/logger/LogFormatterEcs.ts new file mode 100644 index 00000000..1e7e3090 --- /dev/null +++ b/packages/backend-tools/src/logger/LogFormatterEcs.ts @@ -0,0 +1,30 @@ + +import { LogEntry, LogFormatter } from "./interfaces"; +import { toJSON } from "./toJSON"; + +// https://www.elastic.co/guide/en/ecs/8.11/ecs-reference.html +export class LogFormatterEcs implements LogFormatter { + public format(entry: LogEntry): string { + const core = { + "@timestamp": entry.time.toISOString(), + log: { + level: entry.level + }, + service: { + name: entry.service + }, + message: entry.message, + error: entry.resolvedError ? { + message: entry.resolvedError.error, + type: entry.resolvedError.name, + stack_trace: entry.resolvedError.stack + } : undefined, + } + + try { + return toJSON({ ...core, parameters: entry.parameters }) + } catch { + return toJSON({ ...core }) + } + } +} \ No newline at end of file diff --git a/packages/backend-tools/src/logger/LogFormatterJson.ts b/packages/backend-tools/src/logger/LogFormatterJson.ts new file mode 100644 index 00000000..1dd278e7 --- /dev/null +++ b/packages/backend-tools/src/logger/LogFormatterJson.ts @@ -0,0 +1,21 @@ + +import { LogEntry, LogFormatter } from "./interfaces"; +import { toJSON } from "./toJSON"; + +export class LogFormatterJson implements LogFormatter { + public format(entry: LogEntry): string { + const core = { + time: entry.time.toISOString(), + level: entry.level, + service: entry.service, + message: entry.message, + error: entry.resolvedError, + } + + try { + return toJSON({ ...core, parameters: entry.parameters }) + } catch { + return toJSON({ ...core }) + } + } +} \ No newline at end of file diff --git a/packages/backend-tools/src/logger/LogFormatterPretty.ts b/packages/backend-tools/src/logger/LogFormatterPretty.ts new file mode 100644 index 00000000..42c0c76f --- /dev/null +++ b/packages/backend-tools/src/logger/LogFormatterPretty.ts @@ -0,0 +1,144 @@ +import chalk from "chalk"; +import { inspect } from 'util' + +import { LogEntry, LogFormatter } from "./interfaces"; +import { LogLevel } from "./LogLevel"; +import { toJSON } from "./toJSON"; + +const STYLES = { + bigint: 'white', + boolean: 'white', + date: 'white', + module: 'white', + name: 'blue', + null: 'white', + number: 'white', + regexp: 'white', + special: 'white', + string: 'white', + symbol: 'white', + undefined: 'white', +} + +const INDENT_SIZE = 4 +const INDENT = ' '.repeat(INDENT_SIZE) + +export class LogFormatterPretty implements LogFormatter { + constructor(private readonly colors: boolean, private readonly utc: boolean) { + + } + + public format(entry: LogEntry): string { + const timeOut = this.formatTimePretty( + entry.time, + this.utc, + this.colors, + ) + const levelOut = this.formatLevelPretty(entry.level, this.colors) + const serviceOut = this.formatServicePretty(entry.service, this.colors) + const messageOut = entry.message ? ` ${entry.message}` : '' + const paramsOut = this.formatParametersPretty( + this.sanitize( + entry.resolvedError + ? { ...entry.resolvedError, ...entry.parameters } + : entry.parameters ?? {}, + ), + this.colors, + ) + + return `${timeOut} ${levelOut}${serviceOut}${messageOut}${paramsOut}` + } + + + private formatLevelPretty(level: LogLevel, colors: boolean): string { + if (colors) { + switch (level) { + case 'CRITICAL': + case 'ERROR': + return chalk.red(chalk.bold(level.toUpperCase())) + case 'WARN': + return chalk.yellow(chalk.bold(level.toUpperCase())) + case 'INFO': + return chalk.green(chalk.bold(level.toUpperCase())) + case 'DEBUG': + return chalk.magenta(chalk.bold(level.toUpperCase())) + case 'TRACE': + return chalk.gray(chalk.bold(level.toUpperCase())) + } + } + return level.toUpperCase() + } + + private formatTimePretty( + now: Date, + utc: boolean, + colors: boolean, + ): string { + const h = (utc ? now.getUTCHours() : now.getHours()) + .toString() + .padStart(2, '0') + const m = (utc ? now.getUTCMinutes() : now.getMinutes()) + .toString() + .padStart(2, '0') + const s = (utc ? now.getUTCSeconds() : now.getSeconds()) + .toString() + .padStart(2, '0') + const ms = (utc ? now.getUTCMilliseconds() : now.getMilliseconds()) + .toString() + .padStart(3, '0') + + let result = `${h}:${m}:${s}.${ms}` + if (utc) { + result += 'Z' + } + + return colors ? chalk.gray(result) : result + } + + private formatParametersPretty( + parameters: object, + colors: boolean, + ): string { + const oldStyles = inspect.styles + inspect.styles = STYLES + + const inspected = inspect(parameters, { + colors, + breakLength: 80 - INDENT_SIZE, + depth: 5, + }) + + inspect.styles = oldStyles + + if (inspected === '{}') { + return '' + } + + const indented = inspected + .split('\n') + .map((x) => INDENT + x) + .join('\n') + + if (colors) { + return '\n' + chalk.gray(indented) + } + return '\n' + indented + } + + private formatServicePretty( + service: string | undefined, + colors: boolean, + ): string { + if (!service) { + return '' + } + return colors + ? ` ${chalk.gray('[')} ${chalk.yellow(service)} ${chalk.gray(']')}` + : ` [ ${service} ]` + } + + private sanitize(parameters: object): object { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return JSON.parse(toJSON(parameters)) + } +} \ No newline at end of file diff --git a/packages/backend-tools/src/logger/Logger.test.ts b/packages/backend-tools/src/logger/Logger.test.ts index 0e8c41ff..1f4e13fb 100644 --- a/packages/backend-tools/src/logger/Logger.test.ts +++ b/packages/backend-tools/src/logger/Logger.test.ts @@ -1,11 +1,17 @@ import { expect, formatCompact, mockFn } from 'earl' -import { LogEntry, Logger } from './Logger' +import { LogEntry } from './interfaces' +import { LogFormatterJson } from './LogFormatterJson' +import { LogFormatterPretty } from './LogFormatterPretty' +import { Logger } from './Logger' describe(Logger.name, () => { it('calls correct backend', () => { const backend = createTestBackend() - const logger = new Logger({ backend, logLevel: 'TRACE' }) + const logger = new Logger({ backends: [{ + backend, + formatter: new LogFormatterJson() + }], logLevel: 'TRACE' }) logger.trace('foo') logger.debug('foo') @@ -25,9 +31,11 @@ describe(Logger.name, () => { it('supports bigint values in json output', () => { const backend = createTestBackend() const logger = new Logger({ - backend, + backends: [{ + backend, + formatter: new LogFormatterJson() + }], logLevel: 'TRACE', - format: 'json', getTime: () => new Date(0), utc: true, }) @@ -48,9 +56,11 @@ describe(Logger.name, () => { it('supports bigint values in pretty output', () => { const backend = createTestBackend() const logger = new Logger({ - backend, + backends: [{ + backend, + formatter: new LogFormatterPretty(false, true) + }], logLevel: 'TRACE', - format: 'pretty', getTime: () => new Date(0), utc: true, }) @@ -68,9 +78,11 @@ describe(Logger.name, () => { function setup() { const backend = createTestBackend() const baseLogger = new Logger({ - backend, + backends: [{ + backend, + formatter: new LogFormatterPretty(false, true) + }], logLevel: 'TRACE', - format: 'pretty', getTime: () => new Date(0), utc: true, }) @@ -92,7 +104,7 @@ describe(Logger.name, () => { const { backend, baseLogger } = setup() // eslint-disable-next-line @typescript-eslint/no-extraneous-class - class FooService {} + class FooService { } const instance = new FooService() const logger = baseLogger.for(instance) logger.info('hello') @@ -150,14 +162,14 @@ describe(Logger.name, () => { describe('error reporting', () => { const oldConsoleError = console.error beforeEach(() => { - console.error = () => {} + console.error = () => { } }) afterEach(() => { console.error = oldConsoleError }) it('reports error and critical error', () => { - const mockReportError = mockFn((_: unknown) => {}) + const mockReportError = mockFn((_: unknown) => { }) const logger = new Logger({ reportError: mockReportError, }) @@ -299,7 +311,7 @@ describe(Logger.name, () => { for (const [args, expected] of patterns) { it(`supports ${formatCompact(args, 60)}`, () => { - const mockReportError = mockFn((_: unknown) => {}) + const mockReportError = mockFn((_: unknown) => { }) const logger = new Logger({ reportError: mockReportError }) logger.error(...args) @@ -312,9 +324,9 @@ describe(Logger.name, () => { function createTestBackend() { return { - debug: mockFn((_: string): void => {}), - log: mockFn((_: string): void => {}), - warn: mockFn((_: string): void => {}), - error: mockFn((_: string): void => {}), + debug: mockFn((_: string): void => { }), + log: mockFn((_: string): void => { }), + warn: mockFn((_: string): void => { }), + error: mockFn((_: string): void => { }), } } diff --git a/packages/backend-tools/src/logger/Logger.ts b/packages/backend-tools/src/logger/Logger.ts index 0d0e666c..74403b3b 100644 --- a/packages/backend-tools/src/logger/Logger.ts +++ b/packages/backend-tools/src/logger/Logger.ts @@ -2,46 +2,15 @@ import { join } from 'path' import { assertUnreachable } from '../utils/assertUnreachable' -import { formatLevelPretty } from './formatLevelPretty' -import { formatParametersPretty } from './formatParametersPretty' -import { formatServicePretty } from './formatServicePretty' -import { formatTimePretty } from './formatTimePretty' +import { LogEntry, LoggerOptions } from './interfaces' +import { LogFormatterJson } from './LogFormatterJson' +import { LogFormatterPretty } from './LogFormatterPretty' import { LEVEL, LogLevel } from './LogLevel' import { LogThrottle, LogThrottleOptions } from './LogThrottle' import { parseLogArguments } from './parseLogArguments' -import { ResolvedError, resolveError } from './resolveError' +import { resolveError } from './resolveError' import { tagService } from './tagService' -export interface LoggerBackend { - debug(message: string): void - log(message: string): void - warn(message: string): void - error(message: string): void -} - -export interface LoggerOptions { - logLevel: LogLevel - service?: string - tag?: string - format: 'pretty' | 'json' - utc: boolean - colors: boolean - cwd: string - getTime: () => Date - reportError: (entry: LogEntry) => void - backend: LoggerBackend -} - -export interface LogEntry { - level: LogLevel - time: Date - service?: string - message?: string - error?: Error - resolvedError?: ResolvedError - parameters?: object -} - /** * [Read full documentation](https://github.com/l2beat/tools/blob/master/packages/backend-tools/src/logger/docs.md) */ @@ -56,40 +25,67 @@ export class Logger { logLevel: options.logLevel ?? 'INFO', service: options.service, tag: options.tag, - format: options.format ?? 'json', utc: options.utc ?? false, - colors: options.colors ?? false, cwd: options.cwd ?? process.cwd(), getTime: options.getTime ?? (() => new Date()), - reportError: options.reportError ?? (() => {}), - backend: options.backend ?? console, + reportError: options.reportError ?? (() => { }), + backends: options.backends ?? [{ + backend: console, + formatter: new LogFormatterJson() + }], } this.cwd = join(this.options.cwd, '/') this.logLevel = LEVEL[this.options.logLevel] } - static SILENT = new Logger({ logLevel: 'NONE', format: 'pretty' }) + static SILENT = new Logger({ logLevel: 'NONE' }) + static CRITICAL = new Logger({ logLevel: 'CRITICAL', - format: 'pretty', - colors: true, + backends: [{ + backend: console, + formatter: new LogFormatterPretty(true, false) + }] }) + static ERROR = new Logger({ logLevel: 'ERROR', - format: 'pretty', - colors: true, + backends: [{ + backend: console, + formatter: new LogFormatterPretty(true, false) + }] }) - static WARN = new Logger({ logLevel: 'WARN', format: 'pretty', colors: true }) - static INFO = new Logger({ logLevel: 'INFO', format: 'pretty', colors: true }) + + static WARN = new Logger({ + logLevel: 'WARN', + backends: [{ + backend: console, + formatter: new LogFormatterPretty(true, false) + }] + }) + + static INFO = new Logger({ + logLevel: 'INFO', + backends: [{ + backend: console, + formatter: new LogFormatterPretty(true, false) + }] + }) + static DEBUG = new Logger({ logLevel: 'DEBUG', - format: 'pretty', - colors: true, + backends: [{ + backend: console, + formatter: new LogFormatterPretty(true, false) + }] }) + static TRACE = new Logger({ logLevel: 'TRACE', - format: 'pretty', - colors: true, + backends: [{ + backend: console, + formatter: new LogFormatterPretty(true, false) + }] }) configure(options: Partial): Logger { @@ -189,78 +185,28 @@ export class Logger { } private printExactly(entry: LogEntry): void { - const output = - this.options.format === 'json' - ? this.formatJson(entry) - : this.formatPretty(entry) - - switch (entry.level) { - case 'CRITICAL': - case 'ERROR': - this.options.backend.error(output) - break - case 'WARN': - this.options.backend.warn(output) - break - case 'INFO': - this.options.backend.log(output) - break - case 'DEBUG': - case 'TRACE': - this.options.backend.debug(output) - break - case 'NONE': - break - default: - assertUnreachable(entry.level) - } - } - - private formatJson(entry: LogEntry): string { - const core = { - time: entry.time.toISOString(), - level: entry.level, - service: entry.service, - message: entry.message, - error: entry.resolvedError, - } - try { - return toJSON({ ...core, parameters: entry.parameters }) - } catch (e) { - this.error('Unable to log', e) - return JSON.stringify(core) - } - } - - private formatPretty(entry: LogEntry): string { - const timeOut = formatTimePretty( - entry.time, - this.options.utc, - this.options.colors, - ) - const levelOut = formatLevelPretty(entry.level, this.options.colors) - const serviceOut = formatServicePretty(entry.service, this.options.colors) - const messageOut = entry.message ? ` ${entry.message}` : '' - const paramsOut = formatParametersPretty( - sanitize( - entry.resolvedError - ? { ...entry.resolvedError, ...entry.parameters } - : entry.parameters ?? {}, - ), - this.options.colors, - ) - - return `${timeOut} ${levelOut}${serviceOut}${messageOut}${paramsOut}` + this.options.backends.forEach((backendOptions) => { + const output = backendOptions.formatter.format(entry) + switch (entry.level) { + case 'CRITICAL': + case 'ERROR': + backendOptions.backend.error(output) + break + case 'WARN': + backendOptions.backend.warn(output) + break + case 'INFO': + backendOptions.backend.log(output) + break + case 'DEBUG': + case 'TRACE': + backendOptions.backend.debug(output) + break + case 'NONE': + break + default: + assertUnreachable(entry.level) + } + }) } } - -function toJSON(parameters: {}): string { - return JSON.stringify(parameters, (k, v: unknown) => - typeof v === 'bigint' ? v.toString() : v, - ) -} - -function sanitize(parameters: {}): {} { - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return JSON.parse(toJSON(parameters)) -} diff --git a/packages/backend-tools/src/logger/formatLevelPretty.ts b/packages/backend-tools/src/logger/formatLevelPretty.ts deleted file mode 100644 index 50e4ea24..00000000 --- a/packages/backend-tools/src/logger/formatLevelPretty.ts +++ /dev/null @@ -1,22 +0,0 @@ -import chalk from 'chalk' - -import { LogLevel } from './LogLevel' - -export function formatLevelPretty(level: LogLevel, colors: boolean): string { - if (colors) { - switch (level) { - case 'CRITICAL': - case 'ERROR': - return chalk.red(chalk.bold(level.toUpperCase())) - case 'WARN': - return chalk.yellow(chalk.bold(level.toUpperCase())) - case 'INFO': - return chalk.green(chalk.bold(level.toUpperCase())) - case 'DEBUG': - return chalk.magenta(chalk.bold(level.toUpperCase())) - case 'TRACE': - return chalk.gray(chalk.bold(level.toUpperCase())) - } - } - return level.toUpperCase() -} diff --git a/packages/backend-tools/src/logger/formatParametersPretty.ts b/packages/backend-tools/src/logger/formatParametersPretty.ts deleted file mode 100644 index 64d2af76..00000000 --- a/packages/backend-tools/src/logger/formatParametersPretty.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* eslint-disable @typescript-eslint/ban-types */ -import chalk from 'chalk' -import { inspect } from 'util' - -const STYLES = { - bigint: 'white', - boolean: 'white', - date: 'white', - module: 'white', - name: 'blue', - null: 'white', - number: 'white', - regexp: 'white', - special: 'white', - string: 'white', - symbol: 'white', - undefined: 'white', -} - -const INDENT_SIZE = 4 -const INDENT = ' '.repeat(INDENT_SIZE) - -export function formatParametersPretty( - parameters: {}, - colors: boolean, -): string { - const oldStyles = inspect.styles - inspect.styles = STYLES - - const inspected = inspect(parameters, { - colors, - breakLength: 80 - INDENT_SIZE, - depth: 5, - }) - - inspect.styles = oldStyles - - if (inspected === '{}') { - return '' - } - - const indented = inspected - .split('\n') - .map((x) => INDENT + x) - .join('\n') - - if (colors) { - return '\n' + chalk.gray(indented) - } - return '\n' + indented -} diff --git a/packages/backend-tools/src/logger/formatServicePretty.ts b/packages/backend-tools/src/logger/formatServicePretty.ts deleted file mode 100644 index ede9d362..00000000 --- a/packages/backend-tools/src/logger/formatServicePretty.ts +++ /dev/null @@ -1,13 +0,0 @@ -import chalk from 'chalk' - -export function formatServicePretty( - service: string | undefined, - colors: boolean, -): string { - if (!service) { - return '' - } - return colors - ? ` ${chalk.gray('[')} ${chalk.yellow(service)} ${chalk.gray(']')}` - : ` [ ${service} ]` -} diff --git a/packages/backend-tools/src/logger/formatTimePretty.ts b/packages/backend-tools/src/logger/formatTimePretty.ts deleted file mode 100644 index fa065ca7..00000000 --- a/packages/backend-tools/src/logger/formatTimePretty.ts +++ /dev/null @@ -1,27 +0,0 @@ -import chalk from 'chalk' - -export function formatTimePretty( - now: Date, - utc: boolean, - colors: boolean, -): string { - const h = (utc ? now.getUTCHours() : now.getHours()) - .toString() - .padStart(2, '0') - const m = (utc ? now.getUTCMinutes() : now.getMinutes()) - .toString() - .padStart(2, '0') - const s = (utc ? now.getUTCSeconds() : now.getSeconds()) - .toString() - .padStart(2, '0') - const ms = (utc ? now.getUTCMilliseconds() : now.getMilliseconds()) - .toString() - .padStart(3, '0') - - let result = `${h}:${m}:${s}.${ms}` - if (utc) { - result += 'Z' - } - - return colors ? chalk.gray(result) : result -} diff --git a/packages/backend-tools/src/logger/interfaces.ts b/packages/backend-tools/src/logger/interfaces.ts new file mode 100644 index 00000000..b427eb30 --- /dev/null +++ b/packages/backend-tools/src/logger/interfaces.ts @@ -0,0 +1,39 @@ +import { LogLevel } from "./LogLevel" +import { ResolvedError } from "./resolveError" + +export interface LoggerBackend { + debug(message: string): void + log(message: string): void + warn(message: string): void + error(message: string): void + } + + export interface LogFormatter { + format(entry: LogEntry): string + } + + export interface LoggerBackendOptions { + backend: LoggerBackend, + formatter: LogFormatter, + } + + export interface LoggerOptions { + logLevel: LogLevel + service?: string + tag?: string + utc: boolean + cwd: string + getTime: () => Date + reportError: (entry: LogEntry) => void + backends: LoggerBackendOptions[] + } + + export interface LogEntry { + level: LogLevel + time: Date + service?: string + message?: string + error?: Error + resolvedError?: ResolvedError + parameters?: object + } \ No newline at end of file diff --git a/packages/backend-tools/src/logger/toJSON.ts b/packages/backend-tools/src/logger/toJSON.ts new file mode 100644 index 00000000..65c7e4b1 --- /dev/null +++ b/packages/backend-tools/src/logger/toJSON.ts @@ -0,0 +1,5 @@ +export function toJSON(parameters: object): string { + return JSON.stringify(parameters, (k, v: unknown) => + typeof v === 'bigint' ? v.toString() : v, + ) +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 724af882..dc40f5dc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -237,6 +237,26 @@ dependencies: "@jridgewell/trace-mapping" "0.3.9" +"@elastic/elasticsearch@^8.13.1": + version "8.13.1" + resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.13.1.tgz#0fbe8318cf7f21c599165bb901277428639d57ec" + integrity sha512-2G4Vu6OHw4+XTrp7AGIcOEezpPEoVrWg2JTK1v/exEKSLYquZkUdd+m4yOL3/UZ6bTj7hmXwrmYzW76BnLCkJQ== + dependencies: + "@elastic/transport" "~8.4.1" + tslib "^2.4.0" + +"@elastic/transport@~8.4.1": + version "8.4.1" + resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.4.1.tgz#f98c5a5e2156bcb3f01170b4aca7e7de4d8b61b8" + integrity sha512-/SXVuVnuU5b4dq8OFY4izG+dmGla185PcoqgK6+AJMpmOeY1QYVNbWtCwvSvoAANN5D/wV+EBU8+x7Vf9EphbA== + dependencies: + debug "^4.3.4" + hpagent "^1.0.0" + ms "^2.1.3" + secure-json-parse "^2.4.0" + tslib "^2.4.0" + undici "^5.22.1" + "@esbuild/android-arm64@0.17.19": version "0.17.19" resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.17.19.tgz#bafb75234a5d3d1b690e7c2956a599345e84a2fd" @@ -721,6 +741,11 @@ "@ethersproject/properties" "^5.7.0" "@ethersproject/strings" "^5.7.0" +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" @@ -912,6 +937,11 @@ resolved "https://registry.yarnpkg.com/@types/deep-diff/-/deep-diff-1.0.2.tgz#36f1291f0aead8aceb847cde6f07ae613a78ac4f" integrity sha512-WD2O611C7Oz7RSwKbSls8LaznKfWfXh39CHY9Amd8FhQz+NJRe20nUHhYpOopVq9M2oqDZd4L6AzqJIXQycxiA== +"@types/elasticsearch@^5.0.43": + version "5.0.43" + resolved "https://registry.yarnpkg.com/@types/elasticsearch/-/elasticsearch-5.0.43.tgz#a3bbf56922de2d0e24c6117e8de1c9b50029c3c6" + integrity sha512-N+MpzURpDCWd7zaJ7CE1aU+nBSeAABLhDE0lGodQ0LLftx7ku6hjTXLr9OAFZLSXiWL3Xxx8jts485ynrcm5NA== + "@types/is-ci@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/is-ci/-/is-ci-3.0.0.tgz#7e8910af6857601315592436f030aaa3ed9783c3" @@ -1002,6 +1032,11 @@ resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.2.tgz#bf2e02a3dbd4aecaf95942ecd99b7402e03fad5e" integrity sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA== +"@types/uuid@^9.0.8": + version "9.0.8" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.8.tgz#7545ba4fc3c003d6c756f651f3bf163d8f0f29ba" + integrity sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA== + "@types/yargs-parser@*": version "21.0.0" resolved "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz" @@ -2623,6 +2658,11 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== +hpagent@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-1.2.0.tgz#0ae417895430eb3770c03443456b8d90ca464903" + integrity sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA== + http-cache-semantics@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" @@ -3358,7 +3398,7 @@ ms@2.1.2: resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.0.0, ms@^2.1.1: +ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -3942,6 +3982,11 @@ scrypt-js@3.0.1: resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-3.0.1.tgz#d314a57c2aef69d1ad98a138a21fe9eafa9ee312" integrity sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA== +secure-json-parse@^2.4.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" + integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== + "semver@2 || 3 || 4 || 5": version "5.7.2" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" @@ -4396,6 +4441,11 @@ tslib@^1.8.1: resolved "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== +tslib@^2.4.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tsutils@^3.21.0: version "3.21.0" resolved "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz" @@ -4472,6 +4522,13 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" +undici@^5.22.1: + version "5.28.4" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" + integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + dependencies: + "@fastify/busboy" "^2.0.0" + unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" @@ -4503,6 +4560,11 @@ util-deprecate@^1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== +uuid@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz" From fe029b5ab51eace12a012d32d51b15faae51e5a7 Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Wed, 24 Apr 2024 14:21:46 +0200 Subject: [PATCH 02/12] [L2B-4977] Change node version to 18.x in CI --- .github/workflows/ci.yml | 8 ++++---- .github/workflows/release-canary.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96f72dff..9e345ac8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile - run: yarn build @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile - run: yarn build @@ -36,7 +36,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile - run: yarn build @@ -48,7 +48,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile - run: yarn build diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index 33a918fb..069829dc 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1c17534..34835104 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile From 19257a21fb28d69e25e78ab9163680c6b9d26159 Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Wed, 24 Apr 2024 14:24:42 +0200 Subject: [PATCH 03/12] [L2B-4977] Fix formatting --- .../src/logger/ElasticSearchBackend.ts | 181 ++++++------ .../src/logger/LogFormatterEcs.ts | 53 ++-- .../src/logger/LogFormatterJson.ts | 35 ++- .../src/logger/LogFormatterPretty.ts | 257 +++++++++--------- .../backend-tools/src/logger/Logger.test.ts | 59 ++-- packages/backend-tools/src/logger/Logger.ts | 72 +++-- .../backend-tools/src/logger/interfaces.ts | 74 ++--- packages/backend-tools/src/logger/toJSON.ts | 8 +- 8 files changed, 378 insertions(+), 361 deletions(-) diff --git a/packages/backend-tools/src/logger/ElasticSearchBackend.ts b/packages/backend-tools/src/logger/ElasticSearchBackend.ts index dd33c59d..03496779 100644 --- a/packages/backend-tools/src/logger/ElasticSearchBackend.ts +++ b/packages/backend-tools/src/logger/ElasticSearchBackend.ts @@ -1,97 +1,100 @@ import { Client } from '@elastic/elasticsearch' -import { v4 as uuidv4 } from 'uuid'; +import { v4 as uuidv4 } from 'uuid' -import { LoggerBackend } from './interfaces'; +import { LoggerBackend } from './interfaces' export interface ElasticSearchBackendOptions { - node: string - apiKey: string - flushInterval?: number - indexPrefix?: string + node: string + apiKey: string + flushInterval?: number + indexPrefix?: string } export class ElasticSearchBackend implements LoggerBackend { - - private readonly options: Required - private readonly buffer: string[] - private readonly client: Client; - - constructor(options: ElasticSearchBackendOptions) { - this.options = { - ...options, - flushInterval: options.flushInterval ?? 10000, - indexPrefix: options.indexPrefix ?? 'logs' - } - - this.client = new Client({ - node: options.node, - auth: { - apiKey: options.apiKey - } - }); - - this.buffer = [] - this.start() - } - - public debug(message: string): void { - this.buffer.push(message); - } - - public log(message: string): void { - this.buffer.push(message); - } - - public warn(message: string): void { - this.buffer.push(message); - } - - public error(message: string): void { - this.buffer.push(message); - } - - private start(): void { - // eslint-disable-next-line @typescript-eslint/no-misused-promises - setInterval(async () => { - await this.flushLogs() - }, this.options.flushInterval) - } - - private async flushLogs(): Promise { - try { - const index = await this.createIndex(); - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - const documents = this.buffer.map((message) => ({ - id: uuidv4(), - ...JSON.parse(message) - })) - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - const operations = documents.flatMap(doc => [{ index: { _index: index } }, doc]) - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - const bulkResponse = await this.client.bulk({ refresh: true, operations }) - - if (bulkResponse.errors) { - throw new Error('Failed to push liogs to Elastic Search node') - } - - } catch (error) { - console.log(error); - } - } - - private async createIndex(): Promise { - const now = new Date(); - const indexName = `${this.options.indexPrefix}-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` - - const exist = await this.client.indices.exists({ index: indexName }) - if (!exist) { - await this.client.indices.create({ - index: indexName - }) - } - return indexName - } + private readonly options: Required + private readonly buffer: string[] + private readonly client: Client + + constructor(options: ElasticSearchBackendOptions) { + this.options = { + ...options, + flushInterval: options.flushInterval ?? 10000, + indexPrefix: options.indexPrefix ?? 'logs', + } + + this.client = new Client({ + node: options.node, + auth: { + apiKey: options.apiKey, + }, + }) + + this.buffer = [] + this.start() + } + + public debug(message: string): void { + this.buffer.push(message) + } + + public log(message: string): void { + this.buffer.push(message) + } + + public warn(message: string): void { + this.buffer.push(message) + } + + public error(message: string): void { + this.buffer.push(message) + } + + private start(): void { + // eslint-disable-next-line @typescript-eslint/no-misused-promises + setInterval(async () => { + await this.flushLogs() + }, this.options.flushInterval) + } + + private async flushLogs(): Promise { + try { + const index = await this.createIndex() + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const documents = this.buffer.map((message) => ({ + id: uuidv4(), + ...JSON.parse(message), + })) + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const operations = documents.flatMap((doc) => [ + { index: { _index: index } }, + doc, + ]) + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + const bulkResponse = await this.client.bulk({ refresh: true, operations }) + + if (bulkResponse.errors) { + throw new Error('Failed to push liogs to Elastic Search node') + } + } catch (error) { + console.log(error) + } + } + + private async createIndex(): Promise { + const now = new Date() + const indexName = `${ + this.options.indexPrefix + }-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` + + const exist = await this.client.indices.exists({ index: indexName }) + if (!exist) { + await this.client.indices.create({ + index: indexName, + }) + } + return indexName + } } diff --git a/packages/backend-tools/src/logger/LogFormatterEcs.ts b/packages/backend-tools/src/logger/LogFormatterEcs.ts index 1e7e3090..d11ef5d6 100644 --- a/packages/backend-tools/src/logger/LogFormatterEcs.ts +++ b/packages/backend-tools/src/logger/LogFormatterEcs.ts @@ -1,30 +1,31 @@ - -import { LogEntry, LogFormatter } from "./interfaces"; -import { toJSON } from "./toJSON"; +import { LogEntry, LogFormatter } from './interfaces' +import { toJSON } from './toJSON' // https://www.elastic.co/guide/en/ecs/8.11/ecs-reference.html export class LogFormatterEcs implements LogFormatter { - public format(entry: LogEntry): string { - const core = { - "@timestamp": entry.time.toISOString(), - log: { - level: entry.level - }, - service: { - name: entry.service - }, - message: entry.message, - error: entry.resolvedError ? { - message: entry.resolvedError.error, - type: entry.resolvedError.name, - stack_trace: entry.resolvedError.stack - } : undefined, - } + public format(entry: LogEntry): string { + const core = { + '@timestamp': entry.time.toISOString(), + log: { + level: entry.level, + }, + service: { + name: entry.service, + }, + message: entry.message, + error: entry.resolvedError + ? { + message: entry.resolvedError.error, + type: entry.resolvedError.name, + stack_trace: entry.resolvedError.stack, + } + : undefined, + } - try { - return toJSON({ ...core, parameters: entry.parameters }) - } catch { - return toJSON({ ...core }) - } - } -} \ No newline at end of file + try { + return toJSON({ ...core, parameters: entry.parameters }) + } catch { + return toJSON({ ...core }) + } + } +} diff --git a/packages/backend-tools/src/logger/LogFormatterJson.ts b/packages/backend-tools/src/logger/LogFormatterJson.ts index 1dd278e7..c7a20abd 100644 --- a/packages/backend-tools/src/logger/LogFormatterJson.ts +++ b/packages/backend-tools/src/logger/LogFormatterJson.ts @@ -1,21 +1,20 @@ - -import { LogEntry, LogFormatter } from "./interfaces"; -import { toJSON } from "./toJSON"; +import { LogEntry, LogFormatter } from './interfaces' +import { toJSON } from './toJSON' export class LogFormatterJson implements LogFormatter { - public format(entry: LogEntry): string { - const core = { - time: entry.time.toISOString(), - level: entry.level, - service: entry.service, - message: entry.message, - error: entry.resolvedError, - } + public format(entry: LogEntry): string { + const core = { + time: entry.time.toISOString(), + level: entry.level, + service: entry.service, + message: entry.message, + error: entry.resolvedError, + } - try { - return toJSON({ ...core, parameters: entry.parameters }) - } catch { - return toJSON({ ...core }) - } - } -} \ No newline at end of file + try { + return toJSON({ ...core, parameters: entry.parameters }) + } catch { + return toJSON({ ...core }) + } + } +} diff --git a/packages/backend-tools/src/logger/LogFormatterPretty.ts b/packages/backend-tools/src/logger/LogFormatterPretty.ts index 42c0c76f..5300fbf1 100644 --- a/packages/backend-tools/src/logger/LogFormatterPretty.ts +++ b/packages/backend-tools/src/logger/LogFormatterPretty.ts @@ -1,144 +1,133 @@ -import chalk from "chalk"; +import chalk from 'chalk' import { inspect } from 'util' -import { LogEntry, LogFormatter } from "./interfaces"; -import { LogLevel } from "./LogLevel"; -import { toJSON } from "./toJSON"; +import { LogEntry, LogFormatter } from './interfaces' +import { LogLevel } from './LogLevel' +import { toJSON } from './toJSON' const STYLES = { - bigint: 'white', - boolean: 'white', - date: 'white', - module: 'white', - name: 'blue', - null: 'white', - number: 'white', - regexp: 'white', - special: 'white', - string: 'white', - symbol: 'white', - undefined: 'white', + bigint: 'white', + boolean: 'white', + date: 'white', + module: 'white', + name: 'blue', + null: 'white', + number: 'white', + regexp: 'white', + special: 'white', + string: 'white', + symbol: 'white', + undefined: 'white', } const INDENT_SIZE = 4 const INDENT = ' '.repeat(INDENT_SIZE) export class LogFormatterPretty implements LogFormatter { - constructor(private readonly colors: boolean, private readonly utc: boolean) { - - } - - public format(entry: LogEntry): string { - const timeOut = this.formatTimePretty( - entry.time, - this.utc, - this.colors, - ) - const levelOut = this.formatLevelPretty(entry.level, this.colors) - const serviceOut = this.formatServicePretty(entry.service, this.colors) - const messageOut = entry.message ? ` ${entry.message}` : '' - const paramsOut = this.formatParametersPretty( - this.sanitize( - entry.resolvedError - ? { ...entry.resolvedError, ...entry.parameters } - : entry.parameters ?? {}, - ), - this.colors, - ) - - return `${timeOut} ${levelOut}${serviceOut}${messageOut}${paramsOut}` - } - - - private formatLevelPretty(level: LogLevel, colors: boolean): string { - if (colors) { - switch (level) { - case 'CRITICAL': - case 'ERROR': - return chalk.red(chalk.bold(level.toUpperCase())) - case 'WARN': - return chalk.yellow(chalk.bold(level.toUpperCase())) - case 'INFO': - return chalk.green(chalk.bold(level.toUpperCase())) - case 'DEBUG': - return chalk.magenta(chalk.bold(level.toUpperCase())) - case 'TRACE': - return chalk.gray(chalk.bold(level.toUpperCase())) - } - } - return level.toUpperCase() - } - - private formatTimePretty( - now: Date, - utc: boolean, - colors: boolean, - ): string { - const h = (utc ? now.getUTCHours() : now.getHours()) - .toString() - .padStart(2, '0') - const m = (utc ? now.getUTCMinutes() : now.getMinutes()) - .toString() - .padStart(2, '0') - const s = (utc ? now.getUTCSeconds() : now.getSeconds()) - .toString() - .padStart(2, '0') - const ms = (utc ? now.getUTCMilliseconds() : now.getMilliseconds()) - .toString() - .padStart(3, '0') - - let result = `${h}:${m}:${s}.${ms}` - if (utc) { - result += 'Z' - } - - return colors ? chalk.gray(result) : result - } - - private formatParametersPretty( - parameters: object, - colors: boolean, - ): string { - const oldStyles = inspect.styles - inspect.styles = STYLES - - const inspected = inspect(parameters, { - colors, - breakLength: 80 - INDENT_SIZE, - depth: 5, - }) - - inspect.styles = oldStyles - - if (inspected === '{}') { - return '' - } - - const indented = inspected - .split('\n') - .map((x) => INDENT + x) - .join('\n') - - if (colors) { - return '\n' + chalk.gray(indented) - } - return '\n' + indented - } - - private formatServicePretty( - service: string | undefined, - colors: boolean, - ): string { - if (!service) { - return '' - } - return colors - ? ` ${chalk.gray('[')} ${chalk.yellow(service)} ${chalk.gray(']')}` - : ` [ ${service} ]` - } - - private sanitize(parameters: object): object { - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return JSON.parse(toJSON(parameters)) - } -} \ No newline at end of file + constructor( + private readonly colors: boolean, + private readonly utc: boolean, + ) {} + + public format(entry: LogEntry): string { + const timeOut = this.formatTimePretty(entry.time, this.utc, this.colors) + const levelOut = this.formatLevelPretty(entry.level, this.colors) + const serviceOut = this.formatServicePretty(entry.service, this.colors) + const messageOut = entry.message ? ` ${entry.message}` : '' + const paramsOut = this.formatParametersPretty( + this.sanitize( + entry.resolvedError + ? { ...entry.resolvedError, ...entry.parameters } + : entry.parameters ?? {}, + ), + this.colors, + ) + + return `${timeOut} ${levelOut}${serviceOut}${messageOut}${paramsOut}` + } + + private formatLevelPretty(level: LogLevel, colors: boolean): string { + if (colors) { + switch (level) { + case 'CRITICAL': + case 'ERROR': + return chalk.red(chalk.bold(level.toUpperCase())) + case 'WARN': + return chalk.yellow(chalk.bold(level.toUpperCase())) + case 'INFO': + return chalk.green(chalk.bold(level.toUpperCase())) + case 'DEBUG': + return chalk.magenta(chalk.bold(level.toUpperCase())) + case 'TRACE': + return chalk.gray(chalk.bold(level.toUpperCase())) + } + } + return level.toUpperCase() + } + + private formatTimePretty(now: Date, utc: boolean, colors: boolean): string { + const h = (utc ? now.getUTCHours() : now.getHours()) + .toString() + .padStart(2, '0') + const m = (utc ? now.getUTCMinutes() : now.getMinutes()) + .toString() + .padStart(2, '0') + const s = (utc ? now.getUTCSeconds() : now.getSeconds()) + .toString() + .padStart(2, '0') + const ms = (utc ? now.getUTCMilliseconds() : now.getMilliseconds()) + .toString() + .padStart(3, '0') + + let result = `${h}:${m}:${s}.${ms}` + if (utc) { + result += 'Z' + } + + return colors ? chalk.gray(result) : result + } + + private formatParametersPretty(parameters: object, colors: boolean): string { + const oldStyles = inspect.styles + inspect.styles = STYLES + + const inspected = inspect(parameters, { + colors, + breakLength: 80 - INDENT_SIZE, + depth: 5, + }) + + inspect.styles = oldStyles + + if (inspected === '{}') { + return '' + } + + const indented = inspected + .split('\n') + .map((x) => INDENT + x) + .join('\n') + + if (colors) { + return '\n' + chalk.gray(indented) + } + return '\n' + indented + } + + private formatServicePretty( + service: string | undefined, + colors: boolean, + ): string { + if (!service) { + return '' + } + return colors + ? ` ${chalk.gray('[')} ${chalk.yellow(service)} ${chalk.gray(']')}` + : ` [ ${service} ]` + } + + private sanitize(parameters: object): object { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return JSON.parse(toJSON(parameters)) + } +} diff --git a/packages/backend-tools/src/logger/Logger.test.ts b/packages/backend-tools/src/logger/Logger.test.ts index 1f4e13fb..c9e1f8d4 100644 --- a/packages/backend-tools/src/logger/Logger.test.ts +++ b/packages/backend-tools/src/logger/Logger.test.ts @@ -8,10 +8,15 @@ import { Logger } from './Logger' describe(Logger.name, () => { it('calls correct backend', () => { const backend = createTestBackend() - const logger = new Logger({ backends: [{ - backend, - formatter: new LogFormatterJson() - }], logLevel: 'TRACE' }) + const logger = new Logger({ + backends: [ + { + backend, + formatter: new LogFormatterJson(), + }, + ], + logLevel: 'TRACE', + }) logger.trace('foo') logger.debug('foo') @@ -31,10 +36,12 @@ describe(Logger.name, () => { it('supports bigint values in json output', () => { const backend = createTestBackend() const logger = new Logger({ - backends: [{ - backend, - formatter: new LogFormatterJson() - }], + backends: [ + { + backend, + formatter: new LogFormatterJson(), + }, + ], logLevel: 'TRACE', getTime: () => new Date(0), utc: true, @@ -56,10 +63,12 @@ describe(Logger.name, () => { it('supports bigint values in pretty output', () => { const backend = createTestBackend() const logger = new Logger({ - backends: [{ - backend, - formatter: new LogFormatterPretty(false, true) - }], + backends: [ + { + backend, + formatter: new LogFormatterPretty(false, true), + }, + ], logLevel: 'TRACE', getTime: () => new Date(0), utc: true, @@ -78,10 +87,12 @@ describe(Logger.name, () => { function setup() { const backend = createTestBackend() const baseLogger = new Logger({ - backends: [{ - backend, - formatter: new LogFormatterPretty(false, true) - }], + backends: [ + { + backend, + formatter: new LogFormatterPretty(false, true), + }, + ], logLevel: 'TRACE', getTime: () => new Date(0), utc: true, @@ -104,7 +115,7 @@ describe(Logger.name, () => { const { backend, baseLogger } = setup() // eslint-disable-next-line @typescript-eslint/no-extraneous-class - class FooService { } + class FooService {} const instance = new FooService() const logger = baseLogger.for(instance) logger.info('hello') @@ -162,14 +173,14 @@ describe(Logger.name, () => { describe('error reporting', () => { const oldConsoleError = console.error beforeEach(() => { - console.error = () => { } + console.error = () => {} }) afterEach(() => { console.error = oldConsoleError }) it('reports error and critical error', () => { - const mockReportError = mockFn((_: unknown) => { }) + const mockReportError = mockFn((_: unknown) => {}) const logger = new Logger({ reportError: mockReportError, }) @@ -311,7 +322,7 @@ describe(Logger.name, () => { for (const [args, expected] of patterns) { it(`supports ${formatCompact(args, 60)}`, () => { - const mockReportError = mockFn((_: unknown) => { }) + const mockReportError = mockFn((_: unknown) => {}) const logger = new Logger({ reportError: mockReportError }) logger.error(...args) @@ -324,9 +335,9 @@ describe(Logger.name, () => { function createTestBackend() { return { - debug: mockFn((_: string): void => { }), - log: mockFn((_: string): void => { }), - warn: mockFn((_: string): void => { }), - error: mockFn((_: string): void => { }), + debug: mockFn((_: string): void => {}), + log: mockFn((_: string): void => {}), + warn: mockFn((_: string): void => {}), + error: mockFn((_: string): void => {}), } } diff --git a/packages/backend-tools/src/logger/Logger.ts b/packages/backend-tools/src/logger/Logger.ts index 74403b3b..c89cd04d 100644 --- a/packages/backend-tools/src/logger/Logger.ts +++ b/packages/backend-tools/src/logger/Logger.ts @@ -28,11 +28,13 @@ export class Logger { utc: options.utc ?? false, cwd: options.cwd ?? process.cwd(), getTime: options.getTime ?? (() => new Date()), - reportError: options.reportError ?? (() => { }), - backends: options.backends ?? [{ - backend: console, - formatter: new LogFormatterJson() - }], + reportError: options.reportError ?? (() => {}), + backends: options.backends ?? [ + { + backend: console, + formatter: new LogFormatterJson(), + }, + ], } this.cwd = join(this.options.cwd, '/') this.logLevel = LEVEL[this.options.logLevel] @@ -42,50 +44,62 @@ export class Logger { static CRITICAL = new Logger({ logLevel: 'CRITICAL', - backends: [{ - backend: console, - formatter: new LogFormatterPretty(true, false) - }] + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) static ERROR = new Logger({ logLevel: 'ERROR', - backends: [{ - backend: console, - formatter: new LogFormatterPretty(true, false) - }] + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) static WARN = new Logger({ logLevel: 'WARN', - backends: [{ - backend: console, - formatter: new LogFormatterPretty(true, false) - }] + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) static INFO = new Logger({ logLevel: 'INFO', - backends: [{ - backend: console, - formatter: new LogFormatterPretty(true, false) - }] + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) static DEBUG = new Logger({ logLevel: 'DEBUG', - backends: [{ - backend: console, - formatter: new LogFormatterPretty(true, false) - }] + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) static TRACE = new Logger({ logLevel: 'TRACE', - backends: [{ - backend: console, - formatter: new LogFormatterPretty(true, false) - }] + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) configure(options: Partial): Logger { diff --git a/packages/backend-tools/src/logger/interfaces.ts b/packages/backend-tools/src/logger/interfaces.ts index b427eb30..d8404124 100644 --- a/packages/backend-tools/src/logger/interfaces.ts +++ b/packages/backend-tools/src/logger/interfaces.ts @@ -1,39 +1,39 @@ -import { LogLevel } from "./LogLevel" -import { ResolvedError } from "./resolveError" +import { LogLevel } from './LogLevel' +import { ResolvedError } from './resolveError' export interface LoggerBackend { - debug(message: string): void - log(message: string): void - warn(message: string): void - error(message: string): void - } - - export interface LogFormatter { - format(entry: LogEntry): string - } - - export interface LoggerBackendOptions { - backend: LoggerBackend, - formatter: LogFormatter, - } - - export interface LoggerOptions { - logLevel: LogLevel - service?: string - tag?: string - utc: boolean - cwd: string - getTime: () => Date - reportError: (entry: LogEntry) => void - backends: LoggerBackendOptions[] - } - - export interface LogEntry { - level: LogLevel - time: Date - service?: string - message?: string - error?: Error - resolvedError?: ResolvedError - parameters?: object - } \ No newline at end of file + debug(message: string): void + log(message: string): void + warn(message: string): void + error(message: string): void +} + +export interface LogFormatter { + format(entry: LogEntry): string +} + +export interface LoggerBackendOptions { + backend: LoggerBackend + formatter: LogFormatter +} + +export interface LoggerOptions { + logLevel: LogLevel + service?: string + tag?: string + utc: boolean + cwd: string + getTime: () => Date + reportError: (entry: LogEntry) => void + backends: LoggerBackendOptions[] +} + +export interface LogEntry { + level: LogLevel + time: Date + service?: string + message?: string + error?: Error + resolvedError?: ResolvedError + parameters?: object +} diff --git a/packages/backend-tools/src/logger/toJSON.ts b/packages/backend-tools/src/logger/toJSON.ts index 65c7e4b1..45b74ff2 100644 --- a/packages/backend-tools/src/logger/toJSON.ts +++ b/packages/backend-tools/src/logger/toJSON.ts @@ -1,5 +1,5 @@ export function toJSON(parameters: object): string { - return JSON.stringify(parameters, (k, v: unknown) => - typeof v === 'bigint' ? v.toString() : v, - ) -} \ No newline at end of file + return JSON.stringify(parameters, (k, v: unknown) => + typeof v === 'bigint' ? v.toString() : v, + ) +} From d8a975ae9e11cb4bf0b4ac3ea1fb94bebda4dda1 Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Wed, 24 Apr 2024 16:06:57 +0200 Subject: [PATCH 04/12] [L2B-4977] Clear buffer --- packages/backend-tools/src/logger/ElasticSearchBackend.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend-tools/src/logger/ElasticSearchBackend.ts b/packages/backend-tools/src/logger/ElasticSearchBackend.ts index 03496779..7bfc7f30 100644 --- a/packages/backend-tools/src/logger/ElasticSearchBackend.ts +++ b/packages/backend-tools/src/logger/ElasticSearchBackend.ts @@ -78,6 +78,8 @@ export class ElasticSearchBackend implements LoggerBackend { if (bulkResponse.errors) { throw new Error('Failed to push liogs to Elastic Search node') } + + this.buffer.splice(0) } catch (error) { console.log(error) } From 9e7e9c8582a5397921e75644bbe51576a36f24f3 Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Wed, 24 Apr 2024 20:23:24 +0200 Subject: [PATCH 05/12] Minor fixes --- .../backend-tools/src/logger/ElasticSearchBackend.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/backend-tools/src/logger/ElasticSearchBackend.ts b/packages/backend-tools/src/logger/ElasticSearchBackend.ts index 7bfc7f30..60b47139 100644 --- a/packages/backend-tools/src/logger/ElasticSearchBackend.ts +++ b/packages/backend-tools/src/logger/ElasticSearchBackend.ts @@ -57,9 +57,19 @@ export class ElasticSearchBackend implements LoggerBackend { } private async flushLogs(): Promise { + if (!this.buffer.length) { + return + } + try { const index = await this.createIndex() + // copy buffer contents as it may change during async operations below + const batch = [...this.buffer] + + //clear buffer + this.buffer.splice(0) + // eslint-disable-next-line @typescript-eslint/no-unsafe-return const documents = this.buffer.map((message) => ({ id: uuidv4(), @@ -78,8 +88,6 @@ export class ElasticSearchBackend implements LoggerBackend { if (bulkResponse.errors) { throw new Error('Failed to push liogs to Elastic Search node') } - - this.buffer.splice(0) } catch (error) { console.log(error) } From 80f1a6e0bfb3b43e60de61c6aea17ce67d81f11b Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Wed, 24 Apr 2024 23:04:49 +0200 Subject: [PATCH 06/12] Refactor ElasticSearchBackend to make it testable --- .../elastic-search/ElasticSearchBackend.ts | 98 ++++++++++++++++ .../src/elastic-search/ElasticSearchClient.ts | 40 +++++++ packages/backend-tools/src/index.ts | 2 +- .../src/logger/ElasticSearchBackend.ts | 110 ------------------ 4 files changed, 139 insertions(+), 111 deletions(-) create mode 100644 packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts create mode 100644 packages/backend-tools/src/elastic-search/ElasticSearchClient.ts delete mode 100644 packages/backend-tools/src/logger/ElasticSearchBackend.ts diff --git a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts new file mode 100644 index 00000000..eadd7d71 --- /dev/null +++ b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts @@ -0,0 +1,98 @@ +import { v4 as uuidv4 } from 'uuid' + +import { LoggerBackend } from '../logger/interfaces' +import { + ElasticSearchClient, + ElasticSearchClientOptions, +} from './ElasticSearchClient' + +export interface ElasticSearchBackendOptions + extends ElasticSearchClientOptions { + flushInterval?: number + indexPrefix?: string +} + +export type UuidProvider = () => string + +export class ElasticSearchBackend implements LoggerBackend { + private readonly buffer: string[] + + constructor( + private readonly options: ElasticSearchBackendOptions, + private readonly client: ElasticSearchClient = new ElasticSearchClient( + options, + ), + private readonly uuidProvider: UuidProvider = uuidv4, + ) { + this.buffer = [] + this.start() + } + + public debug(message: string): void { + this.buffer.push(message) + } + + public log(message: string): void { + this.buffer.push(message) + } + + public warn(message: string): void { + this.buffer.push(message) + } + + public error(message: string): void { + this.buffer.push(message) + } + + private start(): void { + // eslint-disable-next-line @typescript-eslint/no-misused-promises + setInterval(async () => { + await this.flushLogs() + }, this.options.flushInterval ?? 10000) + } + + private async flushLogs(): Promise { + if (!this.buffer.length) { + return + } + + try { + const index = await this.createIndex() + + // copy buffer contents as it may change during async operations below + const batch = [...this.buffer] + + //clear buffer + this.buffer.splice(0) + + const documents = batch.map( + (log) => + ({ + id: this.uuidProvider(), + ...JSON.parse(log), + } as object), + ) + + const success = await this.client.bulk(documents, index) + + if (!success) { + throw new Error('Failed to push liogs to Elastic Search node') + } + } catch (error) { + console.log(error) + } + } + + private async createIndex(): Promise { + const now = new Date() + const indexName = `${ + this.options.indexPrefix ?? 'logs-' + }-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` + + const exist = await this.client.indexExist(indexName) + if (!exist) { + await this.client.indexCreate(indexName) + } + return indexName + } +} diff --git a/packages/backend-tools/src/elastic-search/ElasticSearchClient.ts b/packages/backend-tools/src/elastic-search/ElasticSearchClient.ts new file mode 100644 index 00000000..b44252e4 --- /dev/null +++ b/packages/backend-tools/src/elastic-search/ElasticSearchClient.ts @@ -0,0 +1,40 @@ +import { Client } from '@elastic/elasticsearch' + +export interface ElasticSearchClientOptions { + node: string + apiKey: string +} + +// hides complexity of ElastiSearch client API +export class ElasticSearchClient { + private readonly client: Client + + constructor(private readonly options: ElasticSearchClientOptions) { + this.client = new Client({ + node: options.node, + auth: { + apiKey: options.apiKey, + }, + }) + } + + public async bulk(documents: object[], index: string): Promise { + const operations = documents.flatMap((doc: object) => [ + { index: { _index: index } }, + doc, + ]) + + const bulkResponse = await this.client.bulk({ refresh: true, operations }) + return bulkResponse.errors + } + + public async indexExist(index: string): Promise { + return await this.client.indices.exists({ index }) + } + + public async indexCreate(index: string): Promise { + await this.client.indices.create({ + index, + }) + } +} diff --git a/packages/backend-tools/src/index.ts b/packages/backend-tools/src/index.ts index 37333619..c1cc01d3 100644 --- a/packages/backend-tools/src/index.ts +++ b/packages/backend-tools/src/index.ts @@ -1,5 +1,5 @@ +export * from './elastic-search/ElasticSearchBackend' export * from './env' -export * from './logger/ElasticSearchBackend' export * from './logger/interfaces' export * from './logger/LogFormatterEcs' export * from './logger/LogFormatterJson' diff --git a/packages/backend-tools/src/logger/ElasticSearchBackend.ts b/packages/backend-tools/src/logger/ElasticSearchBackend.ts deleted file mode 100644 index 60b47139..00000000 --- a/packages/backend-tools/src/logger/ElasticSearchBackend.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { Client } from '@elastic/elasticsearch' -import { v4 as uuidv4 } from 'uuid' - -import { LoggerBackend } from './interfaces' - -export interface ElasticSearchBackendOptions { - node: string - apiKey: string - flushInterval?: number - indexPrefix?: string -} - -export class ElasticSearchBackend implements LoggerBackend { - private readonly options: Required - private readonly buffer: string[] - private readonly client: Client - - constructor(options: ElasticSearchBackendOptions) { - this.options = { - ...options, - flushInterval: options.flushInterval ?? 10000, - indexPrefix: options.indexPrefix ?? 'logs', - } - - this.client = new Client({ - node: options.node, - auth: { - apiKey: options.apiKey, - }, - }) - - this.buffer = [] - this.start() - } - - public debug(message: string): void { - this.buffer.push(message) - } - - public log(message: string): void { - this.buffer.push(message) - } - - public warn(message: string): void { - this.buffer.push(message) - } - - public error(message: string): void { - this.buffer.push(message) - } - - private start(): void { - // eslint-disable-next-line @typescript-eslint/no-misused-promises - setInterval(async () => { - await this.flushLogs() - }, this.options.flushInterval) - } - - private async flushLogs(): Promise { - if (!this.buffer.length) { - return - } - - try { - const index = await this.createIndex() - - // copy buffer contents as it may change during async operations below - const batch = [...this.buffer] - - //clear buffer - this.buffer.splice(0) - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - const documents = this.buffer.map((message) => ({ - id: uuidv4(), - ...JSON.parse(message), - })) - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - const operations = documents.flatMap((doc) => [ - { index: { _index: index } }, - doc, - ]) - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - const bulkResponse = await this.client.bulk({ refresh: true, operations }) - - if (bulkResponse.errors) { - throw new Error('Failed to push liogs to Elastic Search node') - } - } catch (error) { - console.log(error) - } - } - - private async createIndex(): Promise { - const now = new Date() - const indexName = `${ - this.options.indexPrefix - }-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` - - const exist = await this.client.indices.exists({ index: indexName }) - if (!exist) { - await this.client.indices.create({ - index: indexName, - }) - } - return indexName - } -} From 104e517f9d15247437b23ae51a5cfca5228b437d Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Thu, 25 Apr 2024 09:28:51 +0200 Subject: [PATCH 07/12] Add tests --- .../ElasticSearchBackend.test.ts | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts diff --git a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts new file mode 100644 index 00000000..52ab8145 --- /dev/null +++ b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts @@ -0,0 +1,91 @@ +import { MockObject, expect, mockFn, mockObject } from 'earl' +import { + ElasticSearchBackend, + ElasticSearchBackendOptions, + UuidProvider, +} from './ElasticSearchBackend' +import { ElasticSearchClient } from './ElasticSearchClient' + +const flushInterval = 10 +const id = 'some-id' +const indexPrefix = 'logs-' +const indexName = createIndexName() +const log = { + '@timestamp': '2024-04-24T21:02:30.916Z', + log: { + level: 'INFO', + }, + message: 'Update started', +} + +describe(ElasticSearchBackend.name, () => { + it("creates index if doesn't exist", async () => { + const clientMock = createClienMock(false) + const backendMock = createBackendMock(clientMock) + + backendMock.log(JSON.stringify(log)) + + // wait for log flus + await delay(flushInterval + 10) + + expect(clientMock.indexExist).toHaveBeenOnlyCalledWith(indexName) + expect(clientMock.indexCreate).toHaveBeenOnlyCalledWith(indexName) + }) + + it('does nothing if buffer is empty', async () => { + const clientMock = createClienMock(false) + const backendMock = createBackendMock(clientMock) + + // wait for log flush + await delay(flushInterval + 10) + + expect(clientMock.bulk).not.toHaveBeenCalled() + }) + + it('pushes logs to ES if there is something in the buffer', async () => { + const clientMock = createClienMock(false) + const backendMock = createBackendMock(clientMock) + + backendMock.log(JSON.stringify(log)) + + // wait for log flush + await delay(flushInterval + 10) + + expect(clientMock.bulk).toHaveBeenOnlyCalledWith( + [{ id, ...log }], + indexName, + ) + }) +}) + +function createClienMock(indextExist: boolean = true) { + return mockObject({ + indexExist: mockFn(async (_: string): Promise => indextExist), + indexCreate: mockFn(async (_: string): Promise => {}), + bulk: mockFn(async (_: object[]): Promise => true), + }) +} + +function createBackendMock(clientMock: MockObject) { + const uuidProviderMock: UuidProvider = () => id + + const options: ElasticSearchBackendOptions = { + node: 'node', + apiKey: 'apiKey', + indexPrefix, + flushInterval, + } + + return new ElasticSearchBackend(options, clientMock, uuidProviderMock) +} + +function createIndexName() { + const now = new Date() + return `${ + indexPrefix ?? 'logs-' + }-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` +} + +async function delay(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} From 8dd6eff21d7dd3246e9b3f8e80e86b3da7583429 Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Thu, 25 Apr 2024 09:34:03 +0200 Subject: [PATCH 08/12] Fix lint issues --- .../src/elastic-search/ElasticSearchBackend.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts index 52ab8145..8423c32a 100644 --- a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts +++ b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts @@ -1,4 +1,5 @@ -import { MockObject, expect, mockFn, mockObject } from 'earl' +import { expect, mockFn, MockObject, mockObject } from 'earl' + import { ElasticSearchBackend, ElasticSearchBackendOptions, @@ -34,6 +35,7 @@ describe(ElasticSearchBackend.name, () => { it('does nothing if buffer is empty', async () => { const clientMock = createClienMock(false) + // eslint-disable-next-line @typescript-eslint/no-unused-vars const backendMock = createBackendMock(clientMock) // wait for log flush @@ -58,7 +60,7 @@ describe(ElasticSearchBackend.name, () => { }) }) -function createClienMock(indextExist: boolean = true) { +function createClienMock(indextExist = true) { return mockObject({ indexExist: mockFn(async (_: string): Promise => indextExist), indexCreate: mockFn(async (_: string): Promise => {}), @@ -81,9 +83,7 @@ function createBackendMock(clientMock: MockObject) { function createIndexName() { const now = new Date() - return `${ - indexPrefix ?? 'logs-' - }-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` + return `${indexPrefix}-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` } async function delay(ms: number) { From a4cf6189e36caf4201da4c58958e6e2d1d9a4efe Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Thu, 25 Apr 2024 17:48:38 +0200 Subject: [PATCH 09/12] Update docs --- packages/backend-tools/src/logger/docs.md | 42 ++++++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/packages/backend-tools/src/logger/docs.md b/packages/backend-tools/src/logger/docs.md index 8081cf11..44ecb200 100644 --- a/packages/backend-tools/src/logger/docs.md +++ b/packages/backend-tools/src/logger/docs.md @@ -26,13 +26,11 @@ The logger can be configured using the following options, all of them optional: - `logLevel` - minimum level of messages that will be logged, defaults to `INFO`. See more in the [Levels](#levels) section. - `service` - name of the service (class) that is using the logger. See more in the [Services](#services) section. - `tag` - tag that is used to identify the logger. See more in the [Tags](#tags) section. -- `format` - either `pretty` or `json`. It is recommended to use the `pretty` format during development and the `json` format in production. Defaults to `json`. - `utc` - when set to true time is logged in UTC, otherwise in local time. Defaults to `false`. -- `colors` - when set to true colors are used in the `pretty` format, otherwise they are not. Defaults to `false`. - `cwd` - current working directory, used to shorten error stack traces. Defaults to `process.cwd()`. - `getTime` - callback that returns the current time. Defaults to `() => new Date()`. - `reportError` - callback called when a message is logged at the `ERROR` or `CRITICAL` level. See more in the [Error reporting](#error-reporting) section. -- `backend` - object that is used to log messages. Defaults to `console`. +- `backends` - a set of pairs ([backend](#backends) + [formatter](#formatters)) which define where and in what form logs are being outputed. Defaults to `console` and `pretty` formatter ### Services @@ -119,9 +117,27 @@ This is done using the following rules: - non-object arguments are stored as `parameters.value` or `parameters.values` depending on the number of such arguments - object arguments are merged into a single `parameters` object +## Backends + +Currently we support two backends + +- `console` - standard output to console +- `ElasticSearchBackend` - pushes logs ElasticSearch node (should be used together with [ECS formatter](#ecs)) + +## Formatters + +Along with each backend it is required to provide a formatter which will produce an output string for each log entry + ### Pretty -In this format every message is logged on one or more lines with another newline in between the messages. The first line contains the timestamp, log level, service, tag and the message. The following lines contain a representation of the parameters. +Type: `LogFormarretPretty` + +In this format every message is logged on one or more lines with another newline in between the messages. The first line contains the timestamp, log level, service, tag and the message. The following lines contain a representation of the parameters. This form is best suited for local development purposes. + +This formatter accepts two params: + +- `utc` - when set to true time is logged in UTC, otherwise in local time. Defaults to `false`. +- `colors` - when set to true colors are used in the `pretty` format, otherwise they are not. Defaults to `false`. Below is an example log output: @@ -142,7 +158,9 @@ Below is an example log output: ### JSON -In this format every message is logged on a single line as a single JSON object. The object contains the timestamp, log level, service, tag, message, error and parameters. +Type: `LogFormarretJson` + +In this format every message is logged on a single line as a single JSON object. The object contains the timestamp, log level, service, tag, message, error and parameters. This format is best suited for deploment environments. Below is an example log output: @@ -151,6 +169,20 @@ Below is an example log output: {"time":"2023-01-02T12:34:56.002Z","level":"ERROR","service":"PriceService:USD","message":"Error fetching prices","error":{"name":"Error","error":"429: You have been rate limited!","stack":["PriceService.fetchPrices (src/PriceService.ts:12:34)","TaxService.computeTaxes (src/TaxService.ts:56:78)"]}} ``` +### ECS + +Type: `LogFormarretEcs` + +In this format every message is logged on a single line as a single JSON object compatible with [ECS standard](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html). This format is best suited for deploment environments with ElastiSearch enabled + +Below is an example log output: + +``` +{"@timestamp":"2024-04-25T15:47:52.731Z","log":{"level":"INFO"},"service":{"name":"Application"},"message":"Log level","parameters":{"value":"INFO"}} +{"@timestamp":"2024-04-25T15:47:52.733Z","log":{"level":"INFO"},"service":{"name":"ApiServer"},"message":"Listening","parameters":{"port":3000}} +{"@timestamp":"2024-04-25T15:47:52.864Z","log":{"level":"INFO"},"service":{"name":"Database"},"message":"Migrations completed","parameters":{"version":"105"}} +``` + ## Error reporting It might be useful to connect the logger into an error reporting system. This can be done by providing a `reportError` callback that will be called when a message is logged at the `ERROR` or `CRITICAL` level. From f18c1b04b61f94399fa608bbb3e5225c9513fe4c Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Fri, 26 Apr 2024 08:37:43 +0200 Subject: [PATCH 10/12] Add changeset and gnerate new version --- packages/backend-tools/CHANGELOG.md | 6 ++++++ packages/backend-tools/package.json | 2 +- packages/discovery/CHANGELOG.md | 7 +++++++ packages/discovery/package.json | 4 ++-- packages/uif/CHANGELOG.md | 7 +++++++ packages/uif/package.json | 4 ++-- 6 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/backend-tools/CHANGELOG.md b/packages/backend-tools/CHANGELOG.md index f9476a18..896204d9 100644 --- a/packages/backend-tools/CHANGELOG.md +++ b/packages/backend-tools/CHANGELOG.md @@ -1,5 +1,11 @@ # @l2beat/backend-tools +## 0.6.0 + +### Minor Changes + +- Refactor logger to allow for multiple backends and formatters + ## 0.5.2 ### Patch Changes diff --git a/packages/backend-tools/package.json b/packages/backend-tools/package.json index 9f2fab8c..6cbb137e 100644 --- a/packages/backend-tools/package.json +++ b/packages/backend-tools/package.json @@ -1,7 +1,7 @@ { "name": "@l2beat/backend-tools", "description": "Common utilities for L2BEAT projects.", - "version": "0.5.2", + "version": "0.6.0", "license": "MIT", "repository": "https://github.com/l2beat/tools", "bugs": { diff --git a/packages/discovery/CHANGELOG.md b/packages/discovery/CHANGELOG.md index 1c604442..b85a4642 100644 --- a/packages/discovery/CHANGELOG.md +++ b/packages/discovery/CHANGELOG.md @@ -1,5 +1,12 @@ # @l2beat/discovery +## 0.47.2 + +### Patch Changes + +- Updated dependencies + - @l2beat/backend-tools@0.6.0 + ## 0.47.1 ### Patch Changes diff --git a/packages/discovery/package.json b/packages/discovery/package.json index b349b3cc..bcde1321 100644 --- a/packages/discovery/package.json +++ b/packages/discovery/package.json @@ -1,7 +1,7 @@ { "name": "@l2beat/discovery", "description": "L2Beat discovery - engine & tooling utilized for keeping an eye on L2s", - "version": "0.47.1", + "version": "0.47.2", "main": "dist/index.js", "types": "dist/index.d.ts", "bin": { @@ -19,7 +19,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@l2beat/backend-tools": "^0.5.1", + "@l2beat/backend-tools": "^0.6.0", "@l2beat/discovery-types": "^0.8.1", "@mradomski/fast-solidity-parser": "0.1.1", "chalk": "^4.1.2", diff --git a/packages/uif/CHANGELOG.md b/packages/uif/CHANGELOG.md index 1fe3ae26..f88b57f9 100644 --- a/packages/uif/CHANGELOG.md +++ b/packages/uif/CHANGELOG.md @@ -1,5 +1,12 @@ # @l2beat/uif +## 0.5.1 + +### Patch Changes + +- Updated dependencies + - @l2beat/backend-tools@0.6.0 + ## 0.5.0 ### Minor Changes diff --git a/packages/uif/package.json b/packages/uif/package.json index 58a891bc..5c228262 100644 --- a/packages/uif/package.json +++ b/packages/uif/package.json @@ -1,7 +1,7 @@ { "name": "@l2beat/uif", "description": "Universal Indexer Framework.", - "version": "0.5.0", + "version": "0.5.1", "license": "MIT", "repository": "https://github.com/l2beat/tools", "bugs": { @@ -30,7 +30,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@l2beat/backend-tools": "^0.5.0" + "@l2beat/backend-tools": "^0.6.0" }, "devDependencies": { "@sinonjs/fake-timers": "^11.1.0", From 5d376a16d90353d13249dc465f1e82dc95b12f4c Mon Sep 17 00:00:00 2001 From: maciekop-l2b Date: Fri, 26 Apr 2024 10:07:52 +0200 Subject: [PATCH 11/12] Fix interval issue --- .../src/elastic-search/ElasticSearchBackend.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts index eadd7d71..70005875 100644 --- a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts +++ b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts @@ -46,9 +46,13 @@ export class ElasticSearchBackend implements LoggerBackend { private start(): void { // eslint-disable-next-line @typescript-eslint/no-misused-promises - setInterval(async () => { + const interval = setInterval(async () => { await this.flushLogs() }, this.options.flushInterval ?? 10000) + + // object will not require the Node.js event loop to remain active + // nodejs.org/api/timers.html#timers_timeout_unref + interval.unref() } private async flushLogs(): Promise { @@ -70,7 +74,7 @@ export class ElasticSearchBackend implements LoggerBackend { ({ id: this.uuidProvider(), ...JSON.parse(log), - } as object), + }) as object, ) const success = await this.client.bulk(documents, index) From 2425220687ebe3c1df139e8eaaec357d287ba3b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Fri, 26 Apr 2024 10:28:08 +0200 Subject: [PATCH 12/12] fix formatting --- .../backend-tools/src/elastic-search/ElasticSearchBackend.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts index 70005875..f1b0245a 100644 --- a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts +++ b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts @@ -74,7 +74,7 @@ export class ElasticSearchBackend implements LoggerBackend { ({ id: this.uuidProvider(), ...JSON.parse(log), - }) as object, + } as object), ) const success = await this.client.bulk(documents, index)