Add maester v header (#71)

Author: EmptyInfinity

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 3.0.1 (August 12, 2022)
+* Increase default values and limits for envs `API_RETRIES_COUNT` and `API_REQUEST_TIMEOUT`
+* Add `axiosReqWithRetryOnServerError` function to proceed with most commons use cases of making query to external API
+* Updated @elasticio/maester-client to v4.0.2
+
 ## 3.0.0 (July 25, 2022)
 * Updated method `uploadAttachment` from `AttachmentProcessor`
 * Added method `getMaesterAttachmentUrlById` which return url to Maester attachment

README.md

@@ -16,6 +16,7 @@
   - [JSON Schema Converter](#JSON-Schema-Converter)
   - [JSON Transformation](#JSON-Transformation)
   - [Attachment Processor](#Attachment-Processor)
+  - [External API](#external-api)
   - [Logger](#Logger)
 - [License](#license)
 
@@ -229,6 +230,56 @@ const result = await new AttachmentProcessor().getAttachment('http://example.com
 const result = await new AttachmentProcessor().getAttachment('http://example.com?storage_type=maester', 'stream'); // maester storage
 ```
 
+## External API
+
+### Environment variables
+* **API_RETRIES_COUNT** (defaults to 3): maximum amount of retries for 5xx errors. If server still responding 5xx, error will be thrown.
+* **API_REQUEST_TIMEOUT** (defaults to 15000): specifies the number of milliseconds before the request times out. If the request takes longer than timeout, the request will be aborted.
+
+- `axiosReqWithRetryOnServerError` (use with `.call()` to pass context, implement it as a method of class with `logger` and `cfg` (value of configuration object for current action) values in a constructor) - function which makes axios request by specified request-config, making logging and error handling:
+  1. If 5xx error occurred, it will be retried maximum `API_RETRIES_COUNT` times, each retry will be delayed with `exponentialSleep` function. 
+  2. If 4xx error occurred - error will be throw.
+  3. If action `cfg` has `doNotThrow404` set to true: 404 error won't be treated as error. <br>
+  Look on examples below.
+- `getErrMsg` - forms error message from axios-response.
+- `getRetryOptions` - return valid values for envs `API_RETRIES_COUNT` and `API_REQUEST_TIMEOUT`. If values are higher or lower the limit - they'll be overwritten by default values.
+- `sleep` - return promise which resolves after N time.
+- `exponentialDelay` - returns number of milliseconds depending to current retry. See [exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff) to explanation.
+- `exponentialSleep` - return promise which resolves after N time. Where N is number of milliseconds from `exponentialDelay` execution.
+
+Example for `axiosReqWithRetryOnServerError` function:
+```javascript
+class Client {
+  private logger: any;
+
+  private cfg: any;
+
+  constructor(emitter, cfg) {
+    this.logger = emitter.logger;
+    this.cfg = cfg;
+  }
+
+   public async apiRequest(options: AxiosRequestConfig): Promise<any> {
+    try {
+      const response = await axiosReq.axiosReqWithRetryOnServerError(this, requestOptions);
+      return response.data;
+    } catch (error) {
+      if (error.response?.status === 401) {
+       // update token
+      }
+      throw error;
+    }
+  }
+
+  public async getUserById(id) {
+    return this.apiRequest({
+      url: `/users/${id}`,
+      method: 'GET',
+    });
+  }
+}
+```
+
 ## Logger
 The built in logger uses Bunyan Logger as its base implementation. The available logger methods can be found [here](https://github.com/elasticio/component-commons-library/blob/master/src/logger/logger.ts#L19).
 

dist/spec-integration/attachmentProcessor.spec.js

@@ -32,8 +32,8 @@ const common_1 = require("./common");
 const AttachmentProcessor_1 = require("../src/attachment/AttachmentProcessor");
 chai_1.default.use(require('chai-as-promised'));
 describe('AttachmentProcessor', () => {
-    const objectStorage = new maester_client_1.ObjectStorage({ uri: common_1.creds.uri, jwtSecret: common_1.creds.token });
-    const attachmentProcessor = new AttachmentProcessor_1.AttachmentProcessor();
+    const objectStorage = new maester_client_1.ObjectStorage({ uri: common_1.creds.uri, jwtSecret: common_1.creds.token, userAgent: 'userAgent' });
+    const attachmentProcessor = new AttachmentProcessor_1.AttachmentProcessor('userAgent');
     describe('uploadAttachment', () => {
         it('uploadAttachment (/samples/sample.json)', async () => {
             const getFileAsStream = async () => fs_1.default.createReadStream(path_1.default.join(__dirname, './samples/sample.json'));

dist/spec/attachment/AttachmentProcessor.spec.js

@@ -21,7 +21,7 @@ const formStream = (dataString) => {
     return stream;
 };
 describe('AttachmentProcessor', () => {
-    const attachmentProcessor = new AttachmentProcessor_1.AttachmentProcessor();
+    const attachmentProcessor = new AttachmentProcessor_1.AttachmentProcessor('userAgent');
     describe('Steward', () => {
         it('Should successfully retrieve csv', async () => {
             const attachmentOptions = {

dist/src/attachment/AttachmentProcessor.js

@@ -10,14 +10,19 @@ const url_1 = require("url");
 const maester_client_1 = require("@elastic.io/maester-client");
 const interfaces_1 = require("@elastic.io/maester-client/dist/src/interfaces");
 const logger_1 = require("../logger/logger");
+const package_json_1 = __importDefault(require("../../package.json"));
 const logger = (0, logger_1.getLogger)();
+const maesterClientVersion = package_json_1.default.dependencies['@elastic.io/maester-client'];
 exports.STORAGE_TYPE_PARAMETER = 'storage_type';
 exports.DEFAULT_STORAGE_TYPE = 'steward';
 exports.MAESTER_OBJECT_ID_ENDPOINT = '/objects/';
 const { ELASTICIO_OBJECT_STORAGE_TOKEN = '', ELASTICIO_OBJECT_STORAGE_URI = '' } = process.env;
 const maesterCreds = { jwtSecret: ELASTICIO_OBJECT_STORAGE_TOKEN, uri: ELASTICIO_OBJECT_STORAGE_URI };
 const DEFAULT_ATTACHMENT_REQUEST_TIMEOUT = process.env.REQUEST_TIMEOUT ? parseInt(process.env.REQUEST_TIMEOUT, 10) : interfaces_1.REQUEST_TIMEOUT.maxValue; // 20s
 class AttachmentProcessor {
+    constructor(userAgent) {
+        this.userAgent = `${userAgent} maester-client/${maesterClientVersion}`;
+    }
     async getAttachment(url, responseType) {
         const storageType = this.getStorageTypeByUrl(url);
         const axConfig = {
@@ -26,6 +31,7 @@ class AttachmentProcessor {
             method: 'get',
             timeout: DEFAULT_ATTACHMENT_REQUEST_TIMEOUT,
             retry: interfaces_1.RETRIES_COUNT.defaultValue,
+            headers: { 'User-Agent': this.userAgent }
         };
         switch (storageType) {
             case 'steward': return this.getStewardAttachment(axConfig);
@@ -38,7 +44,7 @@ class AttachmentProcessor {
         const headers = {};
         if (contentType)
             headers[interfaces_1.CONTENT_TYPE_HEADER] = contentType;
-        const objectStorage = new maester_client_1.ObjectStorage(maesterCreds);
+        const objectStorage = new maester_client_1.ObjectStorage({ ...maesterCreds, userAgent: this.userAgent });
         return objectStorage.add(getAttachment, {
             headers,
             retryOptions: {
@@ -55,8 +61,7 @@ class AttachmentProcessor {
         return ax(axConfig);
     }
     async getMaesterAttachment({ url, responseType }) {
-        const client = new maester_client_1.StorageClient(maesterCreds);
-        const objectStorage = new maester_client_1.ObjectStorage(maesterCreds, client);
+        const objectStorage = new maester_client_1.ObjectStorage({ ...maesterCreds, userAgent: this.userAgent });
         const maesterAttachmentId = this.getMaesterAttachmentIdByUrl(url);
         const response = await objectStorage.getOne(maesterAttachmentId, { responseType });
         return { data: response };

dist/src/externalApi/index.js

@@ -1,16 +1,20 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getErrMsg = exports.exponentialSleep = exports.sleep = exports.exponentialDelay = exports.getRetryOptions = exports.API_REQUEST_TIMEOUT = exports.API_RETRIES_COUNT = void 0;
+exports.axiosReqWithRetryOnServerError = exports.getErrMsg = exports.exponentialSleep = exports.sleep = exports.exponentialDelay = exports.getRetryOptions = exports.API_REQUEST_TIMEOUT = exports.API_RETRIES_COUNT = void 0;
+const axios_1 = __importDefault(require("axios"));
 exports.API_RETRIES_COUNT = {
     minValue: 0,
-    defaultValue: 2,
-    maxValue: 4
+    defaultValue: 3,
+    maxValue: 5
 };
 const ENV_API_RETRIES_COUNT = process.env.API_RETRIES_COUNT ? parseInt(process.env.API_RETRIES_COUNT, 10) : exports.API_RETRIES_COUNT.defaultValue;
 exports.API_REQUEST_TIMEOUT = {
     minValue: 500,
-    defaultValue: 10000,
-    maxValue: 15000
+    defaultValue: 15000,
+    maxValue: 20000
 };
 const ENV_API_REQUEST_TIMEOUT = process.env.API_REQUEST_TIMEOUT ? parseInt(process.env.API_REQUEST_TIMEOUT, 10) : exports.API_REQUEST_TIMEOUT.defaultValue;
 /**
@@ -27,7 +31,7 @@ const getRetryOptions = () => ({
 });
 exports.getRetryOptions = getRetryOptions;
 const exponentialDelay = (currentRetries) => {
-    const maxBackoff = 10000;
+    const maxBackoff = 15000;
     const delay = (2 ** currentRetries) * 100;
     const randomSum = delay * 0.2 * Math.random(); // 0-20% of the delay
     return Math.min(delay + randomSum, maxBackoff);
@@ -46,3 +50,33 @@ const getErrMsg = (errResponse) => {
     return `Got error "${statusText}", status - "${status}", body: ${JSON.stringify(data)}`;
 };
 exports.getErrMsg = getErrMsg;
+const axiosReqWithRetryOnServerError = async function (options, axiosInstance = axios_1.default) {
+    var _a;
+    const { retriesCount, requestTimeout } = (0, exports.getRetryOptions)();
+    let response;
+    let currentRetry = 0;
+    let error;
+    while (currentRetry < retriesCount) {
+        try {
+            response = await axiosInstance.request({
+                ...options,
+                timeout: requestTimeout,
+                validateStatus: (status) => (status >= 200 && status < 300) || (status === 404 && this.cfg.doNotThrow404)
+            });
+            return response;
+        }
+        catch (err) {
+            error = err;
+            if (((_a = err.response) === null || _a === void 0 ? void 0 : _a.status) < 500) {
+                throw error;
+            }
+            this.logger.info(`URL: "${options.url}", method: ${options.method}, Error message: "${err.message}"`);
+            this.logger.error((0, exports.getErrMsg)(err.response));
+            this.logger.info(`Request failed, retrying(${1 + currentRetry})`);
+            await (0, exports.exponentialSleep)(currentRetry);
+            currentRetry++;
+        }
+    }
+    throw error;
+};
+exports.axiosReqWithRetryOnServerError = axiosReqWithRetryOnServerError;

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@elastic.io/component-commons-library",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "description": "Library for most common component development cases",
   "author": {
     "name": "elastic.io GmbH",
@@ -24,7 +24,7 @@
   },
   "dependencies": {
     "@elastic.io/jsonata-moment": "1.1.4",
-    "@elastic.io/maester-client": "4.0.0",
+    "@elastic.io/maester-client": "4.0.2",
     "@elastic.io/ntlm-client": "1.0.0",
     "async": "3.2.3",
     "axios": "0.27.2",

package.json

@@ -10,8 +10,8 @@ import { AttachmentProcessor } from '../src/attachment/AttachmentProcessor';
 chai.use(require('chai-as-promised'));
 
 describe('AttachmentProcessor', () => {
-  const objectStorage = new ObjectStorage({ uri: creds.uri, jwtSecret: creds.token });
-  const attachmentProcessor = new AttachmentProcessor();
+  const objectStorage = new ObjectStorage({ uri: creds.uri, jwtSecret: creds.token, userAgent: 'userAgent' });
+  const attachmentProcessor = new AttachmentProcessor('userAgent');
   describe('uploadAttachment', () => {
     it('uploadAttachment (/samples/sample.json)', async () => {
       const getFileAsStream = async () => fs.createReadStream(path.join(__dirname, './samples/sample.json'));

spec-integration/attachmentProcessor.spec.ts

@@ -19,7 +19,7 @@ const formStream = (dataString: string): Readable => {
 };
 
 describe('AttachmentProcessor', () => {
-  const attachmentProcessor = new AttachmentProcessor();
+  const attachmentProcessor = new AttachmentProcessor('userAgent');
   describe('Steward', () => {
     it('Should successfully retrieve csv', async () => {
       const attachmentOptions = {