fixed bug, added env variables (#712)

rotem-codefresh · web-flow · commit 2804065390a6 · 2021-08-17T16:32:28.000+03:00
* fixed bug, added env variables

* deleted json files
diff --git a/lib/interface/cli/commands/offline-logs/base.cmd.js b/lib/interface/cli/commands/offline-logs/base.cmd.js
@@ -1,46 +1,33 @@
-const yargs = require('yargs');
-const Command = require('../../Command');
+const yargs = require("yargs");
+const Command = require("../../Command");
 
 const command = new Command({
-    command: 'offline-logs',
-    root: true,
-    description: 'Manages offline logs',
-    webDocs: {
-        category: 'Logs category',
-        subCategory: 'Logs sub category',
-        title: 'Archives old logs to file or collection.',
-    },
-    builder: (yargs) => {
-        // set options which are used in both sub-commands
-        return yargs
-        .option('uri', {
-              describe: "Mongodb URI",
-              demandOption: true,
-              type: "string",
-            })
-        .option('db', {
-          describe: "Database name",
-          demandOption: true,
-          type: "string",
-            })
-        .option('collections', {
-          alias: "c",
-          describe: "Source collections names",
-          default: ["logs", "metadata"],
-          array: true,
-          type: "string",
-        })
-        .option('cutoffDate', {
-          alias: "cod",
-          describe:
-            "The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
-          demandOption: true,
-          type: "string",
-        });
-    },
-    handler: () => {
-        yargs.showHelp();
-    },
+  command: "offline-logs",
+  root: true,
+  description: "Manages offline logs",
+  webDocs: {
+    category: "Logs category",
+    subCategory: "Logs sub category",
+    title: "Archives old logs to file or collection.",
+  },
+  builder: (yargs) => {
+    // set options which are used in both sub-commands
+    return yargs
+      .env("RUNTIME_MONGO")
+      .option("uri", {
+        describe:
+          "Mongodb URI. If not provided, will be parsed from environment variables.",
+        type: "string",
+      })
+      .option("db", {
+        describe:
+          "Database name. If not provided, will be parsed from environment variables.",
+        type: "string",
+      });
+  },
+  handler: () => {
+    yargs.showHelp();
+  },
 });
 
 module.exports = command;
diff --git a/lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js b/lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js
@@ -2,21 +2,21 @@ const { MongoClient, ObjectId } = require("mongodb");
 const moment = require('moment')
 const Command = require('../../Command');
 const cmd = require('./base.cmd');
-const { objectIdFromDate } = require('./utils')
+const { objectIdFromDate, defaultCollections } = require('./utils')
 
 const offloadToCollection = async function(sourceDBObj, collection, targetDB, cutoffDate) {
   const sourceCollectionObj = sourceDBObj.collection(collection)
   const targetCollection = `archive-${collection}`
   
   const cutoffDateObj = moment(cutoffDate)
                         .add(1, 'days')
-                        .startOf("day").toDate()
+                        .startOf("day")
 
   if(!cutoffDateObj.isValid()){
     throw new Error('please enter a valid date in ISO 8601 format')
   }
 
-  const cutoffDateId = objectIdFromDate(cutoffDateObj)
+  const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate())
 
   var result = sourceCollectionObj.aggregate([
       { $match: { _id: { $lte: ObjectId(cutoffDateId) } } },
@@ -31,6 +31,7 @@ const offloadToCollection = async function(sourceDBObj, collection, targetDB, cu
   await result.toArray()
 
   if (!result.cursorState.killed){
+    console.info(`logs from '${collection}' were archived to '${targetCollection}'`)
     await sourceCollectionObj.deleteMany({_id: {$lte: ObjectId(cutoffDateId)}})
   }
   else {
@@ -53,21 +54,28 @@ const command = new Command({
           describe: "Target database name, if none inserted, db will be defined as target.",
           type: "string",
         })
-        .example('codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --c logs foo --cod "2021-07-08" '),
+    .option('cutoffDate', {
+      alias: "cod",
+      describe:
+        "The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
+      demandOption: true,
+      type: "string",
+    })
+    .example('codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --c logs foo --cod "2021-07-08" '),
     handler: async (argv) => {
       const {
         uri,
         db,
-        collections,
         targetDB,
         cutoffDate,
       } = argv
+
       const client = new MongoClient(uri);
       try{
         await client.connect()
         const failedCollections = [];
         const sourceDBObj = client.db(db);
-        const promises = collections.map( async (collection) => {
+        const promises = defaultCollections.map( async (collection) => {
           try{
             await offloadToCollection(sourceDBObj, collection, targetDB || db, cutoffDate);
           } catch (error) {
diff --git a/lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js b/lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js
@@ -3,8 +3,7 @@ const { MongoClient, ObjectId } = require("mongodb");
 const moment = require('moment')
 const Command = require('../../Command');
 const cmd = require('./base.cmd');
-const {objectIdFromDate, writeLogsToFile, checkRemnant} = require('./utils')
-const { boolean } = require("yargs");
+const {objectIdFromDate, writeLogsToFile, checkRemnant, defaultCollections} = require('./utils');
 
 const offloadToFile = async function(database, collection, chunkDuration, cutoffDate, path) {
   const collectionObj = database.collection(collection);
@@ -93,12 +92,34 @@ const command = new Command({
         default: 1,
         type: "number",
       })
+      .option('cutoffDate', {
+        alias: "cod",
+        describe:
+          "The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
+        demandOption: true,
+        type: "string",
+      })
       .option('path', {
         describe: "Directory path to which archive files will be saved.",
         default: ".",
         type: "string",
       })
-      .example('codefresh offline-logs offload-to-file --uri "mongodb://192.168.99.100:27017" --db logs --collections logs foo --cod "2021-07-08" --chdur 3 --path "./" '),
+      .option('collections', {
+        alias: 'c',
+        describe: "Source collections names",
+        default: defaultCollections,
+        array: true,
+        type: "string",
+        coerce(arg) {
+          for (const collection of arg) {
+            if (!collection.includes('logs') && !collection.includes('metadata')) {
+              throw new Error ('invalid collection name')
+            }
+            return arg
+          } 
+        }
+      })
+      .example('codefresh offline-logs offload-to-file --uri "mongodb://192.168.99.100:27017" --db logs --collections archive-logs --cod "2021-07-08" --chdur 3 --path "./" '),
     handler: async (argv) => {
       const {
         uri,
diff --git a/lib/interface/cli/commands/offline-logs/utils.js b/lib/interface/cli/commands/offline-logs/utils.js
@@ -1,24 +1,43 @@
-const moment = require('moment');
+const readline = require("readline");
+const moment = require("moment");
 
-const { join } = require('path');
-const fs = require('fs');
+const { join } = require("path");
+const fs = require("fs");
+
+const defaultCollections = ["logs", "metadata"];
 
 // converts date to objectId for filter purposes
 const objectIdFromDate = function (date) {
-	return Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
+  return Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
 };
 
-const writeLogsToFile = function(upperBound, lowerBound, collection, logsToArchive, path) {
-  const date = moment(upperBound).subtract(1, "days")
-  const fileDateRange = `${moment(lowerBound).format('YYYY-MM-DD')}-${date.format('YYYY-MM-DD')}`
+const writeLogsToFile = function (
+  upperBound,
+  lowerBound,
+  collection,
+  logsToArchive,
+  path
+) {
+  const date = moment(upperBound).subtract(1, "days");
+  const fileDateRange = `${moment(lowerBound).format(
+    "YYYY-MM-DD"
+  )}-${date.format("YYYY-MM-DD")}`;
   const fileName = `${fileDateRange}-${collection}.json`;
   const absPath = join(path, fileName);
   const data = JSON.stringify(logsToArchive);
   fs.writeFileSync(absPath, data);
+  console.info(`logs from collection '${collection}' were offloaded to '${absPath}'`)
+};
+
+const checkRemnant = function (lowerBound, cutoffDateObj) {
+  return Math.ceil(cutoffDateObj.diff(lowerBound, "days"));
 };
 
-const checkRemnant = function(lowerBound, cutoffDateObj) {
-  return Math.ceil(cutoffDateObj.diff(lowerBound , "days" ));
-}
 
-module.exports = {objectIdFromDate, writeLogsToFile, checkRemnant};
+
+module.exports = {
+  objectIdFromDate,
+  writeLogsToFile,
+  checkRemnant,
+  defaultCollections,
+};
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "codefresh",
-  "version": "0.76.0",
+  "version": "0.77.0",
   "description": "Codefresh command line utility",
   "main": "index.js",
   "preferGlobal": true,

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "codefresh",`
`3`		`- "version": "0.76.0",`
	`3`	`+ "version": "0.77.0",`
`4`	`4`	`"description": "Codefresh command line utility",`
`5`	`5`	`"main": "index.js",`
`6`	`6`	`"preferGlobal": true,`