RAINCATCH-652 - file-storage module initial implementation

.eslintrc

@@ -0,0 +1,37 @@
+{
+  "env": {
+    "node": true,
+    "browser": true,
+    "mocha": true
+  },
+  "extends": "eslint:recommended",
+  "rules": {
+    "array-callback-return": "warn",
+    "brace-style": ["error", "1tbs"],
+    "complexity": ["warn", 20],
+    "eqeqeq": "error",
+    "guard-for-in": "error",
+    "indent": ["error", 2],
+    "linebreak-style": ["error", "unix"],
+    "no-array-constructor": "error",
+    "no-console": "warn",
+    "no-lonely-if": "warn",
+    "no-loop-func": "warn",
+    "no-mixed-spaces-and-tabs": ["error"],
+    "no-nested-ternary": "error",
+    "no-spaced-func": "error",
+    "no-trailing-spaces": "error",
+    "semi": ["error", "always"],
+    "space-before-blocks": "error",
+    "space-before-function-paren": ["error", "never"],
+    "keyword-spacing": ["error"],
+    "curly": ["error", "all"]
+  },
+  "globals": {
+    "angular": false,
+    "$fh": false,
+    "FileTransfer": false,
+    "FileUploadOptions": false
+  }
+}
+

.gitignore

@@ -0,0 +1,3 @@
+node_modules
+npm-debug.log
+.idea

.travis.yml

@@ -0,0 +1,25 @@
+language: node_js
+node_js:
+  - '0.10'
+  - '4.4'
+sudo: false
+before_install:
+  - npm install -g [email protected]
+  - npm install -g [email protected]
+  - npm install -g grunt-cli
+install: npm install
+services:
+  - docker
+script:
+  - npm test
+  - bash <(curl https://gist.githubusercontent.com/raincatcher-bot/01ac4cdb3b0770bdb58489dbc17ed6b6/raw/6205a628c3616f6736fd866d5f0fba0a781ec1e4/sonarqube.sh)
+notifications:
+  email: false
+  slack:
+    on_success: change
+    on_failure: always
+    rooms:
+      secure: >-
+        qrcxApjaQtD/vHsvkApXF8KBPncxPfDZmQSSWktGKuReC5MjJjPSHn658/jRNVxzJfL7Bp0TvQmVsBFNnmlXrw97p7r9rM4zOJRFMRXHXO9q7pjS/bDiJAck5nrKMBmCQfkgy+TxIO6aKErso/T8VJSJJgAP8FCbtYGrkqcvHBZka0Migy1+hyJ56+KdogTlGMSyQqAQPLzFLcwr9v7Sf/xAY2Hok0WSkpdcH4AUld9p2N5U+IZ12E2Szb9GtZmsNqL/3OzqOOWomAguXsgzxz1Morg947d9g95EStO7TlwqYu0IKo8JljmKj6k6Mu7Q2KPViPEJfTxMe0F3OJj2pAoBrM4ZZrGmr8NdNbU5YiqiWly2Z5Jwx8P8gC7HEcX89YrBzvAen1MVFLAxjnaRsca2UdwdcPdNHi67Xl6tPm4u8JE3UYzh6Wj0JO5BPDLzzvaTpzqbZTX6EvydOrwByKfRj0cfHUYInhezYoVMQa5s3sRYOTReTOvxoyoLJFY8Tv3gmRC75JIFszDz6XaWjmP8IFEyPnjo6QSTF61c6jRHJwtXgyXKwih9zbVvNgm559bmIYZBnxC36zoWde/o3WynY0EKLxjqkKyu8Lccmo9172s0rQFlC2SCmI8bUaDjClHWNUXIsscUa6NfvTJ6oget978jtdTqdxQQDSq66Do=
+    on_pull_requests: false
+

Gruntfile.js

@@ -0,0 +1,27 @@
+module.exports = function(grunt){
+  'use strict';
+  require('load-grunt-tasks')(grunt);
+  grunt.initConfig({
+    eslint: {
+      options: {
+        configFile: '.eslintrc'
+      },
+      target: ["lib/**/*.js"]
+    },
+    mochify: {
+      options: {
+        reporter: 'spec'
+      },
+      unit: {
+        src: ['lib/**/*-spec.js']
+      }
+    },
+    wfmTemplate: {
+      module: "wfm.file.directives",
+      templateDir: "lib/angular/templates",
+      outputDir: "lib/angular/dist-templates"
+    }
+  });
+  grunt.registerTask('eslint', ['eslint']);
+  grunt.registerTask('test', ['mochify:unit']);
+};

LICENSE.txt

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2016 Red Hat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

lib/client/index.js

@@ -0,0 +1 @@
+// TODO Add code to support offline storage capabilities for client applications.

lib/cloud/file-storage/awsStorage.js

@@ -0,0 +1,91 @@
+var q = require('q');
+var s3 = require('s3');
+var path = require('path');
+
+var config;
+var awsClient;
+
+function init(storageConfiguration) {
+  config = storageConfiguration;
+  validateConfig();
+  awsClient = s3.createClient(config);
+}
+
+function writeFile(namespace, fileName, fileLocation) {
+  var file;
+  if (namespace) {
+    file = path.join(namespace, fileName);
+  } else {
+    file = fileName;
+  }
+
+  var params = {
+    localFile: fileLocation,
+    ACL: 'authenticated-read',
+    s3Params: {
+      Bucket: config.bucket,
+      Key: file
+    }
+  };
+
+  var deferred = q.defer();
+  var uploader = awsClient.uploadFile(params);
+  uploader.on('error', function(err) {
+    console.log(err);
+    deferred.reject(err.stack);
+  });
+  uploader.on('progress', function() {
+  });
+  uploader.on('end', function() {
+    deferred.resolve(fileName);
+  });
+  return deferred.promise;
+}
+
+function streamFile(namespace, fileName) {
+  var file;
+  if (namespace) {
+    file = path.join(namespace, fileName);
+  } else {
+    file = fileName;
+  }
+  var deferred = q.defer();
+  var paramsStream = {
+    Bucket: config.bucket,
+    Key: file
+  };
+  try {
+    var buffer = awsClient.downloadStream(paramsStream);
+    deferred.resolve(buffer);
+  } catch (error) {
+    console.log(error);
+    deferred.reject(error);
+  }
+  return deferred.promise;
+}
+
+function validateConfig() {
+  if (!config.s3Options) {
+    throw Error("Invalid configuration for s3 storage");
+  }
+  if (!config.s3Options.accessKeyId) {
+    throw Error("Invalid configuration for s3 storage: Access Key missing");
+  }
+  if (!config.s3Options.secretAccessKey) {
+    throw Error("Invalid configuration for s3 storage: secretAccessKeymissing");
+  }
+  if (!config.s3Options.region) {
+    throw Error("Invalid configuration for s3 storage: region missing");
+  }
+}
+
+/**
+ * Implements storage interface for AWS S3 storage
+ *
+ * @type {{init: function, writeFile: function, streamFile: function}}
+ */
+module.exports = {
+  init: init,
+  writeFile: writeFile,
+  streamFile: streamFile
+}

lib/cloud/file-storage/gridfsStorage.js

@@ -0,0 +1,73 @@
+var q = require('q');
+var fs = require('fs');
+var mongo = require('mongodb');
+var Grid = require('gridfs-stream');
+var MongoClient = require('mongodb').MongoClient
+
+var gfs;
+
+function init(storageConfiguration) {
+  var config = storageConfiguration;
+  if (!config || !config.mongoUrl) {
+    throw Error("Missing mongoUrl parameter for GridFs storage");
+  }
+  MongoClient.connect(config.mongoUrl, function(err, connection) {
+    if (err) {
+      return console.log("Cannot connect to mongodb server. Gridfs storage will be disabled");
+    }
+    console.log("Connected successfully to storage server");
+    gfs = Grid(connection, mongo);
+  });
+}
+
+function writeFile(namespace, fileName, fileLocation) {
+  if (!gfs) {
+    console.log("Gridfs not initialized");
+    return;
+  }
+  var deferred = q.defer();
+  var options = {
+    filename: fileName
+  };
+  if (namespace) {
+    options.root = namespace;
+  }
+  var writeStream = gfs.createWriteStream(options);
+  writeStream.on('error', function(err) {
+    console.log('An error occurred!', err);
+    deferred.reject(err);
+  });
+
+  writeStream.on('close', function(file) {
+    deferred.resolve(file);
+  });
+  fs.createReadStream(fileLocation).pipe(writeStream);
+  return deferred.promise;
+}
+
+function streamFile(namespace, fileName) {
+  if (!gfs) {
+    console.log("Gridfs not initialized");
+    return;
+  }
+  var deferred = q.defer();
+  var options = {
+    filename: fileName
+  };
+  if (namespace) {
+    options.root = namespace;
+  }
+
+  var readstream = gfs.createReadStream(options);
+  readstream.on('error', function(err) {
+    console.log('An error occurred when reading file from gridfs!', err);
+  });
+  deferred.resolve(readstream);
+  return deferred.promise;
+}
+
+module.exports = {
+  init: init,
+  writeFile: writeFile,
+  streamFile: streamFile
+};

lib/cloud/file-storage/index.js

@@ -0,0 +1,26 @@
+var gridfsStorage = require('./gridfsStorage');
+var s3Storage = require('./awsStorage');
+
+var client;
+
+/**
+ *
+ * Initialising the storage engine
+ *
+ * @param config
+ * @returns {StorageEngine}
+ */
+module.exports = function(config) {
+  if (!client) {
+    if (config.s3) {
+      client = s3Storage;
+      client.init(config.s3);
+    } else if (config.gridFs) {
+      client = gridfsStorage;
+      client.init(config.gridFs);
+    } else {
+      throw Error("Invalid configuration passed to the module", config);
+    }
+  }
+  return client;
+};

lib/cloud/index.js

@@ -0,0 +1,14 @@
+var mediatorSubscribers = require('./mediator-subscribers');
+var StorageEngine = require('./file-storage');
+
+/**
+ * Initialisation of the file storage module.
+ *
+ * @param {Mediator} mediator
+ * @param {object}  config -  module configuration
+ */
+module.exports = function(mediator, config) {
+  //Initialising the subscribers to topics that the module is interested in.
+  var storageEngine = StorageEngine(config);
+  return mediatorSubscribers.init(mediator, storageEngine);
+};

lib/cloud/mediator-subscribers/create.js

@@ -0,0 +1,34 @@
+var _ = require('lodash');
+var CONSTANTS = require('../../constants');
+
+
+/**
+ * Initialising a subscriber for creating a file.
+ *
+ * @param {object} fileEntityTopics
+ * @param fileClient
+ */
+module.exports = function createFileSubscriber(fileEntityTopics, storageClient) {
+
+  /**
+   * Handling the file upload
+   *
+   * @param {object} parameters
+   * @returns {*}
+   */
+  return function handleCreateFileTopic(parameters) {
+    var self = this;
+    parameters = parameters || {};
+    var fileCreateErrorTopic = fileEntityTopics.getTopic(CONSTANTS.TOPICS.CREATE, CONSTANTS.ERROR_PREFIX, parameters.topicUid);
+    if (!parameters.fileName || !parameters.location) {
+      return self.mediator.publish(fileCreateErrorTopic, new Error("Invalid Data To Create A File."));
+    }
+    storageClient.writeFile(parameters.namespace, parameters.fileName, parameters.location)
+      .then(function(response) {
+        self.mediator.publish(fileEntityTopics.getTopic(CONSTANTS.TOPICS.CREATE, CONSTANTS.DONE_PREFIX, parameters.topicUid), response);
+      })
+      .catch(function(error) {
+        self.mediator.publish(fileCreateErrorTopic, error);
+      });
+  };
+};