From 5f963eef69ae062ef11a8b40a501102429c908f2 Mon Sep 17 00:00:00 2001 From: Wojciech Trocki Date: Thu, 30 Mar 2017 00:30:49 +0100 Subject: [PATCH 1/5] RAINCATCH-652 - file-storage module initial implementation --- .eslintrc | 37 ++++++++++++ .gitignore | 3 + .travis.yml | 25 ++++++++ Gruntfile.js | 27 +++++++++ LICENSE.txt | 20 +++++++ lib/client/index.js | 1 + lib/cloud/file-storage/awsStorage.js | 75 ++++++++++++++++++++++++ lib/cloud/file-storage/gridfsStorage.js | 30 ++++++++++ lib/cloud/file-storage/index.js | 26 ++++++++ lib/cloud/index.js | 14 +++++ lib/cloud/mediator-subscribers/create.js | 34 +++++++++++ lib/cloud/mediator-subscribers/get.js | 35 +++++++++++ lib/cloud/mediator-subscribers/index.js | 42 +++++++++++++ lib/constants.js | 12 ++++ package.json | 37 ++++++++++++ 15 files changed, 418 insertions(+) create mode 100644 .eslintrc create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 Gruntfile.js create mode 100644 LICENSE.txt create mode 100644 lib/client/index.js create mode 100644 lib/cloud/file-storage/awsStorage.js create mode 100644 lib/cloud/file-storage/gridfsStorage.js create mode 100644 lib/cloud/file-storage/index.js create mode 100644 lib/cloud/index.js create mode 100644 lib/cloud/mediator-subscribers/create.js create mode 100644 lib/cloud/mediator-subscribers/get.js create mode 100644 lib/cloud/mediator-subscribers/index.js create mode 100644 lib/constants.js create mode 100644 package.json diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..edbccd7 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,37 @@ +{ + "env": { + "node": true, + "browser": true, + "mocha": true + }, + "extends": "eslint:recommended", + "rules": { + "array-callback-return": "warn", + "brace-style": ["error", "1tbs"], + "complexity": ["warn", 20], + "eqeqeq": "error", + "guard-for-in": "error", + "indent": ["error", 2], + "linebreak-style": ["error", "unix"], + "no-array-constructor": "error", + "no-console": "warn", + "no-lonely-if": "warn", + "no-loop-func": "warn", + "no-mixed-spaces-and-tabs": ["error"], + "no-nested-ternary": "error", + "no-spaced-func": "error", + "no-trailing-spaces": "error", + "semi": ["error", "always"], + "space-before-blocks": "error", + "space-before-function-paren": ["error", "never"], + "keyword-spacing": ["error"], + "curly": ["error", "all"] + }, + "globals": { + "angular": false, + "$fh": false, + "FileTransfer": false, + "FileUploadOptions": false + } +} + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d73ef08 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules +npm-debug.log +.idea \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..efc4615 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,25 @@ +language: node_js +node_js: + - '0.10' + - '4.4' +sudo: false +before_install: + - npm install -g npm@2.13.5 + - npm install -g grunt@0.4.5 + - npm install -g grunt-cli +install: npm install +services: + - docker +script: + - npm test + - bash <(curl https://gist.githubusercontent.com/raincatcher-bot/01ac4cdb3b0770bdb58489dbc17ed6b6/raw/6205a628c3616f6736fd866d5f0fba0a781ec1e4/sonarqube.sh) +notifications: + email: false + slack: + on_success: change + on_failure: always + rooms: + secure: >- + qrcxApjaQtD/vHsvkApXF8KBPncxPfDZmQSSWktGKuReC5MjJjPSHn658/jRNVxzJfL7Bp0TvQmVsBFNnmlXrw97p7r9rM4zOJRFMRXHXO9q7pjS/bDiJAck5nrKMBmCQfkgy+TxIO6aKErso/T8VJSJJgAP8FCbtYGrkqcvHBZka0Migy1+hyJ56+KdogTlGMSyQqAQPLzFLcwr9v7Sf/xAY2Hok0WSkpdcH4AUld9p2N5U+IZ12E2Szb9GtZmsNqL/3OzqOOWomAguXsgzxz1Morg947d9g95EStO7TlwqYu0IKo8JljmKj6k6Mu7Q2KPViPEJfTxMe0F3OJj2pAoBrM4ZZrGmr8NdNbU5YiqiWly2Z5Jwx8P8gC7HEcX89YrBzvAen1MVFLAxjnaRsca2UdwdcPdNHi67Xl6tPm4u8JE3UYzh6Wj0JO5BPDLzzvaTpzqbZTX6EvydOrwByKfRj0cfHUYInhezYoVMQa5s3sRYOTReTOvxoyoLJFY8Tv3gmRC75JIFszDz6XaWjmP8IFEyPnjo6QSTF61c6jRHJwtXgyXKwih9zbVvNgm559bmIYZBnxC36zoWde/o3WynY0EKLxjqkKyu8Lccmo9172s0rQFlC2SCmI8bUaDjClHWNUXIsscUa6NfvTJ6oget978jtdTqdxQQDSq66Do= + on_pull_requests: false + diff --git a/Gruntfile.js b/Gruntfile.js new file mode 100644 index 0000000..3248fcb --- /dev/null +++ b/Gruntfile.js @@ -0,0 +1,27 @@ +module.exports = function(grunt){ + 'use strict'; + require('load-grunt-tasks')(grunt); + grunt.initConfig({ + eslint: { + options: { + configFile: '.eslintrc' + }, + target: ["lib/**/*.js"] + }, + mochify: { + options: { + reporter: 'spec' + }, + unit: { + src: ['lib/**/*-spec.js'] + } + }, + wfmTemplate: { + module: "wfm.file.directives", + templateDir: "lib/angular/templates", + outputDir: "lib/angular/dist-templates" + } + }); + grunt.registerTask('eslint', ['eslint']); + grunt.registerTask('test', ['mochify:unit']); +}; diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..6c554c4 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,20 @@ +The MIT License (MIT) +Copyright (c) 2016 Red Hat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/lib/client/index.js b/lib/client/index.js new file mode 100644 index 0000000..5d86756 --- /dev/null +++ b/lib/client/index.js @@ -0,0 +1 @@ +// TODO Add code to support offline storage capabilities for client applications. \ No newline at end of file diff --git a/lib/cloud/file-storage/awsStorage.js b/lib/cloud/file-storage/awsStorage.js new file mode 100644 index 0000000..3e7831a --- /dev/null +++ b/lib/cloud/file-storage/awsStorage.js @@ -0,0 +1,75 @@ +var q = require('q'); +var s3 = require('s3'); +var path = require('path'); + +var config; +var awsClient; + +function init(storageConfiguration) { + config = storageConfiguration; + awsClient = s3.createClient(config); +} + +function writeFile(namespace, fileName, fileLocation) { + var file; + if (namespace) { + file = path.join(namespace, fileName); + } else { + file = fileName; + } + + var params = { + localFile: fileLocation, + ACL: 'authenticated-read', + s3Params: { + Bucket: config.bucket, + Key: file + } + }; + + var deferred = q.defer(); + var uploader = awsClient.uploadFile(params); + uploader.on('error', function(err) { + console.log(err); + deferred.reject(err.stack); + }); + uploader.on('progress', function() { + }); + uploader.on('end', function() { + deferred.resolve(fileName); + }); + return deferred.promise; +} + +function streamFile(namespace, fileName) { + var file; + if (namespace) { + file = path.join(namespace, fileName); + } else { + file = fileName; + } + var deferred = q.defer(); + var paramsStream = { + Bucket: config.bucket, + Key: file + }; + try { + var buffer = awsClient.downloadStream(paramsStream); + deferred.resolve(buffer); + } + catch (error) { + console.log(error); + deferred.reject(error); + } + return deferred.promise; +} +/** + * Implements storage interface for AWS S3 storage + * + * @type {{init: function, writeFile: function, streamFile: function}} + */ +module.exports = { + init: init, + writeFile: writeFile, + streamFile: streamFile +} diff --git a/lib/cloud/file-storage/gridfsStorage.js b/lib/cloud/file-storage/gridfsStorage.js new file mode 100644 index 0000000..4c1f3fc --- /dev/null +++ b/lib/cloud/file-storage/gridfsStorage.js @@ -0,0 +1,30 @@ +var q = require('q'); +var config; + + +function init(storageConfiguration){ + config = storageConfiguration; + //TODO +} + +function writeFile(namespace, fileName, fileLocation) { + var deferred = q.defer(); + //TODO + // deferred.resolve(fileMeta); + // deferred.reject(error); + return deferred.promise; +} + +function streamFile(namespace, fileName) { + var deferred = q.defer(); + //TODO + // deferred.resolve(fileMeta); + // deferred.reject(error); + return deferred.promise; +} + +module.exports = { + init: init, + writeFile: writeFile, + streamFile: streamFile +} diff --git a/lib/cloud/file-storage/index.js b/lib/cloud/file-storage/index.js new file mode 100644 index 0000000..2e16b81 --- /dev/null +++ b/lib/cloud/file-storage/index.js @@ -0,0 +1,26 @@ +var gridfsStorage = require('./gridfsStorage'); +var s3Storage = require('./awsStorage'); + +var client; + +/** + * + * Initialising the storage engine + * + * @param config + * @returns {StorageEngine} + */ +module.exports = function(config) { + if (!client) { + if (config.s3) { + client = s3Storage; + client.init(config.s3); + } else if (config.gridFs) { + client = gridfsStorage; + client.init(config.gridFs); + } else { + throw Error("Invalid configuration passed to the module", config); + } + } + return client; +}; \ No newline at end of file diff --git a/lib/cloud/index.js b/lib/cloud/index.js new file mode 100644 index 0000000..b276b94 --- /dev/null +++ b/lib/cloud/index.js @@ -0,0 +1,14 @@ +var mediatorSubscribers = require('./mediator-subscribers'); +var StorageEngine = require('./file-storage'); + +/** + * Initialisation of the file storage module. + * + * @param {Mediator} mediator + * @param {object} config - module configuration + */ +module.exports = function(mediator, config) { + //Initialising the subscribers to topics that the module is interested in. + var storageEngine = StorageEngine(config); + return mediatorSubscribers.init(mediator, storageEngine); +}; \ No newline at end of file diff --git a/lib/cloud/mediator-subscribers/create.js b/lib/cloud/mediator-subscribers/create.js new file mode 100644 index 0000000..c735813 --- /dev/null +++ b/lib/cloud/mediator-subscribers/create.js @@ -0,0 +1,34 @@ +var _ = require('lodash'); +var CONSTANTS = require('../../constants'); + + +/** + * Initialising a subscriber for creating a file. + * + * @param {object} fileEntityTopics + * @param fileClient + */ +module.exports = function createFileSubscriber(fileEntityTopics, storageClient) { + + /** + * Handling the file upload + * + * @param {object} parameters + * @returns {*} + */ + return function handleCreateFileTopic(parameters) { + var self = this; + parameters = parameters || {}; + var fileCreateErrorTopic = fileEntityTopics.getTopic(CONSTANTS.TOPICS.CREATE, CONSTANTS.ERROR_PREFIX, parameters.topicUid); + if (!parameters.fileName || !parameters.location) { + return self.mediator.publish(fileCreateErrorTopic, new Error("Invalid Data To Create A File.")); + } + storageClient.writeFile(parameters.namespace, parameters.fileName, parameters.location) + .then(function(response) { + self.mediator.publish(fileEntityTopics.getTopic(CONSTANTS.TOPICS.CREATE, CONSTANTS.DONE_PREFIX, parameters.topicUid), response); + }) + .catch(function(error) { + self.mediator.publish(fileCreateErrorTopic, error); + }); + }; +}; \ No newline at end of file diff --git a/lib/cloud/mediator-subscribers/get.js b/lib/cloud/mediator-subscribers/get.js new file mode 100644 index 0000000..0024c73 --- /dev/null +++ b/lib/cloud/mediator-subscribers/get.js @@ -0,0 +1,35 @@ +var CONSTANTS = require('../../constants'); + +/** + * Initialising a subscriber for fetching file. + * + * @param {object} fileEntityTopics + * @param fileClient + */ +module.exports = function getFileSubscribers(fileEntityTopics, storageClient) { + + /** + * Handle fetching file binary data + * + * @param {object} parameters + * @returns {*} + */ + return function handleGetFileTopic(parameters) { + var self = this; + parameters = parameters || {}; + var errorTopic = fileEntityTopics.getTopic(CONSTANTS.TOPICS.GET, CONSTANTS.ERROR_PREFIX, parameters.topicUid); + + if (!parameters.fileName) { + return self.mediator.publish(errorTopic, new Error("Invalid Data To Create A File.")); + } + + storageClient.streamFile(parameters.namespace, parameters.fileName) + .then(function(fileBuffer) { + var doneTopic = fileEntityTopics.getTopic(CONSTANTS.TOPICS.GET, CONSTANTS.DONE_PREFIX, parameters.topicUid); + self.mediator.publish(doneTopic, fileBuffer); + }) + .catch(function(error) { + self.mediator.publish(errorTopic, error); + }); + }; +}; \ No newline at end of file diff --git a/lib/cloud/mediator-subscribers/index.js b/lib/cloud/mediator-subscribers/index.js new file mode 100644 index 0000000..2580fac --- /dev/null +++ b/lib/cloud/mediator-subscribers/index.js @@ -0,0 +1,42 @@ +var _ = require('lodash'); +var topicHandlers = { + create: require('./create'), + get: require('./get') +}; +var CONSTANTS = require('../../constants'); + +var MediatorTopicUtility = require('fh-wfm-mediator/lib/topics'); + +var fileSubscribers; + +module.exports = { + /** + * Initialisation of all the topics that this module is interested in. + * + * @param mediator + * @param storageEngine + */ + init: function(mediator, storageEngine) { + //If there is already a set of subscribers set up, then don't subscribe again. + if (fileSubscribers) { + return fileSubscribers; + } + + fileSubscribers = new MediatorTopicUtility(mediator); + fileSubscribers.prefix(CONSTANTS.TOPIC_PREFIX).entity(CONSTANTS.FILES_STORAGE_ENTITY_NAME); + + //Setting up subscribers to the file topics. + _.each(CONSTANTS.TOPICS, function(topicName) { + if (topicHandlers[topicName]) { + fileSubscribers.on(topicName, topicHandlers[topicName](fileSubscribers, storageEngine)); + } + }); + return fileSubscribers; + }, + tearDown: function() { + if (fileSubscribers) { + fileSubscribers.unsubscribeAll(); + fileSubscribers = null; + } + } +}; \ No newline at end of file diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..4575eaf --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,12 @@ +module.exports = { + TOPIC_PREFIX: "wfm", + FILES_STORAGE_ENTITY_NAME: "files-store", + TOPIC_SEPARATOR: ":", + ERROR_PREFIX: "error", + DONE_PREFIX: "done", + TOPIC_TIMEOUT: 1000, + TOPICS: { + CREATE: "create", + GET: "get" + } +}; \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..15ad8d0 --- /dev/null +++ b/package.json @@ -0,0 +1,37 @@ +{ + "name": "fh-wfm-file-storage", + "version": "0.1.0", + "description": "A WFM module providing file storage support", + "repository": { + "type": "git", + "url": "https://github.com/feedhenry-raincatcher/raincatcher-file-storage.git" + }, + "scripts": { + "test": "grunt test" + }, + "keywords": [ + "wfm", + "file" + ], + "author": "feedhenry-raincatcher@redhat.com", + "license": "MIT", + "dependencies": { + "debug": "^2.6.3", + "fh-wfm-mediator": "0.3.1", + "lodash": "4.7.0", + "q": "1.4.1", + "s3": "^4.4.0" + }, + "devDependencies": { + "grunt": "0.4.5", + "grunt-eslint": "18.1.0", + "grunt-mochify": "^0.3.0", + "load-grunt-tasks": "^3.5.2", + "mochify": "^2.18.1", + "proxyquireify": "^3.2.1", + "sinon": "^1.17.6", + "sinon-as-promised": "^4.0.2", + "chai": "^3.5.0", + "proxyquire": "^1.7.10" + } +} From 8d154d108cf5f8dfc0aa05f418bf3436cf789ae6 Mon Sep 17 00:00:00 2001 From: Wojciech Trocki Date: Fri, 31 Mar 2017 17:50:48 +0100 Subject: [PATCH 2/5] RAINCATCH-654 - gridfs support --- lib/cloud/file-storage/awsStorage.js | 20 +++++++- lib/cloud/file-storage/gridfsStorage.js | 62 ++++++++++++++++++++----- package.json | 1 + 3 files changed, 70 insertions(+), 13 deletions(-) diff --git a/lib/cloud/file-storage/awsStorage.js b/lib/cloud/file-storage/awsStorage.js index 3e7831a..57ee30e 100644 --- a/lib/cloud/file-storage/awsStorage.js +++ b/lib/cloud/file-storage/awsStorage.js @@ -7,6 +7,7 @@ var awsClient; function init(storageConfiguration) { config = storageConfiguration; + validateConfig(); awsClient = s3.createClient(config); } @@ -56,13 +57,28 @@ function streamFile(namespace, fileName) { try { var buffer = awsClient.downloadStream(paramsStream); deferred.resolve(buffer); - } - catch (error) { + } catch (error) { console.log(error); deferred.reject(error); } return deferred.promise; } + +function validateConfig() { + if (!config.s3Options) { + throw Error("Invalid configuration for s3 storage"); + } + if (!config.s3Options.accessKeyId) { + throw Error("Invalid configuration for s3 storage: Access Key missing"); + } + if (!config.s3Options.secretAccessKey) { + throw Error("Invalid configuration for s3 storage: secretAccessKeymissing"); + } + if (!config.s3Options.region) { + throw Error("Invalid configuration for s3 storage: region missing"); + } +} + /** * Implements storage interface for AWS S3 storage * diff --git a/lib/cloud/file-storage/gridfsStorage.js b/lib/cloud/file-storage/gridfsStorage.js index 4c1f3fc..8d70888 100644 --- a/lib/cloud/file-storage/gridfsStorage.js +++ b/lib/cloud/file-storage/gridfsStorage.js @@ -1,25 +1,65 @@ var q = require('q'); -var config; +var fs = require('fs'); +var mongo = require('mongodb'); +var Grid = require('gridfs-stream'); +var MongoClient = require('mongodb').MongoClient +var gfs; -function init(storageConfiguration){ - config = storageConfiguration; - //TODO +function init(storageConfiguration) { + var config = storageConfiguration; + MongoClient.connect(config.mongoUrl, function(err, connection) { + if (err) { + return console.log("Cannot connect to mongodb server. Gridfs storage will be disabled"); + } + console.log("Connected successfully to storage server"); + gfs = Grid(connection, mongo); + }); } function writeFile(namespace, fileName, fileLocation) { + if (!gfs) { + console.log("Gridfs not initialized"); + return; + } var deferred = q.defer(); - //TODO - // deferred.resolve(fileMeta); - // deferred.reject(error); + var options = { + filename: fileName + }; + if (namespace) { + options.root = namespace; + } + var writeStream = gfs.createWriteStream(options); + writeStream.on('error', function(err) { + console.log('An error occurred!', err); + deferred.reject(err); + }); + + writeStream.on('close', function(file) { + deferred.resolve(file); + }); + fs.createReadStream(fileLocation).pipe(writeStream); return deferred.promise; } function streamFile(namespace, fileName) { + if (!gfs) { + console.log("Gridfs not initialized"); + return; + } var deferred = q.defer(); - //TODO - // deferred.resolve(fileMeta); - // deferred.reject(error); + var options = { + filename: fileName + }; + if (namespace) { + options.root = namespace; + } + + var readstream = gfs.createReadStream(options); + readstream.on('error', function(err) { + console.log('An error occurred when reading file from gridfs!', err); + }); + deferred.resolve(readstream); return deferred.promise; } @@ -27,4 +67,4 @@ module.exports = { init: init, writeFile: writeFile, streamFile: streamFile -} +}; \ No newline at end of file diff --git a/package.json b/package.json index 15ad8d0..2a7e5d8 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "dependencies": { "debug": "^2.6.3", "fh-wfm-mediator": "0.3.1", + "gridfs-stream": "^1.1.1", "lodash": "4.7.0", "q": "1.4.1", "s3": "^4.4.0" From c8914689b998bd7ad380ec2209a1a4d8b804d803 Mon Sep 17 00:00:00 2001 From: Wojciech Trocki Date: Fri, 31 Mar 2017 18:29:56 +0100 Subject: [PATCH 3/5] RAINCATCH-539 - Adding missing validation for gridfs --- lib/cloud/file-storage/gridfsStorage.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cloud/file-storage/gridfsStorage.js b/lib/cloud/file-storage/gridfsStorage.js index 8d70888..a0dad8d 100644 --- a/lib/cloud/file-storage/gridfsStorage.js +++ b/lib/cloud/file-storage/gridfsStorage.js @@ -8,6 +8,9 @@ var gfs; function init(storageConfiguration) { var config = storageConfiguration; + if (!config || !config.mongoUrl) { + throw Error("Missing mongoUrl parameter for GridFs storage"); + } MongoClient.connect(config.mongoUrl, function(err, connection) { if (err) { return console.log("Cannot connect to mongodb server. Gridfs storage will be disabled"); From cfcd09194bc2a431461e524d4f080221d825ea1a Mon Sep 17 00:00:00 2001 From: Wojciech Trocki Date: Tue, 4 Apr 2017 14:43:06 +0100 Subject: [PATCH 4/5] Documentation for file storage module. --- README.md | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/README.md b/README.md index 3377bff..7657fdf 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,82 @@ # FeedHenry RainCatcher file storage [![Build Status](https://travis-ci.org/feedhenry-raincatcher/raincatcher-file-storage.png)](https://travis-ci.org/feedhenry-raincatcher/raincatcher-file-storage) A module for FeedHenry RainCatcher that manages file storage + +## Supported storage engines + +### AWS S3 storage + +Allows to store files in AWS S3 buckets. + +Options: + +``` +var storageConfig = { + s3: { + s3Options: { + accessKeyId: process.env.AWS_S3_ACCESS_KEY, + secretAccessKey: process.env.AWS_S3_ACCESS_KEY_SECRET, + region: process.env.AWS_S3_REGION + }, + bucket: "raincatcher-files" + } +} +require('fh-wfm-file-storage/lib/cloud')(mediator, storageConfig); +``` + +### Gridfs MongoDB storage + +Allows to store file in MongoDB database using Gridfs driver + +Options: +``` +var storageConfig = { + gridFs: { + mongoUrl: "mongodb://localhost:27017/files" + } +}; +require('fh-wfm-file-storage/lib/cloud')(mediator, storageConfig); +``` + +### Topic Subscriptions + +Topics are used in file module. + +#### wfm:files:create + +##### Description + +Save file to the storage + +##### Example + + +```javascript +var parameters = { + // File namespace (folder) + namespace:null, + fileName:"test", + location: "/tmp/file" + //Optional topic unique identifier + topicUid: "uniquetopicid" +} + +mediator.publish("wfm:files-store:create", parameters); +``` + +#### wfm:files:get +##### Description + +Retrieve file from the storage (BinaryStream) + +##### Example + +```javascript +var parameters = { + namespace:null, + fileName:"test", + topicUid: "uniquetopicid" +} + +mediator.publish("wfm:files-store:get", parameters); +``` From 32a88127a218b8a234133aa00884806f191587b1 Mon Sep 17 00:00:00 2001 From: Wojciech Trocki Date: Wed, 5 Apr 2017 12:27:20 +0100 Subject: [PATCH 5/5] Minor fixes --- lib/cloud/file-storage/awsStorage.js | 41 +++++++-- lib/cloud/file-storage/gridfsStorage.js | 105 +++++++++++++++--------- lib/constants.js | 1 + package.json | 6 +- 4 files changed, 104 insertions(+), 49 deletions(-) diff --git a/lib/cloud/file-storage/awsStorage.js b/lib/cloud/file-storage/awsStorage.js index 57ee30e..0a0d0f7 100644 --- a/lib/cloud/file-storage/awsStorage.js +++ b/lib/cloud/file-storage/awsStorage.js @@ -1,16 +1,39 @@ var q = require('q'); var s3 = require('s3'); var path = require('path'); +var CONSTANTS = require('../../constants'); var config; var awsClient; +/** + * Init module + * + * @param storageConfiguration configuration that should be passed from client + * + * var storageConfig = { + * s3: { + * s3Options: { + * accessKeyId: process.env.AWS_S3_ACCESS_KEY, + * secretAccessKey: process.env.AWS_S3_ACCESS_KEY_SECRET, + * region: process.env.AWS_S3_REGION + * }, + * bucket: "raincatcher-files" + * } + * } + */ function init(storageConfiguration) { config = storageConfiguration; validateConfig(); awsClient = s3.createClient(config); } - +/** + * Write file to the storage + * + * @param {string} namespace - location (folder) used to place saved file. + * @param {string} fileName - filename that should be unique within namespace + * @param {string} fileLocation - local filesystem location to the file + */ function writeFile(namespace, fileName, fileLocation) { var file; if (namespace) { @@ -21,7 +44,7 @@ function writeFile(namespace, fileName, fileLocation) { var params = { localFile: fileLocation, - ACL: 'authenticated-read', + ACL: CONSTANTS.AWS_BUCKET_PERMISSIONS, s3Params: { Bucket: config.bucket, Key: file @@ -34,14 +57,18 @@ function writeFile(namespace, fileName, fileLocation) { console.log(err); deferred.reject(err.stack); }); - uploader.on('progress', function() { - }); uploader.on('end', function() { deferred.resolve(fileName); }); return deferred.promise; } - +/** + * Retrieve file stream from storage + * + * @param {string} namespace - location (folder) used to place saved file. + * @param {string} fileName - filename that should be unique within namespace + * + */ function streamFile(namespace, fileName) { var file; if (namespace) { @@ -55,8 +82,8 @@ function streamFile(namespace, fileName) { Key: file }; try { - var buffer = awsClient.downloadStream(paramsStream); - deferred.resolve(buffer); + var stream = awsClient.downloadStream(paramsStream); + deferred.resolve(stream); } catch (error) { console.log(error); deferred.reject(error); diff --git a/lib/cloud/file-storage/gridfsStorage.js b/lib/cloud/file-storage/gridfsStorage.js index a0dad8d..4c1c6ca 100644 --- a/lib/cloud/file-storage/gridfsStorage.js +++ b/lib/cloud/file-storage/gridfsStorage.js @@ -4,66 +4,91 @@ var mongo = require('mongodb'); var Grid = require('gridfs-stream'); var MongoClient = require('mongodb').MongoClient -var gfs; +var gfsPromise; +/** + * Init module + * + * @param storageConfiguration configuration that should be passed from client + * + * var storageConfig = { + * bucket: "raincatcher-files" + * } + */ function init(storageConfiguration) { var config = storageConfiguration; if (!config || !config.mongoUrl) { throw Error("Missing mongoUrl parameter for GridFs storage"); } + var deferred = q.defer(); MongoClient.connect(config.mongoUrl, function(err, connection) { if (err) { - return console.log("Cannot connect to mongodb server. Gridfs storage will be disabled"); + console.log("Cannot connect to mongodb server. Gridfs storage will be disabled"); + return deferred.reject(err); } - console.log("Connected successfully to storage server"); - gfs = Grid(connection, mongo); + var gfs = Grid(connection, mongo); + deferred.resolve(gfs); }); + gfsPromise = deferred.promise; } +/** + * Write file to the storage + * + * @param {string} namespace - location (folder) used to place saved file. + * @param {string} fileName - filename that should be unique within namespace + * @param {string} fileLocation - local filesystem location to the file + */ function writeFile(namespace, fileName, fileLocation) { - if (!gfs) { - console.log("Gridfs not initialized"); + if(!gfsPromise){ return; } - var deferred = q.defer(); - var options = { - filename: fileName - }; - if (namespace) { - options.root = namespace; - } - var writeStream = gfs.createWriteStream(options); - writeStream.on('error', function(err) { - console.log('An error occurred!', err); - deferred.reject(err); - }); - - writeStream.on('close', function(file) { - deferred.resolve(file); + return gfsPromise.then(function(gfs) { + var deferred = q.defer(); + var options = { + filename: fileName + }; + if (namespace) { + options.root = namespace; + } + var writeStream = gfs.createWriteStream(options); + writeStream.on('error', function(err) { + console.log('An error occurred!', err); + deferred.reject(err); + }); + + writeStream.on('close', function(file) { + deferred.resolve(file); + }); + fs.createReadStream(fileLocation).pipe(writeStream); + return deferred.promise; }); - fs.createReadStream(fileLocation).pipe(writeStream); - return deferred.promise; } +/** + * Retrieve file stream from storage + * + * @param {string} namespace - location (folder) used to place saved file. + * @param {string} fileName - filename that should be unique within namespace + * + */ function streamFile(namespace, fileName) { - if (!gfs) { - console.log("Gridfs not initialized"); - return; - } - var deferred = q.defer(); - var options = { - filename: fileName - }; - if (namespace) { - options.root = namespace; - } - - var readstream = gfs.createReadStream(options); - readstream.on('error', function(err) { - console.log('An error occurred when reading file from gridfs!', err); + return gfsPromise.then(function(gfs) { + var deferred = q.defer(); + var options = { + filename: fileName + }; + if (namespace) { + options.root = namespace; + } + + var readstream = gfs.createReadStream(options); + readstream.on('error', function(err) { + console.log('An error occurred when reading file from gridfs!', err); + }); + deferred.resolve(readstream); + return deferred.promise; }); - deferred.resolve(readstream); - return deferred.promise; } module.exports = { diff --git a/lib/constants.js b/lib/constants.js index 4575eaf..d60634d 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -5,6 +5,7 @@ module.exports = { ERROR_PREFIX: "error", DONE_PREFIX: "done", TOPIC_TIMEOUT: 1000, + AWS_BUCKET_PERMISSIONS: 'authenticated-read', TOPICS: { CREATE: "create", GET: "get" diff --git a/package.json b/package.json index 2a7e5d8..c656eea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fh-wfm-file-storage", - "version": "0.1.0", + "version": "0.2.0", "description": "A WFM module providing file storage support", "repository": { "type": "git", @@ -17,13 +17,15 @@ "license": "MIT", "dependencies": { "debug": "^2.6.3", - "fh-wfm-mediator": "0.3.1", "gridfs-stream": "^1.1.1", "lodash": "4.7.0", + "mongodb": "^2.2.25", + "fh-wfm-mediator": "0.3.2", "q": "1.4.1", "s3": "^4.4.0" }, "devDependencies": { + "fh-wfm-mediator": "0.3.1", "grunt": "0.4.5", "grunt-eslint": "18.1.0", "grunt-mochify": "^0.3.0",