From 6dd9199d3b9da591cd3ba1136f8c22a5f3256088 Mon Sep 17 00:00:00 2001 From: Andrew Van Tassel Date: Wed, 7 Dec 2016 21:50:47 -0700 Subject: [PATCH 1/3] Added MailHops API for geo routing --- cli.js | 10 +++++++++- lib/mailUtilities.js | 17 +++++++++++++++++ lib/mailin.js | 26 ++++++++++++++++++++++---- package.json | 1 + test/fixtures/test-html-only.eml | 3 +++ test/fixtures/test.eml | 7 +++++-- test/mailUtilitiesSpec.js | 15 +++++++++++++++ test/mailinSpec.js | 4 ++++ 8 files changed, 76 insertions(+), 7 deletions(-) diff --git a/cli.js b/cli.js index 822a8c8..83a4f11 100755 --- a/cli.js +++ b/cli.js @@ -29,7 +29,9 @@ program.version(pkg.version) .option('-l, --log-file [file path]', "The log file path. Default to '/var/log/mailin.log'.") .option('--disable-dkim', 'Disable dkim checking. The dkim field in the webhook payload will be set to false.') .option('--disable-spf', 'Disable spf checking. The spf field in the webhook payload will be set to false.') + .option('--disable-webhook', 'Disable Webhook.') .option('--disable-spam-score', 'Disable spam score computation. The spamScore field in the webhook payload will be set to 0.0.') + .option('--mailhops-api-key', 'Get the mail route from MailHops API. https://mailhops.com') .option('--verbose', 'Set the logging level to verbose.') .option('--debug', 'Printout debug info such as the smtp commands.') .option('--profile', 'Enable basic memory usage profiling.') @@ -55,6 +57,8 @@ mailin.start({ disableDkim: program.disableDkim, disableSpf: program.disableSpf, disableSpamScore: program.disableSpamScore, + disableWebhook: program.disableWebhook, + mailhopsApiKey: program.mailhopsApiKey, verbose: program.verbose, debug: program.debug, profile: program.profile, @@ -63,11 +67,15 @@ mailin.start({ }, function (err) { if (err) process.exit(1); - logger.info('Webhook url: ' + mailin.configuration.webhook); + if(mailin.configuration.disableWebhook) + logger.info('Webhook is disabled'); + else + logger.info('Webhook url: ' + mailin.configuration.webhook); if (mailin.configuration.logFile) logger.info('Log file: ' + mailin.configuration.logFile); if (mailin.configuration.disableDkim) logger.info('Dkim checking is disabled'); if (mailin.configuration.disableSpf) logger.info('Spf checking is disabled'); if (mailin.configuration.disableSpamScore) logger.info('Spam score computation is disabled'); + if (mailin.configuration.mailhopsApiKey) logger.info('Using MailHops'); }); diff --git a/lib/mailUtilities.js b/lib/mailUtilities.js index a054489..ae0c600 100644 --- a/lib/mailUtilities.js +++ b/lib/mailUtilities.js @@ -6,6 +6,7 @@ var logger = require('./logger'); var path = require('path'); var Spamc = require('spamc'); var spamc = new Spamc(); +var MailHops = require("mailhops"); /* Verify Python availability. */ var isPythonAvailable = shell.which('python'); @@ -79,5 +80,21 @@ module.exports = { if (err) return callback(new Error('Unable to compute spam score.')); callback(null, result.spamScore); }); + }, + + getMailHops: function (rawEmail, mailhopsApiKey, callback) { + var mailhops = new MailHops({ + api_key: mailhopsApiKey, + app_name: "Node Mailin" + }); + var headerText = rawEmail.substring(0,rawEmail.indexOf('\n\n')); + var ips = mailhops.getIPsFromHeader(headerText); + if(!ips) + return callback(new Error('Unable to get MailHops.')); + mailhops.lookup(ips,function(err, result, body){ + if (err) logger.error(err); + if (err) return callback(new Error('Unable to get MailHops.')); + callback(null, body); + }); } }; diff --git a/lib/mailin.js b/lib/mailin.js index c4fa3e0..b944b56 100644 --- a/lib/mailin.js +++ b/lib/mailin.js @@ -33,6 +33,7 @@ var Mailin = function () { disableDkim: false, disableSpf: false, disableSpamScore: false, + mailhopsApiKey: false, verbose: false, debug: false, logLevel: 'info', @@ -209,16 +210,18 @@ Mailin.prototype.start = function (options, callback) { validateDkim(connection, rawEmail), validateSpf(connection), computeSpamScore(connection, rawEmail), + getMailHops(connection, rawEmail), parseEmail(connection) ]); }) - .spread(function (rawEmail, isDkimValid, isSpfValid, spamScore, parsedEmail) { + .spread(function (rawEmail, isDkimValid, isSpfValid, spamScore, mailHops, parsedEmail) { return Promise.all([ connection, rawEmail, isDkimValid, isSpfValid, spamScore, + mailHops, parsedEmail, detectLanguage(connection, parsedEmail.text) ]); @@ -283,6 +286,19 @@ Mailin.prototype.start = function (options, callback) { }); } + function getMailHops(connection, rawEmail) { + if (configuration.mailhopsApiKey===false) { + return Promise.resolve(false); + } + + return mailUtilities.getMailHops(rawEmail, configuration.mailhopsApiKey) + .catch(function (err) { + logger.error(connection.id + ' Unable to get MailHops.'); + logger.error(err); + return false; + }); + } + function parseEmail(connection) { return new Promise(function (resolve) { logger.verbose(connection.id + ' Parsing email.'); @@ -335,12 +351,13 @@ Mailin.prototype.start = function (options, callback) { return language; } - function finalizeMessage(connection, rawEmail, isDkimValid, isSpfValid, spamScore, parsedEmail, language) { + function finalizeMessage(connection, rawEmail, isDkimValid, isSpfValid, spamScore, mailHops, parsedEmail, language) { /* Finalize the parsed email object. */ parsedEmail.dkim = isDkimValid ? 'pass' : 'failed'; parsedEmail.spf = isSpfValid ? 'pass' : 'failed'; parsedEmail.spamScore = spamScore; + parsedEmail.mailHops = mailHops.response || null; parsedEmail.language = language; /* Make fields exist, even if empty. That will make @@ -362,6 +379,9 @@ Mailin.prototype.start = function (options, callback) { function postWebhook(connection, finalizedMessage) { return new Promise(function (resolve) { + + logger.debug(finalizedMessage); + if (configuration.disableWebhook) return resolve(); logger.info(connection.id + ' Sending request to webhook ' + configuration.webhook); @@ -378,8 +398,6 @@ Mailin.prototype.start = function (options, callback) { }); logger.profile('Convert attachments to strings'); - logger.debug(finalizedMessage); - var req = request.post(configuration.webhook); req.field('mailinMsg', JSON.stringify(finalizedMessage)); diff --git a/package.json b/package.json index 9f35941..0629cad 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "html-to-text": "^1.3.1", "languagedetect": "^1.1.1", "lodash": "^3.9.3", + "mailhops": "^2.0.0", "mailparser": "^0.5.1", "node-uuid": "^1.4.3", "semver": "^5.0.1", diff --git a/test/fixtures/test-html-only.eml b/test/fixtures/test-html-only.eml index 8d60c8e..7790c48 100644 --- a/test/fixtures/test-html-only.eml +++ b/test/fixtures/test-html-only.eml @@ -2,6 +2,9 @@ X-Mailer: Nodemailer 1.0 From: "Me" To: to@jokund.com Content-Type: text/html +Received: from mta2.e.mozilla.org (mta2.e.mozilla.org [68.232.195.239]) + (using TLSv1 with cipher ECDHE-RSA-AES256-SHA (256/256 bits)) + ; Fri, 2 Dec 2016 09:41:31 -0800 (PST)

Hello World

This is a line that needs to be at least a little longer than 80 characters so that we can check the character wrapping functionality.

diff --git a/test/fixtures/test.eml b/test/fixtures/test.eml index b8ba164..5844b11 100644 --- a/test/fixtures/test.eml +++ b/test/fixtures/test.eml @@ -4,8 +4,11 @@ To: "First Receiver" , second@jokund.com Content-Type: multipart/mixed; boundary="----mailcomposer-?=_1-1402581589619" MIME-Version: 1.0 - -------mailcomposer-?=_1-1402581589619 +Received: from mta2.e.mozilla.org (mta2.e.mozilla.org [68.232.195.239]) + (using TLSv1 with cipher ECDHE-RSA-AES256-SHA (256/256 bits)) + ; Fri, 2 Dec 2016 09:41:31 -0800 (PST) + +------mailcomposer-?=_1-1402581589619 Content-Type: multipart/alternative; boundary="----mailcomposer-?=_2-1402581589620" diff --git a/test/mailUtilitiesSpec.js b/test/mailUtilitiesSpec.js index 08cd964..9023644 100644 --- a/test/mailUtilitiesSpec.js +++ b/test/mailUtilitiesSpec.js @@ -37,3 +37,18 @@ describe('The mail signature verfier', function () { }); }); }); + +describe('The mailhops verfier', function () { + it('should be able to verify the route', + function (done) { + var email = fs.readFileSync('./test/fixtures/test.eml').toString(); + mailUtilities.getMailHops(email, "", function (err, result) { + if (err) console.log(err); + should.not.exist(err); + + result.response.route.length.should.eql(2); + result.response.route[0].ip.should.eql('68.232.195.239'); + done(); + }); + }); +}); diff --git a/test/mailinSpec.js b/test/mailinSpec.js index cda1d54..8ae4125 100644 --- a/test/mailinSpec.js +++ b/test/mailinSpec.js @@ -94,6 +94,7 @@ describe('Mailin', function () { 'mime-version': '1.0' }, priority: 'normal', + receivedDate: '2016-12-02T17:41:31.000Z', from: [{ address: 'me@jokund.com', name: 'Me' @@ -128,6 +129,7 @@ describe('Mailin', function () { spf: 'failed', spamScore: expectedSpamScore, language: 'pidgin', + mailHops: null, cc: [], connection: connData }); @@ -175,6 +177,7 @@ describe('Mailin', function () { 'mime-version': '1.0' }, priority: 'normal', + receivedDate: '2016-12-02T17:41:31.000Z', from: [{ address: 'me@jokund.com', name: 'Me' @@ -208,6 +211,7 @@ describe('Mailin', function () { spf: 'failed', spamScore: expectedSpamScore, language: 'pidgin', + mailHops: null, cc: [], connection: connData }); From 04362e119fcecd24d296be560ab303ae9b3445b2 Mon Sep 17 00:00:00 2001 From: Andrew Van Tassel Date: Wed, 7 Dec 2016 21:57:30 -0700 Subject: [PATCH 2/3] Updated travis badge to mailhops branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d3ed0c0..61e4181 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#Mailin [![Build Status](https://travis-ci.org/Flolagale/mailin.svg?branch=master)](https://travis-ci.org/Flolagale/mailin) +#Mailin [![Build Status](https://travis-ci.org/Flolagale/mailin.svg?branch=master)](https://travis-ci.org/MailHops/mailin.svg?branch=mailhops) __Artisanal inbound emails for every web app__ From d9a898a96ef79bc8d20f54a6e34d0b27bf9cbe00 Mon Sep 17 00:00:00 2001 From: Andrew Van Tassel Date: Wed, 7 Dec 2016 22:55:16 -0700 Subject: [PATCH 3/3] Cleaned up tests and MailHops async --- lib/mailUtilities.js | 1 + lib/mailin.js | 2 +- test/mailinSpec.js | 14 +++++++------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/mailUtilities.js b/lib/mailUtilities.js index ae0c600..ea314c6 100644 --- a/lib/mailUtilities.js +++ b/lib/mailUtilities.js @@ -82,6 +82,7 @@ module.exports = { }); }, + /* @param rawEmail is the full raw mime email as a string. */ getMailHops: function (rawEmail, mailhopsApiKey, callback) { var mailhops = new MailHops({ api_key: mailhopsApiKey, diff --git a/lib/mailin.js b/lib/mailin.js index b944b56..9f53c47 100644 --- a/lib/mailin.js +++ b/lib/mailin.js @@ -291,7 +291,7 @@ Mailin.prototype.start = function (options, callback) { return Promise.resolve(false); } - return mailUtilities.getMailHops(rawEmail, configuration.mailhopsApiKey) + return mailUtilities.getMailHopsAsync(rawEmail, configuration.mailhopsApiKey) .catch(function (err) { logger.error(connection.id + ' Unable to get MailHops.'); logger.error(err); diff --git a/test/mailinSpec.js b/test/mailinSpec.js index 8ae4125..bf75925 100644 --- a/test/mailinSpec.js +++ b/test/mailinSpec.js @@ -36,6 +36,7 @@ before(function (done) { // This checks the webhook; that's why the server must be already up and listening mailin.start({ // verbose: true, + mailhopsApiKey: false, smtpOptions: { secure: false } @@ -75,13 +76,13 @@ describe('Mailin', function () { mailin.on('message', function (connection, data) { console.log("Event 'message' triggered."); - // console.log(data); try { data.attachments[0].content.toString().should.eql('my dummy attachment contents'); /* Delete the headers that include a timestamp. */ delete data.headers.received; + delete data.receivedDate; data.should.eql({ html: 'Hello world!', @@ -94,7 +95,6 @@ describe('Mailin', function () { 'mime-version': '1.0' }, priority: 'normal', - receivedDate: '2016-12-02T17:41:31.000Z', from: [{ address: 'me@jokund.com', name: 'Me' @@ -128,15 +128,15 @@ describe('Mailin', function () { }], spf: 'failed', spamScore: expectedSpamScore, - language: 'pidgin', mailHops: null, + language: 'pidgin', cc: [], connection: connData }); doing--; } catch (e) { - done(e); + done(e); } }); @@ -162,6 +162,7 @@ describe('Mailin', function () { /* Delete the headers that include a timestamp. */ delete mailinMsg.headers.received; + delete mailinMsg.receivedDate; /* And the connection id, which is random. */ delete mailinMsg.data; @@ -177,7 +178,6 @@ describe('Mailin', function () { 'mime-version': '1.0' }, priority: 'normal', - receivedDate: '2016-12-02T17:41:31.000Z', from: [{ address: 'me@jokund.com', name: 'Me' @@ -220,7 +220,7 @@ describe('Mailin', function () { doing--; } catch (e) { - done(e); + done(e); } }); @@ -263,7 +263,7 @@ describe('Mailin', function () { this.timeout(10000); mailin.on('message', function (connection, data) { - // console.log(data); + try { data.text.should.eql('HELLO WORLD\nThis is a line that needs to be at least a little longer than 80 characters so\nthat we can check the character wrapping functionality.\n\nThis is a test of a link [https://github.com/Flolagale/mailin] .'); done();