outbound es6 + bug fix (haraka#2197)

* outbound test organize
* outbound/hmail: use es6 class
* fix truncated JSON queue headers
* outbound: get tls config via tls_socket (vs net_utils)

.gitignore

@@ -6,3 +6,5 @@ Haraka
 bower_components
 coverage
 .idea
+tests/queue/plain
+tests/queue/multibyte

outbound/index.js

@@ -1,9 +1,9 @@
 'use strict';
 
-const async       = require('async');
 const fs          = require('fs');
 const path        = require('path');
 
+const async       = require('async');
 const Address     = require('address-rfc2821').Address;
 const config      = require('haraka-config');
 const constants   = require('haraka-constants');
@@ -283,6 +283,7 @@ exports.process_delivery = function (ok_paths, todo, hmails, cb) {
     const fname = _qfile.name();
     const tmp_path = path.join(queue_dir, _qfile.platformDOT + fname);
     const ws = new FsyncWriteStream(tmp_path, { flags: constants.WRITE_EXCL });
+
     ws.on('close', function () {
         const dest_path = path.join(queue_dir, fname);
         fs.rename(tmp_path, dest_path, function (err) {
@@ -296,14 +297,16 @@ exports.process_delivery = function (ok_paths, todo, hmails, cb) {
                 ok_paths.push(dest_path);
                 cb();
             }
-        });
-    });
+        })
+    })
+
     ws.on('error', function (err) {
         logger.logerror("[outbound] Unable to write queue file (" + fname + "): " + err);
         ws.destroy();
         fs.unlink(tmp_path, function () {});
         cb("Queueing failed");
-    });
+    })
+
     self.build_todo(todo, ws, function () {
         todo.message_stream.pipe(ws, { line_endings: '\r\n', dot_stuffing: true, ending_dot: false });
     });
@@ -320,9 +323,11 @@ exports.build_todo = function (todo, ws, write_more) {
         }
     }
 
-    const todo_json = JSON.stringify(todo, exclude_from_json);
-    const buf = new Buffer(4 + todo_json.length);
-    buf.writeUInt32BE(todo_json.length, 0);
+    const todo_json = JSON.stringify(todo, exclude_from_json)
+    const todo_len = new Buffer.from(todo_json).length
+
+    const buf = new Buffer(4 + todo_len);
+    buf.writeUInt32BE(todo_len, 0);
     buf.write(todo_json, 4);
 
     const continue_writing = ws.write(buf);

outbound/tls.js

@@ -1,15 +1,31 @@
 'use strict';
 
-const net_utils   = require('haraka-net-utils');
+exports.config     = require('haraka-config');
 
-exports.config = require('haraka-config');
+exports.tls_socket = require('../tls_socket');
 
-// TODO: replace this with newer tls_ini loading in tls_socket
 exports.get_tls_options = function (mx) {
 
-    const tls_options = net_utils.tls_ini_section_with_defaults('outbound');
+    const tls_cfg = exports.tls_socket.load_tls_ini();
+    const tls_options = JSON.parse(JSON.stringify(tls_cfg.outbound || {}));
+
+    const inheritable_opts = [
+        'key', 'cert', 'ciphers', 'dhparam',
+        'requestCert', 'honorCipherOrder', 'rejectUnauthorized'
+    ];
+
     tls_options.servername = mx.exchange;
 
+    for (const opt of inheritable_opts) {
+        if (tls_options[opt] === undefined) {
+            // not declared in tls.ini[section]
+            if (tls_cfg.main[opt] !== undefined) {
+                // use value from [main] section
+                tls_options[opt] = tls_cfg.main[opt];
+            }
+        }
+    }
+
     if (tls_options.key) {
         if (Array.isArray(tls_options.key)) {
             tls_options.key = tls_options.key[0];
@@ -30,4 +46,3 @@ exports.get_tls_options = function (mx) {
 
     return tls_options;
 };
-

run_tests

@@ -1,36 +1,31 @@
 #!/usr/bin/env node
 
 "use strict";
-
+let reporter;
 try {
-    var reporter = require('nodeunit').reporters.default;
+    reporter = require('nodeunit').reporters.default;
 }
 catch (e) {
-    console.log("Error: " + e.message);
-    console.log("");
-    console.log("Cannot find nodeunit module.");
-    console.log("Please run the following:");
-    console.log("");
-    console.log("    npm install");
-    console.log("");
+    console.log(`Error: ${e.message}
+
+Cannot find nodeunit module. Please run the following:
+
+    npm install\n`);
+
     process.exit();
 }
 
 process.chdir(__dirname);
 
+let tests = [
+    'tests', 'tests/outbound', 'tests/plugins',
+    'tests/plugins/auth', 'tests/plugins/queue'
+];
+
 if (process.argv[2]) {
     console.log("Running tests: ", process.argv.slice(2));
-    reporter.run(process.argv.slice(2), undefined, function (err) {
-        process.exit(((err) ? 1 : 0));
-    });
-}
-else {
-    reporter.run([
-        'tests',
-        'tests/plugins',
-        'tests/plugins/auth',
-        'tests/plugins/queue'
-    ], undefined, function (err) {
-        process.exit(((err) ? 1 : 0));
-    });
+    tests = process.argv.slice(2);
 }
+reporter.run(tests, undefined, function (err) {
+    process.exit(((err) ? 1 : 0));
+});

tests/outbound/hmail.js

@@ -0,0 +1,21 @@
+
+const Hmail = require('../../outbound/hmail');
+
+exports.HMailItem = {
+    /*
+     * TODO: this just delays the crash...
+    'truncated TODO does not crash': function (test) {
+        this.hmail = new Hmail('1507509981169_1507509981169_0_61403_e0Y0Ym_1_haraka', 'tests/queue/1507509981169_1507509981169_0_61403_e0Y0Ym_1_haraka', {});
+        this.hmail.on('ready', function () {
+            test.expect(1);
+            console.log(this.hmail);
+            test.ok(this.hmail)
+            test.done();
+        });
+        this.hmail.on('error', function (err) {
+            console.log(err);
+            test.done();
+        })
+    },
+    */
+}

tests/outbound.js → tests/outbound/index.js

@@ -4,6 +4,9 @@ const fs   = require('fs');
 const path = require('path');
 const os   = require('os');
 
+const constants = require('haraka-constants');
+const logger = require('../../logger');
+
 const lines = [
     'From: John Johnson <[email protected]>',
     'To: Jane Johnson <[email protected]>',
@@ -40,10 +43,9 @@ exports.outbound = {
         test.done();
     },
     'log_methods added': function (test) {
-        const logger = require('../logger');
         test.expect(Object.keys(logger.levels).length);
 
-        const HMailItem = require('../outbound').HMailItem;
+        const HMailItem = require('../../outbound/hmail');
 
         Object.keys(logger.levels).forEach(function (level) {
             test.ok(HMailItem.prototype['log' + level.toLowerCase()], "Log method for level: " + level);
@@ -54,7 +56,7 @@ exports.outbound = {
 
 exports.qfile = {
     setUp : function (done) {
-        this.qfile = require('../outbound').qfile;
+        this.qfile = require('../../outbound').qfile;
         done();
     },
     'name() basic functions': function (test){
@@ -132,8 +134,8 @@ exports.qfile = {
 exports.get_tls_options = {
     setUp : function (done) {
         process.env.HARAKA_TEST_DIR=path.resolve('tests');
-        this.outbound = require('../outbound');
-        this.obtls = require('../outbound/tls');
+        this.outbound = require('../../outbound');
+        this.obtls = require('../../outbound/tls');
         done();
     },
     tearDown: function (done) {
@@ -145,9 +147,9 @@ exports.get_tls_options = {
 
         // reset config to load from tests directory
         const testDir = path.resolve('tests');
-        this.outbound.net_utils.config = this.outbound.net_utils.config.module_config(testDir);
+        this.obtls.tls_socket.config = this.obtls.tls_socket.config.module_config(testDir);
         this.outbound.config = this.outbound.config.module_config(testDir);
-        this.obtls.config = this.outbound.config;
+        this.obtls.config = this.outbound.config.module_config(testDir);
 
         const tls_config = this.obtls.get_tls_options(
             { exchange: 'mail.example.com'}
@@ -166,3 +168,58 @@ exports.get_tls_options = {
         test.done();
     },
 }
+
+exports.build_todo = {
+    setUp : function (done) {
+        this.outbound = require('../../outbound');
+        try {
+            fs.unlinkSync('tests/queue/multibyte');
+            fs.unlinkSync('tests/queue/plain');
+        }
+        catch (ignore) {}
+        done();
+    },
+    tearDown: function (done) {
+        // fs.unlink('tests/queue/multibyte', done);
+        done();
+    },
+    'saves a file': function (test) {
+        const todo = JSON.parse('{"queue_time":1507509981169,"domain":"redacteed.com","rcpt_to":[{"original":"<[email protected]>","original_host":"redacteed.com","host":"redacteed.com","user":"postmaster"}],"mail_from":{"original":"<[email protected]>","original_host":"tnpi.net","host":"tnpi.net","user":"matt"},"notes":{"authentication_results":["spf=pass smtp.mailfrom=tnpi.net"],"spf_mail_result":"Pass","spf_mail_record":"v=spf1 a mx include:mx.theartfarm.com ?include:forwards._spf.tnpi.net include:lists._spf.tnpi.net -all","attachment_count":0,"attachments":[{"ctype":"application/pdf","filename":"FileWithoutAccent Chars.pdf","extension":".pdf","md5":"6c1d5f5c047cff3f6320b1210970bdf6"}],"attachment_ctypes":["application/pdf","multipart/mixed","text/plain","application/pdf"],"attachment_files":["FileWithoutaccent Chars.pdf"],"attachment_archive_files":[]},"uuid":"1D5483B0-3E00-4280-A961-3AFD2017B4FC.1"}');
+        const fd = fs.openSync('tests/queue/plain', 'w');
+        const ws = new fs.createWriteStream('tests/queue/plain', { fd: fd, flags: constants.WRITE_EXCL });
+        ws.on('close', function () {
+            // console.log(arguments);
+            test.ok(1);
+            test.done();
+        })
+        ws.on('error', (e) => {
+            console.error(e);
+            test.done();
+        })
+        this.outbound.build_todo(todo, ws, () => {
+            ws.write(new Buffer('This is the message body'));
+            fs.fsync(fd, () => { ws.close(); })
+        })
+    },
+    'saves a file with multibyte chars': function (test) {
+        const todo = JSON.parse('{"queue_time":1507509981169,"domain":"redacteed.com","rcpt_to":[{"original":"<[email protected]>","original_host":"redacteed.com","host":"redacteed.com","user":"postmaster"}],"mail_from":{"original":"<[email protected]>","original_host":"tnpi.net","host":"tnpi.net","user":"matt"},"notes":{"authentication_results":["spf=pass smtp.mailfrom=tnpi.net"],"spf_mail_result":"Pass","spf_mail_record":"v=spf1 a mx include:mx.theartfarm.com ?include:forwards._spf.tnpi.net include:lists._spf.tnpi.net -all","attachment_count":0,"attachments":[{"ctype":"application/pdf","filename":"FileWîthÁccent Chars.pdf","extension":".pdf","md5":"6c1d5f5c047cff3f6320b1210970bdf6"}],"attachment_ctypes":["application/pdf","multipart/mixed","text/plain","application/pdf"],"attachment_files":["FileWîthÁccent Chars.pdf"],"attachment_archive_files":[]},"uuid":"1D5483B0-3E00-4280-A961-3AFD2017B4FC.1"}');
+        const fd = fs.openSync('tests/queue/multibyte', 'w');
+        const ws = new fs.WriteStream('tests/queue/multibyte', { fd: fd, flags: constants.WRITE_EXCL });
+        ws.on('close', function () {
+            test.ok(1);
+            test.done();
+        })
+        ws.on('error', (e) => {
+            console.error(e);
+            test.done();
+        })
+        this.outbound.build_todo(todo, ws, () => {
+            ws.write(new Buffer('This is the message body'));
+            fs.fsync(fd, () => { ws.close(); })
+        })
+    },
+    // '': function (test) {
+
+    //     test.done();
+    // },
+}

tests/outbound_bounce_net_errors.js

@@ -170,6 +170,4 @@ exports.bounce_3464 = {
             test.done();
         });
     },
-
 }
-