Merge pull request #552 from kazk/python-no-concat

Add Python 3 unittest runner without concatenation

Author: jhoffner

frameworks/python/codewars.py

@@ -0,0 +1,71 @@
+import unittest
+import traceback
+from time import perf_counter
+
+class CodewarsTestRunner(object):
+    def __init__(self): pass
+    def run(self, test):
+        r = CodewarsTestResult()
+        s = perf_counter()
+        print("\n<DESCRIBE::>Tests")
+        try:
+            test(r)
+        finally:
+            pass
+        print("\n<COMPLETEDIN::>{:.4f}".format(1000*(perf_counter() - s)))
+        return r
+
+__unittest = True
+class CodewarsTestResult(unittest.TestResult):
+    def __init__(self):
+        super().__init__()
+        self.start = 0.0
+
+    def startTest(self, test):
+        print("\n<IT::>" + test._testMethodName)
+        super().startTest(test)
+        self.start = perf_counter()
+
+    def stopTest(self, test):
+        print("\n<COMPLETEDIN::>{:.4f}".format(1000*(perf_counter() - self.start)))
+        super().stopTest(test)
+
+    def addSuccess(self, test):
+        print("\n<PASSED::>Test Passed")
+        super().addSuccess(test)
+
+    def addError(self, test, err):
+        print("\n<ERROR::>Unhandled Exception")
+        print("\n<LOG:ESC:Error>" + esc(''.join(traceback.format_exception_only(err[0], err[1]))))
+        print("\n<LOG:ESC:Traceback>" + esc(self._exc_info_to_string(err, test)))
+        super().addError(test, err)
+
+    def addFailure(self, test, err):
+        print("\n<FAILED::>Test Failed")
+        print("\n<LOG:ESC:Failure>" + esc(''.join(traceback.format_exception_only(err[0], err[1]))))
+        super().addFailure(test, err)
+
+    # from unittest/result.py
+    def _exc_info_to_string(self, err, test):
+        exctype, value, tb = err
+        # Skip test runner traceback levels
+        while tb and self._is_relevant_tb_level(tb):
+            tb = tb.tb_next
+        if exctype is test.failureException:
+            length = self._count_relevant_tb_levels(tb) # Skip assert*() traceback levels
+        else:
+            length = None
+        return ''.join(traceback.format_tb(tb, limit=length))
+
+    def _is_relevant_tb_level(self, tb):
+        return '__unittest' in tb.tb_frame.f_globals
+
+    def _count_relevant_tb_levels(self, tb):
+        length = 0
+        while tb and not self._is_relevant_tb_level(tb):
+            length += 1
+            tb = tb.tb_next
+        return length
+
+def esc(s):
+    return s.replace("\n", "<:LF:>")

lib/runners/python.js

@@ -1,20 +1,23 @@
 "use strict";
 
+const writeFileSync = require('../utils/write-file-sync');
+
 module.exports = {
   solutionOnly(opts, runCode) {
     runVersion(opts, [opts.setup, opts.solution].join("\n"), runCode);
   },
   testIntegration(opts, runCode) {
+    if (isNoConcat(opts)) return python3unittest(opts, runCode);
     var code;
     switch (opts.testFramework) {
       case 'cw':
       case 'cw-2':
         code = opts.projectMode ? cw2Project(opts) : cw2Code(opts);
         break;
       case 'unittest':
-      // TODO: support projectMode for unittest, which should require
-      // improving unittest support so that a specific test case called Test doesn't need
-      // to be used
+        // TODO: support projectMode for unittest, which should require
+        // improving unittest support so that a specific test case called Test doesn't need
+        // to be used
         code = unittestCode(opts);
         break;
       default:
@@ -23,6 +26,7 @@ module.exports = {
     runVersion(opts, code, runCode);
   },
   sanitizeStdErr(opts, err) {
+    if (isNoConcat(opts)) return err;
     // get rid of some of the noisy content. We remove line numbers since
     // they don't match up to what the user sees and will only confuse them.
     return err
@@ -123,3 +127,59 @@ print("<DESCRIBE::>Tests")
 unittest.TestLoader().loadTestsFromTestCase(Test).run(reload(__import__('unittestwrapper')).CwTestResult())
 print("<COMPLETEDIN::>")
 `;
+
+// Returns true if running tests without concatenation is possible.
+function isNoConcat(opts) {
+  return opts.testFramework === 'unittest' && isPython3(opts);
+}
+
+// .
+// |-- setup.py
+// |-- solution.py
+// `-- test
+//     |-- __init__.py
+//     |-- __main__.py
+//     `-- test_solution.py
+// inspired by http://stackoverflow.com/a/27630375
+//
+// for backward compatibility:
+// - prepend `import unittest` to fixture if missing
+// - prepend `from solution import *` to fixture to simulate concatenation
+// - prepend `from setup import *` to solution to simulate concatenation
+function python3unittest(opts, runCode) {
+  let solution = opts.solution;
+  if (opts.setup) {
+    writeFileSync(opts.dir, 'setup.py', opts.setup);
+    if (!/^\s*import setup\s*$/m.test(solution) && !/^\s*from setup\s+/m.test(solution)) {
+      solution = 'from setup import *\n' + solution;
+    }
+  }
+  writeFileSync(opts.dir, 'solution.py', solution);
+  let fixture = opts.fixture;
+  if (!/^\s*import\s+unittest/m.test(fixture)) fixture = 'import unittest\n' + fixture;
+  if (!/^\s*import solution\s*$/m.test(fixture) && !/^\s*from solution\s+/m.test(fixture)) {
+    fixture = 'from solution import *\n' + fixture;
+  }
+  writeFileSync(opts.dir+'/test', 'test_solution.py', fixture);
+  writeFileSync(opts.dir+'/test', '__init__.py', '');
+  writeFileSync(opts.dir+'/test', '__main__.py', [
+    'import unittest',
+    'from codewars import CodewarsTestRunner',
+    '',
+    'def load_tests(loader, tests, pattern):',
+    '    return loader.discover(".")',
+    '',
+    'unittest.main(testRunner=CodewarsTestRunner())',
+    '',
+  ].join('\n'));
+  runCode({
+    name: pythonCmd(opts),
+    args: ['test'],
+    options: {
+      cwd: opts.dir,
+      env: Object.assign({}, process.env, {
+        PYTHONPATH: `/runner/frameworks/python:${process.env.PYTHONPATH}`
+      }),
+    }
+  });
+}

test/runners/python_spec.js

@@ -1,7 +1,20 @@
 var expect = require('chai').expect;
 var runner = require('../runner');
+const exec = require('child_process').exec;
 
 describe('python runner', function() {
+  afterEach(function cleanup(done) {
+    exec([
+      'rm -rf',
+      '/home/codewarrior/*.py',
+      '/home/codewarrior/__pycache__',
+      '/home/codewarrior/test',
+    ].join(' '), function(err) {
+      if (err) return done(err);
+      return done();
+    });
+  });
+
   // These specs are compatible with both Python 2 and 3
   ['2', '3', '3.6'].forEach(lv => {
     describe('.run', function() {
@@ -178,7 +191,12 @@ describe('python runner', function() {
           ].join('\n'),
           testFramework: 'unittest'
         }, function(buffer) {
-          expect(buffer.stdout).to.contain('\n<ERROR::>Unhandled Exception: exceptions are my favorite, I always throw them\n');
+          if (lv.startsWith('3')) {
+            expect(buffer.stdout).to.contain('\n<ERROR::>Unhandled Exception');
+          }
+          else {
+            expect(buffer.stdout).to.contain('\n<ERROR::>Unhandled Exception: exceptions are my favorite, I always throw them\n');
+          }
           done();
         });
       });
@@ -187,6 +205,18 @@ describe('python runner', function() {
 });
 
 describe('Output format commands', function() {
+  afterEach(function cleanup(done) {
+    exec([
+      'rm -rf',
+      '/home/codewarrior/*.py',
+      '/home/codewarrior/__pycache__',
+      '/home/codewarrior/test',
+    ].join(' '), function(err) {
+      if (err) return done(err);
+      return done();
+    });
+  });
+
   for (const v of ['2', '3', '3.6']) {
     it(`should be on independent lines (Python${v} cw-2)`, function(done) {
       runner.run({
@@ -225,3 +255,127 @@ describe('Output format commands', function() {
     });
   }
 });
+
+describe('unittest no concat', function() {
+  afterEach(function cleanup(done) {
+    exec([
+      'rm -rf',
+      '/home/codewarrior/*.py',
+      '/home/codewarrior/__pycache__',
+      '/home/codewarrior/test',
+    ].join(' '), function(err) {
+      if (err) return done(err);
+      return done();
+    });
+  });
+
+  for (const v of ['3.x', '3.6']) {
+    it(`should handle a basic assertion (${v})`, function(done) {
+      runner.run({
+        language: 'python',
+        languageVersion: v,
+        testFramework: 'unittest',
+        solution: [
+          'def add(a, b): return a + b'
+        ].join('\n'),
+        fixture: [
+          // Test class name is no longer restricted.
+          'class TestAddition(unittest.TestCase):',
+          '  def test_add(self):',
+          '    self.assertEqual(add(1, 1), 2)'
+        ].join('\n'),
+      }, function(buffer) {
+        expect(buffer.stdout).to.contain('\n<PASSED::>');
+        done();
+      });
+    });
+
+    it(`should handle a failed assetion (${v})`, function(done) {
+      runner.run({
+        language: 'python',
+        languageVersion: v,
+        testFramework: 'unittest',
+        solution: [
+          'def add(a, b): return a - b'
+        ].join('\n'),
+        fixture: [
+          'class TestAddition(unittest.TestCase):',
+          '  def test_add(self):',
+          '    self.assertEqual(add(1, 1), 2)',
+          ''
+        ].join('\n'),
+      }, function(buffer) {
+        expect(buffer.stdout).to.contain('\n<FAILED::>');
+        done();
+      });
+    });
+
+    it(`syntax error in solution show line numbers (${v})`, function(done) {
+      runner.run({
+        language: 'python',
+        languageVersion: v,
+        testFramework: 'unittest',
+        solution: [
+          'def add(a, b): return a b'
+        ].join('\n'),
+        fixture: [
+          'class TestAddition(unittest.TestCase):',
+          '  def test_add(self):',
+          '    self.assertEqual(add(1, 1), 2)',
+          ''
+        ].join('\n'),
+      }, function(buffer) {
+        expect(buffer.stdout).to.contain('\n<ERROR::>');
+        expect(buffer.stdout).to.contain('solution.py", line 1');
+        expect(buffer.stdout).to.contain('SyntaxError');
+        done();
+      });
+    });
+
+    it(`should handle error (${v})`, function(done) {
+      runner.run({
+        language: 'python',
+        languageVersion: v,
+        testFramework: 'unittest',
+        solution: [
+          'def add(a, b): return a / b'
+        ].join('\n'),
+        fixture: [
+          'class TestAddition(unittest.TestCase):',
+          '  def test_add(self):',
+          '    self.assertEqual(add(1, 0), 1)',
+          ''
+        ].join('\n'),
+      }, function(buffer) {
+        expect(buffer.stdout).to.contain('\n<ERROR::>');
+        done();
+      });
+    });
+
+    it(`should handle tests in multiple suites (${v})`, function(done) {
+      // combined into one suite
+      runner.run({
+        language: 'python',
+        languageVersion: v,
+        testFramework: 'unittest',
+        solution: [
+          'def add(a, b): return a + b'
+        ].join('\n'),
+        fixture: [
+          'class TestAddition1(unittest.TestCase):',
+          '  def test_add1(self):',
+          '    self.assertEqual(add(1, 1), 2)',
+          '',
+          'class TestAddition2(unittest.TestCase):',
+          '  def test_add2(self):',
+          '    self.assertEqual(add(2, 2), 4)',
+          '',
+        ].join('\n'),
+      }, function(buffer) {
+        expect(buffer.stdout).to.contain('\n<IT::>test_add1');
+        expect(buffer.stdout).to.contain('\n<IT::>test_add2');
+        done();
+      });
+    });
+  }
+});