From 6c4dd6b460a05286a74c462b1643a7689b5189e6 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Tue, 21 Mar 2017 13:54:02 -0400 Subject: [PATCH] Share config between util and functional tests Summary: Backport core's PR10331 Depends on D956 Test Plan: make check ./test/functional/test_runner.py Reviewers: schancel, #bitcoin_abc Reviewed By: schancel, #bitcoin_abc Differential Revision: https://reviews.bitcoinabc.org/D959 --- .gitignore | 4 +- Makefile.am | 4 - configure.ac | 8 +- src/Makefile.test.include | 2 +- test/{functional => }/config.ini.in | 2 +- test/functional/test_runner.py | 2 +- test/functional/zmq_test.py | 3 +- test/util/bctest.py | 144 ---------------------- test/util/bitcoin-util-test.py | 180 +++++++++++++++++++++++++--- test/util/buildenv.py.in | 4 - 10 files changed, 172 insertions(+), 181 deletions(-) rename test/{functional => }/config.ini.in (90%) delete mode 100644 test/util/bctest.py delete mode 100644 test/util/buildenv.py.in diff --git a/.gitignore b/.gitignore index 87588a8cfc..f00f2c3a65 100644 --- a/.gitignore +++ b/.gitignore @@ -81,7 +81,6 @@ Bitcoin-Qt.app # Unit-tests Makefile.test bitcoin-qt_test -src/test/buildenv.py # Resources cpp qrc_*.cpp @@ -102,8 +101,7 @@ coverage_percent.txt linux-coverage-build linux-build win32-build -test/functional/config.ini -test/util/buildenv.py +test/config.ini test/cache/* !src/leveldb*/Makefile diff --git a/Makefile.am b/Makefile.am index 5ba9967dca..85b9ca6009 100644 --- a/Makefile.am +++ b/Makefile.am @@ -232,7 +232,6 @@ dist_noinst_SCRIPTS = autogen.sh EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) EXTRA_DIST += \ - test/util/bctest.py \ test/util/bitcoin-util-test.py \ test/util/data/bitcoin-util-test.json \ test/util/data/blanktxv1.hex \ @@ -274,9 +273,6 @@ EXTRA_DIST += \ CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) -# This file is problematic for out-of-tree builds if it exists. -DISTCLEANFILES = test/util/buildenv.pyc - .INTERMEDIATE: $(COVERAGE_INFO) DISTCHECK_CONFIGURE_FLAGS = --enable-man diff --git a/configure.ac b/configure.ac index fa81aaefd6..7651ca8fd3 100644 --- a/configure.ac +++ b/configure.ac @@ -1277,12 +1277,10 @@ AC_SUBST(EVENT_PTHREADS_LIBS) AC_SUBST(ZMQ_LIBS) AC_SUBST(PROTOBUF_LIBS) AC_SUBST(QR_LIBS) -AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/functional/config.ini]) -AC_CONFIG_FILES([test/util/buildenv.py],[chmod +x test/util/buildenv.py]) +AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py]) -AC_CONFIG_LINKS([test/util/bctest.py:test/util/bctest.py]) dnl boost's m4 checks do something really nasty: they export these vars. As a dnl result, they leak into secp256k1's configure and crazy things happen. @@ -1330,8 +1328,8 @@ esac dnl Replace the BUILDDIR path with the correct Windows path if compiling on Native Windows case ${OS} in *Windows*) - sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' test/functional/config.ini > test/functional/config-2.ini - mv test/functional/config-2.ini test/functional/config.ini + sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' test/config.ini > test/config-2.ini + mv test/config-2.ini test/config.ini ;; esac diff --git a/src/Makefile.test.include b/src/Makefile.test.include index dd243897da..9159dd5e3e 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -164,7 +164,7 @@ bitcoin_test_clean : FORCE check-local: @echo "Running test/util/bitcoin-util-test.py..." - $(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py + $(top_builddir)/test/util/bitcoin-util-test.py $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check if EMBEDDED_UNIVALUE $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check diff --git a/test/functional/config.ini.in b/test/config.ini.in similarity index 90% rename from test/functional/config.ini.in rename to test/config.ini.in index 29586c555d..35ee092be4 100644 --- a/test/functional/config.ini.in +++ b/test/config.ini.in @@ -3,7 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # These environment variables are set by the build process and read by -# test/functional/test_runner.py +# test/functional/test_runner.py and test/util/bitcoin-util-test.py [environment] SRCDIR=@abs_top_srcdir@ diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 7aef3c3808..b0e64f4500 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -160,7 +160,7 @@ def main(): # Read config generated by configure. config = configparser.ConfigParser() - configfile = os.path.abspath(os.path.dirname(__file__)) + "/config.ini" + configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" config.read_file(open(configfile)) passon_args.append("--configfile=%s" % configfile) diff --git a/test/functional/zmq_test.py b/test/functional/zmq_test.py index f3fb33a5a8..65521c90bd 100755 --- a/test/functional/zmq_test.py +++ b/test/functional/zmq_test.py @@ -32,7 +32,8 @@ def setup_nodes(self): # Check that bitcoin has been built with ZMQ enabled config = configparser.ConfigParser() if not self.options.configfile: - self.options.configfile = os.path.dirname(__file__) + "/config.ini" + self.options.configfile = os.path.dirname( + __file__) + "/../config.ini" config.read_file(open(self.options.configfile)) if not config["components"].getboolean("ENABLE_ZMQ"): diff --git a/test/util/bctest.py b/test/util/bctest.py deleted file mode 100644 index 2755443644..0000000000 --- a/test/util/bctest.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2014 BitPay Inc. -# Copyright 2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -from __future__ import division, print_function, unicode_literals -import subprocess -import os -import json -import sys -import binascii -import difflib -import logging - - -def parse_output(a, fmt): - """Parse the output according to specified format. - - Raise an error if the output can't be parsed.""" - if fmt == 'json': # json: compare parsed data - return json.loads(a) - elif fmt == 'hex': # hex: parse and compare binary data - return binascii.a2b_hex(a.strip()) - else: - raise NotImplementedError("Don't know how to compare %s" % fmt) - - -def bctest(testDir, testObj, buildenv): - """Runs a single test, comparing output and RC to expected output and RC. - - Raises an error if input can't be read, executable fails, or output/RC - are not as expected. Error is caught by bctester() and reported. - """ - # Get the exec names and arguments - execprog = buildenv.BUILDDIR + "/src/" + testObj['exec'] + buildenv.exeext - execargs = testObj['args'] - execrun = [execprog] + execargs - - # Read the input data (if there is any) - stdinCfg = None - inputData = None - if "input" in testObj: - filename = testDir + "/" + testObj['input'] - inputData = open(filename).read() - stdinCfg = subprocess.PIPE - - # Read the expected output data (if there is any) - outputFn = None - outputData = None - if "output_cmp" in testObj: - outputFn = testObj['output_cmp'] - # output type from file extension (determines how to compare) - outputType = os.path.splitext(outputFn)[1][1:] - try: - outputData = open(testDir + "/" + outputFn).read() - except: - logging.error("Output file " + outputFn + " can not be opened") - raise - if not outputData: - logging.error("Output data missing for " + outputFn) - raise Exception - - # Run the test - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True) - try: - outs = proc.communicate(input=inputData) - except OSError: - logging.error("OSError, Failed to execute " + execprog) - raise - - if outputData: - # Parse command output and expected output - try: - a_parsed = parse_output(outs[0], outputType) - except Exception as e: - logging.error('Error parsing command output as %s: %s' % - (outputType, e)) - raise - try: - b_parsed = parse_output(outputData, outputType) - except Exception as e: - logging.error('Error parsing expected output %s as %s: %s' % - (outputFn, outputType, e)) - raise - # Compare data - if a_parsed != b_parsed: - logging.error("Output data mismatch for " + - outputFn + " (format " + outputType + ")") - raise Exception - # Compare formatting - if outs[0] != outputData: - error_message = "Output formatting mismatch for " + outputFn + ":\n" - error_message += "".join(difflib.context_diff(outputData.splitlines(True), - outs[0].splitlines( - True), - fromfile=outputFn, - tofile="returned")) - logging.error(error_message) - raise Exception - - # Compare the return code to the expected return code - wantRC = 0 - if "return_code" in testObj: - wantRC = testObj['return_code'] - if proc.returncode != wantRC: - logging.error("Return code mismatch for " + outputFn) - raise Exception - - if "error_txt" in testObj: - want_error = testObj["error_txt"] - # Compare error text - # TODO: ideally, we'd compare the strings exactly and also assert - # That stderr is empty if no errors are expected. However, bitcoin-tx - # emits DISPLAY errors when running as a windows application on - # linux through wine. Just assert that the expected error text appears - # somewhere in stderr. - if want_error not in outs[1]: - logging.error("Error mismatch:\n" + "Expected: " + - want_error + "\nReceived: " + outs[1].rstrip()) - raise Exception - - -def bctester(testDir, input_basename, buildenv): - """ Loads and parses the input file, runs all tests and reports results""" - input_filename = testDir + "/" + input_basename - raw_data = open(input_filename).read() - input_data = json.loads(raw_data) - - failed_testcases = [] - - for testObj in input_data: - try: - bctest(testDir, testObj, buildenv) - logging.info("PASSED: " + testObj["description"]) - except: - logging.info("FAILED: " + testObj["description"]) - failed_testcases.append(testObj["description"]) - - if failed_testcases: - logging.error( - "FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]") - sys.exit(1) - else: - sys.exit(0) diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py index 41bfe2a375..5cc8cf2e13 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -1,26 +1,31 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2014 BitPay Inc. -# Copyright 2016 The Bitcoin Core developers +# Copyright 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -from __future__ import division, print_function, unicode_literals -import os -import sys -import argparse -import logging - -help_text = """Test framework for bitcoin utils. +"""Test framework for bitcoin utils. -Runs automatically during `make check`. +Runs automatically during `make check`. Can also be run manually.""" -if __name__ == '__main__': - sys.path.append(os.path.dirname(os.path.abspath(__file__))) - import buildenv - import bctest +import argparse +import binascii +import configparser +import difflib +import json +import logging +import os +import pprint +import subprocess +import sys + + +def main(): + config = configparser.ConfigParser() + config.read_file(open(os.path.dirname(__file__) + "/../config.ini")) - parser = argparse.ArgumentParser(description=help_text) + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--verbose', action='store_true') args = parser.parse_args() verbose = args.verbose @@ -33,5 +38,146 @@ # Add the format/level to the logger logging.basicConfig(format=formatter, level=level) - bctest.bctester(buildenv.SRCDIR + "/test/util/data", - "bitcoin-util-test.json", buildenv) + bctester(config["environment"]["SRCDIR"] + "/test/util/data", + "bitcoin-util-test.json", config["environment"]) + + +def bctester(testDir, input_basename, buildenv): + """ Loads and parses the input file, runs all tests and reports results""" + input_filename = testDir + "/" + input_basename + raw_data = open(input_filename).read() + input_data = json.loads(raw_data) + + failed_testcases = [] + + for testObj in input_data: + try: + bctest(testDir, testObj, buildenv) + logging.info("PASSED: " + testObj["description"]) + except: + logging.info("FAILED: " + testObj["description"]) + failed_testcases.append(testObj["description"]) + + if failed_testcases: + error_message = "FAILED_TESTCASES:\n" + error_message += pprint.pformat(failed_testcases, width=400) + logging.error(error_message) + sys.exit(1) + else: + sys.exit(0) + + +def bctest(testDir, testObj, buildenv): + """Runs a single test, comparing output and RC to expected output and RC. + + Raises an error if input can't be read, executable fails, or output/RC + are not as expected. Error is caught by bctester() and reported. + """ + # Get the exec names and arguments + execprog = buildenv["BUILDDIR"] + "/src/" + \ + testObj['exec'] + buildenv["EXEEXT"] + execargs = testObj['args'] + execrun = [execprog] + execargs + + # Read the input data (if there is any) + stdinCfg = None + inputData = None + if "input" in testObj: + filename = testDir + "/" + testObj['input'] + inputData = open(filename).read() + stdinCfg = subprocess.PIPE + + # Read the expected output data (if there is any) + outputFn = None + outputData = None + if "output_cmp" in testObj: + outputFn = testObj['output_cmp'] + # output type from file extension (determines how to compare) + outputType = os.path.splitext(outputFn)[1][1:] + try: + outputData = open(testDir + "/" + outputFn).read() + except: + logging.error("Output file " + outputFn + " can not be opened") + raise + if not outputData: + logging.error("Output data missing for " + outputFn) + raise Exception + + # Run the test + proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) + try: + outs = proc.communicate(input=inputData) + except OSError: + logging.error("OSError, Failed to execute " + execprog) + raise + + if outputData: + data_mismatch, formatting_mismatch = False, False + # Parse command output and expected output + try: + a_parsed = parse_output(outs[0], outputType) + except Exception as e: + logging.error('Error parsing command output as %s: %s' % + (outputType, e)) + raise + try: + b_parsed = parse_output(outputData, outputType) + except Exception as e: + logging.error('Error parsing expected output %s as %s: %s' % + (outputFn, outputType, e)) + raise + # Compare data + if a_parsed != b_parsed: + logging.error("Output data mismatch for " + + outputFn + " (format " + outputType + ")") + data_mismatch = True + # Compare formatting + if outs[0] != outputData: + error_message = "Output formatting mismatch for " + outputFn + ":\n" + error_message += "".join(difflib.context_diff(outputData.splitlines(True), + outs[0].splitlines( + True), + fromfile=outputFn, + tofile="returned")) + logging.error(error_message) + formatting_mismatch = True + + assert not data_mismatch and not formatting_mismatch + + # Compare the return code to the expected return code + wantRC = 0 + if "return_code" in testObj: + wantRC = testObj['return_code'] + if proc.returncode != wantRC: + logging.error("Return code mismatch for " + outputFn) + raise Exception + + if "error_txt" in testObj: + want_error = testObj["error_txt"] + # Compare error text + # TODO: ideally, we'd compare the strings exactly and also assert + # That stderr is empty if no errors are expected. However, bitcoin-tx + # emits DISPLAY errors when running as a windows application on + # linux through wine. Just assert that the expected error text appears + # somewhere in stderr. + if want_error not in outs[1]: + logging.error("Error mismatch:\n" + "Expected: " + + want_error + "\nReceived: " + outs[1].rstrip()) + raise Exception + + +def parse_output(a, fmt): + """Parse the output according to specified format. + + Raise an error if the output can't be parsed.""" + if fmt == 'json': # json: compare parsed data + return json.loads(a) + elif fmt == 'hex': # hex: parse and compare binary data + return binascii.a2b_hex(a.strip()) + else: + raise NotImplementedError("Don't know how to compare %s" % fmt) + + +if __name__ == '__main__': + main() diff --git a/test/util/buildenv.py.in b/test/util/buildenv.py.in deleted file mode 100644 index 33030b0348..0000000000 --- a/test/util/buildenv.py.in +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python -exeext="@EXEEXT@" -SRCDIR="@abs_top_srcdir@" -BUILDDIR="@abs_top_builddir@"