From c41e6e5399416e0e971ffe896ede3e5913c2104f Mon Sep 17 00:00:00 2001 From: Marius Muench Date: Wed, 10 Feb 2021 10:24:50 +0100 Subject: [PATCH] Dev/pypanda target (#73) * Fix cpu at 100% spinning bug Pygdbmis new API with multilevel timeout allows us to not spin at 100% when waiting for responses, while not experience a performance degration at the same time. * move qemu_cmd_line init into __init__ * allow sublcassing for pandatarget * refactor init to mulitple steps * initial pypanda implementation * use modern pypanda api * add init watch * add callback wrappers * resolve qemutarget executable only if it is qemutarget * Advancements in PyPandaTarget * saner init of PandaTarget (for resolving executable) * disable/enable callbacks * high-performant memory read/write * CI testcase * Fix threading/signal issue * Improved signal handling --- .travis.yml | 3 +- avatar2/targets/__init__.py | 1 + avatar2/targets/panda_target.py | 9 +- avatar2/targets/pypanda_target.py | 122 +++++++++++++++++++++++ avatar2/targets/qemu_target.py | 20 ++-- tests/test_pypandatarget.py | 158 ++++++++++++++++++++++++++++++ 6 files changed, 301 insertions(+), 12 deletions(-) create mode 100644 avatar2/targets/pypanda_target.py create mode 100644 tests/test_pypandatarget.py diff --git a/.travis.yml b/.travis.yml index d4a4eb6777..daf49ecd06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: +python: - "3.8" services: @@ -34,6 +34,7 @@ script: - docker exec avatar2 bash -c 'cd avatar2/ && AVATAR2_GDB_EXECUTABLE=gdb-multiarch AVATAR2_ARCH=MIPS AVATAR2_QEMU_EXECUTABLE=panda-system-mips nosetests-3.4 ./tests/test_qemutarget.py' - docker exec avatar2 bash -c 'cd avatar2/ && AVATAR2_GDB_EXECUTABLE=gdb-multiarch AVATAR2_QEMU_EXECUTABLE=panda-system-arm nosetests-3.4 ./tests/pyperipheral/test_pyperipheral.py' - docker exec avatar2 bash -c 'cd avatar2/ && AVATAR2_GDB_EXECUTABLE=gdb-multiarch AVATAR2_QEMU_EXECUTABLE=panda-system-arm AVATAR2_PANDA_EXECUTABLE=panda-system-arm nosetests-3.4 ./tests/smoke/panda_thumb.py' + - docker exec avatar2 bash -c 'cd avatar2/ && AVATAR2_GDB_EXECUTABLE=gdb-multiarch AVATAR2_QEMU_EXECUTABLE=panda-system-arm AVATAR2_PANDA_EXECUTABLE=panda-system-arm nosetests-3.4 ./tests/test_pypandatarget.py' - docker exec avatar2 bash -c 'cd avatar2/ && python3 ./tests/hello_world.py' - docker exec avatar2 bash -c 'cd avatar2/ && python3 ./tests/gdb_memory_map_loader.py' diff --git a/avatar2/targets/__init__.py b/avatar2/targets/__init__.py index b8766da75b..45e1f18383 100644 --- a/avatar2/targets/__init__.py +++ b/avatar2/targets/__init__.py @@ -5,4 +5,5 @@ from .jlink_target import * from .qemu_target import * from .panda_target import * +from .pypanda_target import * from .unicorn_target import * diff --git a/avatar2/targets/panda_target.py b/avatar2/targets/panda_target.py index 3fc1a3e8cc..063c7ac572 100644 --- a/avatar2/targets/panda_target.py +++ b/avatar2/targets/panda_target.py @@ -5,7 +5,8 @@ class PandaTarget(QemuTarget): def __init__(self, *args, **kwargs): - super(self.__class__, self).__init__(*args, **kwargs) + super(PandaTarget, self).__init__(*args, **kwargs) + executable = kwargs.get('executable') self.executable = (executable if executable is not None else self._arch.get_panda_executable()) @@ -17,7 +18,7 @@ def begin_record(self, record_name): """ Starts recording the execution in PANDA - :param record_name: The name of the record file + :param record_name: The name of the record file """ filename = "%s/%s" % (self.avatar.output_directory, record_name) return self.protocols.monitor.execute_command('begin_record', @@ -59,9 +60,9 @@ def load_plugin(self, plugin_name, plugin_args=None, file_name=None): Loads a PANDA plugin :param plugin_name: The name of the plugin to be loaded - :param plugin_args: Arguments to be passed to the plugin, + :param plugin_args: Arguments to be passed to the plugin, aseperated by commas - :param file_name: Absolute path to the plugin shared object file, + :param file_name: Absolute path to the plugin shared object file, in case that the default one should not be used """ diff --git a/avatar2/targets/pypanda_target.py b/avatar2/targets/pypanda_target.py new file mode 100644 index 0000000000..15c49dcc4c --- /dev/null +++ b/avatar2/targets/pypanda_target.py @@ -0,0 +1,122 @@ +from threading import Thread +from time import sleep +from avatar2.targets import PandaTarget + +from ..watchmen import watch +from .target import action_valid_decorator_factory, TargetStates + + +class PyPandaTarget(PandaTarget): + ''' + The pypanda target is a PANDA target, but uses pypanda to run the framework. + + ''' + def __init__(self, *args, **kwargs): + try: + import pandare + except ImportError: + raise RuntimeError(("PyPanda could not be found! for installation, " + "please follow the steps at https://github.com/" + "panda-re/panda/blob/master/panda/pypanda/docs/USAGE.md")) + + super(PyPandaTarget, self).__init__(*args, **kwargs) + + self.cb_ctx = 0 + self.pypanda = None + self._thread = None + + def shutdown(self): + + + if self._thread.is_alive(): + self.protocols.execution.remote_disconnect() + self.pypanda.end_analysis() + + # Wait for shutdown + while self._thread.is_alive(): + sleep(.01) + + + @watch('TargetInit') + def init(self, **kwargs): + from pandare import Panda + + arch = self.avatar.arch.gdb_name # for now, gdbname and panda-name match + args = self.assemble_cmd_line()[1:] + + + + self.avatar.save_config(file_name=self.qemu_config_file, + config=self.generate_qemu_config()) + + + self.pypanda = Panda(arch=arch, extra_args=args, **kwargs) + + + # adjust panda's signal handler to avatar2-standard + def SigHandler(SIG,a,b): + if self.state == TargetStates.RUNNING: + self.stop() + self.wait() + + self.avatar.sigint_handler() + + + + self.pypanda.setup_internal_signal_handler(signal_handler=SigHandler) + + self._thread = Thread(target=self.pypanda.run, daemon=True) + self._thread.start() + + self._connect_protocols() + + + + def register_callback(self, callback, function, name=None, enabled=True, + procname=None): + pp = self.pypanda + + if hasattr(pp.callback, callback) is False: + raise Exception("Callback %s not found!" % callback) + cb = getattr(pp.callback, callback) + + if name == None: + name = 'avatar_cb_%d' % self.cb_ctx + self.cb_ctx += 1 + + pp.register_callback(cb, cb(function), name, enabled=enabled, + procname=procname) + + return name + + def disable_callback(self, name): + pp = self.pypanda + pp.disable_callback(name) + + def enable_callback(self, name): + pp = self.pypanda + pp.enable_callback(name) + + + @watch('TargetReadMemory') + @action_valid_decorator_factory(TargetStates.STOPPED, 'memory') + def read_memory(self, address, size, num_words=1, raw=False): + if raw == False: + return self.protocols.memory.read_memory(address, size, num_words) + else: + return self.pypanda.physical_memory_read(address,size*num_words) + + + @watch('TargetWriteMemory') + @action_valid_decorator_factory(TargetStates.STOPPED, 'memory') + def write_memory(self, address, size, value, num_words=1, raw=False): + if raw == False: + return self.protocols.memory.write_memory(address, size, num_words) + else: + return self.pypanda.physical_memory_write(address, value) + + + + def delete_callback(self, name): + return self.pypanda.delete_callback(name) + diff --git a/avatar2/targets/qemu_target.py b/avatar2/targets/qemu_target.py index 88eda96123..2ee6d94778 100644 --- a/avatar2/targets/qemu_target.py +++ b/avatar2/targets/qemu_target.py @@ -32,7 +32,7 @@ def __init__(self, avatar, # Qemu parameters self.system_clock_scale = system_clock_scale - if hasattr(self, 'executable') is False: # May be initialized by subclass + if hasattr(self, 'executable') is False and self.__class__ == QemuTarget: self.executable = (executable if executable is not None else self._arch.get_qemu_executable()) self.fw = firmware @@ -58,10 +58,12 @@ def __init__(self, avatar, self._rmem_rx_queue_name = '/{:s}_rx_queue'.format(self.name) self._rmem_tx_queue_name = '/{:s}_tx_queue'.format(self.name) - self.log_items = log_items self.log_file = log_file + self.qemu_config_file = ("%s/%s_conf.json" % + (self.avatar.output_directory, self.name) ) + def assemble_cmd_line(self): if isfile(self.executable + self._arch.qemu_name): @@ -82,7 +84,7 @@ def assemble_cmd_line(self): cmd_line = executable_name + machine + kernel + gdb_option \ + stop_on_startup + self.additional_args + nographic + qmp - + if self.log_items is not None: if isinstance(self.log_items, str): log_items = ['-d', self.log_items] @@ -97,7 +99,7 @@ def assemble_cmd_line(self): log_file = ['-D', '%s/%s' % (self.avatar.output_directory, self.log_file)] else: - log_file = ['-D', '%s/%s_log.txt' % + log_file = ['-D', '%s/%s_log.txt' % (self.avatar.output_directory, self.name)] cmd_line += log_items + log_file @@ -166,8 +168,6 @@ def init(self, cmd_line=None): else: self.log.warning('No cpu_model specified - are you sure?') - self.qemu_config_file = ("%s/%s_conf.json" % - (self.avatar.output_directory, self.name) ) if cmd_line is None: cmd_line = self.assemble_cmd_line() @@ -181,6 +181,12 @@ def init(self, cmd_line=None): self._process = Popen(cmd_line, stdout=out, stderr=err) self.log.debug("QEMU command line: %s" % ' '.join(cmd_line)) self.log.info("QEMU process running") + self._connect_protocols() + + def _connect_protocols(self): + """ + Internal routine to connect the various protocols to a running qemu + """ gdb = GDBProtocol(gdb_executable=self.gdb_executable, arch=self.avatar.arch, @@ -188,7 +194,7 @@ def init(self, cmd_line=None): additional_args=self.gdb_additional_args, avatar=self.avatar, origin=self, ) - qmp = QMPProtocol(self.qmp_port, origin=self) + qmp = QMPProtocol(self.qmp_port, origin=self) if 'avatar-rmemory' in [i[2].qemu_name for i in self._memory_mapping.iter() if diff --git a/tests/test_pypandatarget.py b/tests/test_pypandatarget.py new file mode 100644 index 0000000000..62aaedc0ea --- /dev/null +++ b/tests/test_pypandatarget.py @@ -0,0 +1,158 @@ +import os +from os.path import abspath +from time import sleep + +from nose.tools import * + +import socket + +from avatar2 import * +from avatar2.peripherals.nucleo_usart import * + +from pandare import ffi + + +PORT = 9997 + +panda = None +avatar = None +s = None +test_string = bytearray(b'Hello World !\n') + +sram_dump = './tests/pyperipheral/sram_dump.bin' +rcc_dump = './tests/pyperipheral/rcc_dump.bin' +firmware = './tests/pyperipheral/firmware.bin' + +with open(sram_dump, 'rb') as f: + sram_data = f.read() + +with open(rcc_dump, 'rb') as f: + rcc_data = f.read() + + +@timed(7) +def setup_func(): + # There is only one pypanda instance in the process space allowed. + # Unfortunately, we need a lot of hacks to make this working for CI tests. + # So, beware the globals! + global panda + global avatar + global s + + if panda is None: + # Initiate the avatar-object + avatar = Avatar(output_directory='/tmp/avatar', arch=ARM_CORTEX_M3) + + panda = avatar.add_target(PyPandaTarget, gdb_port=1236, + entry_address=0x08005105) + + # Define the various memory ranges and store references to them + avatar.add_memory_range(0x08000000, 0x1000000, file=firmware) + avatar.add_memory_range(0x20000000, 0x14000) + avatar.add_memory_range(0x40023000, 0x1000) + + avatar.init_targets() + + panda.pypanda.register_pyperipheral(NucleoUSART('USART', 0x40004400, 0x100, + nucleo_usart_port=PORT)) + + time.sleep(.1) + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect(('127.0.0.1', PORT)) + + panda.write_memory(0x20000000, len(sram_data), sram_data, raw=True) + panda.write_memory(0x40023000, len(rcc_data), rcc_data, raw=True) + + panda.regs.pc = 0x08005105 + panda.regs.sp = 0x20014000 + + panda.bp(0x0800419c) # Termination breakpoint to avoid qemu dying + +@timed(8) +def teardown_func(): + #panda.shutdown() + #avatar.shutdown() + #time.sleep(1) + pass + + + +@with_setup(setup_func, teardown_func) +def test_nucleo_usart_read(): + panda.cont() + + data = s.recv(len(test_string), socket.MSG_WAITALL) + assert_equal(data, test_string) + +recv_data = [] + +@with_setup(setup_func, teardown_func) +def test_panda_callback(): + def cb(env, pc, addr, size, buf): + global recv_data + if addr == 0x40004404: + recv_data.append(buf[0]) + + cb_handle = panda.register_callback('mmio_before_write', cb) + + panda.cont() + panda.wait() + + assert_equal(bytearray(recv_data), test_string) + panda.disable_callback(cb_handle) + + + +@timed(10) +@with_setup(setup_func, teardown_func) +def test_nucleo_usart_debug_read(): + + # pyperipherals in debug mode need to be read from/written to directly + pyperiph = panda.pypanda.pyperipherals[0] + + + s.send(b'Hello World') + + reply = bytearray() + time.sleep(.1) + + while pyperiph.read_memory(0x40004400,4) & (1<<5) != 0: + reply.append(pyperiph.read_memory(0x40004404,4)) + + assert_equal(reply, b'Hello World') + + s.send(b'Hello World') + reply = bytearray() + time.sleep(.1) + while pyperiph.read_memory(0x40004400,1) & (1<<5) != 0: + reply.append(pyperiph.read_memory(0x40004404,1)) + + assert_equal(reply, b'Hello World') + + +@timed(11) +@with_setup(setup_func, teardown_func) +def test_nucleo_usart_debug_write(): + pyperiph = panda.pypanda.pyperipherals[0] + + time.sleep(1) + for c in test_string: + pyperiph.write_memory(0x40004404, 1, c) + reply = s.recv(len(test_string), socket.MSG_WAITALL) + assert_equal(reply, test_string) + + time.sleep(.1) + for c in test_string: + pyperiph.write_memory(0x40004404, 4, c) + reply = s.recv(len(test_string), socket.MSG_WAITALL) + assert_equal(reply, test_string) + + +if __name__ == '__main__': + setup_func() + test_nucleo_usart_debug_read() + setup_func() + test_nucleo_usart_debug_write() + teardown_func() +