INTFowarder & HWRunner introduction (#125)

* WIP: Getting coresight to work

* Fixing Coresight protocol and interrupt forwarding TO QEmu

* WIP interrupts fixing my own mistakes

* Added interrupt state transfer

* Reactivated interrupt forwarding, works with timer but the alarm runs out of sync so it only works for a couple of seconds

* Fixing crash due to protocol shutdown order

interrupt protocol depends on memory

* Fixed issues with memory protocols

* Fixed IVT setup in interrupt protocol

* WIP: Added interrupt recording plugin and protocol, not complete

* WIP: Fixed the stub to include an end of buffer flag

* Interrupt recording protocol and plugin v1

* Interrupt recording protocol cleanup

* Added rudimentary support for ARM Cortex M4 (without FPU registers)

* Interrupt recording without manipulating main loop control flow

* WIP: USB rehosting

* Updated interrupt messages to include the IRQ-Address

* IRQ fixing

* IRQ fixing; Reordering of setup code and removal of IRQ init because it might not complete before the first IRQ fires

* IRQ message passing fixes

* Allow plugins to get configuration parameters at load time

* Switch to trace approach for interrupt rehosting, to capture quickly firing interrupts

* Allow avatar peripherals to know on what target they got called

* WIP: Prevent interrupt interleaving

* Fixed wrong program counter

* WIP: picow-blink rehosting, runs much further but crashes because it runs a re-init of the CYW43

* Allow OpenOCD target to have a gdb binary for symbols

* WIP HAL calling

* WIP HAL calling step 2

* WIP HAL calling step 2- THIS IS BROKEN AF

* Finally a working version

* First working HAL implementation for send-uart

* Make HALCaller config more expressive

* Made HAL calls more generic, allow for up more arguments, allow void function returns, allow pointer/mem bocks for function return

* OpenOCD protocol, rework raw memory read/write for higher performance in memory transfers

* Trigger HAL Exit after Hardware reset is complete

* Make protocol usable as a signaled target continue protocol

* WIP: PICOW rehosting

* WIP: Software interrupt rehosting support by using the hardware in the loop as the only source of interrupts

* WIP: USB rehosting, cleanup of software interrupt stuff

* Cleanup interrupt recording

* HAL only deal with interrupt protocol if present

* HALCaller rename to HWRunner

* HWRunner OpenOCD fix; WiFi experiments final

* HWRunner naming cleanup

* Interrupt Recorder naming cleanup

* Interrupt Forwarding naming cleanup

* Cleanup and fixup of refactorings

* INTFowrader & HWRunner summary

* Final cleanup

* Remove union types for type hint

Union in type hint is introduced with PEP 604 in python 3.10.
However, we aim to support python 3.6.

* Remove crashing log

* Remove crashing log line when using pypanda

---------

Co-authored-by: Florian Albrecht <[email protected]>
Co-authored-by: rawsample <[email protected]>

avatar2/archs/arm.py

@@ -44,8 +44,6 @@ class ARM_CORTEX_M3(ARM):
     qemu_name = 'arm'
     gdb_name = 'arm'
 
-    capstone_arch = CS_ARCH_ARM
-    keystone_arch = KS_ARCH_ARM
     capstone_mode = CS_MODE_LITTLE_ENDIAN | CS_MODE_THUMB | CS_MODE_MCLASS
     keystone_arch = KS_ARCH_ARM
     keystone_mode = KS_MODE_LITTLE_ENDIAN | KS_MODE_THUMB
@@ -86,7 +84,8 @@ def init(avatar):
         pass
 
 
-ARMV7M = ARM_CORTEX_M3
+
+ARMV7M = [ARM_CORTEX_M3]
 
 
 class ARMBE(ARM):

avatar2/avatar2.py

@@ -36,7 +36,7 @@ class Avatar(Thread):
     """
 
     def __init__(
-        self, arch=ARM, cpu_model=None, output_directory=None, log_to_stdout=True, configure_logging=True
+            self, arch=ARM, cpu_model=None, output_directory=None, log_to_stdout=True, configure_logging=True
     ):
         super(Avatar, self).__init__()
 
@@ -70,7 +70,7 @@ def __init__(
             makedirs(self.output_directory)
 
         self.log = logging.getLogger("avatar")
-        
+
         if configure_logging:
             format = "%(asctime)s | %(name)s.%(levelname)s | %(message)s"
 
@@ -130,8 +130,8 @@ def load_config(self, file_name=None):
                 if not tname in self.targets:
                     raise Exception(
                         (
-                            "Requested target %s not found in config. "
-                            "Aborting." % tname
+                                "Requested target %s not found in config. "
+                                "Aborting." % tname
                         )
                     )
                 mr["forwarded_to"] = self.targets[tname]
@@ -183,15 +183,15 @@ def sigint_wrapper(self, signal, frame):
         self.log.info("Avatar Received SIGINT")
         self.sigint_handler()
 
-    def load_plugin(self, name, local=False):
+    def load_plugin(self, name, local=False, *args, **kwargs):
         if local is True:
             plugin = __import__(name, fromlist=["."])
         else:
             plugin = __import__(
                 "avatar2.plugins.%s" % name, fromlist=["avatar2.plugins"]
             )
 
-        plugin.load_plugin(self)
+        plugin.load_plugin(self, *args, **kwargs)
         self.loaded_plugins += [name]
 
     @watch("AddTarget")
@@ -233,21 +233,21 @@ def init_targets(self):
             t[1].init()
 
     def add_memory_range(
-        self,
-        address,
-        size,
-        name=None,
-        permissions="rwx",
-        file=None,
-        file_offset=None,
-        file_bytes=None,
-        forwarded=False,
-        forwarded_to=None,
-        emulate=None,
-        interval_tree=None,
-        inline=False,
-        overwrite=False,
-        **kwargs
+            self,
+            address,
+            size,
+            name=None,
+            permissions="rwx",
+            file=None,
+            file_offset=None,
+            file_bytes=None,
+            forwarded=False,
+            forwarded_to=None,
+            emulate=None,
+            interval_tree=None,
+            inline=False,
+            overwrite=False,
+            **kwargs
     ):
         """
         Adds a memory range to avatar
@@ -294,7 +294,7 @@ def add_memory_range(
             **kwargs
         )
 
-        mr_set = memory_ranges[address : address + size]
+        mr_set = memory_ranges[address: address + size]
         if overwrite is True and len(mr_set) > 0:
             start = min(mr_set, key=lambda x: x.begin).begin
             end = max(mr_set, key=lambda x: x.end).end
@@ -312,7 +312,7 @@ def add_memory_range(
                     interval.data.size,
                 )
 
-        memory_ranges[address : address + size] = m
+        memory_ranges[address: address + size] = m
 
         return m
 
@@ -354,8 +354,8 @@ def transfer_state(self, from_target, to_target, sync_regs=True, synced_ranges=[
         """
 
         if (
-            from_target.state & TargetStates.STOPPED == 0
-            or to_target.state & TargetStates.STOPPED == 0
+                from_target.state & TargetStates.STOPPED == 0
+                or to_target.state & TargetStates.STOPPED == 0
         ):
             raise Exception(
                 "Targets must be stopped for State Transfer, \
@@ -403,6 +403,7 @@ def _handle_update_state_message(self, message):
     def _handle_breakpoint_hit_message(self, message):
         self.log.info("Breakpoint hit for Target: %s" % message.origin.name)
         self._handle_update_state_message(message)
+
         # Breakpoints are two stages: SYNCING | STOPPED -> HandleBreakpoint -> STOPPED
         # This makes sure that all handlers are complete before stopping and breaking wait()
 
@@ -446,8 +447,8 @@ def _handle_remote_memory_read_message(self, message):
         try:
             kwargs = {"num_words": message.num_words, "raw": message.raw}
             if (
-                hasattr(range.forwarded_to, "read_supports_pc")
-                and range.forwarded_to.read_supports_pc is True
+                    hasattr(range.forwarded_to, "read_supports_pc")
+                    and range.forwarded_to.read_supports_pc is True
             ):
                 kwargs["pc"] = message.pc
 
@@ -457,13 +458,13 @@ def _handle_remote_memory_read_message(self, message):
             if not message.raw and message.num_words == 1 and not isinstance(mem, int):
                 raise Exception(
                     (
-                        "Forwarded read returned data of type %s "
-                        "(expected: int)" % type(mem)
+                            "Forwarded read returned data of type %s "
+                            "(expected: int)" % type(mem)
                     )
                 )
             success = True
         except Exception as e:
-            self.log.exception("RemoteMemoryRead failed: %s" % e)
+            self.log.exception("RemoteMemoryRead from %s failed: %s" % (range.forwarded_to, e))
             mem = -1
             success = False
 
@@ -484,8 +485,8 @@ def _handle_remote_memory_write_message(self, message):
 
         kwargs = {}
         if (
-            hasattr(mem_range.forwarded_to, "write_supports_pc")
-            and mem_range.forwarded_to.write_supports_pc is True
+                hasattr(mem_range.forwarded_to, "write_supports_pc")
+                and mem_range.forwarded_to.write_supports_pc is True
         ):
             kwargs["pc"] = message.pc
 
@@ -515,7 +516,6 @@ def run(self):
                 "Avatar received %s. Queue-Status: %d/%d"
                 % (message, self.queue.qsize(), self.fast_queue.qsize())
             )
-
             handler = self.message_handlers.get(message.__class__, None)
             if handler is None:
                 raise Exception("No handler for Avatar-message %s registered" % message)

avatar2/memory_range.py

@@ -33,7 +33,7 @@ def __init__(self, address, size, name=None, permissions='rwx',
         self.name = (
             name
             if name is not None else
-            "mem_range_0x{:08x}_0x{:08x}".format(address, address+size))
+            "mem_range_0x{:08x}_0x{:08x}".format(address, address + size))
         self.permissions = permissions
         self.file = abspath(file) if file is not None else None
         self.file_offset = file_offset if file_offset is not None else None
@@ -44,23 +44,25 @@ def __init__(self, address, size, name=None, permissions='rwx',
         self.forwarded_to = forwarded_to
         self.__dict__.update(kwargs)
 
-
     def dictify(self):
         """
         Returns the memory range as *printable* dictionary
         """
-        from .avatar2 import Avatar # we cannot do this import at top-level
+        from .avatar2 import Avatar  # we cannot do this import at top-level
         # Assumption: dicts saved in mrs are of primitive types only
         expected_types = (str, bool, int, dict, AvatarPeripheral, Avatar, list)
-        if version_info < (3, 0): expected_types += (unicode, )
+        if version_info < (3, 0): expected_types += (unicode,)
 
         tmp_dict = dict(self.__dict__)
         mr_dict = {}
         while tmp_dict != {}:
             k, v = tmp_dict.popitem()
-            if v is None or False: continue
-            elif k == 'forwarded_to': v = v.name
-            # TODO handle emulate
+            if v is None or False:
+                continue
+            elif k == 'forwarded_to':
+                v = v.name
+            if k == 'emulate_config':  # Skip config for emulated peripheral
+                continue
             if not isinstance(v, expected_types):
                 raise Exception(
                     "Unsupported type %s for dictifying %s for mem_range at 0x%x"
@@ -71,5 +73,3 @@ def dictify(self):
                 v = v.__class__.__name__
             mr_dict[k] = v
         return mr_dict
-
-

avatar2/message.py

@@ -1,3 +1,4 @@
+from .plugins.arm.hal import HWFunction
 
 
 class AvatarMessage(object):
@@ -23,6 +24,7 @@ def __init__(self, origin, breakpoint_number, address):
         self.breakpoint_number = breakpoint_number
         self.address = address
 
+
 class SyscallCatchedMessage(BreakpointHitMessage):
     def __init__(self, origin, breakpoint_number, address, type='entry'):
         super(self.__class__, self).__init__(origin, breakpoint_number, address)
@@ -40,6 +42,7 @@ def __init__(self, origin, id, pc, address, size, dst=None):
         self.num_words = 1
         self.raw = False
 
+
 class RemoteMemoryWriteMessage(AvatarMessage):
     def __init__(self, origin, id, pc, address, value, size, dst=None):
         super(self.__class__, self).__init__(origin)
@@ -50,12 +53,14 @@ def __init__(self, origin, id, pc, address, value, size, dst=None):
         self.size = size
         self.dst = dst
 
+
 class RemoteInterruptEnterMessage(AvatarMessage):
     def __init__(self, origin, id, interrupt_num):
         super(self.__class__, self).__init__(origin)
         self.id = id
         self.interrupt_num = interrupt_num
 
+
 class RemoteInterruptExitMessage(AvatarMessage):
     def __init__(self, origin, id, transition_type, interrupt_num):
         super(self.__class__, self).__init__(origin)
@@ -64,4 +69,47 @@ def __init__(self, origin, id, transition_type, interrupt_num):
         self.interrupt_num = interrupt_num
 
 
+class TargetInterruptEnterMessage(AvatarMessage):
+    def __init__(self, origin, id, interrupt_num, isr_addr):
+        super(self.__class__, self).__init__(origin)
+        self.id = id
+        self.interrupt_num = interrupt_num
+        self.isr_addr = isr_addr
+
+
+class TargetInterruptExitMessage(AvatarMessage):
+    def __init__(self, origin, id, interrupt_num, isr_addr):
+        super(self.__class__, self).__init__(origin)
+        self.id = id
+        self.interrupt_num = interrupt_num
+        self.isr_addr = isr_addr
+
+
+class HWEnterMessage(AvatarMessage):
+    def __init__(self, origin, function: HWFunction, return_address: int):
+        super(self.__class__, self).__init__(origin)
+        self.function = function
+        self.return_address = return_address
+
+    def __str__(self):
+        return f"{self.__class__.__name__} from {self.origin.name} returning to 0x{self.return_address:x}"
+
+    def __repr__(self):
+        return self.__str__()
+
+
+class HWExitMessage(AvatarMessage):
+    def __init__(self, origin, function: HWFunction, return_val: int, return_address: int):
+        super(self.__class__, self).__init__(origin)
+        self.function = function
+        self.return_val = return_val
+        self.return_address = return_address
+
+    def __str__(self):
+        return f"{self.__class__.__name__} from {self.origin.name} to 0x{self.return_address:x} with return_value 0x{self.return_val:x}"
+
+    def __repr__(self):
+        return self.__str__()
+
+
 from .targets.target import TargetStates

avatar2/peripherals/avatar_peripheral.py

@@ -9,7 +9,6 @@
     from cached_property import cached_property
 
 
-
 class AvatarPeripheral(object):
     def __init__(self, name, address, size, **kwargs):
         self.name = name if name else "%s_%x" % (self.__class__.__name__, address)
@@ -39,7 +38,7 @@ def shutdown(self):
         """
         pass
 
-    def write_memory(self, address, size, value, num_words=1, raw=False, pc=0):
+    def write_memory(self, address, size, value, num_words=1, raw=False, origin=None, pc=0):
 
         if num_words != 1 or raw is True:
             raise Exception(
@@ -48,7 +47,7 @@ def write_memory(self, address, size, value, num_words=1, raw=False, pc=0):
             )
 
         offset = address - self.address
-        intervals = self.write_handler[offset : offset + size]
+        intervals = self.write_handler[offset: offset + size]
         if intervals == set():
             raise Exception(
                 "No write handler for peripheral %s at offset %d \
@@ -62,18 +61,20 @@ def write_memory(self, address, size, value, num_words=1, raw=False, pc=0):
                 % (self.name, offset)
             )
 
-        kwargs = {} if self.write_supports_pc is False else {"pc": pc}
+        kwargs = {'origin': origin}
+        if self.write_supports_pc is True:
+            kwargs["pc"] = pc
         return intervals.pop().data(offset, size, value, **kwargs)
 
-    def read_memory(self, address, size, num_words=1, raw=False, pc=0):
+    def read_memory(self, address, size, num_words=1, raw=False, origin=None, pc=0):
         if num_words != 1 or raw is True:
             raise Exception(
                 "read_memory for AvatarPeripheral does not support \
                              'num_words' or 'raw' kwarg"
             )
 
         offset = address - self.address
-        intervals = self.read_handler[offset : offset + size]
+        intervals = self.read_handler[offset: offset + size]
 
         if intervals == set():
             raise Exception(
@@ -87,5 +88,7 @@ def read_memory(self, address, size, num_words=1, raw=False, pc=0):
                             at offset %d"
                 % (self.name, offset)
             )
-        kwargs = {} if self.write_supports_pc is False else {"pc": pc}
+        kwargs = {'origin': origin}
+        if self.write_supports_pc is True:
+            kwargs["pc"] = pc
         return intervals.pop().data(offset, size, **kwargs)