AI Engine (AIE) Backend support as external plugin

docs/advanced/plugins.rst

@@ -0,0 +1,61 @@
+=======================================
+External Backend and Writer Plugins
+=======================================
+
+``hls4ml`` can discover and load backend implementations from
+external Python packages. This enables specialised flows, such as the AMD AIE backend, to live in
+independent projects that version and iterate at their own cadence while reusing the core
+conversion infrastructure.
+
+Discovery
+=========
+
+Plugin packages advertise themselves through the ``hls4ml.backends`` Python entry point group. Each
+entry either exposes a subclass of :class:`hls4ml.backends.backend.Backend` or a callable that
+receives ``register_backend`` and ``register_writer`` helpers and performs any setup that is
+required. ``hls4ml`` automatically scans for these entry points during ``hls4ml.backends`` import so
+third-party backends become available without additional user configuration.
+
+In addition to entry points, modules listed in the ``HLS4ML_BACKEND_PLUGINS`` environment variable
+are imported and treated as registration callables. The variable accepts an ``os.pathsep`` separated
+list (``:`` on Linux/macOS or ``;`` on Windows):
+
+.. code-block:: bash
+
+   export HLS4ML_BACKEND_PLUGINS=aie4ml.plugin:another_pkg.hls4ml_backend
+
+Authoring a Plugin
+==================
+
+A minimal plugin registers both a backend and an accompanying writer. The example below
+shows how the ``aie4ml`` package exposes its backend via ``pyproject.toml`` and a ``register``
+function:
+
+.. code-block:: toml
+
+   [project.entry-points."hls4ml.backends"]
+   AIE = "aie4ml.plugin:register"
+
+.. code-block:: python
+
+   # aie4ml/plugin.py
+   from aie4ml.aie_backend import AIEBackend
+   from aie4ml.writer import AIEWriter
+
+   def register(*, register_backend, register_writer):
+       register_writer('AIE', AIEWriter)
+       register_backend('AIE', AIEBackend)
+
+When the plugin is installed, ``hls4ml.backends.get_available_backends()`` will report the new
+backend just like the built-in FPGA toolflows.
+
+Packaging Data Files
+====================
+
+Backends often rely on firmware templates or device description files. These assets should be
+packaged alongside the Python sources using the usual ``setuptools`` mechanisms (``package-data`` or
+``include-package-data``) so they are available from the installed distribution.
+
+For an end-to-end example see the companion ``aie4ml`` [https://github.com/dimdano/aie4ml] package that ships alongside this project
+as a standalone distribution; it encapsulates the existing AMD AIE backend as an installable plugin
+depending on ``hls4ml``.

docs/index.rst

@@ -53,6 +53,7 @@
     advanced/extension
     advanced/model_optimization
     advanced/bramfactor
+    advanced/plugins
 
 .. toctree::
     :hidden:

hls4ml/backends/__init__.py

@@ -1,6 +1,7 @@
 from hls4ml.backends.backend import Backend, get_available_backends, get_backend, register_backend  # noqa: F401
 from hls4ml.backends.fpga.fpga_backend import FPGABackend  # noqa: F401
 from hls4ml.backends.oneapi.oneapi_backend import OneAPIBackend
+from hls4ml.backends.plugin_loader import load_backend_plugins
 from hls4ml.backends.quartus.quartus_backend import QuartusBackend
 from hls4ml.backends.symbolic.symbolic_backend import SymbolicExpressionBackend
 from hls4ml.backends.vivado.vivado_backend import VivadoBackend
@@ -11,10 +12,16 @@
 
 from hls4ml.backends.vitis.vitis_backend import VitisBackend  # isort: skip
 
-register_backend('Vivado', VivadoBackend)
-register_backend('VivadoAccelerator', VivadoAcceleratorBackend)
-register_backend('Vitis', VitisBackend)
-register_backend('Quartus', QuartusBackend)
-register_backend('Catapult', CatapultBackend)
-register_backend('SymbolicExpression', SymbolicExpressionBackend)
-register_backend('oneAPI', OneAPIBackend)
+
+def _register_builtin_backends():
+    register_backend('Vivado', VivadoBackend)
+    register_backend('VivadoAccelerator', VivadoAcceleratorBackend)
+    register_backend('Vitis', VitisBackend)
+    register_backend('Quartus', QuartusBackend)
+    register_backend('Catapult', CatapultBackend)
+    register_backend('SymbolicExpression', SymbolicExpressionBackend)
+    register_backend('oneAPI', OneAPIBackend)
+
+
+_register_builtin_backends()
+load_backend_plugins()

hls4ml/backends/plugin_loader.py

@@ -0,0 +1,133 @@
+"""Utilities for discovering and loading external hls4ml backend plugins."""
+
+from __future__ import annotations
+
+import inspect
+import logging
+import os
+from collections.abc import Callable, Iterable
+from importlib import import_module
+from typing import Any
+
+try:  # pragma: no cover - fall back for older Python versions
+    from importlib.metadata import entry_points
+except ImportError:  # pragma: no cover
+    from importlib_metadata import entry_points  # type: ignore
+
+from hls4ml.backends.backend import Backend, register_backend
+from hls4ml.writer.writers import register_writer
+
+ENTRY_POINT_GROUP = 'hls4ml.backends'
+ENV_PLUGIN_MODULES = 'HLS4ML_BACKEND_PLUGINS'
+
+_plugins_loaded = False
+
+
+def load_backend_plugins(logger: logging.Logger | None = None) -> None:
+    """Discover and register backend plugins.
+
+    This function loads plugins published via Python entry points under the
+    ``hls4ml.backends`` group as well as modules listed in the
+    ``HLS4ML_BACKEND_PLUGINS`` environment variable. The environment variable
+    accepts a separator compatible with :data:`os.pathsep`.
+
+    Args:
+        logger (logging.Logger, optional): Optional logger used for diagnostics.
+            When omitted, a module-local logger will be used.
+    """
+
+    global _plugins_loaded
+    if _plugins_loaded:
+        return
+
+    logger = logger or logging.getLogger(__name__)
+
+    _load_entry_point_plugins(logger)
+    _load_env_plugins(logger)
+
+    _plugins_loaded = True
+
+
+def _load_entry_point_plugins(logger: logging.Logger) -> None:
+    eps = entry_points()
+
+    if hasattr(eps, 'select'):
+        group_eps = eps.select(group=ENTRY_POINT_GROUP)
+    else:  # pragma: no cover - legacy importlib_metadata API
+        group_eps = eps.get(ENTRY_POINT_GROUP, [])
+
+    for ep in group_eps:
+        try:
+            obj = ep.load()
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.warning(
+                'Failed to load backend plugin entry %s: %s', ep.name, exc, exc_info=logger.isEnabledFor(logging.DEBUG)
+            )
+            continue
+        _register_plugin_object(ep.name, obj, logger)
+
+
+def _load_env_plugins(logger: logging.Logger) -> None:
+    raw_modules = os.environ.get(ENV_PLUGIN_MODULES, '')
+    if not raw_modules:
+        return
+
+    for module_name in filter(None, raw_modules.split(os.pathsep)):
+        try:
+            module = import_module(module_name)
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.warning(
+                'Failed to import backend plugin module %s: %s',
+                module_name,
+                exc,
+                exc_info=logger.isEnabledFor(logging.DEBUG),
+            )
+            continue
+
+        register_callable: Any = getattr(module, 'register', module)
+        _register_plugin_object(module_name, register_callable, logger)
+
+
+def _register_plugin_object(name: str, obj: Any, logger: logging.Logger) -> None:
+    """Interpret the plugin object and register provided backends."""
+
+    if inspect.isclass(obj) and issubclass(obj, Backend):
+        _safe_register_backend(name, obj, logger)
+        return
+
+    if isinstance(obj, Iterable) and not isinstance(obj, (str, bytes)):
+        for item in obj:
+            _register_plugin_object(name, item, logger)
+        return
+
+    if callable(obj):
+        _invoke_registration_callable(name, obj, logger)
+        return
+
+    logger.warning('Plugin entry %s did not provide a usable backend registration (got %r)', name, obj)
+
+
+def _invoke_registration_callable(name: str, func: Callable[..., Any], logger: logging.Logger) -> None:
+    try:
+        func(register_backend=register_backend, register_writer=register_writer)
+    except TypeError:
+        try:
+            func(register_backend, register_writer)
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.warning('Backend plugin callable %s failed: %s', name, exc, exc_info=logger.isEnabledFor(logging.DEBUG))
+        else:
+            return
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.warning('Backend plugin callable %s failed: %s', name, exc, exc_info=logger.isEnabledFor(logging.DEBUG))
+        return
+    else:
+        return
+
+
+def _safe_register_backend(name: str, backend_cls: type[Backend], logger: logging.Logger) -> None:
+    try:
+        register_backend(name, backend_cls)
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.warning(
+            'Failed to register backend %s from plugin: %s', name, exc, exc_info=logger.isEnabledFor(logging.DEBUG)
+        )