From ca540528f818da9b94e0f1bf7a443aec5193e352 Mon Sep 17 00:00:00 2001 From: rawsample <39617899+rawsample@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:38:45 +0100 Subject: [PATCH] Docker compose (#109) * Add docker-compose commands and a slimmer container image without PANDA avatar2-lite * Update building instructions --- .dockerignore | 105 ++++++++++++++++++++++++++++++++++++++++ README.md | 48 ++++++++++-------- avatar2-lite.Dockerfile | 58 ++++++++++++++++++++++ avatar2.Dockerfile | 4 +- docker-compose.yml | 79 ++++++++++++++++++++++++++++++ generate_dockerfile.py | 13 +++-- 6 files changed, 281 insertions(+), 26 deletions(-) create mode 100644 .dockerignore create mode 100644 avatar2-lite.Dockerfile create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..0822e547df --- /dev/null +++ b/.dockerignore @@ -0,0 +1,105 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# Emacs backup files +*~ +\#*\# +.\#* + +# Vim swap files +*.swp + +# Pycharm/IntelliJ files +.idea \ No newline at end of file diff --git a/README.md b/README.md index 22d9de3b1a..9c5c369369 100644 --- a/README.md +++ b/README.md @@ -9,55 +9,58 @@ Welcome to avatar², the target orchestration framework with focus on dynamic Avatar² is developed and maintained by [Eurecom's S3 Group](http://s3.eurecom.fr/). -# Building -Building avatar² is easy! +# Building -First, make sure that all the dependencies are present: +### Docker +Two Dockerfile are present: `avatar2-lite.Dockerfile` builds avatar² with QEmu target endpoints, while `avatar2.Dockerfile` builds both QEmu and PANDA target endpoints. +For example, to build and run a container with avatar2 and PANDA configured for the ARM architecture: ``` -sudo apt-get install python-pip python-setuptools python-dev cmake +$ COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build avatar2-arm +$ docker-compose run avatar2-arm ``` -Afterwards, use python-pip to install avatar2: - +Alternately, `generate_dockerfile.py` helps you generate the docker image with the endpoints you want: ``` -pip install avatar2 +$ python3 generate_dockerfile.py --endpoint_list avatar-qemu --qemu_targets arm-softmmu +$ docker build -t avatar2 . ``` -Now you are all ready to go. Additionally, if you want to install specific -target entpoints, please run the avatar2-installer, which tries to fetch and -install the endpoints automatically. +### PyPI + +Avatar² can be installed from PyPI. +First, make sure that all the dependencies are present: ``` -python -m avatar2.installer +sudo apt-get install python-pip python-setuptools python-dev cmake ``` -### Building with Docker - -A Dockerfile is present which build by default avatar² with QEmu and PANDA target endpoints: +We recommands you to use a [Python virtual environment](https://virtualenvwrapper.readthedocs.io/en/latest/). +Afterwards, use python-pip to install avatar2: ``` -$ docker build -t avatar2 . -$ docker run --rm avatar2 python3 -c "import avatar2" +python3 -m pip install avatar2 ``` -Alternately, you can use `generate_dockerfile.py` to build a docker image with only the target endpoints your need: +Now you will need to install the supported endpoints you would like to use, such as debuggers (`gdb-multiarch`, `openocd`) and emulators (`qemu-system-*`, `panda-system-*`, etc.). +The old avatar2-installer tries to fetch and install them automatically, but is not actively supported. ``` -$ python3 generate_dockerfile.py --endpoint_list avatar-qemu --qemu_targets arm-softmmu -$ docker build -t avatar2 . +python -m avatar2.installer ``` ### Building manually Avatar² can also be built manually. + The following three commands are enough to install the core. ``` $ git clone https://github.com/avatartwo/avatar2.git $ cd avatar2 -$ sudo python setup.py install +$ python setup.py install ``` + Afterwards, the different target endpoints can be built, such as QEmu or PANDA. For doing so, we are providing build-scripts for Ubuntu 20.04 - while other distributions are not officially supported (yet), the scripts are known to @@ -72,7 +75,9 @@ $ ./build_*.sh with this, please consider building avatar² in a VM/Container or install the dependencies manually and adjust the scripts. + # Getting started + For discovering the power of avatar² and getting a feeling of its usage, we recommend highly checking out the [handbook](https://github.com/avatartwo/avatar2/tree/master/handbook) here on @@ -94,7 +99,9 @@ vivid means of communication - if you want an invite, just send us a mail! # Publications + The following publications describe, use, or extend the avatar² framework: + 1. M. Muench, D. Nisi, A. Francillon, D. Balzarotti. "Avatar²: A Multi-target Orchestration Platform." Workshop on Binary Analysis Research (BAR), San Diego, California, February 2018. - [Paper](http://s3.eurecom.fr/docs/bar18_muench.pdf) - [Code](https://github.com/avatartwo/bar18_avatar2) 2. M. Muench, J. Stijohann, F. Kargl, A. Francillon, D. Balzarotti. "What You Corrupt Is Not What You Crash: Challenges in Fuzzing Embedded Devices." Network and Distributed System Security Symposium (NDSS), San Diego, California, 2018. @@ -112,4 +119,5 @@ The following publications describe, use, or extend the avatar² framework: # Acknowledgements + The avatar² project was partially funded through, and supported by, SIEMENS AG - Technology. diff --git a/avatar2-lite.Dockerfile b/avatar2-lite.Dockerfile new file mode 100644 index 0000000000..c93988d207 --- /dev/null +++ b/avatar2-lite.Dockerfile @@ -0,0 +1,58 @@ +### Stage 0: the base avatar2-core image +FROM ubuntu:20.04 AS base + +# avatar2 run-time dependencies +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-setuptools ipython3 libcapstone3 gdb gdbserver gdb-multiarch udev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + + + +### Stage 1: The avatar2-core build image +FROM base AS build-core + +# avatar2 build dependencies +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends git cmake pkg-config build-essential python3-dev python3-pip libcapstone-dev && \ + pip3 install --upgrade --no-cache-dir pip + +RUN git clone https://github.com/avatartwo/avatar2 /root/avatar2/ +RUN cd /root/avatar2 && \ + python3 setup.py install + + + +### Stage 2: Build avatar-qemu +FROM base AS build-avatar-qemu +ARG QEMU_TARGETS="arm-softmmu,mips-softmmu,i386-softmmu,x86_64-softmmu" + +RUN sed -i '/deb-src .*-security main restricted/s/^#//g' /etc/apt/sources.list +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get build-dep -y qemu && \ + apt-get install -y git ninja-build + +RUN git clone https://github.com/avatartwo/avatar-qemu /root/avatar-qemu/ +RUN cd /root/avatar-qemu/ && \ + git checkout dev/qemu-6.2 +RUN mkdir -p /root/avatar-qemu/build && cd /root/avatar-qemu/build && \ + ../configure \ + --disable-sdl \ + --prefix=/usr/local/ \ + --target-list="${QEMU_TARGETS}" && \ + make -j "$(nproc)" +RUN cd /root/avatar-qemu/build/ && make install + + + +### Stage 3: Assemble the final image +FROM base AS avatar2 + +COPY --from=build-core /usr/local /usr/local + +RUN apt-get update && \ + apt-get install -y --no-install-recommends libpulse0 + +COPY --from=build-avatar-qemu /usr/local /usr/local +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + diff --git a/avatar2.Dockerfile b/avatar2.Dockerfile index a799c0d993..e9683cbe76 100644 --- a/avatar2.Dockerfile +++ b/avatar2.Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:20.04 AS base # avatar2 run-time dependencies RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-setuptools libcapstone3 gdb gdbserver gdb-multiarch && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-setuptools ipython3 libcapstone3 gdb gdbserver gdb-multiarch udev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -25,7 +25,7 @@ RUN cd /root/avatar2 && \ ### Stage 2: Build avatar-qemu FROM base AS build-avatar-qemu -ARG QEMU_TARGETS="arm-softmmu,mips-softmmu" +ARG QEMU_TARGETS="arm-softmmu,mips-softmmu,i386-softmmu,x86_64-softmmu" RUN sed -i '/deb-src .*-security main restricted/s/^#//g' /etc/apt/sources.list RUN apt-get update && \ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..8417ce3a58 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,79 @@ +version: "3.7" + + +x-options: &options + command: ipython3 + stdin_open: true + tty: true + privileged: true + volumes: + - /run/udev:/run/udev:ro + - /dev/bus:/dev/bus:ro + +x-avatar: &avatar + image: avatartwo/avatar2 + build: + dockerfile: avatar2.Dockerfile + context: . + +x-avatar-lite: &avatar-lite + image: avatartwo/avatar2-lite + build: + dockerfile: avatar2-lite.Dockerfile + context: . + + +services: + + avatar2-lite-arm: + <<: *avatar-lite + <<: *options + environment: + AVATAR2_ARCH: ARM + AVATAR2_GDB_EXECUTABLE: gdb-multiarch + AVATAR2_QEMU_EXECUTABLE: qemu-system-arm + + avatar2-lite-mips: + <<: *avatar-lite + <<: *options + environment: + AVATAR2_ARCH: MIPS + AVATAR2_GDB_EXECUTABLE: gdb-multiarch + AVATAR2_QEMU_EXECUTABLE: qemu-system-mips + + avatar2-lite-x86: + <<: *avatar-lite + <<: *options + environment: + AVATAR2_ARCH: X86 + AVATAR2_GDB_EXECUTABLE: gdb-multiarch + AVATAR2_QEMU_EXECUTABLE: qemu-system-x86_64 + + + avatar2-arm: + <<: *avatar + <<: *options + environment: + AVATAR2_ARCH: ARM + AVATAR2_GDB_EXECUTABLE: gdb-multiarch + AVATAR2_QEMU_EXECUTABLE: qemu-system-arm + AVATAR2_PANDA_EXECUTABLE: panda-system-arm + + avatar2-mips: + <<: *avatar + <<: *options + environment: + AVATAR2_ARCH: MIPS + AVATAR2_GDB_EXECUTABLE: gdb-multiarch + AVATAR2_QEMU_EXECUTABLE: qemu-system-mips + AVATAR2_PANDA_EXECUTABLE: panda-system-mips + + avatar2-x86: + <<: *avatar + <<: *options + environment: + AVATAR2_ARCH: X86 + AVATAR2_GDB_EXECUTABLE: gdb-multiarch + AVATAR2_QEMU_EXECUTABLE: qemu-system-x86_64 + AVATAR2_PANDA_EXECUTABLE: panda-system-x86_64 + diff --git a/generate_dockerfile.py b/generate_dockerfile.py index 9d7b4813db..e514df2d82 100644 --- a/generate_dockerfile.py +++ b/generate_dockerfile.py @@ -17,10 +17,12 @@ avatar2_runtime_dependencies=[ 'python3', 'python3-setuptools', + 'ipython3', 'libcapstone3', 'gdb', 'gdbserver', - 'gdb-multiarch'] + 'gdb-multiarch', + 'udev'] avatar2_build_dependencies=[ 'git', 'cmake', 'pkg-config', @@ -88,12 +90,12 @@ -def generate(endpoint_list, qemu_targets=['arm-softmmu']): +def generate(filename, endpoint_list, qemu_targets=['arm-softmmu']): print(f'[*] Generate avatar2 Dockerfile with the following endpoints: {endpoint_list}') stage = 0 - with open('./Dockerfile', 'w') as f: + with open(filename, 'w') as f: # avatar2 base images @@ -149,6 +151,9 @@ def generate(endpoint_list, qemu_targets=['arm-softmmu']): parser = argparse.ArgumentParser(description=DESCRIPTION, usage=USAGE) + parser.add_argument('-o', '--output_file', type=str, default='Dockerfile', + help='Name for the outputed Dockerfile') + parser.add_argument('-e', '--endpoint_list', nargs='+', default=None, choices=['avatar-qemu', 'panda'], help='list of endpoints to build with avatar2') @@ -160,5 +165,5 @@ def generate(endpoint_list, qemu_targets=['arm-softmmu']): args = parser.parse_args() - generate(args.endpoint_list, args.qemu_targets) + generate(args.output_file, args.endpoint_list, args.qemu_targets)