AvatarForge ready to go. Pushed full codebase.

.env

@@ -0,0 +1,16 @@
+##### GENERAL ENVS #####
+GPU_SUPPORTED=True
+GPU_ENABLED=True
+
+##### AUTH ENVS #####
+AUTH_ENABLED=False
+REQUIRED_AUTH_ROLES=video-synthesis
+
+##### BACKEND ENVS #####
+FFMPEG_DOCKER=True # Python will use a different version of ffmpeg, therefore we need to redirect to /usr/bin/ffmpeg
+BACKEND_URL=http://localhost:5000
+
+#### AVATAR SETTINGS ####
+AVAILABLE_AVATARS="" # Comma separated list of all Avatar names
+AVATAR_MODELS=neuralVoice,motionGan # All options: "neuralVoice,motionGan"
+

.gitattributes

@@ -0,0 +1,13 @@
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.mat filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+

.gitignore

@@ -0,0 +1,34 @@
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
+
+.idea
+
+# Logging
+error.log
+
+# Visual Studio 2015/2017 cache/options directory
+.vscode/
+
+#Pycharm directory
+avatar_backend_api/.idea/
+
+# log folder
+sbatch_log/
+.out
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# train cluster
+*.job
+# *.sh
+
+# all mac os files ._
+._*
+*.DS_Store
+
+# Data dir
+data/**

AUTHORS.txt

@@ -0,0 +1,7 @@
+Contributions:
+
+* Alberto Pennino <[email protected]>: Research and backend developement.
+* Thomas Steinmann <[email protected]>: Frontend and API developement.
+* Marc Willhaus <[email protected]>: Frontend developement.
+* Clara Labrador-Fernandez <[email protected]>: Supervision.
+

Dockerfile-motion-gan

@@ -0,0 +1,89 @@
+FROM nvidia/cuda:11.4.1-base-ubuntu20.04 as base
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV PATH /opt/conda/bin:$PATH
+
+# Update & install packages
+RUN apt-get -qq update --fix-missing && \
+    apt-get -qq install -y bzip2 ca-certificates wget curl git build-essential && \
+    apt-get -qq clean && \
+    rm -rf /var/lib/apt/lists/*
+
+SHELL ["/bin/bash", "-c"]
+WORKDIR /app
+
+# Install miniconda
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.12.0-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    /opt/conda/bin/conda clean --quiet -tipsy && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
+    echo "conda activate base" >> ~/.bashrc
+
+# Set python env vars
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ENV FORCE_CUDA 1
+ENV PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:512
+
+RUN apt-get -qq update && \
+    apt-get -qq install -y python3-opencv libsm6 libxext6 && \
+    apt-get -qq clean && \
+    rm -rf /var/lib/apt/lists/*
+
+FROM base as backend
+WORKDIR /app
+
+### deepspeech ###
+RUN conda create -n deepspeech python=3.7
+# Make run command use the new environment
+SHELL ["conda", "run", "-n", "deepspeech", "/bin/bash", "-c"]
+
+RUN conda install pysoundfile -c conda-forge && \
+  pip install torch==1.9.1+cu111 torchvision==0.10.1+cu111 torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html && \
+  conda install x264=='1!152.20180717' ffmpeg=4.0.2 -c conda-forge
+
+COPY requirements_deepspeech.txt .
+RUN pip install -r requirements_deepspeech.txt
+
+##############################################
+
+### pyenv ###
+RUN conda create -n pyenv python=3.9
+
+# Make run command use the new environment
+SHELL ["conda", "run", "-n", "pyenv", "/bin/bash", "-c"]
+
+RUN conda install pysoundfile -c conda-forge && \
+    conda install pytorch=1.13.0 torchvision torchaudio pytorch-cuda=11.6 -c pytorch -c nvidia && \
+    conda install -c fvcore -c iopath -c conda-forge fvcore iopath && \
+    conda install x264=='1!152.20180717' ffmpeg=4.0.2 -c conda-forge && \
+    conda install pytorch3d -c pytorch3d
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+# Make run command use the new environment
+SHELL ["conda", "run", "-n", "base", "/bin/bash", "-c"]
+
+COPY requirements.api.txt .
+RUN pip install -r requirements.api.txt
+
+COPY avatar-api ./avatar-api
+RUN pip install ./avatar-api --use-feature=in-tree-build && \
+    rm -rf avatar-api
+
+# continue
+COPY motion-gan-pipeline/GFPGAN ./GFPGAN
+COPY motion-gan-pipeline/preprocessing ./preprocessing
+COPY motion-gan-pipeline/motion-generation ./motion-generation
+COPY motion-gan-pipeline/ImageToImage ./ImageToImage
+
+COPY motion-gan-pipeline/gunicorn.conf.py .
+COPY motion-gan-pipeline/motion_gan_backend_api ./motion_gan_backend_api
+COPY motion-gan-pipeline/full_pipeline* ./
+COPY motion-gan-pipeline/*.py ./
+
+CMD [ "conda", "run", "--no-capture-output", "-n", "base", "gunicorn", "-c", "gunicorn.conf.py", "--chdir", "./motion_gan_backend_api", "-k", "uvicorn.workers.UvicornWorker", "app:app" ]

Dockerfile-neural-voice

@@ -0,0 +1,83 @@
+FROM nvidia/cuda:11.4.1-base-ubuntu20.04 as base
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV PATH /opt/conda/bin:$PATH
+
+# Update & install packages
+RUN apt-get -qq update --fix-missing && \
+    apt-get -qq install -y bzip2 ca-certificates wget curl git build-essential && \
+    apt-get -qq clean && \
+    rm -rf /var/lib/apt/lists/*
+
+SHELL ["/bin/bash", "-c"]
+WORKDIR /app
+
+# Install miniconda
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.12.0-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    /opt/conda/bin/conda clean --quiet -tipsy && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
+    echo "conda activate base" >> ~/.bashrc
+
+# Set python env vars
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ENV FORCE_CUDA 1
+ENV PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:512
+
+RUN apt-get -qq update && \
+    apt-get -qq install -y python3-opencv libsm6 libxext6 && \
+    apt-get -qq clean && \
+    rm -rf /var/lib/apt/lists/*
+
+FROM base as backend
+WORKDIR /app
+
+### deepspeech ###
+RUN conda create -n deepspeech python=3.7
+# Make run command use the new environment
+SHELL ["conda", "run", "-n", "deepspeech", "/bin/bash", "-c"]
+
+RUN conda install pysoundfile -c conda-forge && \
+  pip install torch==1.9.1+cu111 torchvision==0.10.1+cu111 torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html && \
+  conda install x264=='1!152.20180717' ffmpeg=4.0.2 -c conda-forge
+
+COPY requirements_deepspeech.txt .
+RUN pip install -r requirements_deepspeech.txt
+
+##############################################
+
+### pyenv ###
+RUN conda create -n pyenv python=3.9
+
+# Make run command use the new environment
+SHELL ["conda", "run", "-n", "pyenv", "/bin/bash", "-c"]
+
+RUN conda install pysoundfile -c conda-forge && \
+    conda install pytorch=1.13.0 torchvision torchaudio pytorch-cuda=11.6 -c pytorch -c nvidia && \
+    conda install -c fvcore -c iopath -c conda-forge fvcore iopath && \
+    conda install x264=='1!152.20180717' ffmpeg=4.0.2 -c conda-forge && \
+    conda install pytorch3d -c pytorch3d
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+# Make run command use the new environment
+SHELL ["conda", "run", "-n", "base", "/bin/bash", "-c"]
+
+COPY requirements.api.txt .
+RUN pip install -r requirements.api.txt 
+
+COPY avatar-api ./avatar-api
+RUN pip install ./avatar-api --use-feature=in-tree-build && \
+    rm -rf avatar-api
+
+COPY NeuralVoicePuppetry/neural-code ./neural-code
+
+COPY NeuralVoicePuppetry/neural_voice_backend_api ./neural_voice_backend_api
+
+COPY NeuralVoicePuppetry/gunicorn.conf.py .
+CMD [ "conda", "run", "--no-capture-output", "-n", "base", "gunicorn", "-c", "gunicorn.conf.py", "--chdir", "./neural_voice_backend_api", "-k", "uvicorn.workers.UvicornWorker", "app:app" ]

NeuralVoicePuppetry/.dockerignore

@@ -0,0 +1,5 @@
+.idea
+
+**__pycache__**
+
+**.git

NeuralVoicePuppetry/.env

@@ -0,0 +1,13 @@
+##### GENERAL ENVS #####
+GPU_SUPPORTED=True
+GPU_ENABLED=True
+
+##### AUTH ENVS #####
+AUTH_ENABLED=False
+REQUIRED_AUTH_ROLES=video-synthesis
+
+##### BACKEND ENVS #####
+FFMPEG_DOCKER=True # Python will use a different version of ffmpeg, therefore we need to redirect to /usr/bin/ffmpeg
+
+##### REACT ENVS #####
+REACT_APP_ENVIRONMENT=prod

NeuralVoicePuppetry/AUTHORS.txt

@@ -0,0 +1,7 @@
+Contributions:
+
+* Alberto Pennino <[email protected]>: Research and backend developement.
+* Thomas Steinmann <[email protected]>: Frontend and API developement.
+* Marc Willhaus <[email protected]>: Frontend developement.
+* Clara Labrador-Fernandez <[email protected]>: Supervision.
+

NeuralVoicePuppetry/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 ETH Zurich
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.