Week 1: Containerize Training Environment with LoRA and Wav2Vec2 Support

.dockerignore.training

@@ -0,0 +1,73 @@
+# Dockerignore for training container
+# Exclude unnecessary files to reduce image size
+
+# Development files
+.git/
+.gitignore
+*.md
+!README.md
+!docs/**/*.md
+
+# Python cache
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# IDE files
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Test files (not needed in training container)
+tests/
+experiments_old/
+
+# Frontend (not needed for training)
+frontend/
+
+# API files (not needed for training)
+src/inference_api.py
+src/agent_api.py
+src/control_panel_api.py
+
+# Data files (will be mounted or downloaded)
+data/raw/
+data/processed/
+data/production/
+data/test_audio/
+data/recordings_for_test/
+
+# Output files
+*.log
+*.json
+*.png
+*.jpg
+*.pdf
+!requirements.txt
+!requirements-ci.txt
+
+# Documentation (keep minimal)
+docs/submission/
+*.mp4
+
+# CI/CD
+.github/
+
+# Docker files (except this one)
+Dockerfile
+docker-compose.yml
+cloudbuild.yaml
+
+# Temporary files
+*.tmp
+*.bak
+*.swp

Dockerfile.training

@@ -0,0 +1,144 @@
+# Dockerfile for Training Environment
+# Week 1 Task: Dockerize local training environment with LoRA and Wav2Vec2 support
+# Optimized for GCP Deep Learning VMs with GPU support
+
+# Use NVIDIA CUDA base image for GPU training
+# Compatible with GCP Deep Learning VM images (T4, L4, A100)
+# Note: For local testing without GPU, use python:3.10-slim and install CPU-only PyTorch
+FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04
+
+# Alternative for CPU-only local testing (uncomment if GPU not available):
+# FROM python:3.10-slim
+
+# Set environment variables
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+ENV CUDA_VISIBLE_DEVICES=0
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies for audio processing and training
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3.10 \
+    python3-pip \
+    python3.10-dev \
+    gcc \
+    g++ \
+    make \
+    libsndfile1 \
+    ffmpeg \
+    sox \
+    curl \
+    git \
+    wget \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create symlink for python3 -> python
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+# Upgrade pip
+RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel
+
+# Install PyTorch with CUDA support (matching CUDA 11.8)
+# This ensures compatibility with GCP GPU instances
+RUN pip3 install --no-cache-dir \
+    torch==2.0.1 \
+    torchvision==0.15.2 \
+    torchaudio==2.0.1 \
+    --index-url https://download.pytorch.org/whl/cu118
+
+# Install core ML libraries for training
+RUN pip3 install --no-cache-dir \
+    transformers>=4.35.0 \
+    accelerate>=0.24.0 \
+    datasets>=2.14.0 \
+    peft>=0.8.0 \
+    bitsandbytes>=0.43.0
+
+# Install audio processing libraries
+RUN pip3 install --no-cache-dir \
+    librosa>=0.10.0 \
+    soundfile>=0.12.0 \
+    pydub>=0.25.0 \
+    audioread>=3.0.0 \
+    sox>=1.4.1
+
+# Install evaluation metrics
+RUN pip3 install --no-cache-dir \
+    jiwer>=3.0.0
+
+# Install Google Cloud libraries for GCS integration
+RUN pip3 install --no-cache-dir \
+    google-cloud-storage>=2.10.0 \
+    gcsfs>=2023.6.0 \
+    google-auth>=2.23.0
+
+# Install data processing libraries
+RUN pip3 install --no-cache-dir \
+    pandas>=2.0.0 \
+    numpy>=1.24.0 \
+    scikit-learn>=1.3.0 \
+    scipy>=1.11.0
+
+# Install visualization libraries (for training monitoring)
+RUN pip3 install --no-cache-dir \
+    matplotlib>=3.7.0 \
+    seaborn>=0.12.0
+
+# Install experiment tracking
+RUN pip3 install --no-cache-dir \
+    wandb>=0.16.0
+
+# Install utilities
+RUN pip3 install --no-cache-dir \
+    tqdm>=4.65.0 \
+    pyyaml>=6.0 \
+    python-dotenv>=1.0.0 \
+    importlib-metadata
+
+# Copy requirements.txt for any additional dependencies
+COPY requirements.txt /app/requirements.txt
+
+# Install any remaining dependencies from requirements.txt
+# Skip FastAPI/uvicorn for training container (not needed)
+RUN pip3 install --no-cache-dir -r requirements.txt || \
+    pip3 install --no-cache-dir $(grep -v "fastapi\|uvicorn" requirements.txt | grep -v "^#" | grep -v "^$") || true
+
+# Copy source code
+COPY src/ /app/src/
+COPY scripts/ /app/scripts/
+# Create experiments directory (will be mounted or created at runtime)
+RUN mkdir -p /app/experiments
+
+# Create necessary directories for training
+RUN mkdir -p \
+    /app/data/raw \
+    /app/data/processed \
+    /app/data/finetuning \
+    /app/data/checkpoints \
+    /app/data/models \
+    /app/data/logs \
+    /app/data/cache
+
+# Set up training scripts as executable
+RUN chmod +x /app/scripts/finetune_wav2vec2.py
+
+# Verify installations
+RUN python3 -c "import torch; print(f'PyTorch: {torch.__version__}'); print(f'CUDA Available: {torch.cuda.is_available()}'); print(f'CUDA Version: {torch.version.cuda if torch.cuda.is_available() else \"N/A\"}')" && \
+    python3 -c "import transformers; print(f'Transformers: {transformers.__version__}')" && \
+    python3 -c "from peft import LoraConfig; print('PEFT/LoRA: Available')" && \
+    python3 -c "from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor; print('Wav2Vec2: Available')" && \
+    python3 -c "import librosa; print(f'Librosa: {librosa.__version__}')" && \
+    python3 -c "import jiwer; print('jiwer: Available')"
+
+# Default command (can be overridden)
+# This allows the container to be used interactively or with training scripts
+CMD ["/bin/bash"]
+
+# Health check for training container
+HEALTHCHECK --interval=60s --timeout=30s --start-period=120s --retries=3 \
+    CMD python3 -c "import torch; import transformers; import peft; print('Training environment ready')" || exit 1

Dockerfile.training.cpu

@@ -0,0 +1,129 @@
+# Dockerfile for Training Environment (CPU-only version)
+# Alternative version for local testing without GPU
+# Week 1 Task: Dockerize local training environment with LoRA and Wav2Vec2 support
+
+# Use Python base image for CPU-only training
+FROM python:3.10-slim
+
+# Set environment variables
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+ENV CUDA_VISIBLE_DEVICES=""
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies for audio processing and training
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    g++ \
+    make \
+    libsndfile1 \
+    ffmpeg \
+    sox \
+    curl \
+    git \
+    wget \
+    && rm -rf /var/lib/apt/lists/*
+
+# Upgrade pip
+RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel
+
+    # Install PyTorch CPU-only version (for local testing)
+    # Use compatible versions: torchaudio 2.0.1 requires torch 2.0.0
+    RUN pip3 install --no-cache-dir \
+        torch==2.0.0 \
+        torchvision==0.15.1 \
+        torchaudio==2.0.1 \
+        --index-url https://download.pytorch.org/whl/cpu
+
+# Install core ML libraries for training
+RUN pip3 install --no-cache-dir \
+    transformers>=4.35.0 \
+    accelerate>=0.24.0 \
+    datasets>=2.14.0 \
+    peft>=0.8.0
+
+# Install audio processing libraries
+RUN pip3 install --no-cache-dir \
+    librosa>=0.10.0 \
+    soundfile>=0.12.0 \
+    pydub>=0.25.0 \
+    audioread>=3.0.0 \
+    sox>=1.4.1
+
+# Install evaluation metrics
+RUN pip3 install --no-cache-dir \
+    jiwer>=3.0.0
+
+# Install Google Cloud libraries for GCS integration
+RUN pip3 install --no-cache-dir \
+    google-cloud-storage>=2.10.0 \
+    gcsfs>=2023.6.0 \
+    google-auth>=2.23.0
+
+# Install data processing libraries
+RUN pip3 install --no-cache-dir \
+    pandas>=2.0.0 \
+    numpy>=1.24.0 \
+    scikit-learn>=1.3.0 \
+    scipy>=1.11.0
+
+# Install visualization libraries (for training monitoring)
+RUN pip3 install --no-cache-dir \
+    matplotlib>=3.7.0 \
+    seaborn>=0.12.0
+
+# Install experiment tracking
+RUN pip3 install --no-cache-dir \
+    wandb>=0.16.0
+
+# Install utilities
+RUN pip3 install --no-cache-dir \
+    tqdm>=4.65.0 \
+    pyyaml>=6.0 \
+    python-dotenv>=1.0.0 \
+    importlib-metadata
+
+# Copy requirements.txt for any additional dependencies
+COPY requirements.txt /app/requirements.txt
+
+# Install any remaining dependencies from requirements.txt
+# Skip FastAPI/uvicorn for training container (not needed)
+RUN pip3 install --no-cache-dir -r requirements.txt || \
+    pip3 install --no-cache-dir $(grep -v "fastapi\|uvicorn\|bitsandbytes" requirements.txt | grep -v "^#" | grep -v "^$") || true
+
+# Copy source code
+COPY src/ /app/src/
+COPY scripts/ /app/scripts/
+# Create experiments directory (will be mounted or created at runtime)
+RUN mkdir -p /app/experiments
+
+# Create necessary directories for training
+RUN mkdir -p \
+    /app/data/raw \
+    /app/data/processed \
+    /app/data/finetuning \
+    /app/data/checkpoints \
+    /app/data/models \
+    /app/data/logs \
+    /app/data/cache
+
+# Set up training scripts as executable
+RUN chmod +x /app/scripts/finetune_wav2vec2.py
+
+# Verify installations (CPU version)
+RUN python3 -c "import torch; print(f'PyTorch: {torch.__version__}'); print(f'CUDA Available: {torch.cuda.is_available()}')" && \
+    python3 -c "import transformers; print(f'Transformers: {transformers.__version__}')" && \
+    python3 -c "from peft import LoraConfig; print('PEFT/LoRA: Available')" && \
+    python3 -c "from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor; print('Wav2Vec2: Available')" && \
+    python3 -c "import librosa; print(f'Librosa: {librosa.__version__}')" && \
+    python3 -c "import jiwer; print('jiwer: Available')"
+
+# Default command (can be overridden)
+CMD ["/bin/bash"]
+
+# Health check for training container
+HEALTHCHECK --interval=60s --timeout=30s --start-period=120s --retries=3 \
+    CMD python3 -c "import torch; import transformers; import peft; print('Training environment ready')" || exit 1

docker-compose.training.yml

@@ -0,0 +1,46 @@
+# Docker Compose for Training Environment
+# Week 1: Training container with GPU support
+version: '3.8'
+
+services:
+  training:
+    build:
+      context: .
+      dockerfile: Dockerfile.training
+    image: adaptive-stt-training:latest
+    container_name: adaptive-stt-training
+    # GPU support (requires nvidia-docker or Docker with GPU support)
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+    environment:
+      - PYTHONUNBUFFERED=1
+      - PYTHONPATH=/app
+      - CUDA_VISIBLE_DEVICES=0
+      - WANDB_API_KEY=${WANDB_API_KEY:-}
+      - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS:-}
+    volumes:
+      # Mount data directory for training data
+      - ./data:/app/data
+      # Mount checkpoints directory for model persistence
+      - ./data/checkpoints:/app/data/checkpoints
+      # Mount models directory for saved models
+      - ./data/models:/app/data/models
+      # Mount logs directory
+      - ./data/logs:/app/data/logs
+      # Mount GCP credentials if available
+      - ${GOOGLE_APPLICATION_CREDENTIALS:-./gcp-credentials.json}:/app/gcp-credentials.json:ro
+    working_dir: /app
+    # Keep container running for interactive use
+    stdin_open: true
+    tty: true
+    networks:
+      - training-network
+
+networks:
+  training-network:
+    driver: bridge