From 462f96720fea97c139c1c692422d5f4d09cb56ad Mon Sep 17 00:00:00 2001 From: Hunia Fatima Date: Fri, 25 Oct 2024 18:35:21 +0500 Subject: [PATCH 1/3] chore: remove dockerfile setup --- .dockerignore | 152 ------------- .github/workflows/publish-ci-docker-image.yml | 43 ---- Dockerfile | 200 ------------------ 3 files changed, 395 deletions(-) delete mode 100644 .dockerignore delete mode 100644 .github/workflows/publish-ci-docker-image.yml delete mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index c3873d33a595..000000000000 --- a/.dockerignore +++ /dev/null @@ -1,152 +0,0 @@ -# .dockerignore for edx-platform. -# There's a lot here, please try to keep it organized. - -### Files that are not needed in the docker file - -/test_root/ -.git - -### Files private to developers - -# Files that should be git-ignored, but are hand-edited or otherwise valued, -# and so should not be destroyed by "make clean". -# start-noclean -requirements/private.txt -requirements/edx/private.in -requirements/edx/private.txt -lms/envs/private.py -cms/envs/private.py -# end-noclean - -### Python artifacts -**/*.pyc - -### Editor and IDE artifacts -**/*~ -**/*.swp -**/*.orig -**/nbproject -**/.idea/ -**/.redcar/ -**/codekit-config.json -**/.pycharm_helpers/ -**/_mac/* -**/IntelliLang.xml -**/conda_packages.xml -**/databaseSettings.xml -**/diff.xml -**/debugger.xml -**/editor.xml -**/ide.general.xml -**/inspection/Default.xml -**/other.xml -**/packages.xml -**/web-browsers.xml - -### NFS artifacts -**/.nfs* - -### OS X artifacts -**/*.DS_Store -**/.AppleDouble -**/:2e_* -**/:2e# - -### Internationalization artifacts -**/*.mo -**/*.po -**/*.prob -**/*.dup -!**/django.po -!**/django.mo -!**/djangojs.po -!**/djangojs.mo -conf/locale/en/LC_MESSAGES/*.mo -conf/locale/fake*/LC_MESSAGES/*.po -conf/locale/fake*/LC_MESSAGES/*.mo - -### Testing artifacts -**/.testids/ -**/.noseids -**/nosetests.xml -**/.cache/ -**/.coverage -**/.coverage.* -**/coverage.xml -**/cover/ -**/cover_html/ -**/reports/ -**/jscover.log -**/jscover.log.* -**/.pytest_cache/ -**/pytest_task*.txt -**/.tddium* -common/test/data/test_unicode/static/ -test_root/courses/ -test_root/data/test_bare.git/ -test_root/export_course_repos/ -test_root/paver_logs/ -test_root/uploads/ -**/django-pyfs -**/.tox/ -common/test/data/badges/*.png - -### Installation artifacts -**/*.egg-info -**/.pip_download_cache/ -**/.prereqs_cache -**/.vagrant/ -**/node_modules -**/bin/ - -### Static assets pipeline artifacts -**/*.scssc -lms/static/css/ -lms/static/certificates/css/ -cms/static/css/ -common/static/common/js/vendor/ -common/static/common/css/vendor/ -common/static/bundles -**/webpack-stats.json - -### Styling generated from templates -lms/static/sass/*.css -lms/static/sass/*.css.map -lms/static/certificates/sass/*.css -lms/static/themed_sass/ -cms/static/css/ -cms/static/sass/*.css -cms/static/sass/*.css.map -cms/static/themed_sass/ -themes/**/css - -### Logging artifacts -**/log/ -**/logs -**/chromedriver.log -**/ghostdriver.log - -### Celery artifacts ### -**/celerybeat-schedule - -### Unknown artifacts -**/database.sqlite -**/courseware/static/js/mathjax/* -**/flushdb.sh -**/build -/src/ -\#*\# -**/.env/ -openedx/core/djangoapps/django_comment_common/comment_client/python -**/autodeploy.properties -**/.ws_migrations_complete -**/dist -**/*.bak - -# Visual Studio Code -**/.vscode - -# Locally generated PII reports -**/pii_report - -/Dockerfile diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml deleted file mode 100644 index 6a0f3768b7e6..000000000000 --- a/.github/workflows/publish-ci-docker-image.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Push CI Runner Docker Image - -on: - workflow_dispatch: - schedule: - - cron: "0 1 * * 3" - -jobs: - push: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - # This has to happen after checkout in order for gh to work. - - name: "Cancel scheduled job on forks" - if: github.repository != 'openedx/edx-platform' && github.event_name == 'schedule' - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - run: | - gh run cancel "${{ github.run_id }}" - gh run watch "${{ github.run_id }}" - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Log in to ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build, tag, and push image to Amazon ECR - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: actions-runner - IMAGE_TAG: latest - run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f scripts/ci-runner.Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 75a716177fdc..000000000000 --- a/Dockerfile +++ /dev/null @@ -1,200 +0,0 @@ -FROM ubuntu:focal as minimal-system - -# Warning: This file is experimental. -# -# Short-term goals: -# * Be a suitable replacement for the `edxops/edxapp` image in devstack (in progress). -# * Take advantage of Docker caching layers: aim to put commands in order of -# increasing cache-busting frequency. -# * Related to ^, use no Ansible or Paver. -# Long-term goal: -# * Be a suitable base for production LMS and CMS images (THIS IS NOT YET THE CASE!). - -ARG DEBIAN_FRONTEND=noninteractive -ARG SERVICE_VARIANT -ARG SERVICE_PORT - -# Env vars: paver -# We intentionally don't use paver in this Dockerfile, but Devstack may invoke paver commands -# during provisioning. Enabling NO_PREREQ_INSTALL tells paver not to re-install Python -# requirements for every paver command, potentially saving a lot of developer time. -ARG NO_PREREQ_INSTALL='1' - -# Env vars: locale -ENV LANG='en_US.UTF-8' -ENV LANGUAGE='en_US:en' -ENV LC_ALL='en_US.UTF-8' - -# Env vars: configuration -ENV CONFIG_ROOT='/edx/etc' -ENV LMS_CFG="$CONFIG_ROOT/lms.yml" -ENV CMS_CFG="$CONFIG_ROOT/cms.yml" - -# Env vars: path -ENV VIRTUAL_ENV="/edx/app/edxapp/venvs/edxapp" -ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" -ENV PATH="/edx/app/edxapp/edx-platform/node_modules/.bin:${PATH}" -ENV PATH="/edx/app/edxapp/edx-platform/bin:${PATH}" -ENV PATH="/edx/app/edxapp/nodeenv/bin:${PATH}" - -WORKDIR /edx/app/edxapp/edx-platform - -# Create user before assigning any directory ownership to it. -RUN useradd -m --shell /bin/false app - -# Use debconf to set locales to be generated when the locales apt package is installed later. -RUN echo "locales locales/default_environment_locale select en_US.UTF-8" | debconf-set-selections -RUN echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8" | debconf-set-selections - -# Setting up ppa deadsnakes to get python 3.11 -RUN apt-get update && \ - apt-get install -y software-properties-common && \ - apt-add-repository -y ppa:deadsnakes/ppa - -# Install requirements that are absolutely necessary -RUN apt-get update && \ - apt-get -y dist-upgrade && \ - apt-get -y install --no-install-recommends \ - python3-pip \ - python3.11 \ - # python3-dev: required for building mysqlclient python package - python3.11-dev \ - python3.11-venv \ - libpython3.11 \ - libpython3.11-stdlib \ - libmysqlclient21 \ - # libmysqlclient-dev: required for building mysqlclient python package - libmysqlclient-dev \ - pkg-config \ - libssl1.1 \ - libxmlsec1-openssl \ - # lynx: Required by https://github.com/openedx/edx-platform/blob/b489a4ecb122/openedx/core/lib/html_to_text.py#L16 - lynx \ - ntp \ - git \ - build-essential \ - gettext \ - gfortran \ - graphviz \ - locales \ - swig \ - && \ - apt-get clean all && \ - rm -rf /var/lib/apt/* - -RUN mkdir -p /edx/var/edxapp -RUN mkdir -p /edx/etc -RUN chown app:app /edx/var/edxapp - -# The builder-production stage is a temporary stage that installs required packages and builds the python virtualenv, -# installs nodejs and node_modules. -# The built artifacts from this stage are then copied to the base stage. -FROM minimal-system as builder-production - -RUN apt-get update && \ - apt-get -y install --no-install-recommends \ - curl \ - libssl-dev \ - libffi-dev \ - libfreetype6-dev \ - libgeos-dev \ - libgraphviz-dev \ - libjpeg8-dev \ - liblapack-dev \ - libpng-dev \ - libsqlite3-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev - -# Setup python virtual environment -# It is already 'activated' because $VIRTUAL_ENV/bin was put on $PATH -RUN python3.11 -m venv "${VIRTUAL_ENV}" - -# Install python requirements -# Requires copying over requirements files, but not entire repository -COPY requirements requirements -RUN pip install -r requirements/pip.txt -RUN pip install -r requirements/edx/base.txt - -# Install node and npm -RUN nodeenv /edx/app/edxapp/nodeenv --node=18.19.0 --prebuilt -RUN npm install -g npm@10.5.x - -# This script is used by an npm post-install hook. -# We copy it into the image now so that it will be available when we run `npm install` in the next step. -# The script itself will copy certain modules into some uber-legacy parts of edx-platform which still use RequireJS. -COPY scripts/copy-node-modules.sh scripts/copy-node-modules.sh - -# Install node modules -COPY package.json package.json -COPY package-lock.json package-lock.json -RUN npm set progress=false && npm ci - -# The builder-development stage is a temporary stage that installs python modules required for development purposes -# The built artifacts from this stage are then copied to the development stage. -FROM builder-production as builder-development - -RUN pip install -r requirements/edx/development.txt - -# base stage -FROM minimal-system as base - -# Copy python virtual environment, nodejs and node_modules -COPY --from=builder-production /edx/app/edxapp/venvs/edxapp /edx/app/edxapp/venvs/edxapp -COPY --from=builder-production /edx/app/edxapp/nodeenv /edx/app/edxapp/nodeenv -COPY --from=builder-production /edx/app/edxapp/edx-platform/node_modules /edx/app/edxapp/edx-platform/node_modules - -# Copy over remaining parts of repository (including all code) -COPY . . - -# Install Python requirements again in order to capture local projects -RUN pip install -e . - -# Setting edx-platform directory as safe for git commands -RUN git config --global --add safe.directory /edx/app/edxapp/edx-platform - -# Production target -FROM base as production - -USER app - -ENV EDX_PLATFORM_SETTINGS='docker-production' -ENV SERVICE_VARIANT="${SERVICE_VARIANT}" -ENV SERVICE_PORT="${SERVICE_PORT}" -ENV DJANGO_SETTINGS_MODULE="${SERVICE_VARIANT}.envs.$EDX_PLATFORM_SETTINGS" -EXPOSE ${SERVICE_PORT} - -CMD gunicorn \ - -c /edx/app/edxapp/edx-platform/${SERVICE_VARIANT}/docker_${SERVICE_VARIANT}_gunicorn.py \ - --name ${SERVICE_VARIANT} \ - --bind=0.0.0.0:${SERVICE_PORT} \ - --max-requests=1000 \ - --access-logfile \ - - ${SERVICE_VARIANT}.wsgi:application - -# Development target -FROM base as development - -RUN apt-get update && \ - apt-get -y install --no-install-recommends \ - # wget is used in Makefile for common_constraints.txt - wget \ - && \ - apt-get clean all && \ - rm -rf /var/lib/apt/* - -COPY --from=builder-development /edx/app/edxapp/venvs/edxapp /edx/app/edxapp/venvs/edxapp - -RUN ln -s "$(pwd)/lms/envs/devstack-experimental.yml" "$LMS_CFG" -RUN ln -s "$(pwd)/cms/envs/devstack-experimental.yml" "$CMS_CFG" -# Temporary compatibility hack while devstack is supporting both the old `edxops/edxapp` image and this image. -# * Add in a dummy ../edxapp_env file -# * devstack sets /edx/etc/studio.yml as CMS_CFG. -RUN ln -s "$(pwd)/cms/envs/devstack-experimental.yml" "/edx/etc/studio.yml" -RUN touch ../edxapp_env - -ENV EDX_PLATFORM_SETTINGS='devstack_docker' -ENV SERVICE_VARIANT="${SERVICE_VARIANT}" -EXPOSE ${SERVICE_PORT} -CMD ./manage.py ${SERVICE_VARIANT} runserver 0.0.0.0:${SERVICE_PORT} From 6954cddd3dd6c9d44a7952b9f33f3e92f1bb3772 Mon Sep 17 00:00:00 2001 From: Hunia Fatima Date: Tue, 29 Oct 2024 23:11:42 +0500 Subject: [PATCH 2/3] test: add back workflow to push image to ECR --- .github/workflows/publish-ci-docker-image.yml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/publish-ci-docker-image.yml diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml new file mode 100644 index 000000000000..8a35289ab1e9 --- /dev/null +++ b/.github/workflows/publish-ci-docker-image.yml @@ -0,0 +1,42 @@ +name: Push CI Runner Docker Image + +on: + workflow_dispatch: + schedule: + - cron: "0 1 * * 3" + +jobs: + push: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + # This has to happen after checkout in order for gh to work. + - name: "Cancel scheduled job on forks" + if: github.repository != 'openedx/edx-platform' && github.event_name == 'schedule' + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + run: | + gh run cancel "${{ github.run_id }}" + gh run watch "${{ github.run_id }}" + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Log in to ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push image to Amazon ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: actions-runner + IMAGE_TAG: latest + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f scripts/ci-runner.Dockerfile . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG From 50ed55fd4dea134e196cbf3b5a143f3c4e610799 Mon Sep 17 00:00:00 2001 From: Hunia Fatima Date: Thu, 31 Oct 2024 11:26:43 +0500 Subject: [PATCH 3/3] chore: remove docker publish workflow --- .github/workflows/docker-publish.yml | 43 ---------------------------- 1 file changed, 43 deletions(-) delete mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index 6831e3563d81..000000000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Push Docker Images - -on: - push: - branches: - - master - -jobs: - # Push image to GitHub Packages. - # See also https://docs.docker.com/docker-hub/builds/ - push: - runs-on: ubuntu-latest - if: github.event_name == 'push' - - strategy: - matrix: - variant: - - "lms_dev" - - "cms_dev" - - "cms" - - "lms" - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push lms/cms base docker images - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - run: make docker_tag_build_push_${{matrix.variant}}