-> Swap to Python community image

ImagingDataCommons · Oct 4, 2023 · 03a8ffb · 03a8ffb
1 parent 0b12d5e
commit 03a8ffb
Show file tree

Hide file tree

Showing 6 changed files with 69 additions and 80 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -7,7 +7,13 @@ commands:
           command: |
             TIER=DEV
             if [[ ${CIRCLE_BRANCH} =~ idc-(prod|uat|test).* ]]; then
+              echo "Saw deployment tier branch."
               TIER=$(awk -F- '{print toupper($2)}' \<<< ${CIRCLE_BRANCH})
+            elif [[ ${CIRCLE_BRANCH} =~ expr.* ]]; then
+              echo "Saw experimental tier branch."
+              TIER=EXPR
+            else
+              echo "Saw dev tier branch."
             fi
             echo "export TIER=${TIER}" >> $BASH_ENV
             echo "Tier was identified as ${TIER} for branch ${CIRCLE_BRANCH}"
@@ -24,6 +30,7 @@ commands:
             CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_IDC_DEV}
             PROJECT_ID=${DEPLOYMENT_PROJECT_ID_IDC_DEV}
             BUCKET=${DEPLOYMENT_BUCKET_IDC_DEV}
+            DEPLOYMENT_CONFIG=deployment_config.txt
             if [[ ${TIER} == "PROD" ]]; then
               KEY=${DEPLOYMENT_KEY_IDC_PROD}
               CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_IDC_PROD}
@@ -42,6 +49,9 @@ commands:
               PROJECT_ID=${DEPLOYMENT_PROJECT_ID_IDC_TEST}
               BUCKET=${DEPLOYMENT_BUCKET_IDC_TEST}
               echo "Using test project and related deployment SA and bucket."
+            elif [[ ${TIER} == "EXPR" ]]; then
+              DEPLOYMENT_CONFIG=expr.deployment_config.txt
+              echo "Using experimental project and related deployment SA and bucket."
             elif [[ ${TIER} == "DEV" ]]; then
               echo "Using development/default project and related deployment SA and bucket."
             else
@@ -54,6 +64,7 @@ commands:
             echo "export DEPLOYMENT_CLIENT_EMAIL=${CLIENT_EMAIL}" >> $BASH_ENV
             echo "export DEPLOYMENT_PROJECT_ID=${PROJECT_ID}" >> $BASH_ENV
             echo "export DEPLOYMENT_BUCKET=${BUCKET}" >> $BASH_ENV
+            echo "export DEPLOYMENT_CONFIG=${DEPLOYMENT_CONFIG}" >> $BASH_ENV
 
             echo "Project settings assigned:"
             echo "Deployment client: ${CLIENT_EMAIL}"
@@ -66,7 +77,7 @@ commands:
           name: "Set tier-specific configuration file"
           command: |
             sudo -E /bin/bash ./shell/pull_config.sh
-            xargs -a deployment_config.txt -I{} echo "export {}" >> $BASH_ENV
+            xargs -a ${DEPLOYMENT_CONFIG} -I{} echo "export {}" >> $BASH_ENV
             # Set up test/deployment database values
             echo "export MYSQL_ROOT_PASSWORD_BUILD=idc" >> $BASH_ENV
             echo "export DATABASE_NAME_BUILD=build_database" >> $BASH_ENV
@@ -107,8 +118,8 @@ jobs:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
     working_directory: ~/IDC-WebApp
     docker:
-      - image: cimg/python:3.8.14
-      - image: cimg/mysql:5.7.36
+      - image: cimg/python:3.9.2
+      - image: cimg/mysql:8.0
         environment:
           MYSQL_ROOT_HOST: "%"
           MYSQL_USER: ubuntu
@@ -183,7 +194,7 @@ jobs:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
     working_directory: ~/IDC-WebApp
     docker:
-      - image: cimg/python:3.8.14
+      - image: cimg/python:3.9.2
     steps:
       - restore_cache:
           keys:
@@ -212,3 +223,4 @@ workflows:
                 - idc-test
                 - idc-uat
                 - idc-prod
+                - expr
diff --git a/Dockerfile b/Dockerfile
@@ -16,37 +16,28 @@
 #
 ###
 
-# Dockerfile extending the generic Python image with application files for a
+# Dockerfile extending the Python Community image from Dockerhub with application files for a
 # single application.
-FROM gcr.io/google_appengine/python
+FROM python:3.9-bullseye
 
-# Create a virtualenv for dependencies. This isolates these packages from
-# system-level packages.
-# Use -p python3 or -p python3.7 to select python version. Default is version 2.
-RUN virtualenv /env -p python3
+SHELL ["/bin/bash", "-c"]
 
-# Setting these environment variables are the same as running
-# source /env/bin/activate.
-ENV VIRTUAL_ENV /env
-ENV PATH /env/bin:$PATH
+ENV PYTHONUNBUFFERED=1
+ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get install -y wget
-RUN wget "http://repo.mysql.com/mysql-apt-config_0.8.9-1_all.deb" -P /tmp
+RUN wget "http://repo.mysql.com/mysql-apt-config_0.8.26-1_all.deb" -P /tmp
 
 # install lsb-release (a dependency of mysql-apt-config), since dpkg doesn't
 # do dependency resolution
 RUN apt-get install -y lsb-release
-# add a debconf entry to select mysql-5.7 as the server version when we install
-# the mysql config package
-RUN echo "mysql-apt-config mysql-apt-config/select-server select mysql-5.7" | debconf-set-selections
-# having 'selected' mysql-5.7 for 'server', install the mysql config package
-RUN echo 'download mysql public build key'
+# TODO: we need to start using the keyring instead
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 467B942D3A79BD29
-RUN dpkg --install /tmp/mysql-apt-config_0.8.9-1_all.deb
+RUN dpkg --install /tmp/mysql-apt-config_0.8.26-1_all.deb
 
-# fetch the updated package metadata (in particular, mysql-server-5.7)
+# fetch the updated package metadata (in particular, mysql-server)
 RUN apt-get update
 
 # aaaand now let's install mysql-server
@@ -56,23 +47,25 @@ RUN apt-get install -y mysql-server
 RUN curl --silent https://bootstrap.pypa.io/get-pip.py | python3
 
 RUN apt-get -y install build-essential
-RUN apt-get -y install --reinstall python-m2crypto python3-crypto
+RUN apt-get -y install --reinstall python3-m2crypto python3-cryptography
 RUN apt-get -y install libxml2-dev libxmlsec1-dev swig
-RUN pip3 install pexpect
+RUN pip install pexpect
 
 RUN apt-get -y install unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl
 RUN easy_install -U distribute
 
 ADD . /app
 
 # We need to recompile some of the items because of differences in compiler versions
-RUN pip3 install -r /app/requirements.txt -t /app/lib/ --upgrade
-RUN pip3 install gunicorn==19.6.0
+RUN pip install -r /app/requirements.txt -t /app/lib/ --upgrade
+RUN pip install gunicorn==21.2.0
+
+ENV PYTHONPATH="/app:/app/lib:/app/IDC-Common:${PYTHONPATH}"
 
-ENV PYTHONPATH=/app:/app/lib:/app/IDC-Common:${PYTHONPATH}
+WORKDIR /app/
 
-# Check Axes config
-RUN python3 manage.py check
+# Quick config check
+RUN python manage.py check
 
 # Until we figure out a way to do it in CircleCI without whitelisting IPs this has to be done by a dev from
 # ISB

diff --git a/Vagrantfile b/Vagrantfile
@@ -18,8 +18,8 @@ Vagrant.configure(2) do |config|
 
   config.vagrant.plugins = ["vagrant-vbguest"]
 
-  config.vm.box_url = "https://app.vagrantup.com/ubuntu/boxes/bionic64"
-  config.vm.box = "ubuntu/bionic64"
+  config.vm.box_url = "https://app.vagrantup.com/debian/boxes/bullseye64"
+  config.vm.box = "debian/bullseye64"
 
   config.vbguest.installer_options = { allow_kernel_upgrade: true }
   config.vbguest.installer_hooks[:before_install] = [

diff --git a/shell/create-database.sh b/shell/create-database.sh
@@ -13,11 +13,18 @@ else
     export MYSQL_DB_HOST=localhost
 fi
 
+if [ -z "$MYSQL_ROOT_PASSWORD" ]; then
+  echo "No MySQL root password found! Can't install MySQL without a root password. Exiting..."
+  exit 1
+fi
+
 # MySQL Install
-echo "Installing MySQL..."
-sudo debconf-set-selections <<< "mysql-server-5.7 mysql-server/root_password password $MYSQL_ROOT_PASSWORD"
-sudo debconf-set-selections <<< "mysql-server-5.7 mysql-server/root_password_again password $MYSQL_ROOT_PASSWORD"
-sudo DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install mysql-server-5.7
+echo "Installing MySQL"
+echo "Note that you can ignore the \"root@localhost is created with an empty password\" message, which is due to root"
+echo "  being created before the debconf password has been set."
+sudo debconf-set-selections <<< "mysql-community-server mysql-community-server/root-pass password ${MYSQL_ROOT_PASSWORD}"
+sudo debconf-set-selections <<< "mysql-community-server mysql-community-server/re-root-pass password ${MYSQL_ROOT_PASSWORD}"
+sudo DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install mysql-server
 
 echo "Creating Databases..."
-mysql -u$MYSQL_ROOT_USER -p$MYSQL_ROOT_PASSWORD -h $MYSQL_DB_HOST -e "CREATE DATABASE $DATABASE_NAME"
+mysql -u${MYSQL_ROOT_USER} -p${MYSQL_ROOT_PASSWORD} -h ${MYSQL_DB_HOST} -e "CREATE DATABASE ${DATABASE_NAME}"
diff --git a/shell/install-deps.sh b/shell/install-deps.sh
@@ -11,6 +11,8 @@ if [ -n "$CI" ]; then
     COMMON_BRANCH=master
     if [[ ${CIRCLE_BRANCH} =~ idc-(prod|uat|test).* ]]; then
         COMMON_BRANCH=$(awk -F- '{print $1"-"$2}' <<< ${CIRCLE_BRANCH})
+    elif [[ ${CIRCLE_BRANCH} == "expr" ]]; then
+        COMMON_BRANCH=expr
     fi
     echo "Cloning IDC-Common branch ${COMMON_BRANCH}..."
     git clone -b ${COMMON_BRANCH} https://github.com/ImagingDataCommons/IDC-Common.git
@@ -24,6 +26,8 @@ else
     export HOMEROOT=/home/vagrant/www
 fi
 
+export DEBIAN_FRONTEND=noninteractive
+
 # Remove .pyc files; these can sometimes stick around and if a
 # model has changed names it will cause various load failures
 find . -type f -name '*.pyc' -delete
@@ -32,44 +36,21 @@ apt-get update -qq
 
 # Install and update apt-get info
 echo "Preparing System..."
-apt-get -y --force-yes install software-properties-common
-if [ -n "$CI" ]; then
-    # Use these next 4 lines to update mysql public build key
-    echo 'download mysql public build key'
-    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 5072E1F5
-    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 467B942D3A79BD29
-    echo 'mysql build key update done.'
-    wget https://dev.mysql.com/get/mysql-apt-config_0.8.9-1_all.deb
-    apt-get install -y lsb-release
-    dpkg -i mysql-apt-config_0.8.9-1_all.deb
-fi
+apt-get -y --force-yes install software-properties-common ca-certificates
+
+apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 467B942D3A79BD29
+wget "https://repo.mysql.com/mysql-apt-config_0.8.26-1_all.deb" -P /tmp
+dpkg --install /tmp/mysql-apt-config_0.8.26-1_all.deb
 
 apt-get update -qq
-apt-get install ca-certificates
+
+apt-get install mysql-client
 
 # Install apt-get dependencies
 echo "Installing Dependencies..."
 apt-get install -y --force-yes unzip libffi-dev libssl-dev git ruby g++ curl dos2unix
-# CircleCI provides a Python 3.8 image, but locally, we use 3.7 to mimic the Dockerfile
-if [ -z "${CI}" ]; then
-    # Update to Python 3.7
-    add-apt-repository ppa:deadsnakes/ppa
-    apt update
-    apt install -y --force-yes python3.7
-    # Set Python 3.7 as the python3 version
-    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
-    apt-get install -y --force-yes python3.7-venv python3.7-distutils python3.7-dev
-else
-  apt-get install -y --force-yes python3-distutils
-fi
-apt-get install -y --force-yes python3-mysqldb libmysqlclient-dev libpython3-dev build-essential
-apt-get install -y --force-yes mysql-client
-
-if [ -z "${CI}" ]; then
-  # Per https://stackoverflow.com/questions/13708180/python-dev-installation-error-importerror-no-module-named-apt-pkg
-  # there's an issue with Python 3.7 and deadsnakes.
-  cp -v /usr/lib/python3/dist-packages/apt_pkg.cpython-36m-x86_64-linux-gnu.so /usr/lib/python3/dist-packages/apt_pkg.so
-fi
+apt-get install -y --force-yes python3-distutils python3-mysqldb libmysqlclient-dev libpython3-dev build-essential
+apt-get install -y --force-yes python3-pip
 
 echo "Dependencies Installed"
 
@@ -83,32 +64,28 @@ if [ -z "${CI}" ]; then
     ls ${HOMEROOT}/lib/
 fi
 
-# Install PIP + Dependencies
-echo "Installing pip3..."
-curl --silent https://bootstrap.pypa.io/get-pip.py | python3
-
 # Install our primary python libraries
 # If we're not on CircleCI, or we are but the lib directory isn't there (cache miss), install lib
 if [ -z "${CI}" ] || [ ! -d "lib" ]; then
     echo "Installing Python Libraries..."
-    pip3 install -r ${HOMEROOT}/requirements.txt -t ${HOMEROOT}/lib --upgrade --only-binary all
+    pip install -r ${HOMEROOT}/requirements.txt -t ${HOMEROOT}/lib --upgrade --only-binary all
 else
     echo "Using restored cache for Python Libraries"
 fi
 
 if [ -z "${CI}" ]; then
     echo "Installing responses library for unit tests, but not for deployment..."
-    pip3 install -q responses -t ${HOMEROOT}/lib --only-binary all
+    pip install -q responses -t ${HOMEROOT}/lib --only-binary all
 fi
 
 if [ "$DEBUG" = "True" ] && [ "$DEBUG_TOOLBAR" = "True" ]; then
     echo "Installing Django Debug Toolbar for local dev..."
-    pip3 install -q django-debug-toolbar==3.2.4 -t ${HOMEROOT}/lib --only-binary all
+    pip install -q django-debug-toolbar==3.2.4 -t ${HOMEROOT}/lib --only-binary all
 fi
 
 if [ "$IS_DEV" = "True" ]; then
     echo "Installing GitPython for local dev version display..."
-    pip3 install -q gitpython -t ${HOMEROOT}/lib --only-binary all
+    pip install -q gitpython -t ${HOMEROOT}/lib --only-binary all
 fi
 
 echo "Libraries Installed"

diff --git a/shell/pull_config.sh b/shell/pull_config.sh
@@ -1,7 +1,7 @@
-if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/deployment_config.txt" ]; then
-    gsutil cp gs://${DEPLOYMENT_BUCKET}/deployment_config.txt /home/circleci/${CIRCLE_PROJECT_REPONAME}/
-    chmod ugo+r /home/circleci/${CIRCLE_PROJECT_REPONAME}/deployment_config.txt
-    if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/deployment_config.txt" ]; then
+if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/${DEPLOYMENT_CONFIG}" ]; then
+    gsutil cp gs://${DEPLOYMENT_BUCKET}/${DEPLOYMENT_CONFIG} /home/circleci/${CIRCLE_PROJECT_REPONAME}/
+    chmod ugo+r /home/circleci/${CIRCLE_PROJECT_REPONAME}/${DEPLOYMENT_CONFIG}
+    if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/${DEPLOYMENT_CONFIG}" ]; then
       echo "[ERROR] Couldn't assign deployment configuration file - exiting."
       exit 1
     fi