diff --git a/.circleci/config.yml b/.circleci/config.yml index 36e56acf0..32987151e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,13 @@ commands: command: | TIER=DEV if [[ ${CIRCLE_BRANCH} =~ idc-(prod|uat|test).* ]]; then + echo "Saw deployment tier branch." TIER=$(awk -F- '{print toupper($2)}' \<<< ${CIRCLE_BRANCH}) + elif [[ ${CIRCLE_BRANCH} =~ expr.* ]]; then + echo "Saw experimental tier branch." + TIER=EXPR + else + echo "Saw dev tier branch." fi echo "export TIER=${TIER}" >> $BASH_ENV echo "Tier was identified as ${TIER} for branch ${CIRCLE_BRANCH}" @@ -24,6 +30,7 @@ commands: CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_IDC_DEV} PROJECT_ID=${DEPLOYMENT_PROJECT_ID_IDC_DEV} BUCKET=${DEPLOYMENT_BUCKET_IDC_DEV} + DEPLOYMENT_CONFIG=deployment_config.txt if [[ ${TIER} == "PROD" ]]; then KEY=${DEPLOYMENT_KEY_IDC_PROD} CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_IDC_PROD} @@ -42,6 +49,9 @@ commands: PROJECT_ID=${DEPLOYMENT_PROJECT_ID_IDC_TEST} BUCKET=${DEPLOYMENT_BUCKET_IDC_TEST} echo "Using test project and related deployment SA and bucket." + elif [[ ${TIER} == "EXPR" ]]; then + DEPLOYMENT_CONFIG=expr.deployment_config.txt + echo "Using experimental project and related deployment SA and bucket." elif [[ ${TIER} == "DEV" ]]; then echo "Using development/default project and related deployment SA and bucket." else @@ -54,6 +64,7 @@ commands: echo "export DEPLOYMENT_CLIENT_EMAIL=${CLIENT_EMAIL}" >> $BASH_ENV echo "export DEPLOYMENT_PROJECT_ID=${PROJECT_ID}" >> $BASH_ENV echo "export DEPLOYMENT_BUCKET=${BUCKET}" >> $BASH_ENV + echo "export DEPLOYMENT_CONFIG=${DEPLOYMENT_CONFIG}" >> $BASH_ENV echo "Project settings assigned:" echo "Deployment client: ${CLIENT_EMAIL}" @@ -66,7 +77,7 @@ commands: name: "Set tier-specific configuration file" command: | sudo -E /bin/bash ./shell/pull_config.sh - xargs -a deployment_config.txt -I{} echo "export {}" >> $BASH_ENV + xargs -a ${DEPLOYMENT_CONFIG} -I{} echo "export {}" >> $BASH_ENV # Set up test/deployment database values echo "export MYSQL_ROOT_PASSWORD_BUILD=idc" >> $BASH_ENV echo "export DATABASE_NAME_BUILD=build_database" >> $BASH_ENV @@ -107,8 +118,8 @@ jobs: TZ: "/usr/share/zoneinfo/America/Los_Angeles" working_directory: ~/IDC-WebApp docker: - - image: cimg/python:3.8.14 - - image: cimg/mysql:5.7.36 + - image: cimg/python:3.9.2 + - image: cimg/mysql:8.0 environment: MYSQL_ROOT_HOST: "%" MYSQL_USER: ubuntu @@ -183,7 +194,7 @@ jobs: TZ: "/usr/share/zoneinfo/America/Los_Angeles" working_directory: ~/IDC-WebApp docker: - - image: cimg/python:3.8.14 + - image: cimg/python:3.9.2 steps: - restore_cache: keys: @@ -212,3 +223,4 @@ workflows: - idc-test - idc-uat - idc-prod + - expr diff --git a/Dockerfile b/Dockerfile index 4c1291d26..cab0438a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,37 +16,28 @@ # ### -# Dockerfile extending the generic Python image with application files for a +# Dockerfile extending the Python Community image from Dockerhub with application files for a # single application. -FROM gcr.io/google_appengine/python +FROM python:3.9-bullseye -# Create a virtualenv for dependencies. This isolates these packages from -# system-level packages. -# Use -p python3 or -p python3.7 to select python version. Default is version 2. -RUN virtualenv /env -p python3 +SHELL ["/bin/bash", "-c"] -# Setting these environment variables are the same as running -# source /env/bin/activate. -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH +ENV PYTHONUNBUFFERED=1 +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update ENV DEBIAN_FRONTEND=noninteractive RUN apt-get install -y wget -RUN wget "http://repo.mysql.com/mysql-apt-config_0.8.9-1_all.deb" -P /tmp +RUN wget "http://repo.mysql.com/mysql-apt-config_0.8.26-1_all.deb" -P /tmp # install lsb-release (a dependency of mysql-apt-config), since dpkg doesn't # do dependency resolution RUN apt-get install -y lsb-release -# add a debconf entry to select mysql-5.7 as the server version when we install -# the mysql config package -RUN echo "mysql-apt-config mysql-apt-config/select-server select mysql-5.7" | debconf-set-selections -# having 'selected' mysql-5.7 for 'server', install the mysql config package -RUN echo 'download mysql public build key' +# TODO: we need to start using the keyring instead RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 467B942D3A79BD29 -RUN dpkg --install /tmp/mysql-apt-config_0.8.9-1_all.deb +RUN dpkg --install /tmp/mysql-apt-config_0.8.26-1_all.deb -# fetch the updated package metadata (in particular, mysql-server-5.7) +# fetch the updated package metadata (in particular, mysql-server) RUN apt-get update # aaaand now let's install mysql-server @@ -56,9 +47,9 @@ RUN apt-get install -y mysql-server RUN curl --silent https://bootstrap.pypa.io/get-pip.py | python3 RUN apt-get -y install build-essential -RUN apt-get -y install --reinstall python-m2crypto python3-crypto +RUN apt-get -y install --reinstall python3-m2crypto python3-cryptography RUN apt-get -y install libxml2-dev libxmlsec1-dev swig -RUN pip3 install pexpect +RUN pip install pexpect RUN apt-get -y install unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl RUN easy_install -U distribute @@ -66,13 +57,15 @@ RUN easy_install -U distribute ADD . /app # We need to recompile some of the items because of differences in compiler versions -RUN pip3 install -r /app/requirements.txt -t /app/lib/ --upgrade -RUN pip3 install gunicorn==19.6.0 +RUN pip install -r /app/requirements.txt -t /app/lib/ --upgrade +RUN pip install gunicorn==21.2.0 + +ENV PYTHONPATH="/app:/app/lib:/app/IDC-Common:${PYTHONPATH}" -ENV PYTHONPATH=/app:/app/lib:/app/IDC-Common:${PYTHONPATH} +WORKDIR /app/ -# Check Axes config -RUN python3 manage.py check +# Quick config check +RUN python manage.py check # Until we figure out a way to do it in CircleCI without whitelisting IPs this has to be done by a dev from # ISB diff --git a/Vagrantfile b/Vagrantfile index b68b5024c..ef3631d2c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -18,8 +18,8 @@ Vagrant.configure(2) do |config| config.vagrant.plugins = ["vagrant-vbguest"] - config.vm.box_url = "https://app.vagrantup.com/ubuntu/boxes/bionic64" - config.vm.box = "ubuntu/bionic64" + config.vm.box_url = "https://app.vagrantup.com/debian/boxes/bullseye64" + config.vm.box = "debian/bullseye64" config.vbguest.installer_options = { allow_kernel_upgrade: true } config.vbguest.installer_hooks[:before_install] = [ diff --git a/shell/create-database.sh b/shell/create-database.sh index 049c0e611..966750730 100755 --- a/shell/create-database.sh +++ b/shell/create-database.sh @@ -13,11 +13,18 @@ else export MYSQL_DB_HOST=localhost fi +if [ -z "$MYSQL_ROOT_PASSWORD" ]; then + echo "No MySQL root password found! Can't install MySQL without a root password. Exiting..." + exit 1 +fi + # MySQL Install -echo "Installing MySQL..." -sudo debconf-set-selections <<< "mysql-server-5.7 mysql-server/root_password password $MYSQL_ROOT_PASSWORD" -sudo debconf-set-selections <<< "mysql-server-5.7 mysql-server/root_password_again password $MYSQL_ROOT_PASSWORD" -sudo DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install mysql-server-5.7 +echo "Installing MySQL" +echo "Note that you can ignore the \"root@localhost is created with an empty password\" message, which is due to root" +echo " being created before the debconf password has been set." +sudo debconf-set-selections <<< "mysql-community-server mysql-community-server/root-pass password ${MYSQL_ROOT_PASSWORD}" +sudo debconf-set-selections <<< "mysql-community-server mysql-community-server/re-root-pass password ${MYSQL_ROOT_PASSWORD}" +sudo DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install mysql-server echo "Creating Databases..." -mysql -u$MYSQL_ROOT_USER -p$MYSQL_ROOT_PASSWORD -h $MYSQL_DB_HOST -e "CREATE DATABASE $DATABASE_NAME" +mysql -u${MYSQL_ROOT_USER} -p${MYSQL_ROOT_PASSWORD} -h ${MYSQL_DB_HOST} -e "CREATE DATABASE ${DATABASE_NAME}" diff --git a/shell/install-deps.sh b/shell/install-deps.sh index e857aa720..e72177572 100755 --- a/shell/install-deps.sh +++ b/shell/install-deps.sh @@ -11,6 +11,8 @@ if [ -n "$CI" ]; then COMMON_BRANCH=master if [[ ${CIRCLE_BRANCH} =~ idc-(prod|uat|test).* ]]; then COMMON_BRANCH=$(awk -F- '{print $1"-"$2}' <<< ${CIRCLE_BRANCH}) + elif [[ ${CIRCLE_BRANCH} == "expr" ]]; then + COMMON_BRANCH=expr fi echo "Cloning IDC-Common branch ${COMMON_BRANCH}..." git clone -b ${COMMON_BRANCH} https://github.com/ImagingDataCommons/IDC-Common.git @@ -24,6 +26,8 @@ else export HOMEROOT=/home/vagrant/www fi +export DEBIAN_FRONTEND=noninteractive + # Remove .pyc files; these can sometimes stick around and if a # model has changed names it will cause various load failures find . -type f -name '*.pyc' -delete @@ -32,44 +36,21 @@ apt-get update -qq # Install and update apt-get info echo "Preparing System..." -apt-get -y --force-yes install software-properties-common -if [ -n "$CI" ]; then - # Use these next 4 lines to update mysql public build key - echo 'download mysql public build key' - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 5072E1F5 - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 467B942D3A79BD29 - echo 'mysql build key update done.' - wget https://dev.mysql.com/get/mysql-apt-config_0.8.9-1_all.deb - apt-get install -y lsb-release - dpkg -i mysql-apt-config_0.8.9-1_all.deb -fi +apt-get -y --force-yes install software-properties-common ca-certificates + +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 467B942D3A79BD29 +wget "https://repo.mysql.com/mysql-apt-config_0.8.26-1_all.deb" -P /tmp +dpkg --install /tmp/mysql-apt-config_0.8.26-1_all.deb apt-get update -qq -apt-get install ca-certificates + +apt-get install mysql-client # Install apt-get dependencies echo "Installing Dependencies..." apt-get install -y --force-yes unzip libffi-dev libssl-dev git ruby g++ curl dos2unix -# CircleCI provides a Python 3.8 image, but locally, we use 3.7 to mimic the Dockerfile -if [ -z "${CI}" ]; then - # Update to Python 3.7 - add-apt-repository ppa:deadsnakes/ppa - apt update - apt install -y --force-yes python3.7 - # Set Python 3.7 as the python3 version - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 - apt-get install -y --force-yes python3.7-venv python3.7-distutils python3.7-dev -else - apt-get install -y --force-yes python3-distutils -fi -apt-get install -y --force-yes python3-mysqldb libmysqlclient-dev libpython3-dev build-essential -apt-get install -y --force-yes mysql-client - -if [ -z "${CI}" ]; then - # Per https://stackoverflow.com/questions/13708180/python-dev-installation-error-importerror-no-module-named-apt-pkg - # there's an issue with Python 3.7 and deadsnakes. - cp -v /usr/lib/python3/dist-packages/apt_pkg.cpython-36m-x86_64-linux-gnu.so /usr/lib/python3/dist-packages/apt_pkg.so -fi +apt-get install -y --force-yes python3-distutils python3-mysqldb libmysqlclient-dev libpython3-dev build-essential +apt-get install -y --force-yes python3-pip echo "Dependencies Installed" @@ -83,32 +64,28 @@ if [ -z "${CI}" ]; then ls ${HOMEROOT}/lib/ fi -# Install PIP + Dependencies -echo "Installing pip3..." -curl --silent https://bootstrap.pypa.io/get-pip.py | python3 - # Install our primary python libraries # If we're not on CircleCI, or we are but the lib directory isn't there (cache miss), install lib if [ -z "${CI}" ] || [ ! -d "lib" ]; then echo "Installing Python Libraries..." - pip3 install -r ${HOMEROOT}/requirements.txt -t ${HOMEROOT}/lib --upgrade --only-binary all + pip install -r ${HOMEROOT}/requirements.txt -t ${HOMEROOT}/lib --upgrade --only-binary all else echo "Using restored cache for Python Libraries" fi if [ -z "${CI}" ]; then echo "Installing responses library for unit tests, but not for deployment..." - pip3 install -q responses -t ${HOMEROOT}/lib --only-binary all + pip install -q responses -t ${HOMEROOT}/lib --only-binary all fi if [ "$DEBUG" = "True" ] && [ "$DEBUG_TOOLBAR" = "True" ]; then echo "Installing Django Debug Toolbar for local dev..." - pip3 install -q django-debug-toolbar==3.2.4 -t ${HOMEROOT}/lib --only-binary all + pip install -q django-debug-toolbar==3.2.4 -t ${HOMEROOT}/lib --only-binary all fi if [ "$IS_DEV" = "True" ]; then echo "Installing GitPython for local dev version display..." - pip3 install -q gitpython -t ${HOMEROOT}/lib --only-binary all + pip install -q gitpython -t ${HOMEROOT}/lib --only-binary all fi echo "Libraries Installed" diff --git a/shell/pull_config.sh b/shell/pull_config.sh index 8980d7283..84da3a5b2 100755 --- a/shell/pull_config.sh +++ b/shell/pull_config.sh @@ -1,7 +1,7 @@ -if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/deployment_config.txt" ]; then - gsutil cp gs://${DEPLOYMENT_BUCKET}/deployment_config.txt /home/circleci/${CIRCLE_PROJECT_REPONAME}/ - chmod ugo+r /home/circleci/${CIRCLE_PROJECT_REPONAME}/deployment_config.txt - if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/deployment_config.txt" ]; then +if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/${DEPLOYMENT_CONFIG}" ]; then + gsutil cp gs://${DEPLOYMENT_BUCKET}/${DEPLOYMENT_CONFIG} /home/circleci/${CIRCLE_PROJECT_REPONAME}/ + chmod ugo+r /home/circleci/${CIRCLE_PROJECT_REPONAME}/${DEPLOYMENT_CONFIG} + if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/${DEPLOYMENT_CONFIG}" ]; then echo "[ERROR] Couldn't assign deployment configuration file - exiting." exit 1 fi