Small updates to system declaration

.github/workflows/main.yml

@@ -17,7 +17,7 @@ on:
 jobs:
   # This workflow contains a single job called "build"
   build:
-    name: cgap-portal tests
+    name: cgap-portal
 
     # The type of runner that the job will run on
     runs-on: ubuntu-20.04
@@ -27,7 +27,7 @@ jobs:
       matrix:
         test_type: ['UNIT', 'INDEX', 'DOCKER']
         # We are really not set up for these next two to be multiplicative, so be careful adding more.
-        python_version: ['3.8']
+        python_version: ['3.9']
         node_version: ['16']
 
     # Steps represent a sequence of tasks that will be executed as part of the job
@@ -108,7 +108,7 @@ jobs:
           poetry run wipe-test-indices $TEST_JOB_ID search-cgap-unit-testing-opensearch-tcs45cjpwgdzoi7pafr6oewq6u.us-east-1.es.amazonaws.com:443
 
       - name: Docker Build
-        if: ${{ matrix.test_type == 'DOCKER' && matrix.node_version == '16' && matrix.python_version == '3.8' }}
+        if: ${{ matrix.test_type == 'DOCKER' && matrix.node_version == '16' && matrix.python_version == '3.9' }}
         run: |
           touch deploy/docker/local/docker_development.ini  # cheap substitute for prepare-docker to make ignored file
           docker build .

CHANGELOG.rst

@@ -7,8 +7,17 @@ Change Log
 ----------
 
 
+14.0.0
+======
+
+* Add support for Python 3.9
+
+* Remove inclusion of unused loremipsum library from pyproject.toml
+
+
 13.4.0
 ======
+
 `PR 711: Generic QualityMetric <https://github.com/dbmi-bgm/cgap-portal/pull/711>`_
 
 * Add new QualityMetricGeneric item

Dockerfile

@@ -1,7 +1,7 @@
 # CGAP-Portal (Production) Dockerfile
 
-# Debian Buster with Python 3.8.13
-FROM python:3.8.13-slim-buster
+# Debian Buster with Python 3.9.16
+FROM python:3.9.16-slim-buster
 
 MAINTAINER William Ronchetti "[email protected]"
 

Makefile

@@ -36,14 +36,9 @@ npm-setup:  # runs all front-end setup
 	make aws-ip-ranges
 
 moto-setup:  # optional moto setup that must be done separately
-	@# This setup was needed here because there was no bracket syntax in pypoetry.com.
-	@# Now that we're using a higher version of moto, and not using the server parts, we don't need this here.
-	@# It's now all done in pyproject.toml, getting a higher version as well.
-	@# This comment and this make target can go away once that's proven effective. -kmp 23-Mar-2023
-	@# pip install "moto[server]==1.3.7"
 	@echo "'moto[server]' not being installed here. Regular 'moto' will be installed by pyproject.toml."
 
-macpoetry-install:  # Same as 'poetry install' except that on OSX Catalina, an environment variable wrapper is needed
+macpoetry-install:
 	bin/macpoetry-install
 
 configure:  # does any pre-requisite installs
@@ -53,8 +48,8 @@ configure:  # does any pre-requisite installs
 	pip install wheel
 ifeq ($(shell uname -s), Darwin)
 ifeq ($(shell uname -m), arm64)
-	pip install pysam=="0.21.0"
-	pip install matplotlib=="3.3.4"
+	pip install pysam=="0.21.0"      # Must be kept consistent with version pinned in pyproject.toml
+	pip install matplotlib=="3.3.4"  # Must be kept consistent with version pinned in pyproject.toml
 endif
 endif
 	poetry config virtualenvs.create false --local # do not create a virtualenv - the user should have already done this -wrr 20-Sept-2021
@@ -72,9 +67,12 @@ ifeq ($(shell uname -s), Darwin)
 	@echo "Looks like this is Mac so executing: make macbuild"
 	make macbuild
 else
+	make anybuild
+endif
+
+anybuild:
 	make build-poetry
 	make build-after-poetry
-endif
 
 macbuild:  # Builds for MacOS (see: bin/macpoetry-install)
 	make macbuild-poetry
@@ -154,11 +152,6 @@ kibana-start-test:  # starts a test version of kibana (port chosen for active te
 kibana-stop:
 	scripts/kibana-stop
 
-kill:  # kills back-end processes associated with the application. Use with care.
-	pkill -f postgres &
-	pkill -f elasticsearch &
-	pkill -f moto_server &
-
 clean-python:
 	@echo -n "Are you sure? This will wipe all libraries installed on this virtualenv [y/N] " && read ans && [ $${ans:-N} = y ]
 	pip uninstall encoded
@@ -181,7 +174,6 @@ test:
 	@git log -1 --decorate | head -1
 	@date
 
-
 BASE_MARKERS = not manual and not sloppy and not static and not broken
 # Unit tests
 NORM_MARKERS = not performance and not integratedx
@@ -191,35 +183,35 @@ PERF_MARKERS = performance and not integratedx
 INTG_MARKERS = not performance and (integrated or integratedx)
 
 retest:
-	poetry run python -m pytest -vv -r w --last-failed
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -vv -r w --last-failed
 
 test-any:
-	poetry run python -m pytest -xvv -r w --timeout=200
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -xvv -r w --timeout=200
 
 test-unit:
-	poetry run python -m pytest -xvv -r w --durations=25 --timeout=600 -m "${BASE_MARKERS} and ${NORM_MARKERS} and not indexing"
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -xvv -r w --durations=25 --timeout=600 -m "${BASE_MARKERS} and ${NORM_MARKERS} and not indexing"
 
 test-unit-full:
-	poetry run python -m pytest -vv  -r w --durations=25 --timeout=600 -m "${BASE_MARKERS} and ${NORM_MARKERS} and not indexing"
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -vv  -r w --durations=25 --timeout=600 -m "${BASE_MARKERS} and ${NORM_MARKERS} and not indexing"
 
 test-indexing-full:
 	make test-indexing-es-full
 	make test-indexing-not-es-full
 
 test-indexing-es-full:
-	poetry run python -m pytest -vv  -r w --durations=25 --timeout=300 -m "${BASE_MARKERS} and ${NORM_MARKERS} and indexing and es"
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -vv  -r w --durations=25 --timeout=300 -m "${BASE_MARKERS} and ${NORM_MARKERS} and indexing and es"
 
 test-indexing-not-es-full:
-	poetry run python -m pytest -vv  -r w --durations=25 --timeout=300 -m "${BASE_MARKERS} and ${NORM_MARKERS} and indexing and not es"
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -vv  -r w --durations=25 --timeout=300 -m "${BASE_MARKERS} and ${NORM_MARKERS} and indexing and not es"
 
 test-indexing:
 	make test-indexing-es && make test-indexing-not-es
 
 test-indexing-es:
-	poetry run python -m pytest -xvv -r w --durations=25 --timeout=200 -m "${BASE_MARKERS} and ${NORM_MARKERS} and indexing and es"
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -xvv -r w --durations=25 --timeout=200 -m "${BASE_MARKERS} and ${NORM_MARKERS} and indexing and es"
 
 test-indexing-not-es:
-	poetry run python -m pytest -xvv -r w --durations=25 --timeout=200 -m "${BASE_MARKERS} and ${NORM_MARKERS} and indexing and not es"
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -xvv -r w --durations=25 --timeout=200 -m "${BASE_MARKERS} and ${NORM_MARKERS} and indexing and not es"
 
 test-performance:
 	poetry run python -m pytest -xvv -r w --timeout=200 -m "${BASE_MARKERS} and ${PERF_MARKERS}"
@@ -256,16 +248,16 @@ remote-test-unit:
 	make remote-test-indexing-not-es
 
 remote-test-not-indexing:  # Note this only does the 'not indexing' tests
-	poetry run python -m pytest -xvv -r w --durations=25 --timeout=600 -m "${REMOTE_MARKERS} and not indexing" --aws-auth --es ${REMOTE_ES} --instafail --force-flaky --max-runs=2 --durations=20 --cov src/encoded
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -xvv -r w --durations=25 --timeout=600 -m "${REMOTE_MARKERS} and not indexing" --aws-auth --es ${REMOTE_ES} --instafail --force-flaky --max-runs=2 --durations=20 --cov src/encoded
 
 remote-test-indexing:
 	make remote-test-indexing-es && make remote-test-not-indexing
 
 remote-test-indexing-es:
-	poetry run python -m pytest -xvv -r w --durations=25 --timeout=300 -m "${REMOTE_MARKERS} and indexing and es" --aws-auth --es ${REMOTE_ES} --cov-append
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -xvv -r w --durations=25 --timeout=300 -m "${REMOTE_MARKERS} and indexing and es" --aws-auth --es ${REMOTE_ES} --cov-append
 
 remote-test-indexing-not-es:
-	poetry run python -m pytest -xvv -r w --durations=25 --timeout=300 -m "${REMOTE_MARKERS} and indexing and not es" --aws-auth --es ${REMOTE_ES} --cov-append
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest -xvv -r w --durations=25 --timeout=300 -m "${REMOTE_MARKERS} and indexing and not es" --aws-auth --es ${REMOTE_ES} --cov-append
 
 update:  # updates dependencies
 	poetry update
@@ -326,12 +318,18 @@ tag-and-push-docker-production:
 	docker push ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${ENV_NAME}:latest
 	date
 
+kill:  # kills back-end processes associated with the application. Use with care.
+	pkill -f postgres &
+	pkill -f elasticsearch &
+	pkill -f opensearch &
+	pkill -f moto_server &
+
 lint-full:
-	@flake8 deploy/ || echo "flake8 failed for deploy/"
-	@flake8 src/encoded/ || echo "flake8 failed for src/encoded"
+	poetry run flake8 deploy/ || echo "flake8 failed for deploy/"
+	poetry run flake8 src/encoded/ || echo "flake8 failed for src/encoded/"
 
 lint:
-	@flake8 deploy/ && flake8 src/encoded/
+	poetry run flake8 deploy/ && poetry run flake8 src/encoded/
 
 help:
 	@make info
@@ -345,13 +343,13 @@ info:
 	   $(info - Use 'make clean' to clear out (non-python) dependencies.)
 	   $(info - Use 'make clean-python' to clear python virtualenv for fresh poetry install.)
 	   $(info - Use 'make clear-poetry-cache' to clear the poetry pypi cache if in a bad state. (Safe, but later recaching can be slow.))
-	   $(info - Use 'make configure' to install poetry. You should not have to do this directly.)
+	   $(info - Use 'make configure' to install poetry, though 'make build' will do it automatically.)
 	   $(info - Use 'make deploy1' to spin up postgres/elasticsearch and load inserts.)
 	   $(info - Use 'make deploy2' to spin up the application server.)
 	   $(info - Use 'make deploy3' to load variants and genes.)
 	   $(info - Use 'make kibana-start' to start kibana on the default local ES port, and 'make kibana-stop' to stop it.)
 	   $(info - Use 'make kibana-start-test' to start kibana on the port being used for active testing, and 'make kibana-stop' to stop it.)
-	   $(info - Use 'make kill' to kill postgres and elasticsearch proccesses. Please use with care.)
+	   $(info - Use 'make kill' to kill postgres and opensearch proccesses. Please use with care.)
 	   $(info - Use 'make moto-setup' to install moto, for less flaky tests. Implied by 'make build'.)
 	   $(info - Use 'make npm-setup' to build the front-end. Implied by 'make build'.)
 	   $(info - Use 'make psql-dev' to start psql on data associated with an active 'make deploy1'.)

deploy/docker/local/docker_development.ini.template

@@ -9,7 +9,7 @@ use = config:base.ini#app
 session.secret = %(here)s/session-secret.b64
 env.name = cgap-docker-${USER}-test
 sqlalchemy.url = postgres://postgres:postgres@db:5432/postgres
-elasticsearch.server = search-cgap-unit-testing-6-8-2p5zrlkmxif4bayh5y6kxzlvx4.us-east-1.es.amazonaws.com:443
+elasticsearch.server = search-cgap-unit-testing-opensearch-tcs45cjpwgdzoi7pafr6oewq6u.us-east-1.es.amazonaws.com:443
 elasticsearch.aws_auth = true
 blob_bucket = cgap-unit-testing-blobs
 metadata_bundles_bucket = cgap-unit-testing-metadata-bundles

docs/public/help/accessioning_3.jsx

@@ -54,7 +54,7 @@
         accessioning spreadsheets to the CGAP portal, called
         SubmitCGAP. The current version is still in beta. It can be
         installed by running <code>pip install submit-cgap</code>. We recommend
-        doing this inside a virtual environment running python 3.7.
+        doing this inside a virtual environment running Python 3.7, 3.8 or 3.9.
         Further instructions for submission with this method can be
         found in the <a href="https://submitcgap.readthedocs.io/en/latest/getting_started.html#family-history">SubmitCGAP documentation</a>.
         Briefly, the basic commands are as follows:

docs/public/help/family_history_3.jsx

@@ -46,7 +46,7 @@
         SubmitCGAP. The current version is still in beta. It can be
         installed by running `pip install submit-cgap`. We
         recommend doing this inside a virtual environment running
-        python 3.7. Further instructions for submission with this
+        Python 3.7, 3.8 or 3.9. Further instructions for submission with this
         method can be found in the <a href="https://submitcgap.readthedocs.io/en/latest/getting_started.html#family-history">SubmitCGAP documentation</a>.
     </p>
 

docs/source/local_installation.rst

@@ -11,11 +11,13 @@ Legacy Instructions
 
 The following instructions are for running a CGAP deployment with macOS and homebrew.
 
-Note that as of summer 2021, these instructions are out of date. Please refer to the Docker setup. There are no guarantees the legacy instructions will work from this point forward.
+Note that as of summer 2021, these instructions are out of date. Please refer to the Docker setup.
+There are no guarantees the legacy instructions will work from this point forward.
 
-CGAP is known to work with Python 3.7.x and 3.8.x and will not work with Python 3.9 or greater.
-If part of the HMS team, it is recommended to use a high patch version, such as Python 3.8.13,
-since that's what we try to do with our servers, but any version of 3.8 should work if you
+CGAP is known to work with 3.8.1 through 3.9.x and will not work with Python 3.10 or greater.
+If part of the HMS team, it is recommended to use a high patch version, such as Python 3.9.16,
+since that's what we try to do with our servers, but any version of 3.8.x (starting with 3.8.1)
+or any version of 3.9.x should work if you
 find you are unable to install that particular patch version.
 It is best practice to create a fresh Python
 virtualenv using one of these versions before proceeding to the following steps.