diff --git a/.env b/.env new file mode 100644 index 0000000..1b42d1e --- /dev/null +++ b/.env @@ -0,0 +1,13 @@ +# For local builds, use "host.docker.internal" instead of "localhost" or "127.0.0.1" +HYDROSHARE_DOMAIN=localhost +DATA_SERVICES_DOMAIN=localhost +DATA_SERVICES_PORT=8090 +HS_PORT=8000 +DATA_SERVICES_URL=http://${DATA_SERVICES_DOMAIN}:${DATA_SERVICES_PORT}/his +HYDROSHARE_REST_URL=http://${HYDROSHARE_DOMAIN}:${HS_PORT}/hsapi +GEOSERVER_REST_URL=http://${DATA_SERVICES_DOMAIN}:${DATA_SERVICES_PORT}/geoserver/rest +GEOSERVER_USERNAME=admin +GEOSERVER_PASSWORD=geoserver +SECRET_KEY=secret +IRODS_LOCAL_DIRECTORY=/projects/iRODS/Vault/home/wwwHydroProxy +WORKSPACE_PREFIX=HS \ No newline at end of file diff --git a/.gitignore b/.gitignore index 71d1b68..b3bb116 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__/ local_settings.py .DS_Store *.pyc +*.sqlite3 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 90ca03a..5971da7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,6 @@ RUN conda env create -f $DS_HOME/hs_data_services/environment.yml # Place Application Files --------------------------------------------------------------------------------# -COPY hs_data_services/ $DS_HOME/hs_data_services COPY conf/ $DS_HOME/conf RUN sudo chmod -R +x $DS_HOME/conf diff --git a/README.md b/README.md index 63b5f84..574d5fb 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ These instructions will help you install and run this application in a productio ##### Edit Django settings: -Rename hs_data_services/hs_data_services/hs_data_services/local_settings.template to local_settings.py. +Rename [hs_data_services/hs_data_services/hs_data_services/local_settings.template](hs_data_services/hs_data_services/hs_data_services/local_settings.template) to local_settings.py. Edit the following settings in local_settings.py: * {{HYDROSHARE_DOMAIN}} - The domain for the connected HydroShare instance e.g. 'hydroshare.org' @@ -30,7 +30,7 @@ Edit the following settings in local_settings.py: ##### Edit Docker settings: -Edit the following settings in hs_data_services/docker-compose.yml +Edit the following settings in [hs_data_services/docker-compose.yml](hs_data_services/docker-compose.yml): * {{IRODS_ACCESS_UID}} - The UID of a user on the host system with iRODS read access. * {{HYDROSHARE_DOMAIN}} - The domain for the connected HydroShare instance e.g. 'hydroshare.org' @@ -49,7 +49,30 @@ $ sudo docker-compose up -d By default, all services will be exposed locally on port 8000 and static files will be located in /static/his/. -##### Post-Installation steps: + +### Running locally for development + +##### Django settings for dev builds: + +Copy [hs_data_services/hs_data_services/hs_data_services/local_settings.template](hs_data_services/hs_data_services/hs_data_services/local_settings.template) to local_settings.py. + +For local builds, you shouldn't need to edit local_settings.py, you just need to create it. + +For local builds, instead of editing [hs_data_services/docker-compose.yml](hs_data_services/docker-compose.yml), you can edit the [.env](.env). +The [.env](.env) is where you can modify ports if needed in order to avoid conflicts on your host. + +##### Start Docker containers (dev build): + +From hs_data_services directory: +``` +IRODS_ACCESS_UID=${UID} docker-compose build +IRODS_ACCESS_UID=${UID} docker-compose up -d +``` +The `${UID}` will get your user's UID on the host system and make it available during the docker build + +Services will be exposed locally on DATA_SERVICES_PORT (default 8090), for example: http://localhost:8090/his/admin/ + +##### Post-Installation steps (dev build): Log in to the Django site at {host_url}/his/admin with default username and password: 'admin' and 'default' From the admin settings page, change the admin password. @@ -67,6 +90,27 @@ Update or add the following settings to HydroShare: * HSWS_TIMEOUT - This can be fairly short e.g. 10 * HSWS_ACTIVATED - True or False +Here is an example of settings used in local development: +``` +HSWS_URL = "http://host.docker.internal:8090/his/services/update" +HSWS_API_TOKEN = "fba02c9f6e9a0c269681ece8bd330a9c314923f3" +HSWS_TIMEOUT = 10 +HSWS_PUBLISH_URLS = True +HSWS_ACTIVATED = True + +HSWS_GEOSERVER_URL = "http://host.docker.internal:8090/geoserver" +HSWS_GEOSERVER_ESCAPE = { + '/': ' ' +} +``` +Note that `host.docker.internal` is used for local builds (instead of `localhost` or `127.0.0.1`). This is allows the different docker services to communicate on the host. +Ensure that Hydroshare is up and running locally on whatever HS_PORT you set in the [.env](.env) (default 8000). +The gunicorn container will hot-restart, but the celery worker and celery beat services will not. To restart those you can use, for example, `docker restart hs_data_services-celery-worker-1`. + +Unless you have the `idata_vault_vol` volume from Hydroshare linked to a directory on the host (not the default) you will likely have to modify permissions so that the geoserver can read from the iRods vault: +``` +docker exec hs_data_services-geoserver-1 chmod -R o+rx /projects +``` Once everything is set up, HydroShare should start sending update requests to this data services app. You can check that this app is receiving those requests by going to {host_url}/flower and clicking on 'Tasks'. You can test it manually by sending a POST request to {host_url}/his/services/update/{resource_id}/ and adding {'Authorization': 'Token HSWS_API_TOKEN'} to the request's headers. ##### Troubleshooting steps: diff --git a/conf/gunicorn/gunicorn.sh b/conf/gunicorn/gunicorn.sh index 89fe9ab..c9bb755 100644 --- a/conf/gunicorn/gunicorn.sh +++ b/conf/gunicorn/gunicorn.sh @@ -22,4 +22,5 @@ echo Starting Gunicorn. exec gunicorn hs_data_services.wsgi:application \ --bind 0.0.0.0:8060 \ --workers 3 \ - --timeout 3600 + --timeout 3600 \ + --reload diff --git a/docker-compose.yml b/docker-compose.yml index 4cb239f..4be9a57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,25 +12,27 @@ services: - data_services_volume:/var/lib/postgresql/data/ ports: - 5432:5432 - networks: - - data_services_network geoserver: - image: oscarfonts/geoserver:2.14.1 + image: kartoza/geoserver:2.22.0 environment: - - CUSTOM_UID={{IRODS_ACCESS_UID}} # A user ID on the host with read permissions to the iRODS directory - - GEOSERVER_CSRF_WHITELIST={{HYDROSHARE_DOMAIN}} # e.g. hydroshare.org + - GEOSERVER_UID=${IRODS_ACCESS_UID:-501} # A user ID on the host with read permissions to the iRODS directory + - GEOSERVER_CSRF_WHITELIST=${HYDROSHARE_DOMAIN:-hydroshare.org} + - HTTP_PORT=8080 + - HTTP_PROXY_PORT=${DATA_SERVICES_PORT:-8090} + - HTTP_PROXY_NAME=${DATA_SERVICES_DOMAIN} + - GEOSERVER_ADMIN_PASSWORD=${GEOSERVER_PASSWORD:-geoserver} + - GEOSERVER_ADMIN_USER=${GEOSERVER_USERNAME:-admin} + - CATALINA_OPTS=-server -Djava.awt.headless=true -Xms2g -Xmx10g -XX:+UseConcMarkSweepGC -XX:NewSize=48m -DGEOSERVER_DATA_DIR=/var/local/geoserver + # https://github.com/oscarfonts/docker-geoserver/blob/master/2.24.1/Dockerfile#L35-L37 + # https://docs.geoserver.org/main/en/user/production/container.html volumes: - geoserver_volume:/var/local/geoserver - ./conf/geoserver/geoserver.sh:/usr/local/bin/start.sh # Overwrite the default startup script to allow anonymous GET access - - /projects:/projects:ro # Mount projects as a read-only volume to grant access to resources - networks: - - data_services_network + - idata_vault_vol:/projects redis: image: redis:6.0.4-alpine - networks: - - data_services_network flower: image: mher/flower:0.9.5 @@ -38,8 +40,6 @@ services: - CELERY_BROKER_URL=redis://redis:6379/0 - FLOWER_PORT=8040 - FLOWER_URL_PREFIX=flower - networks: - - data_services_network depends_on: - redis @@ -47,26 +47,23 @@ services: build: . command: /usr/bin/supervisord -c /home/dsuser/conf/gunicorn/gunicorn.conf volumes: - - /static:/static # Static files will be stored locally at root. - networks: - - data_services_network + - static:/static + - "./hs_data_services:/home/dsuser/hs_data_services" depends_on: - postgres celery-worker: build: . command: /usr/bin/supervisord -c /home/dsuser/conf/celery-worker/celery_worker.conf - networks: - - data_services_network depends_on: - redis - postgres + volumes: + - "./hs_data_services:/home/dsuser/hs_data_services" celery-beat: build: . command: /usr/bin/supervisord -c /home/dsuser/conf/celery-beat/celery_beat.conf - networks: - - data_services_network depends_on: - redis - celery-worker @@ -75,24 +72,21 @@ services: nginx: image: nginx:1.19 ports: - - 8000:80 # Expose all services locally on port 8000 + - ${DATA_SERVICES_PORT:-8090}:80 # Expose all services locally on port volumes: - ./conf/nginx:/etc/nginx/conf.d - - /static:/static # Static files will be stored locally at root. + - static:/static #- /opt/hs-certs:/opt/hs-certs # Used to configure SSL if available. - networks: - - data_services_network depends_on: - gunicorn - geoserver - flower -networks: - data_services_network: - driver: bridge - - volumes: + idata_vault_vol: + external: true + name: hydroshare_idata_vault_vol + static: data_services_volume: geoserver_volume: diff --git a/hs_data_services/environment.yml b/hs_data_services/environment.yml index 4e3042d..74692df 100644 --- a/hs_data_services/environment.yml +++ b/hs_data_services/environment.yml @@ -18,3 +18,4 @@ dependencies: - drf-yasg==1.17.0 - redis==3.5.3 - future==0.18.2 + - python-dotenv==0.21.0 diff --git a/hs_data_services/hs_data_services/local_settings.template b/hs_data_services/hs_data_services/local_settings.template index 6997599..4d01abd 100644 --- a/hs_data_services/hs_data_services/local_settings.template +++ b/hs_data_services/hs_data_services/local_settings.template @@ -11,6 +11,9 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ """ import os +from dotenv import dotenv_values + +config = dotenv_values(".env") # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -18,13 +21,12 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [{{HYDROSHARE_DOMAIN}}, 'gnupstream', 'localhost', '127.0.0.1'] # e.g. "hydroshare.org" - +ALLOWED_HOSTS = [config.get('HYDROSHARE_DOMAIN', 'hydroshare.org'), 'gnupstream', 'localhost', '127.0.0.1'] # e.g. "hydroshare.org" # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = {{SECRET_KEY}} +SECRET_KEY = config.get('SECRET_KEY', 'secret') # Database # https://docs.djangoproject.com/en/3.0/ref/settings/#databases @@ -42,7 +44,7 @@ DATABASES = { STATIC_URL = '/static/his/' STATIC_ROOT = '/static/his/' -PROXY_BASE_URL = {{DATA_SERVICES_URL}} # e.g. "https://geoserver-beta.hydroshare.org/his" +PROXY_BASE_URL = config.get('DATA_SERVICES_URL', 'http://host.docker.internal:8090/his') # e.g. "https://geoserver-beta.hydroshare.org/his" # Celery settings @@ -50,15 +52,14 @@ CELERY_BROKER_URL = "redis://redis:6379/0" CELERY_BEAT_SCHEDULE = {} # HydroShare Data Services Connection settings - -HYDROSHARE_URL = {{HYDROSHARE_REST_URL}} # e.g. "https://beta.hydroshare.org/hsapi" +HYDROSHARE_URL = config.get('HYDROSHARE_REST_URL', 'http://host.docker.internal:8000/hsapi') # e.g. "https://beta.hydroshare.org/hsapi" DATA_SERVICES = { 'geoserver': { - 'URL': {{GEOSERVER_REST_URL}}, # e.g. "https://geoserver-beta.hydroshare.org/geoserver/rest" - 'USER': {{GEOSERVER_USERNAME}}, - 'PASSWORD': {{GEOSERVER_PASSWORD}}, - 'IRODS_DIR': {{IRODS_LOCAL_DIRECTORY}}, # e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy" - 'NAMESPACE': {{WORKSPACE_PREFIX}} # e.g. "HS" + 'URL': config.get('GEOSERVER_REST_URL', 'http://host.docker.internal:8090/geoserver/rest'), # e.g. "https://geoserver-beta.hydroshare.org/geoserver/rest" + 'USER': config.get('GEOSERVER_USERNAME', 'admin'), + 'PASSWORD': config.get('GEOSERVER_PASSWORD', 'geoserver'), + 'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/projects/iRODS/Vault/home/wwwHydroProxy'),# e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy" + 'NAMESPACE': config.get('WORKSPACE_PREFIX', 'HS') # e.g. "HS" } }