From 68d333f179845e9d34f968f944346964f9da48af Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 08:58:41 -0500 Subject: [PATCH 01/14] use a .env for local development --- .env | 10 +++ .gitignore | 1 - README.md | 20 +++++- docker-compose.yml | 35 +++------- hs_data_services/environment.yml | 1 + .../hs_data_services/local_settings.py | 67 +++++++++++++++++++ 6 files changed, 103 insertions(+), 31 deletions(-) create mode 100644 .env create mode 100644 hs_data_services/hs_data_services/local_settings.py diff --git a/.env b/.env new file mode 100644 index 0000000..9b07134 --- /dev/null +++ b/.env @@ -0,0 +1,10 @@ +HYDROSHARE_DOMAIN=localhost +SECRET_KEY=secret +DATA_SERVICES_URL=http://localhost:8080/his +HYDROSHARE_REST_URL=http://localhost:8000/hsapi +GEOSERVER_REST_URL=http://localhost:8080/geoserver/rest +GEOSERVER_USERNAME=admin +GEOSERVER_PASSWORD=geoserver +IRODS_LOCAL_DIRECTORY=/tmp +WORKSPACE_PREFIX=HS +DATA_SERVICES_PORT=8080 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 71d1b68..671af51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ irods_vault/ static/ __pycache__/ -local_settings.py .DS_Store *.pyc diff --git a/README.md b/README.md index 63b5f84..1fbb670 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,11 @@ These instructions will help you install and run this application in a productio ### Installing -##### Edit Django settings: +##### Edit Django settings (for prod): -Rename hs_data_services/hs_data_services/hs_data_services/local_settings.template to local_settings.py. +For local development, skip this step and use the local_settings.py file that exists in this repo. + +Copy hs_data_services/hs_data_services/hs_data_services/local_settings.template to overwrite local_settings.py. Edit the following settings in local_settings.py: * {{HYDROSHARE_DOMAIN}} - The domain for the connected HydroShare instance e.g. 'hydroshare.org' @@ -30,12 +32,14 @@ Edit the following settings in local_settings.py: ##### Edit Docker settings: +For local development, skip this step and use the .env values that exist in this repo. + Edit the following settings in hs_data_services/docker-compose.yml * {{IRODS_ACCESS_UID}} - The UID of a user on the host system with iRODS read access. * {{HYDROSHARE_DOMAIN}} - The domain for the connected HydroShare instance e.g. 'hydroshare.org' -##### Start Docker containers: +##### Start Docker containers (for non-local builds): From hs_data_services directory, run the following command to build Docker images: ``` @@ -49,6 +53,16 @@ $ sudo docker-compose up -d By default, all services will be exposed locally on port 8000 and static files will be located in /static/his/. +##### OR Start Docker containers (for local dev): + +Ensure that Hydroshare is up and running locally on port 8000 + +From hs_data_services directory: +`UID=${UID} docker-compose build` +`UID=${UID} docker-compose up -d` + +Services will be exposed locally on port 8080, for example: http://localhost:8080/his/admin/ + ##### Post-Installation steps: Log in to the Django site at {host_url}/his/admin with default username and password: 'admin' and 'default' diff --git a/docker-compose.yml b/docker-compose.yml index 4cb239f..19dd271 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,25 +12,19 @@ services: - data_services_volume:/var/lib/postgresql/data/ ports: - 5432:5432 - networks: - - data_services_network geoserver: image: oscarfonts/geoserver:2.14.1 environment: - - CUSTOM_UID={{IRODS_ACCESS_UID}} # A user ID on the host with read permissions to the iRODS directory - - GEOSERVER_CSRF_WHITELIST={{HYDROSHARE_DOMAIN}} # e.g. hydroshare.org + - CUSTOM_UID=${UID:-501} # A user ID on the host with read permissions to the iRODS directory + - GEOSERVER_CSRF_WHITELIST=${HYDROSHARE_DOMAIN:-hydroshare.org} # e.g. hydroshare.org volumes: - geoserver_volume:/var/local/geoserver - ./conf/geoserver/geoserver.sh:/usr/local/bin/start.sh # Overwrite the default startup script to allow anonymous GET access - - /projects:/projects:ro # Mount projects as a read-only volume to grant access to resources - networks: - - data_services_network + - projects:/projects:ro # Mount projects as a read-only volume to grant access to resources redis: image: redis:6.0.4-alpine - networks: - - data_services_network flower: image: mher/flower:0.9.5 @@ -38,8 +32,6 @@ services: - CELERY_BROKER_URL=redis://redis:6379/0 - FLOWER_PORT=8040 - FLOWER_URL_PREFIX=flower - networks: - - data_services_network depends_on: - redis @@ -47,17 +39,13 @@ services: build: . command: /usr/bin/supervisord -c /home/dsuser/conf/gunicorn/gunicorn.conf volumes: - - /static:/static # Static files will be stored locally at root. - networks: - - data_services_network + - static:/static # Static files will be stored locally at root. depends_on: - postgres celery-worker: build: . command: /usr/bin/supervisord -c /home/dsuser/conf/celery-worker/celery_worker.conf - networks: - - data_services_network depends_on: - redis - postgres @@ -65,8 +53,6 @@ services: celery-beat: build: . command: /usr/bin/supervisord -c /home/dsuser/conf/celery-beat/celery_beat.conf - networks: - - data_services_network depends_on: - redis - celery-worker @@ -75,24 +61,19 @@ services: nginx: image: nginx:1.19 ports: - - 8000:80 # Expose all services locally on port 8000 + - ${DATA_SERVICES_PORT:-8080}:80 # Expose all services locally on port volumes: - ./conf/nginx:/etc/nginx/conf.d - - /static:/static # Static files will be stored locally at root. + - static:/static # Static files will be stored locally at root. #- /opt/hs-certs:/opt/hs-certs # Used to configure SSL if available. - networks: - - data_services_network depends_on: - gunicorn - geoserver - flower -networks: - data_services_network: - driver: bridge - - volumes: + projects: + static: data_services_volume: geoserver_volume: diff --git a/hs_data_services/environment.yml b/hs_data_services/environment.yml index 4e3042d..74692df 100644 --- a/hs_data_services/environment.yml +++ b/hs_data_services/environment.yml @@ -18,3 +18,4 @@ dependencies: - drf-yasg==1.17.0 - redis==3.5.3 - future==0.18.2 + - python-dotenv==0.21.0 diff --git a/hs_data_services/hs_data_services/local_settings.py b/hs_data_services/hs_data_services/local_settings.py new file mode 100644 index 0000000..6416560 --- /dev/null +++ b/hs_data_services/hs_data_services/local_settings.py @@ -0,0 +1,67 @@ +""" +Django local settings for hs_data_services project. + +Generated by 'django-admin startproject' using Django 3.0.1. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os +from dotenv import dotenv_values + +config = dotenv_values(".env") + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +# ALLOWED_HOSTS = [config.get('HYDROSHARE_DOMAIN', 'localhost'), 'gnupstream', 'localhost', '127.0.0.1'] +ALLOWED_HOSTS = ['*'] +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = config.get('SECRET_KEY', 'secret') + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ + +STATIC_URL = '/static/his/' +STATIC_ROOT = '/static/his/' + +PROXY_BASE_URL = config.get('DATA_SERVICES_URL', 'http://localhost/his') + +# Celery settings + +CELERY_BROKER_URL = "redis://redis:6379/0" +CELERY_BEAT_SCHEDULE = {} + +# HydroShare Data Services Connection settings + +HYDROSHARE_URL = config.get('HYDROSHARE_REST_URL', 'http://localhost:8000/hsapi') + +DATA_SERVICES = { + 'geoserver': { + 'URL': config.get('GEOSERVER_REST_URL', 'http://localhost:8080/geoserver/rest'), + 'USER': config.get('GEOSERVER_USERNAME', 'admin'), + 'PASSWORD': config.get('GEOSERVER_PASSWORD', 'geoserver'), + 'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/tmp'), + 'NAMESPACE': config.get('WORKSPACE_PREFIX', 'HS') + } +} From e3e5db06b15c21b6442f44dd37f2a676685e510e Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 09:27:01 -0500 Subject: [PATCH 02/14] simplify port selection --- .env | 11 +++++---- README.md | 13 ++++++++++- docker-compose.yml | 2 +- .../hs_data_services/local_settings.py | 9 ++++---- .../hs_data_services/local_settings.template | 23 ++++++++++--------- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/.env b/.env index 9b07134..c796ca7 100644 --- a/.env +++ b/.env @@ -1,10 +1,11 @@ HYDROSHARE_DOMAIN=localhost SECRET_KEY=secret -DATA_SERVICES_URL=http://localhost:8080/his -HYDROSHARE_REST_URL=http://localhost:8000/hsapi -GEOSERVER_REST_URL=http://localhost:8080/geoserver/rest +DATA_SERVICES_PORT=8090 +HS_PORT=8000 +DATA_SERVICES_URL=http://localhost:${DATA_SERVICES_PORT}/his +HYDROSHARE_REST_URL=http://localhost:${HS_PORT}/hsapi +GEOSERVER_REST_URL=http://localhost:${DATA_SERVICES_PORT}/geoserver/rest GEOSERVER_USERNAME=admin GEOSERVER_PASSWORD=geoserver IRODS_LOCAL_DIRECTORY=/tmp -WORKSPACE_PREFIX=HS -DATA_SERVICES_PORT=8080 \ No newline at end of file +WORKSPACE_PREFIX=HS \ No newline at end of file diff --git a/README.md b/README.md index 1fbb670..70f7b01 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,18 @@ From hs_data_services directory: `UID=${UID} docker-compose build` `UID=${UID} docker-compose up -d` -Services will be exposed locally on port 8080, for example: http://localhost:8080/his/admin/ +Services will be exposed locally on DATA_SERVICES_PORT (default 8090), for example: http://localhost:8090/his/admin/ + +HSWS_URL = "http://host.docker.internal:8080/his/services/update" +HSWS_API_TOKEN = "fba02c9f6e9a0c269681ece8bd330a9c314923f3" +HSWS_TIMEOUT = 10 +HSWS_PUBLISH_URLS = True +HSWS_ACTIVATED = True + +HSWS_GEOSERVER_URL = "http://host.docker.internal:8080/geoserver" +HSWS_GEOSERVER_ESCAPE = { + '/': ' ' +} ##### Post-Installation steps: diff --git a/docker-compose.yml b/docker-compose.yml index 19dd271..6a2d0a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,7 +61,7 @@ services: nginx: image: nginx:1.19 ports: - - ${DATA_SERVICES_PORT:-8080}:80 # Expose all services locally on port + - ${DATA_SERVICES_PORT:-8090}:80 # Expose all services locally on port volumes: - ./conf/nginx:/etc/nginx/conf.d - static:/static # Static files will be stored locally at root. diff --git a/hs_data_services/hs_data_services/local_settings.py b/hs_data_services/hs_data_services/local_settings.py index 6416560..eb218a8 100644 --- a/hs_data_services/hs_data_services/local_settings.py +++ b/hs_data_services/hs_data_services/local_settings.py @@ -21,8 +21,8 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -# ALLOWED_HOSTS = [config.get('HYDROSHARE_DOMAIN', 'localhost'), 'gnupstream', 'localhost', '127.0.0.1'] -ALLOWED_HOSTS = ['*'] +ALLOWED_HOSTS = [config.get('HYDROSHARE_DOMAIN', 'hydroshare.org'), 'gnupstream', 'localhost', '127.0.0.1'] + # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ @@ -45,7 +45,7 @@ STATIC_URL = '/static/his/' STATIC_ROOT = '/static/his/' -PROXY_BASE_URL = config.get('DATA_SERVICES_URL', 'http://localhost/his') +PROXY_BASE_URL = config.get('DATA_SERVICES_URL', 'http://localhost:8090/his') # Celery settings @@ -53,12 +53,11 @@ CELERY_BEAT_SCHEDULE = {} # HydroShare Data Services Connection settings - HYDROSHARE_URL = config.get('HYDROSHARE_REST_URL', 'http://localhost:8000/hsapi') DATA_SERVICES = { 'geoserver': { - 'URL': config.get('GEOSERVER_REST_URL', 'http://localhost:8080/geoserver/rest'), + 'URL': config.get('GEOSERVER_REST_URL', 'http://localhost:8090/geoserver/rest'), 'USER': config.get('GEOSERVER_USERNAME', 'admin'), 'PASSWORD': config.get('GEOSERVER_PASSWORD', 'geoserver'), 'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/tmp'), diff --git a/hs_data_services/hs_data_services/local_settings.template b/hs_data_services/hs_data_services/local_settings.template index 6997599..c47a816 100644 --- a/hs_data_services/hs_data_services/local_settings.template +++ b/hs_data_services/hs_data_services/local_settings.template @@ -11,6 +11,9 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ """ import os +from dotenv import dotenv_values + +config = dotenv_values(".env") # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -18,13 +21,12 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [{{HYDROSHARE_DOMAIN}}, 'gnupstream', 'localhost', '127.0.0.1'] # e.g. "hydroshare.org" - +ALLOWED_HOSTS = [config.get('HYDROSHARE_DOMAIN', 'hydroshare.org'), 'gnupstream', 'localhost', '127.0.0.1'] # e.g. "hydroshare.org" # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = {{SECRET_KEY}} +SECRET_KEY = config.get('SECRET_KEY', 'secret') # Database # https://docs.djangoproject.com/en/3.0/ref/settings/#databases @@ -42,7 +44,7 @@ DATABASES = { STATIC_URL = '/static/his/' STATIC_ROOT = '/static/his/' -PROXY_BASE_URL = {{DATA_SERVICES_URL}} # e.g. "https://geoserver-beta.hydroshare.org/his" +PROXY_BASE_URL = config.get('DATA_SERVICES_URL', 'http://localhost:8090/his') # e.g. "https://geoserver-beta.hydroshare.org/his" # Celery settings @@ -50,15 +52,14 @@ CELERY_BROKER_URL = "redis://redis:6379/0" CELERY_BEAT_SCHEDULE = {} # HydroShare Data Services Connection settings - -HYDROSHARE_URL = {{HYDROSHARE_REST_URL}} # e.g. "https://beta.hydroshare.org/hsapi" +HYDROSHARE_URL = config.get('HYDROSHARE_REST_URL', 'http://localhost:8000/hsapi') # e.g. "https://beta.hydroshare.org/hsapi" DATA_SERVICES = { 'geoserver': { - 'URL': {{GEOSERVER_REST_URL}}, # e.g. "https://geoserver-beta.hydroshare.org/geoserver/rest" - 'USER': {{GEOSERVER_USERNAME}}, - 'PASSWORD': {{GEOSERVER_PASSWORD}}, - 'IRODS_DIR': {{IRODS_LOCAL_DIRECTORY}}, # e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy" - 'NAMESPACE': {{WORKSPACE_PREFIX}} # e.g. "HS" + 'URL': config.get('GEOSERVER_REST_URL', 'http://localhost:8090/geoserver/rest'), # e.g. "https://geoserver-beta.hydroshare.org/geoserver/rest" + 'USER': config.get('GEOSERVER_USERNAME', 'admin'), + 'PASSWORD': config.get('GEOSERVER_PASSWORD', 'geoserver'), + 'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/tmp'), # e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy" + 'NAMESPACE': config.get('WORKSPACE_PREFIX', 'HS') # e.g. "HS" } } From 65a208ac23fd526a090c26e47cea0c4bb5220ff6 Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 09:33:39 -0500 Subject: [PATCH 03/14] rm local_settings --- .gitignore | 1 + .../hs_data_services/local_settings.py | 66 ------------------- 2 files changed, 1 insertion(+), 66 deletions(-) delete mode 100644 hs_data_services/hs_data_services/local_settings.py diff --git a/.gitignore b/.gitignore index 671af51..71d1b68 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ irods_vault/ static/ __pycache__/ +local_settings.py .DS_Store *.pyc diff --git a/hs_data_services/hs_data_services/local_settings.py b/hs_data_services/hs_data_services/local_settings.py deleted file mode 100644 index eb218a8..0000000 --- a/hs_data_services/hs_data_services/local_settings.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Django local settings for hs_data_services project. - -Generated by 'django-admin startproject' using Django 3.0.1. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/3.0/ref/settings/ -""" - -import os -from dotenv import dotenv_values - -config = dotenv_values(".env") - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [config.get('HYDROSHARE_DOMAIN', 'hydroshare.org'), 'gnupstream', 'localhost', '127.0.0.1'] - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = config.get('SECRET_KEY', 'secret') - -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.0/howto/static-files/ - -STATIC_URL = '/static/his/' -STATIC_ROOT = '/static/his/' - -PROXY_BASE_URL = config.get('DATA_SERVICES_URL', 'http://localhost:8090/his') - -# Celery settings - -CELERY_BROKER_URL = "redis://redis:6379/0" -CELERY_BEAT_SCHEDULE = {} - -# HydroShare Data Services Connection settings -HYDROSHARE_URL = config.get('HYDROSHARE_REST_URL', 'http://localhost:8000/hsapi') - -DATA_SERVICES = { - 'geoserver': { - 'URL': config.get('GEOSERVER_REST_URL', 'http://localhost:8090/geoserver/rest'), - 'USER': config.get('GEOSERVER_USERNAME', 'admin'), - 'PASSWORD': config.get('GEOSERVER_PASSWORD', 'geoserver'), - 'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/tmp'), - 'NAMESPACE': config.get('WORKSPACE_PREFIX', 'HS') - } -} From b2dcea11c87c6f7c7384f03d04939e524dbeec12 Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 09:59:34 -0500 Subject: [PATCH 04/14] update readme and UID --- README.md | 59 ++++++++++++++++++++++++++++------------------ docker-compose.yml | 2 +- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 70f7b01..41668e2 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,9 @@ These instructions will help you install and run this application in a productio ### Installing -##### Edit Django settings (for prod): +##### Edit Django settings: -For local development, skip this step and use the local_settings.py file that exists in this repo. - -Copy hs_data_services/hs_data_services/hs_data_services/local_settings.template to overwrite local_settings.py. +Rename [hs_data_services/hs_data_services/hs_data_services/local_settings.template](hs_data_services/hs_data_services/hs_data_services/local_settings.template) to local_settings.py. Edit the following settings in local_settings.py: * {{HYDROSHARE_DOMAIN}} - The domain for the connected HydroShare instance e.g. 'hydroshare.org' @@ -32,14 +30,12 @@ Edit the following settings in local_settings.py: ##### Edit Docker settings: -For local development, skip this step and use the .env values that exist in this repo. - -Edit the following settings in hs_data_services/docker-compose.yml +Edit the following settings in [hs_data_services/docker-compose.yml](hs_data_services/docker-compose.yml): * {{IRODS_ACCESS_UID}} - The UID of a user on the host system with iRODS read access. * {{HYDROSHARE_DOMAIN}} - The domain for the connected HydroShare instance e.g. 'hydroshare.org' -##### Start Docker containers (for non-local builds): +##### Start Docker containers: From hs_data_services directory, run the following command to build Docker images: ``` @@ -53,26 +49,27 @@ $ sudo docker-compose up -d By default, all services will be exposed locally on port 8000 and static files will be located in /static/his/. -##### OR Start Docker containers (for local dev): -Ensure that Hydroshare is up and running locally on port 8000 +### Running locally for development -From hs_data_services directory: -`UID=${UID} docker-compose build` -`UID=${UID} docker-compose up -d` +##### Django settings for local builds: -Services will be exposed locally on DATA_SERVICES_PORT (default 8090), for example: http://localhost:8090/his/admin/ +Copy [hs_data_services/hs_data_services/hs_data_services/local_settings.template](hs_data_services/hs_data_services/hs_data_services/local_settings.template) to local_settings.py. -HSWS_URL = "http://host.docker.internal:8080/his/services/update" -HSWS_API_TOKEN = "fba02c9f6e9a0c269681ece8bd330a9c314923f3" -HSWS_TIMEOUT = 10 -HSWS_PUBLISH_URLS = True -HSWS_ACTIVATED = True +For local builds, you shouldn't need to edit local_settings.py, you just need to create it. -HSWS_GEOSERVER_URL = "http://host.docker.internal:8080/geoserver" -HSWS_GEOSERVER_ESCAPE = { - '/': ' ' -} +For local builds, instead of editing [hs_data_services/docker-compose.yml](hs_data_services/docker-compose.yml), you can edit the [.env](.env). +The [.env](.env) is where you can modify ports if needed in order to avoid conflicts on your host. + +##### Start Docker containers for local build: + +From hs_data_services directory: +`IRODS_ACCESS_UID=${UID} docker-compose build` +`IRODS_ACCESS_UID=${UID} docker-compose up -d` + +The `${UID}` will get your user's UID on the host system and make it available during the docker build + +Services will be exposed locally on DATA_SERVICES_PORT (default 8090), for example: http://localhost:8090/his/admin/ ##### Post-Installation steps: @@ -92,6 +89,22 @@ Update or add the following settings to HydroShare: * HSWS_TIMEOUT - This can be fairly short e.g. 10 * HSWS_ACTIVATED - True or False +Here is an example of settings used in local development: +``` +HSWS_URL = "http://host.docker.internal:8090/his/services/update" +HSWS_API_TOKEN = "fba02c9f6e9a0c269681ece8bd330a9c314923f3" +HSWS_TIMEOUT = 10 +HSWS_PUBLISH_URLS = True +HSWS_ACTIVATED = True + +HSWS_GEOSERVER_URL = "http://host.docker.internal:8090/geoserver" +HSWS_GEOSERVER_ESCAPE = { + '/': ' ' +} +``` +Note that `host.docker.internal` is used for local builds (instead of `localhost` or `127.0.0.1`). This is allows the different docker services to communicate on the host. +Ensure that Hydroshare is up and running locally on whatever HS_PORT you set in the [.env](.env) (default 8000) + Once everything is set up, HydroShare should start sending update requests to this data services app. You can check that this app is receiving those requests by going to {host_url}/flower and clicking on 'Tasks'. You can test it manually by sending a POST request to {host_url}/his/services/update/{resource_id}/ and adding {'Authorization': 'Token HSWS_API_TOKEN'} to the request's headers. ##### Troubleshooting steps: diff --git a/docker-compose.yml b/docker-compose.yml index 6a2d0a8..ae2312a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: geoserver: image: oscarfonts/geoserver:2.14.1 environment: - - CUSTOM_UID=${UID:-501} # A user ID on the host with read permissions to the iRODS directory + - CUSTOM_UID=${IRODS_ACCESS_UID:-501} # A user ID on the host with read permissions to the iRODS directory - GEOSERVER_CSRF_WHITELIST=${HYDROSHARE_DOMAIN:-hydroshare.org} # e.g. hydroshare.org volumes: - geoserver_volume:/var/local/geoserver From 68c4eddda3911e7138067b0bc5a765bdfce21c6a Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 11:46:48 -0500 Subject: [PATCH 05/14] host.docker.internal --- .env | 12 +++++++----- .../hs_data_services/local_settings.template | 10 +++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.env b/.env index c796ca7..b37694e 100644 --- a/.env +++ b/.env @@ -1,11 +1,13 @@ -HYDROSHARE_DOMAIN=localhost -SECRET_KEY=secret +# For local builds, use "host.docker.internal" instead of "localhost" or "127.0.0.1" +HYDROSHARE_DOMAIN=host.docker.internal +DATA_SERVICES_DOMAIN=host.docker.internal DATA_SERVICES_PORT=8090 HS_PORT=8000 -DATA_SERVICES_URL=http://localhost:${DATA_SERVICES_PORT}/his -HYDROSHARE_REST_URL=http://localhost:${HS_PORT}/hsapi -GEOSERVER_REST_URL=http://localhost:${DATA_SERVICES_PORT}/geoserver/rest +DATA_SERVICES_URL=http://${DATA_SERVICES_DOMAIN}:${DATA_SERVICES_PORT}/his +HYDROSHARE_REST_URL=http://${HYDROSHARE_DOMAIN}:${HS_PORT}/hsapi +GEOSERVER_REST_URL=http://${DATA_SERVICES_DOMAIN}:${DATA_SERVICES_PORT}/geoserver/rest GEOSERVER_USERNAME=admin GEOSERVER_PASSWORD=geoserver +SECRET_KEY=secret IRODS_LOCAL_DIRECTORY=/tmp WORKSPACE_PREFIX=HS \ No newline at end of file diff --git a/hs_data_services/hs_data_services/local_settings.template b/hs_data_services/hs_data_services/local_settings.template index c47a816..6d5f5b5 100644 --- a/hs_data_services/hs_data_services/local_settings.template +++ b/hs_data_services/hs_data_services/local_settings.template @@ -44,7 +44,7 @@ DATABASES = { STATIC_URL = '/static/his/' STATIC_ROOT = '/static/his/' -PROXY_BASE_URL = config.get('DATA_SERVICES_URL', 'http://localhost:8090/his') # e.g. "https://geoserver-beta.hydroshare.org/his" +PROXY_BASE_URL = config.get('DATA_SERVICES_URL', 'http://host.docker.internal:8090/his') # e.g. "https://geoserver-beta.hydroshare.org/his" # Celery settings @@ -52,14 +52,14 @@ CELERY_BROKER_URL = "redis://redis:6379/0" CELERY_BEAT_SCHEDULE = {} # HydroShare Data Services Connection settings -HYDROSHARE_URL = config.get('HYDROSHARE_REST_URL', 'http://localhost:8000/hsapi') # e.g. "https://beta.hydroshare.org/hsapi" +HYDROSHARE_URL = config.get('HYDROSHARE_REST_URL', 'http://host.docker.internal:8000/hsapi') # e.g. "https://beta.hydroshare.org/hsapi" DATA_SERVICES = { 'geoserver': { - 'URL': config.get('GEOSERVER_REST_URL', 'http://localhost:8090/geoserver/rest'), # e.g. "https://geoserver-beta.hydroshare.org/geoserver/rest" + 'URL': config.get('GEOSERVER_REST_URL', 'http://host.docker.internal:8090/geoserver/rest'), # e.g. "https://geoserver-beta.hydroshare.org/geoserver/rest" 'USER': config.get('GEOSERVER_USERNAME', 'admin'), 'PASSWORD': config.get('GEOSERVER_PASSWORD', 'geoserver'), - 'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/tmp'), # e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy" - 'NAMESPACE': config.get('WORKSPACE_PREFIX', 'HS') # e.g. "HS" + 'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/tmp'), # e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy" + 'NAMESPACE': config.get('WORKSPACE_PREFIX', 'HS') # e.g. "HS" } } From 5865b54b8fb5de3d23e24984bbc247ee8eeaa862 Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 13:45:08 -0500 Subject: [PATCH 06/14] use kartoza geoserver image --- docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ae2312a..a59a7d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,10 +14,12 @@ services: - 5432:5432 geoserver: - image: oscarfonts/geoserver:2.14.1 + image: kartoza/geoserver:2.22.0 environment: - - CUSTOM_UID=${IRODS_ACCESS_UID:-501} # A user ID on the host with read permissions to the iRODS directory - - GEOSERVER_CSRF_WHITELIST=${HYDROSHARE_DOMAIN:-hydroshare.org} # e.g. hydroshare.org + - GEOSERVER_UID=${IRODS_ACCESS_UID:-501} # A user ID on the host with read permissions to the iRODS directory + - HTTP_PORT=8080 + - HTTP_PROXY_PORT=${DATA_SERVICES_PORT:-8090} + - HTTP_PROXY_NAME=${DATA_SERVICES_DOMAIN} volumes: - geoserver_volume:/var/local/geoserver - ./conf/geoserver/geoserver.sh:/usr/local/bin/start.sh # Overwrite the default startup script to allow anonymous GET access From 9ca897effcfb6a9cbafe86a6ac14658f8207c0cc Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 13:52:56 -0500 Subject: [PATCH 07/14] fix geoserver redirect --- .env | 4 ++-- docker-compose.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.env b/.env index b37694e..ab43b91 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ # For local builds, use "host.docker.internal" instead of "localhost" or "127.0.0.1" -HYDROSHARE_DOMAIN=host.docker.internal -DATA_SERVICES_DOMAIN=host.docker.internal +HYDROSHARE_DOMAIN=localhost +DATA_SERVICES_DOMAIN=localhost DATA_SERVICES_PORT=8090 HS_PORT=8000 DATA_SERVICES_URL=http://${DATA_SERVICES_DOMAIN}:${DATA_SERVICES_PORT}/his diff --git a/docker-compose.yml b/docker-compose.yml index a59a7d4..a402dc0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,9 +17,12 @@ services: image: kartoza/geoserver:2.22.0 environment: - GEOSERVER_UID=${IRODS_ACCESS_UID:-501} # A user ID on the host with read permissions to the iRODS directory + - GEOSERVER_CSRF_WHITELIST=${HYDROSHARE_DOMAIN:-hydroshare.org} - HTTP_PORT=8080 - HTTP_PROXY_PORT=${DATA_SERVICES_PORT:-8090} - HTTP_PROXY_NAME=${DATA_SERVICES_DOMAIN} + - GEOSERVER_ADMIN_PASSWORD=${GEOSERVER_PASSWORD:-geoserver} + - GEOSERVER_ADMIN_USER=${GEOSERVER_USERNAME:-admin} volumes: - geoserver_volume:/var/local/geoserver - ./conf/geoserver/geoserver.sh:/usr/local/bin/start.sh # Overwrite the default startup script to allow anonymous GET access From 5000c7b4ec0584784c1b7bd2cfaf56ba2d96cc72 Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 14:56:50 -0500 Subject: [PATCH 08/14] sync local dev to volume --- .gitignore | 1 + Dockerfile | 1 - docker-compose.yml | 3 +++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 71d1b68..b3bb116 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__/ local_settings.py .DS_Store *.pyc +*.sqlite3 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 90ca03a..5971da7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,6 @@ RUN conda env create -f $DS_HOME/hs_data_services/environment.yml # Place Application Files --------------------------------------------------------------------------------# -COPY hs_data_services/ $DS_HOME/hs_data_services COPY conf/ $DS_HOME/conf RUN sudo chmod -R +x $DS_HOME/conf diff --git a/docker-compose.yml b/docker-compose.yml index a402dc0..27753d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,7 @@ services: command: /usr/bin/supervisord -c /home/dsuser/conf/gunicorn/gunicorn.conf volumes: - static:/static # Static files will be stored locally at root. + - "./hs_data_services:/home/dsuser/hs_data_services" depends_on: - postgres @@ -54,6 +55,8 @@ services: depends_on: - redis - postgres + volumes: + - "./hs_data_services:/home/dsuser/hs_data_services" celery-beat: build: . From 76f53198f404af045a39559d7e2274b7c166d754 Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 15:55:58 -0500 Subject: [PATCH 09/14] gunicorn reload --- conf/gunicorn/gunicorn.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/gunicorn/gunicorn.sh b/conf/gunicorn/gunicorn.sh index 89fe9ab..c9bb755 100644 --- a/conf/gunicorn/gunicorn.sh +++ b/conf/gunicorn/gunicorn.sh @@ -22,4 +22,5 @@ echo Starting Gunicorn. exec gunicorn hs_data_services.wsgi:application \ --bind 0.0.0.0:8060 \ --workers 3 \ - --timeout 3600 + --timeout 3600 \ + --reload From 4f031c917683cf61382bb331dfa0fad7bd3f783c Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Fri, 13 Jan 2023 20:27:18 -0500 Subject: [PATCH 10/14] mount HS irods into geoserver --- .env | 2 +- README.md | 11 ++++++++--- docker-compose.yml | 6 ++++-- .../hs_data_services/local_settings.template | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.env b/.env index ab43b91..1b42d1e 100644 --- a/.env +++ b/.env @@ -9,5 +9,5 @@ GEOSERVER_REST_URL=http://${DATA_SERVICES_DOMAIN}:${DATA_SERVICES_PORT}/geoserve GEOSERVER_USERNAME=admin GEOSERVER_PASSWORD=geoserver SECRET_KEY=secret -IRODS_LOCAL_DIRECTORY=/tmp +IRODS_LOCAL_DIRECTORY=/projects/iRODS/Vault/home/wwwHydroProxy WORKSPACE_PREFIX=HS \ No newline at end of file diff --git a/README.md b/README.md index 41668e2..7572807 100644 --- a/README.md +++ b/README.md @@ -64,9 +64,10 @@ The [.env](.env) is where you can modify ports if needed in order to avoid confl ##### Start Docker containers for local build: From hs_data_services directory: -`IRODS_ACCESS_UID=${UID} docker-compose build` -`IRODS_ACCESS_UID=${UID} docker-compose up -d` - +``` +IRODS_ACCESS_UID=${UID} docker-compose build +IRODS_ACCESS_UID=${UID} docker-compose up -d +``` The `${UID}` will get your user's UID on the host system and make it available during the docker build Services will be exposed locally on DATA_SERVICES_PORT (default 8090), for example: http://localhost:8090/his/admin/ @@ -105,6 +106,10 @@ HSWS_GEOSERVER_ESCAPE = { Note that `host.docker.internal` is used for local builds (instead of `localhost` or `127.0.0.1`). This is allows the different docker services to communicate on the host. Ensure that Hydroshare is up and running locally on whatever HS_PORT you set in the [.env](.env) (default 8000) +Unless you have the `idata_vault_vol` volume from Hydroshare linked to a directory on the host (not the default) you will likely have to modify permissions so that the geoserver can read from the iRods vault: +``` +docker exec hs_data_services-geoserver-1 chmod -R o+r /projects +``` Once everything is set up, HydroShare should start sending update requests to this data services app. You can check that this app is receiving those requests by going to {host_url}/flower and clicking on 'Tasks'. You can test it manually by sending a POST request to {host_url}/his/services/update/{resource_id}/ and adding {'Authorization': 'Token HSWS_API_TOKEN'} to the request's headers. ##### Troubleshooting steps: diff --git a/docker-compose.yml b/docker-compose.yml index 27753d7..4289b00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: volumes: - geoserver_volume:/var/local/geoserver - ./conf/geoserver/geoserver.sh:/usr/local/bin/start.sh # Overwrite the default startup script to allow anonymous GET access - - projects:/projects:ro # Mount projects as a read-only volume to grant access to resources + - idata_vault_vol:/projects:ro # Mount projects as a read-only volume to grant access to resources! redis: image: redis:6.0.4-alpine @@ -81,7 +81,9 @@ services: volumes: - projects: + idata_vault_vol: + external: true + name: hydroshare_idata_vault_vol static: data_services_volume: geoserver_volume: diff --git a/hs_data_services/hs_data_services/local_settings.template b/hs_data_services/hs_data_services/local_settings.template index 6d5f5b5..4d01abd 100644 --- a/hs_data_services/hs_data_services/local_settings.template +++ b/hs_data_services/hs_data_services/local_settings.template @@ -59,7 +59,7 @@ DATA_SERVICES = { 'URL': config.get('GEOSERVER_REST_URL', 'http://host.docker.internal:8090/geoserver/rest'), # e.g. "https://geoserver-beta.hydroshare.org/geoserver/rest" 'USER': config.get('GEOSERVER_USERNAME', 'admin'), 'PASSWORD': config.get('GEOSERVER_PASSWORD', 'geoserver'), - 'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/tmp'), # e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy" + 'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/projects/iRODS/Vault/home/wwwHydroProxy'),# e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy" 'NAMESPACE': config.get('WORKSPACE_PREFIX', 'HS') # e.g. "HS" } } From dccc5e5e304c2c9abcb851bcea5136fa64a68439 Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Mon, 16 Jan 2023 13:22:18 -0500 Subject: [PATCH 11/14] mount irods as rw for dev --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4289b00..e734e28 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: volumes: - geoserver_volume:/var/local/geoserver - ./conf/geoserver/geoserver.sh:/usr/local/bin/start.sh # Overwrite the default startup script to allow anonymous GET access - - idata_vault_vol:/projects:ro # Mount projects as a read-only volume to grant access to resources! + - idata_vault_vol:/projects redis: image: redis:6.0.4-alpine From 3a50919d122fd9de254945dee231a57f374c6ec7 Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Mon, 16 Jan 2023 13:28:59 -0500 Subject: [PATCH 12/14] readme --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7572807..574d5fb 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ By default, all services will be exposed locally on port 8000 and static files w ### Running locally for development -##### Django settings for local builds: +##### Django settings for dev builds: Copy [hs_data_services/hs_data_services/hs_data_services/local_settings.template](hs_data_services/hs_data_services/hs_data_services/local_settings.template) to local_settings.py. @@ -61,7 +61,7 @@ For local builds, you shouldn't need to edit local_settings.py, you just need to For local builds, instead of editing [hs_data_services/docker-compose.yml](hs_data_services/docker-compose.yml), you can edit the [.env](.env). The [.env](.env) is where you can modify ports if needed in order to avoid conflicts on your host. -##### Start Docker containers for local build: +##### Start Docker containers (dev build): From hs_data_services directory: ``` @@ -72,7 +72,7 @@ The `${UID}` will get your user's UID on the host system and make it available d Services will be exposed locally on DATA_SERVICES_PORT (default 8090), for example: http://localhost:8090/his/admin/ -##### Post-Installation steps: +##### Post-Installation steps (dev build): Log in to the Django site at {host_url}/his/admin with default username and password: 'admin' and 'default' From the admin settings page, change the admin password. @@ -104,11 +104,12 @@ HSWS_GEOSERVER_ESCAPE = { } ``` Note that `host.docker.internal` is used for local builds (instead of `localhost` or `127.0.0.1`). This is allows the different docker services to communicate on the host. -Ensure that Hydroshare is up and running locally on whatever HS_PORT you set in the [.env](.env) (default 8000) +Ensure that Hydroshare is up and running locally on whatever HS_PORT you set in the [.env](.env) (default 8000). +The gunicorn container will hot-restart, but the celery worker and celery beat services will not. To restart those you can use, for example, `docker restart hs_data_services-celery-worker-1`. Unless you have the `idata_vault_vol` volume from Hydroshare linked to a directory on the host (not the default) you will likely have to modify permissions so that the geoserver can read from the iRods vault: ``` -docker exec hs_data_services-geoserver-1 chmod -R o+r /projects +docker exec hs_data_services-geoserver-1 chmod -R o+rx /projects ``` Once everything is set up, HydroShare should start sending update requests to this data services app. You can check that this app is receiving those requests by going to {host_url}/flower and clicking on 'Tasks'. You can test it manually by sending a POST request to {host_url}/his/services/update/{resource_id}/ and adding {'Authorization': 'Token HSWS_API_TOKEN'} to the request's headers. From 16c2f0d3d309f8566fb27295a8d75c54638e4742 Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Wed, 25 Jan 2023 14:14:29 -0500 Subject: [PATCH 13/14] use docker volume for static files --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e734e28..5dd8c90 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: build: . command: /usr/bin/supervisord -c /home/dsuser/conf/gunicorn/gunicorn.conf volumes: - - static:/static # Static files will be stored locally at root. + - static:/static - "./hs_data_services:/home/dsuser/hs_data_services" depends_on: - postgres @@ -72,7 +72,7 @@ services: - ${DATA_SERVICES_PORT:-8090}:80 # Expose all services locally on port volumes: - ./conf/nginx:/etc/nginx/conf.d - - static:/static # Static files will be stored locally at root. + - static:/static #- /opt/hs-certs:/opt/hs-certs # Used to configure SSL if available. depends_on: - gunicorn From 01654a7368535137f3f69552d3f96636f8fa8707 Mon Sep 17 00:00:00 2001 From: Devin Cowan Date: Thu, 7 Dec 2023 13:19:54 -0500 Subject: [PATCH 14/14] update JVM heap allotment --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5dd8c90..4be9a57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,9 @@ services: - HTTP_PROXY_NAME=${DATA_SERVICES_DOMAIN} - GEOSERVER_ADMIN_PASSWORD=${GEOSERVER_PASSWORD:-geoserver} - GEOSERVER_ADMIN_USER=${GEOSERVER_USERNAME:-admin} + - CATALINA_OPTS=-server -Djava.awt.headless=true -Xms2g -Xmx10g -XX:+UseConcMarkSweepGC -XX:NewSize=48m -DGEOSERVER_DATA_DIR=/var/local/geoserver + # https://github.com/oscarfonts/docker-geoserver/blob/master/2.24.1/Dockerfile#L35-L37 + # https://docs.geoserver.org/main/en/user/production/container.html volumes: - geoserver_volume:/var/local/geoserver - ./conf/geoserver/geoserver.sh:/usr/local/bin/start.sh # Overwrite the default startup script to allow anonymous GET access