Local dev

.env

@@ -0,0 +1,13 @@
+# For local builds, use "host.docker.internal" instead of "localhost" or "127.0.0.1"
+HYDROSHARE_DOMAIN=localhost
+DATA_SERVICES_DOMAIN=localhost
+DATA_SERVICES_PORT=8090
+HS_PORT=8000
+DATA_SERVICES_URL=http://${DATA_SERVICES_DOMAIN}:${DATA_SERVICES_PORT}/his
+HYDROSHARE_REST_URL=http://${HYDROSHARE_DOMAIN}:${HS_PORT}/hsapi
+GEOSERVER_REST_URL=http://${DATA_SERVICES_DOMAIN}:${DATA_SERVICES_PORT}/geoserver/rest
+GEOSERVER_USERNAME=admin
+GEOSERVER_PASSWORD=geoserver
+SECRET_KEY=secret
+IRODS_LOCAL_DIRECTORY=/projects/iRODS/Vault/home/wwwHydroProxy
+WORKSPACE_PREFIX=HS

.gitignore

@@ -4,3 +4,4 @@ __pycache__/
 local_settings.py
 .DS_Store
 *.pyc
+*.sqlite3

Dockerfile

@@ -45,7 +45,6 @@ RUN conda env create -f $DS_HOME/hs_data_services/environment.yml
 
 # Place Application Files --------------------------------------------------------------------------------#
 
-COPY hs_data_services/ $DS_HOME/hs_data_services
 COPY conf/ $DS_HOME/conf
 
 RUN sudo chmod -R +x $DS_HOME/conf

README.md

@@ -15,7 +15,7 @@ These instructions will help you install and run this application in a productio
 
 ##### Edit Django settings:
 
-Rename hs_data_services/hs_data_services/hs_data_services/local_settings.template to local_settings.py.
+Rename [hs_data_services/hs_data_services/hs_data_services/local_settings.template](hs_data_services/hs_data_services/hs_data_services/local_settings.template) to local_settings.py.
 
 Edit the following settings in local_settings.py:
 * {{HYDROSHARE_DOMAIN}}     - The domain for the connected HydroShare instance e.g. 'hydroshare.org'
@@ -30,7 +30,7 @@ Edit the following settings in local_settings.py:
 
 ##### Edit Docker settings:
 
-Edit the following settings in hs_data_services/docker-compose.yml
+Edit the following settings in [hs_data_services/docker-compose.yml](hs_data_services/docker-compose.yml):
 
 * {{IRODS_ACCESS_UID}}      - The UID of a user on the host system with iRODS read access.
 * {{HYDROSHARE_DOMAIN}}     - The domain for the connected HydroShare instance e.g. 'hydroshare.org'
@@ -49,7 +49,30 @@ $ sudo docker-compose up -d
 
 By default, all services will be exposed locally on port 8000 and static files will be located in /static/his/. 
 
-##### Post-Installation steps:
+
+### Running locally for development
+
+##### Django settings for dev builds:
+
+Copy [hs_data_services/hs_data_services/hs_data_services/local_settings.template](hs_data_services/hs_data_services/hs_data_services/local_settings.template) to local_settings.py.
+
+For local builds, you shouldn't need to edit local_settings.py, you just need to create it.
+
+For local builds, instead of editing [hs_data_services/docker-compose.yml](hs_data_services/docker-compose.yml), you can edit the [.env](.env).
+The [.env](.env) is where you can modify ports if needed in order to avoid conflicts on your host.
+
+##### Start Docker containers (dev build):
+
+From hs_data_services directory:
+```
+IRODS_ACCESS_UID=${UID} docker-compose build
+IRODS_ACCESS_UID=${UID} docker-compose up -d
+```
+The `${UID}` will get your user's UID on the host system and make it available during the docker build
+
+Services will be exposed locally on DATA_SERVICES_PORT (default 8090), for example: http://localhost:8090/his/admin/
+
+##### Post-Installation steps (dev build):
 
 Log in to the Django site at {host_url}/his/admin with default username and password: 'admin' and 'default'
 From the admin settings page, change the admin password.
@@ -67,6 +90,27 @@ Update or add the following settings to HydroShare:
 * HSWS_TIMEOUT              - This can be fairly short e.g. 10
 * HSWS_ACTIVATED            - True or False
 
+Here is an example of settings used in local development:
+```
+HSWS_URL = "http://host.docker.internal:8090/his/services/update"
+HSWS_API_TOKEN = "fba02c9f6e9a0c269681ece8bd330a9c314923f3"
+HSWS_TIMEOUT = 10
+HSWS_PUBLISH_URLS = True
+HSWS_ACTIVATED = True
+
+HSWS_GEOSERVER_URL = "http://host.docker.internal:8090/geoserver"
+HSWS_GEOSERVER_ESCAPE = {
+    '/': ' '
+}
+```
+Note that `host.docker.internal` is used for local builds (instead of `localhost` or `127.0.0.1`). This is allows the different docker services to communicate on the host.
+Ensure that Hydroshare is up and running locally on whatever HS_PORT you set in the [.env](.env) (default 8000).
+The gunicorn container will hot-restart, but the celery worker and celery beat services will not. To restart those you can use, for example, `docker restart hs_data_services-celery-worker-1`.
+
+Unless you have the `idata_vault_vol` volume from Hydroshare linked to a directory on the host (not the default) you will likely have to modify permissions so that the geoserver can read from the iRods vault:
+```
+docker exec hs_data_services-geoserver-1 chmod -R o+rx /projects
+```
 Once everything is set up, HydroShare should start sending update requests to this data services app. You can check that this app is receiving those requests by going to {host_url}/flower and clicking on 'Tasks'. You can test it manually by sending a POST request to {host_url}/his/services/update/{resource_id}/ and adding {'Authorization': 'Token HSWS_API_TOKEN'} to the request's headers.
 
 ##### Troubleshooting steps:

conf/gunicorn/gunicorn.sh

@@ -22,4 +22,5 @@ echo Starting Gunicorn.
 exec gunicorn hs_data_services.wsgi:application \
     --bind 0.0.0.0:8060 \
     --workers 3 \
-    --timeout 3600
+    --timeout 3600 \
+    --reload

docker-compose.yml

@@ -12,61 +12,58 @@ services:
       - data_services_volume:/var/lib/postgresql/data/
     ports:
       - 5432:5432
-    networks:
-      - data_services_network
 
   geoserver:
-    image: oscarfonts/geoserver:2.14.1
+    image: kartoza/geoserver:2.22.0
     environment: 
-      - CUSTOM_UID={{IRODS_ACCESS_UID}}                        # A user ID on the host with read permissions to the iRODS directory
-      - GEOSERVER_CSRF_WHITELIST={{HYDROSHARE_DOMAIN}}         # e.g. hydroshare.org
+      - GEOSERVER_UID=${IRODS_ACCESS_UID:-501}                                # A user ID on the host with read permissions to the iRODS directory
+      - GEOSERVER_CSRF_WHITELIST=${HYDROSHARE_DOMAIN:-hydroshare.org}
+      - HTTP_PORT=8080
+      - HTTP_PROXY_PORT=${DATA_SERVICES_PORT:-8090}
+      - HTTP_PROXY_NAME=${DATA_SERVICES_DOMAIN}
+      - GEOSERVER_ADMIN_PASSWORD=${GEOSERVER_PASSWORD:-geoserver}
+      - GEOSERVER_ADMIN_USER=${GEOSERVER_USERNAME:-admin}
+      - CATALINA_OPTS=-server -Djava.awt.headless=true 	-Xms2g -Xmx10g -XX:+UseConcMarkSweepGC -XX:NewSize=48m 	-DGEOSERVER_DATA_DIR=/var/local/geoserver
+      # https://github.com/oscarfonts/docker-geoserver/blob/master/2.24.1/Dockerfile#L35-L37
+      # https://docs.geoserver.org/main/en/user/production/container.html
     volumes:
       - geoserver_volume:/var/local/geoserver
       - ./conf/geoserver/geoserver.sh:/usr/local/bin/start.sh  # Overwrite the default startup script to allow anonymous GET access
-      - /projects:/projects:ro                                 # Mount projects as a read-only volume to grant access to resources
-    networks:
-      - data_services_network
+      - idata_vault_vol:/projects
 
   redis:
     image: redis:6.0.4-alpine
-    networks:
-      - data_services_network
 
   flower:  
     image: mher/flower:0.9.5
     environment:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - FLOWER_PORT=8040
       - FLOWER_URL_PREFIX=flower
-    networks:
-      - data_services_network
     depends_on:
       - redis
 
   gunicorn:
     build: .
     command: /usr/bin/supervisord -c /home/dsuser/conf/gunicorn/gunicorn.conf
     volumes:
-      - /static:/static                                        # Static files will be stored locally at root.
-    networks:
-      - data_services_network
+      - static:/static
+      - "./hs_data_services:/home/dsuser/hs_data_services"
     depends_on:
       - postgres
 
   celery-worker:
     build: .
     command: /usr/bin/supervisord -c /home/dsuser/conf/celery-worker/celery_worker.conf
-    networks:
-      - data_services_network
     depends_on:
       - redis
       - postgres
+    volumes:
+      - "./hs_data_services:/home/dsuser/hs_data_services"
 
   celery-beat:
     build: .
     command: /usr/bin/supervisord -c /home/dsuser/conf/celery-beat/celery_beat.conf
-    networks:
-      - data_services_network
     depends_on:
       - redis
       - celery-worker
@@ -75,24 +72,21 @@ services:
   nginx:
     image: nginx:1.19
     ports:
-      - 8000:80                                                # Expose all services locally on port 8000
+      - ${DATA_SERVICES_PORT:-8090}:80                              # Expose all services locally on port
     volumes:
       - ./conf/nginx:/etc/nginx/conf.d
-      - /static:/static                                        # Static files will be stored locally at root.
+      - static:/static
       #- /opt/hs-certs:/opt/hs-certs                            # Used to configure SSL if available.
-    networks:
-      - data_services_network
     depends_on:
       - gunicorn
       - geoserver
       - flower
 
 
-networks:
-  data_services_network:
-    driver: bridge
-
-
 volumes:
+  idata_vault_vol:
+    external: true
+    name: hydroshare_idata_vault_vol
+  static:
   data_services_volume:
   geoserver_volume:

hs_data_services/environment.yml

@@ -18,3 +18,4 @@ dependencies:
     - drf-yasg==1.17.0
     - redis==3.5.3
     - future==0.18.2
+    - python-dotenv==0.21.0

hs_data_services/hs_data_services/local_settings.template

@@ -11,20 +11,22 @@ https://docs.djangoproject.com/en/3.0/ref/settings/
 """
 
 import os
+from dotenv import dotenv_values
+
+config = dotenv_values(".env")
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
-ALLOWED_HOSTS = [{{HYDROSHARE_DOMAIN}}, 'gnupstream', 'localhost', '127.0.0.1']    # e.g. "hydroshare.org"
-
+ALLOWED_HOSTS = [config.get('HYDROSHARE_DOMAIN', 'hydroshare.org'), 'gnupstream', 'localhost', '127.0.0.1']   # e.g. "hydroshare.org"
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = {{SECRET_KEY}}
+SECRET_KEY = config.get('SECRET_KEY', 'secret')
 
 # Database
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
@@ -42,23 +44,22 @@ DATABASES = {
 STATIC_URL = '/static/his/'
 STATIC_ROOT = '/static/his/'
 
-PROXY_BASE_URL = {{DATA_SERVICES_URL}}            # e.g. "https://geoserver-beta.hydroshare.org/his"
+PROXY_BASE_URL = config.get('DATA_SERVICES_URL', 'http://host.docker.internal:8090/his')             # e.g. "https://geoserver-beta.hydroshare.org/his"
 
 # Celery settings
 
 CELERY_BROKER_URL = "redis://redis:6379/0"
 CELERY_BEAT_SCHEDULE = {}
 
 # HydroShare Data Services Connection settings
-
-HYDROSHARE_URL = {{HYDROSHARE_REST_URL}}          # e.g. "https://beta.hydroshare.org/hsapi"
+HYDROSHARE_URL = config.get('HYDROSHARE_REST_URL', 'http://host.docker.internal:8000/hsapi')         # e.g. "https://beta.hydroshare.org/hsapi"
 
 DATA_SERVICES = {
     'geoserver': {
-        'URL': {{GEOSERVER_REST_URL}},            # e.g. "https://geoserver-beta.hydroshare.org/geoserver/rest"
-        'USER': {{GEOSERVER_USERNAME}},
-        'PASSWORD': {{GEOSERVER_PASSWORD}},
-        'IRODS_DIR': {{IRODS_LOCAL_DIRECTORY}},   # e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy"
-        'NAMESPACE': {{WORKSPACE_PREFIX}}         # e.g. "HS"
+        'URL': config.get('GEOSERVER_REST_URL', 'http://host.docker.internal:8090/geoserver/rest'),  # e.g. "https://geoserver-beta.hydroshare.org/geoserver/rest"
+        'USER': config.get('GEOSERVER_USERNAME', 'admin'),
+        'PASSWORD': config.get('GEOSERVER_PASSWORD', 'geoserver'),
+        'IRODS_DIR': config.get('IRODS_LOCAL_DIRECTORY', '/projects/iRODS/Vault/home/wwwHydroProxy'),# e.g. "/projects/hydroshare/vaults/hydrotest/home/betaDataProxy"
+        'NAMESPACE': config.get('WORKSPACE_PREFIX', 'HS')                                            # e.g. "HS"
     }
 }