From d0d0612d8963c3a6c2c5000e82ca54183ac2b6ee Mon Sep 17 00:00:00 2001 From: David Marteau Date: Mon, 18 Nov 2024 18:31:03 +0100 Subject: [PATCH] Use QJazz in tests --- .github/workflows/e2e_tests.yml | 5 +- tests/Makefile | 35 ++++++------- tests/add_server_plugins.sh | 36 ------------- tests/docker-compose.yml | 50 ++++++++++--------- tests/docker-conf/phpfpm/lizmapConfig.ini.php | 4 +- tests/docker-conf/qgis-server.toml | 41 +++++++++++++++ tests/run-docker | 6 +-- 7 files changed, 89 insertions(+), 88 deletions(-) delete mode 100755 tests/add_server_plugins.sh create mode 100644 tests/docker-conf/qgis-server.toml diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 51c44b7491..9e8f7ef58b 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -27,12 +27,12 @@ jobs: - name: "LEGACY" php: "8.2" pg-postgis: "14-3" - qgis-server: "3.40" + qgis-server: "qgis-3.40" update-projects: "FALSE" - name: "BLEEDING_EDGE" php: "8.4" pg-postgis: "17-3" - qgis-server: "3.44" + qgis-server: "qgis-3.44" update-projects: "TRUE" env: CYPRESS_CI: TRUE @@ -337,6 +337,7 @@ jobs: mkdir -p /tmp/e2e/lwc mkdir -p /tmp/e2e/docker docker logs lizmap${{ env.LZMBRANCH }}_test_qgis &> /tmp/e2e/docker/qgis-server.log | true + docker logs lizmap${{ env.LZMBRANCH }}_test_qgis_rpc &> /tmp/e2e/docker/qgis-server-rpc.log | true docker logs lizmap${{ env.LZMBRANCH }}_test_php &> /tmp/e2e/docker/php.log | true docker logs lizmap${{ env.LZMBRANCH }}_test_nginx &> /tmp/e2e/docker/nginx.log | true cp -r ../lizmap/var/log /tmp/e2e/lwc/ diff --git a/tests/Makefile b/tests/Makefile index 949b6faafb..150cf34d9f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -15,10 +15,11 @@ LIZMAP_GROUP_ID ?= $(shell id -g) # Note that GH Action E2E has its own version config for PHP, PG and QGIS PHP_VERSION ?= 8.2 LZMPOSTGISVERSION ?= 14-3 -LZMQGSRVVERSION ?= 3.40 +LZMQGSRVVERSION ?= qgis-ltr-eager QGSRVAPIPORT ?= 8134 LZMPGPORT ?= 8132 LZMQGSRVPORT ?= 8131 +LZMQGSADMINPORT ?= 9876 LZMWEBPORT ?= 8130 LIZMAP_ADMIN_LOGIN ?= admin LIZMAP_ADMIN_EMAIL ?= admin@localhost.local @@ -49,32 +50,24 @@ env: EOF build-plugins: - docker run -q \ - -u $(LIZMAP_USER_ID):$(LIZMAP_GROUP_ID) \ - --rm -i \ - -e QGSRV_SERVER_PLUGINPATH=/srv/plugins \ - -e QGIS_PLUGIN_MANAGER_SOURCES_FILE=/src/docker-conf/unstable-plugin-sources.list \ - -e QGIS_PLUGIN_MANAGER_CACHE_DIR=/tmp/cache-plugin-manager \ - -v $(shell pwd)/qgis-server-plugins:/srv/plugins \ - -v $(shell pwd)/:/src \ - --entrypoint /src/add_server_plugins.sh \ - 3liz/qgis-map-server:${LZMQGSRVVERSION} \ + echo "Plugins are auto installed at qgis worker startup" show-qgis-server-versions: @docker run -q \ --rm -i \ - --entrypoint qgisserver \ - 3liz/qgis-map-server:${LZMQGSRVVERSION} \ - --version - @docker run -q \ + --entrypoint qjazz-config \ + 3liz/qjazz:${LZMQGSRVVERSION} \ + version + docker run \ -u $(LIZMAP_USER_ID):$(LIZMAP_GROUP_ID) \ --rm -i \ - -e QGSRV_SERVER_PLUGINPATH=/srv/plugins \ + -e QGIS_PLUGINPATH=/srv/plugins \ -e QGIS_PLUGIN_MANAGER_SKIP_SOURCES_FILE=True \ -v $(shell pwd)/qgis-server-plugins:/srv/plugins \ -v $(shell pwd)/:/src \ + --workdir /srv/plugins \ --entrypoint qgis-plugin-manager \ - 3liz/qgis-map-server:${LZMQGSRVVERSION} \ + 3liz/qjazz:${LZMQGSRVVERSION} \ list upgrade-projects: @@ -85,19 +78,19 @@ upgrade-projects: -v $(shell pwd)/qgis-server-plugins:/srv/plugins \ -v $(shell pwd)/qgis-projects/tests:/tmp/qgis-projects \ --entrypoint /srv/plugins/upgrade_projects.py \ - 3liz/qgis-map-server:${LZMQGSRVVERSION} \ + 3liz/qjazz:${LZMQGSRVVERSION} \ up: env - docker compose up -V --force-recreate -d + docker compose --profile=dev up -V --force-recreate -d # Alias for 'up' run: up stop: - docker compose stop + docker compose --profile=dev stop reset: - docker compose down -v --remove-orphans + docker compose --profile=dev down -v --remove-orphans ./lizmap-ctl clean # install or update base images, to be sure we have the latest images diff --git a/tests/add_server_plugins.sh b/tests/add_server_plugins.sh deleted file mode 100755 index 7b9194138a..0000000000 --- a/tests/add_server_plugins.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [[ ! -f "$QGIS_PLUGIN_MANAGER_SOURCES_FILE" ]]; then - echo "Please set QGIS_PLUGIN_MANAGER_SOURCES_FILE to a valid file (found \"$QGIS_PLUGIN_MANAGER_SOURCES_FILE\")" - exit 1 -fi - -# Remove legacy folders about qgis-plugin-manager -if [ -d /srv/plugins/.cache_qgis_plugin_manager ]; then - rm -rf /srv/plugins/.cache_qgis_plugin_manager -fi -if [ -f /srv/plugins/sources.list ]; then - rm -f /srv/plugins/sources.list -fi - -echo "QGIS Server Lizmap and WfsOutputExtension plugins" -echo "Loading from $(cat $QGIS_PLUGIN_MANAGER_SOURCES_FILE)" -qgis-plugin-manager update -qgis-plugin-manager install -f "Lizmap server" -qgis-plugin-manager install -f wfsOutputExtension -qgis-plugin-manager install -f atlasprint - -# echo "QGIS Server Lizmap plugin" -# Latest commit -# echo "Latest commit from https://packages.3liz.org" -# wget https://packages.3liz.org/pub/lizmap-server-qgis-plugin/lizmap_server.master.zip -O /tmp/lizmap_server.master.zip - -# Latest release -# VERSION=1.1.1 -# echo "Stable release from GitHub" -# wget https://github.com/3liz/qgis-lizmap-server-plugin/releases/latest/download/lizmap_server.${VERSION}.zip -O /tmp/lizmap_server.master.zip - -# unzip -o /tmp/lizmap_server.master.zip -d /srv/plugins/ -# rm /tmp/lizmap_server.master.zip diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 43cc6d051c..a751c094de 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -76,36 +76,40 @@ services: ports: - ${LZMWEBPORT}:80 depends_on: - lizmap: - condition: service_healthy - map: - container_name: "lizmap${LZMBRANCH}_test_qgis" - image: 3liz/qgis-map-server:${LZMQGSRVVERSION} + - lizmap + # QGIS RPC backend + map-rpc: + container_name: "lizmap${LZMBRANCH}_test_qgis_rpc" + image: 3liz/qjazz:${LZMQGSRVVERSION} + command: ["qjazz-rpc", "serve", "-C", "/srv/etc/qgis-server.toml"] environment: - PGSERVICEFILE: /srv/etc/pg_service.conf + CONF_LOGGING__LEVEL: trace + CONF_WORKER__NUM_PROCESSES: 4 + CONF_DISPLAY_XVFB: ON + CONF_USER: ${LIZMAP_USER_ID}:${LIZMAP_GROUP_ID} + QGIS_PLUGINPATH: /srv/qgis-server-plugins QGIS_SERVER_LIZMAP_REVEAL_SETTINGS: yes - QGIS_SERVER_TRUST_LAYER_METADATA: yes - QGIS_SERVER_FORCE_READONLY_LAYERS: yes - QGSRV_API_ENABLED_LIZMAP: yes - QGSRV_CACHE_ROOTDIR: /srv/projects - QGSRV_CACHE_SIZE: '20' - # For testing purpose, I think it's better to have a strict check of layers - QGSRV_CACHE_STRICT_CHECK: yes - QGSRV_LOGGING_LEVEL: DEBUG - QGSRV_SERVER_PLUGINPATH: /srv/qgis-server-plugins - QGSRV_MANAGEMENT_ENABLED: yes - QGSRV_MANAGEMENT_INTERFACES: "0.0.0.0" - QGSRV_SERVER_WORKERS: 4 - QGSRV_USER: ${LIZMAP_USER_ID}:${LIZMAP_GROUP_ID} - ROUTER_HOST: map + QGIS_PLUGIN_MANAGER_SOURCES_FILE: /srv/etc/plugin-sources.list + PGSERVICEFILE: /srv/etc/pg_service.conf volumes: - { type: bind, source: ./qgis-projects, target: /srv/projects } - { type: bind, source: ./qgis-server-plugins, target: /srv/qgis-server-plugins } - - { type: bind, source: ./wps-data, target: /srv/data } - { type: bind, source: ./docker-conf/pg_service.conf, target: /srv/etc/pg_service.conf } + - { type: bind, source: ./docker-conf/qgis-server.toml, target: /srv/etc/qgis-server.toml } + - type: bind + source: ./docker-conf/unstable-plugin-sources.list + target: /srv/etc/plugin-sources.list + # QGIS workers frontend + map: + container_name: "lizmap${LZMBRANCH}_test_qgis" + image: 3liz/qjazz:${LZMQGSRVVERSION} + command: ["qjazz-map", "serve", "-C", "/srv/etc/qgis-server.toml"] + environment: + CONF_LOGGING__LEVEL: trace + volumes: + - { type: bind, source: ./docker-conf/qgis-server.toml, target: /srv/etc/qgis-server.toml } ports: - - ${LZMQGSRVPORT}:8080 - - ${QGSRVAPIPORT}:19876 + - ${LZMQGSRVPORT}:9080 webdav: container_name: lizmap${LZMBRANCH}_test_webdav image: ugeek/webdav:amd64 diff --git a/tests/docker-conf/phpfpm/lizmapConfig.ini.php b/tests/docker-conf/phpfpm/lizmapConfig.ini.php index c6b6a65432..705838db9d 100644 --- a/tests/docker-conf/phpfpm/lizmapConfig.ini.php +++ b/tests/docker-conf/phpfpm/lizmapConfig.ini.php @@ -5,11 +5,11 @@ ;list the different map services (servers, generic parameters, etc.) [services] ;URL to QGIS Server for OGC web services -wmsServerURL="http://map:8080/ows/" +wmsServerURL="http://map:9080/" ;WMS subdomain URLs list (optional) wmsPublicUrlList= ;URL to the API exposed by the Lizmap plugin for QGIS Server -lizmapPluginAPIURL="http://map:8080/lizmap/" +lizmapPluginAPIURL="http://map:9080/lizmap/" onlyMaps=off defaultRepository=testsrepository diff --git a/tests/docker-conf/qgis-server.toml b/tests/docker-conf/qgis-server.toml new file mode 100644 index 0000000000..c3e42eb0f6 --- /dev/null +++ b/tests/docker-conf/qgis-server.toml @@ -0,0 +1,41 @@ +# +# Worker backend configuration +# + + +[worker] +name = "Server" + +[worker.qgis] +max_projects = 20 +use_default_server_handler = true + +[worker.qgis.projects] +ignore_bad_layers = false + +[worker.qgis.projects.search_paths] +'/' = "/srv/projects" + +# Plugins allowed to install from plugin paths +[worker.qgis.plugins] +install = [ + "Lizmap server", + "atlasprint", + "wfsOutputExtension", +] +# Auto install plugins from embedded plugin manager +install_mode = "auto" + +# +# HTTP Front end configuration +# + +[backends.root] +title = "Root backend" +host = "map-rpc" +route = "/" + +[[backends.root.api]] +endpoint = "lizmap" +name = "Lizmap" + diff --git a/tests/run-docker b/tests/run-docker index 686f7577da..cdd2ac6e06 100755 --- a/tests/run-docker +++ b/tests/run-docker @@ -18,9 +18,7 @@ if [ "$CMD" == "reset" ]; then exit 0 elif [ "$CMD" == "build" ]; then # install or update base images, to be sure we have the latest images - docker compose pull - # install or update plugins, to be sure we have the latest plugins - make build-plugins + docker compose --profile=webdav --profile=swagger pull fi -docker compose $CMD "$@" +docker compose --profile=webdav --profile=swagger $CMD "$@"