diff --git a/.dockerignore b/.dockerignore index b84543fe10..31172b5a9a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,12 +1,41 @@ +# Git and GitHub .git/ -Dockerfile -filestore* +.gitignore +.github/ + +# Docker and Devcontainer +Dockerfile* +docker-compose*.yml +.dockerignore +.devcontainer/ +devcontainer.json hooks/ -coverage/ -docker/solr/ + +# SEEK +filestore* db/*.sqlite3 vendor/bundle/ public/unused_images +public/api/definitions/openapi-v3-resolved.* +public/help_images +public/javascripts/* +public/stylesheets +public/system +public/sitemaps +public/assets +public/assets_dev log/ tmp/ test/ + +# Environment files +.env* + +# IDEs and editors +.idea/ +.vscode/ + +# Ruby +.bundle +coverage/ +.codeclimate.yml diff --git a/.github/workflows/ansible-install.yml b/.github/workflows/ansible-install.yml index 0e18a2ef33..fbb1f7231e 100644 --- a/.github/workflows/ansible-install.yml +++ b/.github/workflows/ansible-install.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Configure ansible for local install working-directory: /home/runner/work/seek/seek/script/ansible/ diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 396e962e19..0f5e11259b 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -7,7 +7,7 @@ on: - workflowhub - rails-7-hacking-rebased - full-test-suite - - seek-1.16 + - seek-1.17 pull_request: @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Create volumes run: | docker volume create --name=seek-filestore @@ -28,8 +28,5 @@ jobs: - name: Build and run the Docker containers run: docker compose --file docker-compose.yml --file docker-compose.build.yml up --build --detach --wait continue-on-error: true - - name: Wait and check containers - run: sleep 120 && docker ps -a - continue-on-error: true - name: Check app status run: ruby script/check_deployment.rb diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a542e7b413..01e0e1a105 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -64,9 +64,9 @@ jobs: sudo apt install -y libmagick++-dev graphviz libcurl4-gnutls-dev libreoffice poppler-utils build-essential \ git imagemagick libgmp-dev python3.9-dev python3.9-distutils python3-pip - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: '11' # The JDK version to make available on the path. diff --git a/Dockerfile b/Dockerfile index 611438da9e..ecb3f7103f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,82 +1,127 @@ -FROM ruby:3.3-slim-bullseye - +FROM ruby:3.3-slim-trixie AS base LABEL maintainer="Stuart Owen , Finn Bacall" ARG SOURCE_COMMIT -ENV APP_DIR=/seek -ENV RAILS_ENV=production + +ENV APP_DIR=/seek \ + RAILS_ENV=production \ + BUNDLE_PATH="/usr/local/bundle" \ + BUNDLE_WITHOUT="development" + +# Supercronic variables +ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.12/supercronic-linux-amd64 \ + SUPERCRONIC=supercronic-linux-amd64 \ + SUPERCRONIC_SHA1SUM=048b95b48b708983effb2e5c935a1ef8483d9e3e + # need to set the locale, otherwise some gems file to install ENV LANG="en_US.UTF-8" LANGUAGE="en_US:UTF-8" LC_ALL="C.UTF-8" +# Install base dependencies RUN apt-get update -qq && \ - apt-get install -y --no-install-recommends build-essential cmake curl default-mysql-client gettext graphviz git \ - libcurl4-gnutls-dev libmagick++-dev libmariadb-dev libpq-dev libreadline-dev \ - libreoffice libsqlite3-dev libssl-dev libxml++2.6-dev \ - libxslt1-dev libyaml-dev locales nginx nodejs openjdk-11-jdk-headless \ - python3.9-dev python3.9-distutils python3-pip \ - poppler-utils postgresql-client shared-mime-info sqlite3 links telnet vim-tiny zip && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - locale-gen en_US.UTF-8 + apt-get install --no-install-recommends -y \ + curl default-mysql-client gettext git graphviz libjemalloc2 libvips links locales \ + openjdk-21-jre poppler-utils postgresql-client python3.13 shared-mime-info sqlite3 telnet vim-tiny zip +# Prepare app directory RUN mkdir -p $APP_DIR -RUN chown -R www-data $APP_DIR /var/www +RUN chown www-data:www-data $APP_DIR +WORKDIR $APP_DIR -USER www-data +# Disable ssl from the mysql client +RUN echo "[client]\nskip-ssl" > /etc/mysql/conf.d/disable-ssl.cnf -WORKDIR $APP_DIR +FROM base AS builder -# Bundle install throw errors if Gemfile has been modified since Gemfile.lock -COPY Gemfile* ./ +# Install build dependencies +RUN apt-get update -qq && \ + apt-get install -y --no-install-recommends build-essential cmake \ + libcurl4-gnutls-dev libmagick++-dev libmariadb-dev libpq-dev libreadline-dev \ + libreoffice libsqlite3-dev libssl-dev libxml++2.6-dev \ + libxslt1-dev libyaml-dev nodejs \ + python3.13-dev python3-setuptools python3-pip python3.13-venv && \ + apt-get clean && rm -rf /var/lib/apt/lists /var/cache/apt/archives && \ + locale-gen en_US.UTF-8 + +# create and use a dedicated python virtualenv +RUN python3.13 -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Copy over App code from local filesystem +COPY . . + +# Export the commit hash if provided at build time +RUN if [ -n "$SOURCE_COMMIT" ] ; then echo $SOURCE_COMMIT > config/.git-revision ; fi + +# Install Ruby gems RUN bundle config --local frozen 1 && \ bundle config set deployment 'true' && \ bundle config set without 'development test' && \ bundle install -# App code -COPY . . -RUN mkdir log tmp +RUN rm -rf ~/.bundle/ "${BUNDLE_PATH}"/ruby/*/cache "${BUNDLE_PATH}"/ruby/*/bundler/gems/*/.git + +# Install supercronic - a cron alternative +RUN curl -fsSLO "$SUPERCRONIC_URL" \ + && echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - \ + && chmod +x "$SUPERCRONIC" \ + && mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" \ + && ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic + +# Copy over virtuoso config COPY docker/virtuoso_settings.docker.yml config/virtuoso_settings.yml -USER root -RUN if [ -n "$SOURCE_COMMIT" ] ; then echo $SOURCE_COMMIT > config/.git-revision ; fi -RUN chown -R www-data solr config docker public /var/www db/schema.rb -USER www-data -RUN touch config/using-docker #allows us to see within SEEK we are running in a container +# Allows us to see within SEEK we are running in a container +RUN touch config/using-docker + +# SQLite Database (for asset compilation) +RUN mkdir sqlite3-db +COPY --chown=www-data:www-data docker/database.docker.sqlite3.yml config/database.yml + +# Create /var/www folder for bundler to compile dependencies into +RUN mkdir -p /var/www + +# Fix permissions +RUN chown www-data:www-data config/initializers public sqlite3-db /var/www +RUN chown -R www-data:www-data config public/api +RUN chmod -R 755 docker/upgrade.sh docker/start_workers.sh # Python dependencies from requirements.txt -ENV PATH="/var/www/.local/bin:$PATH" -RUN python3.9 -m pip install --upgrade pip -RUN python3.9 -m pip install setuptools==58 -RUN python3.9 -m pip install -r requirements.txt +RUN python3.13 -m pip install --upgrade pip +RUN python3.13 -m pip install setuptools==58 +RUN python3.13 -m pip install -r requirements.txt -# SQLite Database (for asset compilation) -RUN mkdir sqlite3-db && \ - cp docker/database.docker.sqlite3.yml config/database.yml && \ - chmod +x docker/upgrade.sh docker/start_workers.sh && \ - bundle exec rake db:setup +USER www-data + +RUN bundle exec rake db:setup +# Create log and tmp directories +RUN mkdir -p log tmp RUN bundle exec rake assets:precompile && \ rm -rf tmp/cache/* -#root access needed for next couple of steps -USER root +FROM base AS runtime -# Install supercronic - a cron alternative -ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.12/supercronic-linux-amd64 \ - SUPERCRONIC=supercronic-linux-amd64 \ - SUPERCRONIC_SHA1SUM=048b95b48b708983effb2e5c935a1ef8483d9e3e +# Set PATH to include python virtualenv +ENV PATH="/opt/venv/bin:$PATH" -RUN curl -fsSLO "$SUPERCRONIC_URL" \ - && echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - \ - && chmod +x "$SUPERCRONIC" \ - && mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" \ - && ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic +# Install nginx +RUN apt-get update -qq && \ + apt-get install -y --no-install-recommends nginx && \ + apt-get clean && rm -rf /var/lib/apt/lists /var/cache/apt/archives + +# Bring over build time dependencies +COPY --from=builder "${BUNDLE_PATH}" "${BUNDLE_PATH}" +COPY --from=builder $APP_DIR $APP_DIR +COPY --from=builder /opt/venv /opt/venv +COPY --from=builder /usr/local/bin/supercronic /usr/local/bin/supercronic # Cleanup and remove default nginx index page RUN rm -rf /tmp/* /var/tmp/* /usr/share/nginx/html/index.html +# Bundler uses /var/www as path +RUN chown -R www-data:www-data /var/www + USER www-data # Network diff --git a/config/routes.rb b/config/routes.rb index f5ea08dd8d..d3906df86f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -886,6 +886,9 @@ # feedback get '/home/feedback' => 'homes#feedback', as: :feedback + # Healthcheck + get "up" => "rails/health#show", as: :rails_health_check + # error rendering get '/404' => 'errors#error_404' get '/406' => 'errors#error_406' diff --git a/docker-compose-relative-root.yml b/docker-compose-relative-root.yml index b7cdb92259..809bd76191 100644 --- a/docker-compose-relative-root.yml +++ b/docker-compose-relative-root.yml @@ -1,6 +1,31 @@ +x-shared: + seek_base: &seek_base + # build: . + image: fairdom/seek:main + + restart: always + environment: &seek_base_env + RAILS_ENV: production + SOLR_PORT: 8983 + SOLR_HOST: solr + RAILS_LOG_LEVEL: info # debug, info, warn, error or fatal + RAILS_RELATIVE_URL_ROOT: '/seek' + env_file: + - docker/db.env + volumes: + - seek-filestore:/seek/filestore + - seek-cache:/seek/tmp/cache + depends_on: + db: + condition: service_healthy + solr: + condition: service_started + + services: - db: # Database implementation, in this case MySQL - image: mysql:8.0 + db: + # Database implementation, in this case MySQL + image: mysql:8.4 container_name: seek-mysql command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --log-error-verbosity=1 restart: always @@ -9,59 +34,39 @@ services: - docker/db.env volumes: - seek-mysql-db:/var/lib/mysql + healthcheck: + test: [ "CMD-SHELL", "mysqladmin ping -h localhost -u $$MYSQL_USER -p$$MYSQL_PASSWORD --silent" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s - seek: # The SEEK application - #build: . - - image: fairdom/seek:main + seek: + # The SEEK application + <<: *seek_base container_name: seek command: docker/entrypoint.sh - restart: always - environment: - RAILS_ENV: production - SOLR_PORT: 8983 - SOLR_HOST: solr - NO_ENTRYPOINT_WORKERS: 1 - RAILS_RELATIVE_URL_ROOT: '/seek' - env_file: - - docker/db.env - volumes: - - seek-filestore:/seek/filestore - - seek-cache:/seek/tmp/cache - - seek-assets:/seek/public/assets ports: - "3000:3000" - depends_on: - - db - - solr - links: - - db - - solr - - seek_workers: # The SEEK delayed job workers - #build: . + environment: + <<: *seek_base_env + NO_ENTRYPOINT_WORKERS: 1 + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:3000/up" ] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s - image: fairdom/seek:main - container_name: seek-workers - command: docker/start_workers.sh - restart: always - environment: - RAILS_ENV: production - SOLR_PORT: 8983 - SOLR_HOST: solr - RAILS_RELATIVE_URL_ROOT: '/seek' - env_file: - - docker/db.env - volumes: - - seek-filestore:/seek/filestore - - seek-cache:/seek/tmp/cache - depends_on: - - db - - solr - links: - - db - - solr + seek_workers: + # The SEEK delayed job workers + <<: *seek_base + container_name: seek-workers + environment: + <<: *seek_base_env + QUIET_SUPERCRONIC: 1 + command: docker/start_workers.sh solr: image: fairdom/seek-solr:8.11 diff --git a/docker-compose-virtuoso.yml b/docker-compose-virtuoso.yml index bc4a0e3923..d6e19a002d 100644 --- a/docker-compose-virtuoso.yml +++ b/docker-compose-virtuoso.yml @@ -1,66 +1,72 @@ -services: - db: # Database implementation, in this case MySQL - image: mysql:8.0 - container_name: seek-mysql - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --log-error-verbosity=1 - restart: always - env_file: - - docker/db.env - volumes: - - seek-mysql-db:/var/lib/mysql - - seek: # The SEEK application - #build: . +x-shared: + seek_base: &seek_base + # build: . image: fairdom/seek:main - container_name: seek - command: docker/entrypoint.sh + restart: always - environment: + environment: &seek_base_env RAILS_ENV: production SOLR_PORT: 8983 SOLR_HOST: solr - NO_ENTRYPOINT_WORKERS: 1 + RAILS_LOG_LEVEL: info # debug, info, warn, error or fatal env_file: - docker/db.env - docker/virtuoso.env volumes: - seek-filestore:/seek/filestore - seek-cache:/seek/tmp/cache - ports: - - "3000:3000" depends_on: - - db - - solr - - virtuoso - links: - - db - - solr - - virtuoso + db: + condition: service_healthy + solr: + condition: service_started + virtuoso: + condition: service_started - seek_workers: # The SEEK delayed job workers - #build: . - image: fairdom/seek:main - container_name: seek-workers - command: docker/start_workers.sh +services: + db: + # Database implementation, in this case MySQL + image: mysql:8.4 + container_name: seek-mysql + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --log-error-verbosity=1 restart: always - environment: - RAILS_ENV: production - SOLR_PORT: 8983 - SOLR_HOST: solr + stop_grace_period: 1m30s env_file: - docker/db.env - - docker/virtuoso.env volumes: - - seek-filestore:/seek/filestore - - seek-cache:/seek/tmp/cache - depends_on: - - db - - solr - - virtuoso - links: - - db - - solr - - virtuoso + - seek-mysql-db:/var/lib/mysql + healthcheck: + test: [ "CMD-SHELL", "mysqladmin ping -h localhost -u $$MYSQL_USER -p$$MYSQL_PASSWORD --silent" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + + seek: + # The SEEK application + <<: *seek_base + container_name: seek + command: docker/entrypoint.sh + ports: + - "3000:3000" + environment: + <<: *seek_base_env + NO_ENTRYPOINT_WORKERS: 1 + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:3000/up" ] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s + + seek_workers: + # The SEEK delayed job workers + <<: *seek_base + container_name: seek-workers + environment: + <<: *seek_base_env + QUIET_SUPERCRONIC: 1 + command: docker/start_workers.sh solr: image: fairdom/seek-solr:8.11 diff --git a/docker-compose-with-email.yml b/docker-compose-with-email.yml deleted file mode 100644 index 1742d4c5bb..0000000000 --- a/docker-compose-with-email.yml +++ /dev/null @@ -1,94 +0,0 @@ -services: - db: # Database implementation, in this case MySQL - image: mysql:8.0 - container_name: seek-mysql - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --log-error-verbosity=1 - restart: always - stop_grace_period: 1m30s - env_file: - - docker/db.env - volumes: - - seek-mysql-db:/var/lib/mysql - - seek: # The SEEK application - #build: . - - image: fairdom/seek:main - - container_name: seek - command: docker/entrypoint.sh - restart: always - environment: - RAILS_ENV: production - SOLR_PORT: 8983 - SOLR_HOST: solr - NO_ENTRYPOINT_WORKERS: 1 - env_file: - - docker/db.env - volumes: - - seek-filestore:/seek/filestore - - seek-cache:/seek/tmp/cache - ports: - - "3000:3000" - depends_on: - - db - - solr - - mail - links: - - db - - solr - - mail - - seek_workers: # The SEEK delayed job workers - #build: . - - image: fairdom/seek:main - container_name: seek-workers - command: docker/start_workers.sh - restart: always - environment: - RAILS_ENV: production - SOLR_PORT: 8983 - SOLR_HOST: solr - env_file: - - docker/db.env - volumes: - - seek-filestore:/seek/filestore - - seek-cache:/seek/tmp/cache - depends_on: - - db - - solr - - mail - links: - - db - - solr - - mail - - solr: - image: fairdom/seek-solr:8.11 - container_name: seek-solr - restart: always - environment: - SOLR_JAVA_MEM: -Xms512m -Xmx1024m - volumes: - - seek-solr-data:/var/solr/ - entrypoint: - - docker-entrypoint.sh - - solr-precreate - - seek - - /opt/solr/server/solr/configsets/seek_config - - mail: - image: eeacms/postfix - environment: - MTP_HOST: fair-dom.org - -volumes: - seek-filestore: - external: true - seek-mysql-db: - external: true - seek-solr-data: - external: true - seek-cache: - external: true diff --git a/docker-compose.yml b/docker-compose.yml index 693ad23b65..169fb2c8b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,30 @@ +x-shared: + seek_base: &seek_base + # build: . + image: fairdom/seek:main + + restart: always + environment: &seek_base_env + RAILS_ENV: production + SOLR_PORT: 8983 + SOLR_HOST: solr + RAILS_LOG_LEVEL: info # debug, info, warn, error or fatal + env_file: + - docker/db.env + volumes: + - seek-filestore:/seek/filestore + - seek-cache:/seek/tmp/cache + depends_on: + db: + condition: service_healthy + solr: + condition: service_started + + services: - db: # Database implementation, in this case MySQL - image: mysql:8.0 + db: + # Database implementation, in this case MySQL + image: mysql:8.4 container_name: seek-mysql command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --log-error-verbosity=1 restart: always @@ -9,59 +33,39 @@ services: - docker/db.env volumes: - seek-mysql-db:/var/lib/mysql + healthcheck: + test: [ "CMD-SHELL", "mysqladmin ping -h localhost -u $$MYSQL_USER -p$$MYSQL_PASSWORD --silent" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s - seek: # The SEEK application - #build: . - - image: fairdom/seek:main + seek: + # The SEEK application + <<: *seek_base container_name: seek command: docker/entrypoint.sh - restart: always - environment: - RAILS_ENV: production - SOLR_PORT: 8983 - SOLR_HOST: solr - NO_ENTRYPOINT_WORKERS: 1 - RAILS_LOG_LEVEL: info # debug, info, warn, error or fatal - env_file: - - docker/db.env - volumes: - - seek-filestore:/seek/filestore - - seek-cache:/seek/tmp/cache ports: - "3000:3000" - depends_on: - - db - - solr - links: - - db - - solr - - seek_workers: # The SEEK delayed job workers - #build: . + environment: + <<: *seek_base_env + NO_ENTRYPOINT_WORKERS: 1 + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:3000/up" ] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s - image: fairdom/seek:main - container_name: seek-workers - command: docker/start_workers.sh - restart: always - environment: - RAILS_ENV: production - SOLR_PORT: 8983 - SOLR_HOST: solr - RAILS_LOG_LEVEL: info # debug, info, warn, error or fatal - QUIET_SUPERCRONIC: 1 # remove to show supercronic activity logs - env_file: - - docker/db.env - volumes: - - seek-filestore:/seek/filestore - - seek-cache:/seek/tmp/cache - depends_on: - - db - - solr - links: - - db - - solr + seek_workers: + # The SEEK delayed job workers + <<: *seek_base + container_name: seek-workers + environment: + <<: *seek_base_env + QUIET_SUPERCRONIC: 1 + command: docker/start_workers.sh solr: image: fairdom/seek-solr:8.11 @@ -75,8 +79,7 @@ services: - docker-entrypoint.sh - solr-precreate - seek - - /opt/solr/server/solr/configsets/seek_config - + - /opt/solr/server/solr/configsets/seek_config volumes: seek-filestore: diff --git a/requirements.txt b/requirements.txt index 5bd999d84f..e90c707589 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ psutil==5.9.3 -cwltool==3.1.20230601100705 +cwltool==3.1.20230906142556 html5lib==1.0.1 galaxy2cwl==0.1.4 #isatools==0.11.0 gxformat2==0.15.0 -nbconvert==7.4.0 -ipython==8.10.0 +nbconvert==7.16.6 +ipython==8.18.1