FIX: Slow startup and performance on NFS or slow volume mounts #188

Author: capile

README.md

@@ -109,6 +109,17 @@ If it is a setting controlled by an environment variable which is meant to overr
     -   `./gnupg`: `/var/www/MISP/.gnupg/`
 -   If you need to automatically run additional steps each time the container starts, create a new file `files/customize_misp.sh`, and replace the variable `${CUSTOM_PATH}` inside `docker-compose.yml` with its parent path.
 
+#### Using slow disks as volume mounts
+
+Using a slow disk as the mounted volume or a volume with high latency like NFS, EFS or S3 might significantly increase the startup time and downgrade the performance of the service. To address this we will mount the bare minimum that needs to be persisted.
+
+- Remove the `/var/www/MISP/app/files/` volume mount.
+- Add the following volume mounts instead:
+    - `./img/`: `/var/www/MISP/app/files/img`
+    - `./terms`: `/var/www/MISP/app/files/terms`
+    - `./attachments`: `/var/www/MISP/app/attachments`
+- Set the environment variable `ATTACHMENTS_DIR` to the above folder location (it is important that it doesn't replace the `/var/www/MISP/app/files/` folder). 
+
 ## Installing custom root CA certificates
 
 Custom root CA certificates can be mounted under `/usr/local/share/ca-certificates` and will be installed during the `misp-core` container start.

core/Dockerfile

@@ -253,6 +253,7 @@ EOF
 
     RUN <<-EOF
         cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist
+        echo "${CORE_COMMIT:-${CORE_TAG}}" > /var/www/MISP/app/files/VERSION
         cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist
         find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data '{}' +;
         find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 '{}' +;

core/files/entrypoint.sh

@@ -25,6 +25,7 @@ export SMTP_FQDN=${SMTP_FQDN:-mail}
 export ADMIN_EMAIL=${ADMIN_EMAIL:-admin@admin.test}
 export GPG_PASSPHRASE=${GPG_PASSPHRASE:-passphrase}
 export MISP_MODULES_FQDN=${MISP_MODULES_FQDN:-http://misp-modules}
+export ATTACHMENTS_DIR=${ATTACHMENTS_DIR:-/var/www/MISP/app/files}
 
 export AUTOCONF_GPG=${AUTOCONF_GPG:-true}
 export AUTOCONF_ADMIN_KEY=${AUTOCONF_ADMIN_KEY:-true}

core/files/entrypoint_nginx.sh

@@ -138,6 +138,18 @@ EOT
 }
 
 update_misp_data_files(){
+    # If $MISP_APP_FILES_PATH was not changed since the build, skip file updates there
+    FILES_VERSION=
+    MISP_APP_FILES_PATH=/var/www/MISP/app/files
+    CORE_COMMIT=${CORE_COMMIT:-${CORE_TAG}}
+    if [ -f ${MISP_APP_FILES_PATH}/VERSION ]; then
+        FILES_VERSION=$(cat ${MISP_APP_FILES_PATH}/VERSION)
+        echo "... found local files/VERSION:" $FILES_VERSION
+        if [ "$FILES_VERSION" = "${CORE_COMMIT:-$(jq -r '"v\(.major).\(.minor).\(.hotfix)"' /var/www/MISP/VERSION.json)}" ]; then
+            echo "... local files/ match distribution version, skipping file sync"
+            return 0;
+        fi
+    fi
     for DIR in $(ls /var/www/MISP/app/files.dist); do
         if [ "$DIR" = "certs" ] || [ "$DIR" = "img" ] || [ "$DIR" == "taxonomies" ] ; then
             echo "... rsync -azh \"/var/www/MISP/app/files.dist/$DIR\" \"/var/www/MISP/app/files/\""
@@ -150,21 +162,28 @@ update_misp_data_files(){
 }
 
 enforce_misp_data_permissions(){
-    echo "... chown -R www-data:www-data /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} +
-    # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory
-    echo "... chmod -R 0550 files /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp -not -perm 550 -type f -exec chmod 0550 {} +
-    # Directories are also writable, because there seems to be a requirement to add new files every once in a while
-    echo "... chmod -R 0770 directories /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp -not -perm 770 -type d -exec chmod 0770 {} +
-    # We make 'files' and 'tmp' (logs) directories and files user and group writable (we removed the SGID bit)
-    echo "... chmod -R u+w,g+w /var/www/MISP/app/tmp" && chmod -R u+w,g+w /var/www/MISP/app/tmp
-    
-    echo "... chown -R www-data:www-data /var/www/MISP/app/files" && find /var/www/MISP/app/files \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} +
-    # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory
-    echo "... chmod -R 0550 files /var/www/MISP/app/files" && find /var/www/MISP/app/files -not -perm 550 -type f -exec chmod 0550 {} +
-    # Directories are also writable, because there seems to be a requirement to add new files every once in a while
-    echo "... chmod -R 0770 directories /var/www/MISP/app/files" && find /var/www/MISP/app/files -not -perm 770 -type d -exec chmod 0770 {} +
-    # We make 'files' and 'tmp' (logs) directories and files user and group writable (we removed the SGID bit)
-    echo "... chmod -R u+w,g+w /var/www/MISP/app/files" && chmod -R u+w,g+w /var/www/MISP/app/files
+    # If $MISP_APP_FILES_PATH was not changed since the build, skip file updates there
+    MISP_APP_FILES_PATH=/var/www/MISP/app/files
+    CORE_COMMIT=${CORE_COMMIT:-${CORE_TAG}}
+    if [ -f "${MISP_APP_FILES_PATH}/VERSION" ] && [ "$(cat ${MISP_APP_FILES_PATH}/VERSION)" = "${CORE_COMMIT:-$(jq -r '"v\(.major).\(.minor).\(.hotfix)"' /var/www/MISP/VERSION.json)}" ]; then
+        echo "... local files/ match distribution version, skipping data permissions in files/"
+    else
+        echo "... chown -R www-data:www-data /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} +
+        # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory
+        echo "... chmod -R 0550 files /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp -not -perm 550 -type f -exec chmod 0550 {} +
+        # Directories are also writable, because there seems to be a requirement to add new files every once in a while
+        echo "... chmod -R 0770 directories /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp -not -perm 770 -type d -exec chmod 0770 {} +
+        # We make 'files' and 'tmp' (logs) directories and files user and group writable (we removed the SGID bit)
+        echo "... chmod -R u+w,g+w /var/www/MISP/app/tmp" && chmod -R u+w,g+w /var/www/MISP/app/tmp
+        
+        echo "... chown -R www-data:www-data /var/www/MISP/app/files" && find /var/www/MISP/app/files \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} +
+        # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory
+        echo "... chmod -R 0550 files /var/www/MISP/app/files" && find /var/www/MISP/app/files -not -perm 550 -type f -exec chmod 0550 {} +
+        # Directories are also writable, because there seems to be a requirement to add new files every once in a while
+        echo "... chmod -R 0770 directories /var/www/MISP/app/files" && find /var/www/MISP/app/files -not -perm 770 -type d -exec chmod 0770 {} +
+        # We make 'files' and 'tmp' (logs) directories and files user and group writable (we removed the SGID bit)
+        echo "... chmod -R u+w,g+w /var/www/MISP/app/files" && chmod -R u+w,g+w /var/www/MISP/app/files
+    fi
 
     echo "... chown -R www-data:www-data /var/www/MISP/app/Config" && find /var/www/MISP/app/Config \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} +
     # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory

core/files/etc/misp-docker/initialisation.envars.json

@@ -12,6 +12,9 @@
   "MISP.contact": {
     "default_value": "${SETTING_CONTACT}"
   },
+  "MISP.attachments_dir": {
+    "default_value": "${ATTACHMENTS_DIR}"
+  },
   "Security.encryption_key": {
     "default_value": "${ENCRYPTION_KEY}",
     "command_args": "-f"

docker-compose.yml

@@ -114,6 +114,7 @@ services:
       - "ADMIN_ORG=${ADMIN_ORG}"
       - "ADMIN_ORG_UUID=${ADMIN_ORG_UUID}"
       - "GPG_PASSPHRASE=${GPG_PASSPHRASE}"
+      - "ATTACHMENTS_DIR=${ATTACHMENTS_DIR}"
       # OIDC authentication settings
       - "OIDC_ENABLE=${OIDC_ENABLE}"
       - "OIDC_PROVIDER_URL=${OIDC_PROVIDER_URL}"

template.env

@@ -53,6 +53,8 @@ ENABLE_DB_SETTINGS=
 ENCRYPTION_KEY=
 # enable background updates. defaults to false
 ENABLE_BACKGROUND_UPDATES=
+# use a different attachments_dir. defaults to /var/www/MISP/app/files
+ATTACHMENTS_DIR=
 
 # defines the FQDN of the mail sub-system (defaults to 'mail')
 # SMTP_FQDN=