SLT-17: Periodical drush SQL-Dump using gdpr-dump; Fallback to standard drush sql-dump when the gdpr.json file can't be located; Documentation;

Author: Jancis

.circleci/config.yml

@@ -22,7 +22,10 @@ jobs:
           keys:
             - v1-dependencies-{{ checksum "composer.lock" }}
 
-      - run: composer install -n --prefer-dist --ignore-platform-reqs --no-dev
+      - run: |
+          composer install -n --prefer-dist --ignore-platform-reqs --no-dev
+          composer config repositories.gdpr-dump-mods git https://github.com/Jancis/gdpr-dump
+          composer require machbarmacher/gdpr-dump:dev-mods -n --ignore-platform-reqs
 
       - save_cache:
           paths:

Dockerfile

@@ -1,6 +1,10 @@
 # Dockerfile for the Drupal container.
 FROM wunderio/drupal-php-fpm:v0.1
 
+USER root
+RUN mkdir -p /var/backups/db
+RUN chown www-data:www-data /var/backups/db
+
 COPY --chown=www-data:www-data . /var/www/html
 USER www-data
 RUN mkdir -p -m +w /var/www/html/web/sites/default/files

README.md

@@ -8,7 +8,7 @@ This project template is an opinionated fork of the popular [Drupal-composer tem
 
 - Copy this repository and push it to our organization. 
 - Log in to CircleCI using your Github account and add the new project.
-
+- Create and maintain a Personal Data mapping list for automatic data sanitization in `gdpr.json` file. See GDPR sanitization section for more information.
 
 ## How it works
 
@@ -21,3 +21,35 @@ Have a look at the file for details, but in short this is how it works:
 - Create a custom docker image for Drupal and nginx, and push those to a docker registry (typically that of your cloud provider).
 - Install or update our helm chart while passing our custom images as parameters.
 - The helm chart executes the usual drush deployment commands.
+
+## GDPR sanitization
+
+SQL data dump for developers is parsed with [GDPR Tools](https://github.com/machbarmacher/gdpr-dump) project.
+You can create a `/gdpr.json` file with [Faker](https://packagist.org/packages/fzaninotto/faker) formatters that will allow replacing data as it's dumped from database using `mysqldump` / `drush sql-dump` command.  
+
+```
+{
+  "users_field_data": {
+    "name": {"formatter": "name"},
+    "pass": {"formatter": "password"},
+    "mail": {"formatter": "email"},
+    "init": {"formatter": "clear"}
+  }
+} 
+```
+Available formatters:
+```
+**name** - generates a name
+**phoneNumber** - generates a phone number
+**username** - generates a random user name
+**password** - generates a random password
+**email** - generates a random email address
+**date** - generates a date
+**longText** - generates a sentence
+**number** - generates a number
+**randomText** - generates a sentence
+**text** - generates a paragraph
+**uri** - generates a URI
+**clear** - generates an empty string
+```
+You can also add extra elements and attributes, like `_cookies`, `_description` or `_purpose` to enrich the Personal Data information. Just make sure it's marked or prefixed so that it does not mess up GDPR dump when it looks for table data replacements.

chart/templates/_helpers.tpl

@@ -21,6 +21,8 @@ volumeMounts:
   - name: drupal-private-files
     mountPath: /var/www/html/private
   {{- end }}
+  - name: drupal-dbdump-volume
+    mountPath: /var/backups/db
   - name: php-conf
     mountPath: /etc/php7/php.ini
     readOnly: true
@@ -44,6 +46,9 @@ volumeMounts:
   persistentVolumeClaim:
     claimName: {{ .Release.Name }}-private-files
 {{- end }}
+- name: drupal-dbdump-volume
+  persistentVolumeClaim:
+    claimName: {{ .Release.Namespace }}-dbdump
 - name: php-conf
   configMap:
     name: {{ .Release.Name }}-php-conf
@@ -64,6 +69,10 @@ imagePullSecrets:
 {{- end }}
 
 {{- define "drupal.env" }}
+- name: BRANCHNAME
+  value: {{ .Values.branchname }}
+- name: PRODUCTION_BRANCHNAME
+  value: {{ .Values.production_branchname | default "production" }}
 {{- if .Values.mariadb.enabled }}
 - name: DB_USER
   value: "{{ .Values.mariadb.db.user }}"

chart/values.yaml

@@ -13,7 +13,24 @@ drupal:
 
   cron:
     - schedule: '0 * * * *'
-      command: 'drush cron'
+      command: 'bootstrapped=$(drush status --field=bootstrap);
+        if [[ $bootstrapped = "Successful" ]];
+        then
+          drush cron;
+        fi;'
+    - schedule: '0 3 * * *'
+      command: 'bootstrapped=$(drush status --field=bootstrap);
+        if [[ $bootstrapped = "Successful" ]] && [[ "$BRANCHNAME" = "$PRODUCTION_BRANCHNAME" ]];
+        then
+          DUMP_PATH=/var/backups/db/${BRANCHNAME//[^[:alnum:]]/-}-latest.sql;
+          rm $DUMP_PATH;
+          if [ -f ../gdpr.json ]; then
+            export PATH=../vendor/bin:$PATH;
+            drush sql-dump --extra-dump="--gdpr-replacements-file=../gdpr.json" --result-file $DUMP_PATH;
+          else
+            drush sql-dump --result-file $DUMP_PATH;
+          fi
+        fi;'
 
   # will be generated random hashsalt if not provided here
   # need to be base64 encoded
@@ -75,4 +92,5 @@ clusterDomain: "silta.wdr.io"
 # See https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets: []
 
-branchname: "default"
+branchname: "default"
+production_branchname: "master"

gdpr.json

@@ -0,0 +1,8 @@
+{
+  "users_field_data": {
+    "name": {"formatter": "name"},
+    "pass": {"formatter": "password"},
+    "mail": {"formatter": "email"},
+    "init": {"formatter": "clear"}
+  }
+}