Release

group_vars/all.yml

@@ -1245,6 +1245,11 @@ magento_media_storage_strategy: "{{ aws_use | ternary('aws-s3', 'local-fs') }}"
 # Any extra buckets you want for you app
 magento_s3fs_buckets_extra: []
 
+# Which S3 mount backend should be used when `magento_media_storage_strategy`
+# or extra mounts require S3-backed storage.
+# Possible values: s3fs, goofys, rclone
+s3_mount_backend: s3fs
+
 # -------------------------------------------------------------
 # --------  Magento Import/Export AWS EFS file shares  --------
 # -------------------------------------------------------------
@@ -1425,8 +1430,8 @@ php_xdebug_remote_port: 9000
 php_max_execution_time: "600"
 php_max_input_time: "600"
 php_max_input_vars: "2000"
-php_realpath_cache_size: "4M"
-php_realpath_cache_ttl: "3600"
+php_realpath_cache_size: "10M"
+php_realpath_cache_ttl: "7200"
 php_upload_max_filesize: "64M"
 php_post_max_size: "32M"
 php_session_name: "PHPSESSID"
@@ -1475,7 +1480,7 @@ php_cli_opcache_file_cache_enable: no
 php_fpm_pm_type: static
 php_fpm_pm_max_children: ""
 php_fpm_pm_max_requests: 100
-
+php_fpm_slowlog_trace_depth: 30
 # -----------------------------
 # --------  Blackfire  --------
 # -----------------------------
@@ -2152,3 +2157,8 @@ aws_pio_ebs_volume_size: "{{ aws_app_node_ebs_volume_size }}"
 new_relic_app_name: "{{ mageops_app_name }}"
 mageops_new_relic_enabled: no
 # new_relic_license need to be set up
+
+# ---------------------
+# ----- Google Tag Manger -----
+# ---------------------
+gtm_enabled: no

requirements-galaxy.yml

@@ -28,4 +28,5 @@ collections:
 - name: ansible.netcommon
   version: 5.1.2
 - community.crypto
-- ansible.posix
+- ansible.posix
+- containers.podman

roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml

@@ -5,6 +5,7 @@
       varnish_backend_probe_endpoint: "{{ varnish_backend_probe_endpoint }}"
       varnish_backend_max_conns: "{{ varnish_backend_max_conns }}"
       varnish_backend_first_byte_timeout: "{{ varnish_backend_first_byte_timeout }}"
+      gtm_enabled: "{{ gtm_enabled | default(false) }}"
 
 - name: Check if lambda function exists
   community.aws.lambda_info:

roles/cs.aws-logs-retention/tasks/main.yml

@@ -14,3 +14,6 @@
 - name: Set log retention for groups without retention
   shell: "aws logs put-retention-policy --log-group-name={{ item }} --retention-in-days=7 --region={{ aws_region }}"
   loop: "{{ aws_log_retention_groups.ansible_facts.data.groups }}"
+  ignore_errors: true
+  when: aws_log_retention_groups.ansible_facts.data.groups | length > 0
+

roles/cs.aws-security-group/tasks/main.yml

@@ -77,7 +77,7 @@
     name: "{{ aws_security_group_app_name }}"
     description: "{{ mageops_app_name }} Webnodes security group"
     region: "{{ aws_region }}"
-    rules: "{{ aws_security_group_app_rules_base + aws_security_group_app_rules + aws_security_group_app_extra_rules }}"
+    rules: "{{ aws_security_group_app_rules_base + aws_security_group_app_rules + aws_security_group_app_extra_rules + (aws_security_group_app_gtm_rules if gtm_enabled | default(false) else []) }}"
     vpc_id: "{{ aws_vpc_id }}"
     tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}"
   vars:
@@ -88,7 +88,12 @@
         ports:
           - "{{ mageops_varnish_backend_port }}"
         group_name: "{{ aws_security_group_varnish_name }}"
-
+    aws_security_group_app_gtm_rules:
+      - proto: tcp
+        ports:
+          - 8080
+          - 8081
+        group_name: "{{ aws_security_group_varnish_name }}"
   register: aws_security_group_app
 
 - name: Create security group for persistant node

roles/cs.gtm/tasks/main.yml

@@ -0,0 +1,57 @@
+- name: Ensure Podman is installed (RHEL/CentOS/Amazon Linux 2023)
+  ansible.builtin.dnf:
+    name:
+      - podman
+      - containers-common
+      - slirp4netns
+      - fuse-overlayfs
+    state: present
+
+- name: Pull Google Tag Manager Server-Side image
+  containers.podman.podman_image:
+    name: gcr.io/cloud-tagging-10302018/gtm-cloud-image
+    tag: stable
+
+# MAIN (tagging) container
+- name: Run GTM SS main container and create systemd service
+  containers.podman.podman_container:
+    name: gtm
+    image: gcr.io/cloud-tagging-10302018/gtm-cloud-image:stable
+    state: started
+    restart_policy: always
+    publish:
+      - "8080:8080"
+    env:
+      PORT: "8080"
+      PREVIEW_SERVER_URL: "https://{{ mageops_gtm_preview_domain }}"
+      CONTAINER_CONFIG: "{{ gtm_container_config }}"
+    generate_systemd:
+      path: /etc/systemd/system
+      restart_policy: always
+
+# PREVIEW container (separate instance)
+- name: Run GTM SS preview container and create systemd service
+  containers.podman.podman_container:
+    name: gtm-preview
+    image: gcr.io/cloud-tagging-10302018/gtm-cloud-image:stable
+    state: started
+    restart_policy: always
+    publish:
+      - "8081:8080"
+    env:
+      PORT: "8080"
+      RUN_AS_PREVIEW_SERVER: "true"
+      CONTAINER_CONFIG: "{{ gtm_container_config }}"
+    generate_systemd:
+      path: /etc/systemd/system
+      restart_policy: always
+
+- name: Reload systemd and enable services
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  loop:
+    - container-gtm.service
+    - container-gtm-preview.service

roles/cs.magento-configure/defaults/main/app-etc.yml

@@ -75,6 +75,7 @@ magento_app_etc_config:
     vertex: 1
     elasticsuite: 1
     graphql_query_resolver_result: 1
+    search_ajax_suggest: 1
 
   install:
     date: "Tue, 11 Nov 2016 11:11:00 +0000"

roles/cs.magento-configure/tasks/035-clean-old-static-content.yml

@@ -1,7 +1,7 @@
 ### 
 # Magento does it anyway during setup:upgrade, but lets make sure anyway.
 # - Use `rm` because ansible cannot easily delete only contents in a
-#   performant way, especially that this might be an s3fs mount.
+#   performant way, especially that this might be an S3-backed mount.
 # - Caution: This will not remove dotfiles, but we don't care.
 ###
 - name: Make sure previous Mangeto static cache files are gone
@@ -13,4 +13,4 @@
   retries: 5
   delay: 10 
   until: not _magento_install_clean_statics is failed
-
+

roles/cs.magento-shared-storage/defaults/main.yml

@@ -1,6 +1,6 @@
 # What to use for storing Magento media that must be shared across nodes
 # Possible values:
-#   - aws-s3 (default if `aws_use`) - store on AWS S3 media bucket and mount it via s3fs
+#   - aws-s3 (default if `aws_use`) - store on AWS S3 media bucket and mount it via the selected S3 backend
 #   - aws-efs - store on AWS EFS and mount it as NFS share on the node
 #   - local-fs (default if not `aws_use`) - do nothing, just store the files on the local fs
 # Note: If you set this to `aws-efs` then the media S3 bucket will not be created, but

roles/cs.magento-shared-storage/tasks/main.yml

@@ -17,6 +17,10 @@
       ----------- Media storage strategy  ---------
       *** {{ magento_media_storage_strategy }} ***
       {% if magento_shared_storage_s3_enabled %}
+      --------------  S3 Backend  ----------------
+
+      {{ s3_mount_backend | default('s3fs') }}
+
       -----------------  S3 Mounts  ---------------
 
       {{ magento_s3_app_mounts | to_nice_yaml }}
@@ -50,8 +54,8 @@
     efs_name: "{{ magento_efs_app_node_name }}"
     efs_tags: "{{ aws_tags_default | combine(aws_tags_role_storage, aws_tags_role_shared_storage) }}"
     efs_mounts: "{{ magento_efs_app_mounts }}"
-    # This is a workaround for EFS mounts overlaying the crappy S3FS mounts
+    # This is a workaround for EFS mounts overlaying the S3-backed mounts.
     efs_mount_watchdog_cron_enable: "{{ magento_media_storage_strategy == 'aws-s3' }}"
 
 - name: Clean up shared Magento asset cache
-  include: cleanup-static-cache-releases.yml
+  include: cleanup-static-cache-releases.yml

roles/cs.magerun/defaults/main.yml

@@ -1,2 +1,3 @@
-magerun_version: 7.4.0
-magerun_checksum: sha256:35377402bd94c8ee19c3aecbc52f9bb6f6f9b4970447d1d7884fb6cfbb8cdf9b
+magerun_version: 9.3.0
+magerun_checksum: sha256:15041cdf99466d80691d9f2ed5f37330115ea37b5ca19b4f1883f1c9a18bd26c
+

roles/cs.php-fpm/defaults/main.yml

@@ -14,6 +14,7 @@ php_fpm_pm_min_spare_servers: 5
 php_fpm_pm_max_spare_servers: 5
 php_fpm_pm_max_requests: 500
 php_fpm_terminate_timeout: 600
+php_fpm_slowlog_trace_depth: 30
 
 # Used for naming files, directories and services
 php_fpm_pool_name: "app"

roles/cs.php-fpm/templates/php-fpm.pool.conf

@@ -26,6 +26,9 @@ ping.response = "{{ php_fpm_pool_ping_response }}"
 
 request_terminate_timeout = {{ php_fpm_terminate_timeout }}
 request_slowlog_timeout = 5
+request_slowlog_trace_depth = {{ php_fpm_slowlog_trace_depth }}
+
+
 slowlog = {{ php_fpm_log_dir_path }}/{{ php_fpm_pool_name }}.slow.log
 
 catch_workers_output = yes

roles/cs.php/defaults/main.yml

@@ -22,8 +22,9 @@ php_session_gc_maxlifetime: 10800
 php_max_input_time: 600
 php_max_input_vars: 2000
 
-php_realpath_cache_size: "4M"
-php_realpath_cache_ttl: "3600"
+# https://experienceleague.adobe.com/en/docs/commerce-operations/implementation-playbook/best-practices/planning/realpath-cache-size
+php_realpath_cache_size: "10M"
+php_realpath_cache_ttl: "7200"
 php_upload_max_filesize: "64M"
 php_post_max_size: "32M"
 php_session_name: PHPSESSID

roles/cs.s3-fuse-rclone/defaults/main.yml

@@ -0,0 +1,2 @@
+s3_mount_rclone_remote_name: mageops-s3
+s3_mount_rclone_config_path: /etc/rclone.conf

roles/cs.s3-fuse-rclone/meta/main.yml

@@ -0,0 +1,3 @@
+allow_duplicates: no
+dependencies:
+  - role: cs.repo-epel

roles/cs.s3-fuse-rclone/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: Install rclone and FUSE runtime packages
+  dnf:
+    name:
+      - rclone
+      - fuse3
+    state: present
+
+- name: Add read permission to other users
+  file:
+    path: /etc/fuse.conf
+    state: file
+    mode: "o=r"
+
+- name: Allow non-root users to specify the allow_other or allow_root mount options
+  lineinfile:
+    dest: /etc/fuse.conf
+    regexp: "^#\\s*user_allow_other$"
+    line: "user_allow_other"
+    state: present
+    backrefs: yes
+
+- name: Write rclone config
+  template:
+    src: rclone.conf.j2
+    dest: "{{ s3_mount_rclone_config_path }}"
+    owner: root
+    mode: "0644"

roles/cs.s3-fuse-rclone/templates/rclone.conf.j2

@@ -0,0 +1,5 @@
+[{{ s3_mount_rclone_remote_name }}]
+type = s3
+provider = AWS
+env_auth = true
+region = {{ aws_region }}

roles/cs.s3-fuse-s3fs/tasks/main.yml

@@ -12,7 +12,7 @@
 - name: Allow non-root users to specify the allow_other or allow_root mount options
   lineinfile:
     dest: "/etc/fuse.conf"
-    regexp: "^#user_allow_other$"
+    regexp: "^#\\s*user_allow_other$"
     line: "user_allow_other"
     state: present
     backrefs: yes

roles/cs.s3-mount/defaults/main.yml

@@ -1,13 +1,33 @@
-s3fs_mount_use_goofys: false
+s3_mount_backend: s3fs
+s3_mount_backends_all:
+  - s3fs
+  - goofys
+  - rclone
 
-s3fs_mount_fstype: "{{ s3fs_mount_use_goofys | ternary('fuse', 'fuse.s3fs') }}"
-s3fs_mount_fuse_binary: "{{ s3fs_mount_use_goofys | ternary('goofys', 's3fs') }}"
+# These are the helper fstypes written to fstab.
+s3_mount_fstype_map:
+  s3fs: fuse
+  goofys: fuse
+  rclone: rclone
+
+s3_mount_binary_map:
+  s3fs: s3fs
+  goofys: goofys
+  rclone: rclone
+
+s3_mount_uses_cache_map:
+  s3fs: true
+  goofys: false
+  rclone: true
 
 s3fs_cache_dir: /tmp/s3
 s3fs_use_cache: yes
 
 s3fs_ensure_diskfree_mb: 1024
 
+s3_mount_rclone_remote_name: mageops-s3
+s3_mount_rclone_config_path: /etc/rclone.conf
+
 s3fs_options_s3fs:
     - "ahbe_conf=/etc/s3fs/ahbe.conf"
     - "auto_unmount"
@@ -27,6 +47,16 @@ s3fs_options_goofys:
     - "--uid={{ s3fs_owner_uid }}"
     - "--gid={{ s3fs_owner_gid }}"
 
-s3fs_mount_opts: "{{ s3fs_mount_use_goofys | ternary(s3fs_options_goofys, s3fs_options_s3fs) }}"
+s3_mount_options_rclone:
+    - "args2env"
+    - "config={{ s3_mount_rclone_config_path }}"
+    - "uid={{ s3fs_owner_uid }}"
+    - "gid={{ s3fs_owner_gid }}"
+    - "umask=0002"
+
+s3_mount_options_map:
+  s3fs: "{{ s3fs_options_s3fs }}"
+  goofys: "{{ s3fs_options_goofys }}"
+  rclone: "{{ s3_mount_options_rclone }}"
 
 s3fs_premount_bind_mounts: []

roles/cs.s3-mount/meta/main.yml

@@ -1,5 +1,7 @@
 dependencies:
   - role: cs.s3-fuse-s3fs
-    when: not s3fs_mount_use_goofys
+    when: s3_mount_backend == 's3fs'
   - role: cs.s3-fuse-goofys
-    when: s3fs_mount_use_goofys
+    when: s3_mount_backend == 'goofys'
+  - role: cs.s3-fuse-rclone
+    when: s3_mount_backend == 'rclone'

roles/cs.s3-mount/tasks/000-validate-config.yml

@@ -0,0 +1,36 @@
+- name: Validate S3 mount backend selector
+  assert:
+    that:
+      - s3_mount_backend in s3_mount_backends_all
+    fail_msg: >-
+      Unsupported S3 mount backend "{{ s3_mount_backend }}".
+      Use one of: {{ s3_mount_backends_all | join(', ') }}
+
+- name: Prepare backend-specific S3 mount facts
+  set_fact:
+    s3_mount_fstype: "{{ s3_mount_fstype_map[s3_mount_backend] }}"
+    s3_mount_binary: "{{ s3_mount_binary_map[s3_mount_backend] }}"
+    s3_mount_opts_base: "{{ s3_mount_options_map[s3_mount_backend] }}"
+    s3_mount_uses_cache: "{{ s3_mount_uses_cache_map[s3_mount_backend] }}"
+    s3_mount_buckets_resolved: []
+
+- name: Resolve configured S3 mount sources
+  set_fact:
+    s3_mount_buckets_resolved: "{{ s3_mount_buckets_resolved + [_s3_mount_bucket_resolved] }}"
+  vars:
+    _s3_mount_remote_path: "{{ s3fs_bucket.bucket | regex_replace('^([^:]+):/(.*)$', '\\1/\\2') }}"
+    _s3_mount_bucket_resolved: >-
+      {{
+        s3fs_bucket | combine({
+          'remote_path': _s3_mount_remote_path,
+          'mount_src': (
+            s3_mount_backend == 'rclone'
+          ) | ternary(
+            s3_mount_rclone_remote_name ~ ':' ~ _s3_mount_remote_path,
+            s3_mount_binary ~ '#' ~ s3fs_bucket.bucket
+          )
+        })
+      }}
+  loop: "{{ s3fs_buckets }}"
+  loop_control:
+    loop_var: s3fs_bucket