From 382e09d8eaf737faed6a0a9c14c67b23b25f1182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3=C5=BAwiak?= Date: Fri, 28 Nov 2025 13:57:41 +0100 Subject: [PATCH 01/15] Ignore errors when setting log retention --- roles/cs.aws-logs-retention/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/cs.aws-logs-retention/tasks/main.yml b/roles/cs.aws-logs-retention/tasks/main.yml index 073ebfac6..7b32aeddb 100644 --- a/roles/cs.aws-logs-retention/tasks/main.yml +++ b/roles/cs.aws-logs-retention/tasks/main.yml @@ -14,3 +14,6 @@ - name: Set log retention for groups without retention shell: "aws logs put-retention-policy --log-group-name={{ item }} --retention-in-days=7 --region={{ aws_region }}" loop: "{{ aws_log_retention_groups.ansible_facts.data.groups }}" + ignore_errors: true + when: aws_log_retention_groups.ansible_facts.data.groups | length > 0 + From bbee46afb2610d81a810b8d701237daaf4dfbb44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3=C5=BAwiak?= Date: Mon, 2 Mar 2026 11:10:11 +0100 Subject: [PATCH 02/15] Update magerun to 9.3.0 --- roles/cs.magerun/defaults/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/cs.magerun/defaults/main.yml b/roles/cs.magerun/defaults/main.yml index 37f92572e..d697d3533 100644 --- a/roles/cs.magerun/defaults/main.yml +++ b/roles/cs.magerun/defaults/main.yml @@ -1,2 +1,3 @@ -magerun_version: 7.4.0 -magerun_checksum: sha256:35377402bd94c8ee19c3aecbc52f9bb6f6f9b4970447d1d7884fb6cfbb8cdf9b \ No newline at end of file +magerun_version: 9.3.0 +magerun_checksum: sha256:15041cdf99466d80691d9f2ed5f37330115ea37b5ca19b4f1883f1c9a18bd26c + From 0485f0123d5e31501611fa5a539bd38e756ac9eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3=C5=BAwiak?= Date: Wed, 7 Jan 2026 15:44:16 +0100 Subject: [PATCH 03/15] Add Google Tag Manager server-side --- group_vars/all.yml | 5 ++ requirements-galaxy.yml | 3 +- .../create-update-varnish-backends-lambda.yml | 1 + roles/cs.aws-security-group/tasks/main.yml | 8 ++- roles/cs.gtm/tasks/main.yml | 57 ++++++++++++++++ roles/cs.varnish/defaults/main.yml | 1 + .../cs.varnish/templates/vcl/backends.vcl.j2 | 65 +++++++++++++++++-- .../templates/vcl/subroutines/recv.vcl.j2 | 53 +++++++++------ site.step-45-app-deploy.yml | 3 + 9 files changed, 169 insertions(+), 27 deletions(-) create mode 100644 roles/cs.gtm/tasks/main.yml diff --git a/group_vars/all.yml b/group_vars/all.yml index c6339776f..84f5625a7 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -2151,3 +2151,8 @@ aws_pio_ebs_volume_size: "{{ aws_app_node_ebs_volume_size }}" new_relic_app_name: "{{ mageops_app_name }}" mageops_new_relic_enabled: no # new_relic_license need to be set up + +# --------------------- +# ----- Google Tag Manger ----- +# --------------------- +gtm_enabled: no \ No newline at end of file diff --git a/requirements-galaxy.yml b/requirements-galaxy.yml index 10ccc8377..90b73254e 100644 --- a/requirements-galaxy.yml +++ b/requirements-galaxy.yml @@ -28,4 +28,5 @@ collections: - name: ansible.netcommon version: 5.1.2 - community.crypto -- ansible.posix \ No newline at end of file +- ansible.posix +- containers.podman diff --git a/roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml b/roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml index a4f5f1747..935f3150b 100644 --- a/roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml +++ b/roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml @@ -5,6 +5,7 @@ varnish_backend_probe_endpoint: "{{ varnish_backend_probe_endpoint }}" varnish_backend_max_conns: "{{ varnish_backend_max_conns }}" varnish_backend_first_byte_timeout: "{{ varnish_backend_first_byte_timeout }}" + gtm_enabled: "{{ gtm_enabled | default(false) }}" - name: Check if lambda function exists community.aws.lambda_info: diff --git a/roles/cs.aws-security-group/tasks/main.yml b/roles/cs.aws-security-group/tasks/main.yml index c1290821e..a9df3bc8b 100644 --- a/roles/cs.aws-security-group/tasks/main.yml +++ b/roles/cs.aws-security-group/tasks/main.yml @@ -77,7 +77,7 @@ name: "{{ aws_security_group_app_name }}" description: "{{ mageops_app_name }} Webnodes security group" region: "{{ aws_region }}" - rules: "{{ aws_security_group_app_rules_base + aws_security_group_app_rules + aws_security_group_app_extra_rules }}" + rules: "{{ aws_security_group_app_rules_base + aws_security_group_app_rules + aws_security_group_app_extra_rules + (aws_security_group_app_gtm_rules if gtm_enabled | default(false) else []) }}" vpc_id: "{{ aws_vpc_id }}" tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" vars: @@ -88,7 +88,11 @@ ports: - "{{ mageops_varnish_backend_port }}" group_name: "{{ aws_security_group_varnish_name }}" - + aws_security_group_app_gtm_rules: + - proto: tcp + ports: + - 8080 + - 8081 register: aws_security_group_app - name: Create security group for persistant node diff --git a/roles/cs.gtm/tasks/main.yml b/roles/cs.gtm/tasks/main.yml new file mode 100644 index 000000000..0d59ba144 --- /dev/null +++ b/roles/cs.gtm/tasks/main.yml @@ -0,0 +1,57 @@ +- name: Ensure Podman is installed (RHEL/CentOS/Amazon Linux 2023) + ansible.builtin.dnf: + name: + - podman + - containers-common + - slirp4netns + - fuse-overlayfs + state: present + +- name: Pull Google Tag Manager Server-Side image + containers.podman.podman_image: + name: gcr.io/cloud-tagging-10302018/gtm-cloud-image + tag: stable + +# MAIN (tagging) container +- name: Run GTM SS main container and create systemd service + containers.podman.podman_container: + name: gtm + image: gcr.io/cloud-tagging-10302018/gtm-cloud-image:stable + state: started + restart_policy: always + publish: + - "8080:8080" + env: + PORT: "8080" + PREVIEW_SERVER_URL: "https://{{ mageops_gtm_preview_domain }}" + CONTAINER_CONFIG: "{{ gtm_container_config }}" + generate_systemd: + path: /etc/systemd/system + restart_policy: always + +# PREVIEW container (separate instance) +- name: Run GTM SS preview container and create systemd service + containers.podman.podman_container: + name: gtm-preview + image: gcr.io/cloud-tagging-10302018/gtm-cloud-image:stable + state: started + restart_policy: always + publish: + - "8081:8080" + env: + PORT: "8080" + RUN_AS_PREVIEW_SERVER: "true" + CONTAINER_CONFIG: "{{ gtm_container_config }}" + generate_systemd: + path: /etc/systemd/system + restart_policy: always + +- name: Reload systemd and enable services + ansible.builtin.systemd: + name: "{{ item }}" + enabled: yes + state: started + daemon_reload: yes + loop: + - container-gtm.service + - container-gtm-preview.service \ No newline at end of file diff --git a/roles/cs.varnish/defaults/main.yml b/roles/cs.varnish/defaults/main.yml index 23b6dfb0b..c2aa2d918 100644 --- a/roles/cs.varnish/defaults/main.yml +++ b/roles/cs.varnish/defaults/main.yml @@ -239,3 +239,4 @@ varnish_gzip_bypass_extensions: # Each url should start with `/` varnish_auth_bypass_urls: [] varnish_custom_bypass: [] +gtm_enabled: false diff --git a/roles/cs.varnish/templates/vcl/backends.vcl.j2 b/roles/cs.varnish/templates/vcl/backends.vcl.j2 index 423465647..11a535486 100644 --- a/roles/cs.varnish/templates/vcl/backends.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/backends.vcl.j2 @@ -14,17 +14,60 @@ probe app_probe { .window = 3; } +{% if (gtm_enabled | default(false)) %} +probe gtm_probe { + .request = + "GET /healthy HTTP/1.1" + "Host: localhost" + "Connection: close"; + .interval = 1s; + .timeout = 1s; + .threshold = 2; + .window = 3; +} +{% endif %} + + + {% for instance in (varnish_backend_instances_app + varnish_backend_instances_extra) %} backend {{ instance.instance_id | replace('-','') }} { .host = "{{ instance.private_ip_address }}"; .port = "{{ varnish_backend_port }}"; .max_connections = {{ varnish_backend_max_conns }}; .probe = app_probe; - .first_byte_timeout = {{ varnish_backend_first_byte_timeout }}; # How long to wait before we receive a first byte from our backend? - .connect_timeout = 5s; # How long to wait for a backend connection? - .between_bytes_timeout = 60s; # How long to wait between bytes received from our backend? + .first_byte_timeout = {{ varnish_backend_first_byte_timeout }}; + .connect_timeout = 5s; + .between_bytes_timeout = 60s; +} +{% endfor %} + +{% if (gtm_enabled | default(false)) %} +{% for instance in (varnish_backend_instances_app + varnish_backend_instances_extra) %} +backend gtm_{{ instance.instance_id | replace('-','') }} { + .host = "{{ instance.private_ip_address }}"; + .port = "8080"; + .max_connections = {{ varnish_backend_max_conns }}; + .probe = gtm_probe; + .first_byte_timeout = {{ varnish_backend_first_byte_timeout }}; + .connect_timeout = 5s; + .between_bytes_timeout = 60s; +} +{% endfor %} +{% endif %} + +{% if (gtm_enabled | default(false)) %} +{% for instance in (varnish_backend_instances_app + varnish_backend_instances_extra) %} +backend gtm_preview{{ instance.instance_id | replace('-','') }} { + .host = "{{ instance.private_ip_address }}"; + .port = "8081"; + .probe = gtm_probe; + .max_connections = {{ varnish_backend_max_conns }}; + .first_byte_timeout = {{ varnish_backend_first_byte_timeout }}; + .connect_timeout = 5s; + .between_bytes_timeout = 60s; } {% endfor %} +{% endif %} sub backends_init { new app_director = directors.round_robin(); @@ -32,8 +75,22 @@ sub backends_init { app_director.add_backend({{ instance.instance_id | replace('-','') }}); {% endfor %} +{% if (gtm_enabled | default(false)) %} + new gtm_director = directors.round_robin(); +{% for instance in varnish_backend_instances_app %} + gtm_director.add_backend(gtm_{{ instance.instance_id | replace('-','') }}); +{% endfor %} + + new gtm_preview_director = directors.round_robin(); +{% for instance in varnish_backend_instances_app %} + gtm_preview_director.add_backend(gtm_preview{{ instance.instance_id | replace('-','') }}); +{% endfor %} +{% endif %} + + + new extra_director = directors.round_robin(); {% for instance in varnish_backend_instances_extra %} extra_director.add_backend({{ instance.instance_id | replace('-','') }}); {% endfor %} -} +} \ No newline at end of file diff --git a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 index 488f5ecf5..5988ce84d 100644 --- a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 @@ -27,30 +27,43 @@ {% if varnish_standalone %} {# This has to be first line, remember this is not declarative config, we always need the director! #} + {% if (gtm_enabled | default(false)) %} + if (req.http.Host == "{{ mageops_gtm_domain }}" ) { + set req.backend_hint = gtm_director.backend(); + return (pass); + } elseif (req.http.Host == "{{ mageops_gtm_preview_domain }}" ) { + set req.backend_hint = gtm_preview_director.backend(); + return (pass); + } else { + set req.backend_hint = app_director.backend(); + } + {% else %} set req.backend_hint = app_director.backend(); + {% endif %} - if (req.http.X-Use-Extra-Instance - {% if aws_extra_app_asg_passthrough_uagent_pattern %} - || req.http.User-Agent ~ "{{ aws_extra_app_asg_passthrough_uagent_pattern }}" - {% endif %} - {% if aws_extra_app_asg_passthrough_url_pattern %} - || req.url ~ "{{ aws_extra_app_asg_passthrough_url_pattern }}" - {% endif %} - {% if aws_extra_app_asg_passthrough_ip_pattern %} - || req.http.x-forwarded-for ~ "{{ aws_extra_app_asg_passthrough_ip_pattern }}" - {% endif %} - ) { - set req.backend_hint = extra_director.backend(); +if (req.http.X-Use-Extra-Instance +{% if aws_extra_app_asg_passthrough_uagent_pattern %} + || req.http.User-Agent ~ "{{ aws_extra_app_asg_passthrough_uagent_pattern }}" +{% endif %} +{% if aws_extra_app_asg_passthrough_url_pattern %} + || req.url ~ "{{ aws_extra_app_asg_passthrough_url_pattern }}" +{% endif %} +{% if aws_extra_app_asg_passthrough_ip_pattern %} + || req.http.x-forwarded-for ~ "{{ aws_extra_app_asg_passthrough_ip_pattern }}" +{% endif %} +) { + set req.backend_hint = extra_director.backend(); - {% if varnish_extra_instance_failover_enable %} - if (!std.healthy(req.backend_hint)) { - set req.backend_hint = app_director.backend(); - } - {% else %} - # Extra instance failover is disabled - requests will return 5xx - # in case all of the the extra instances are not healthy. - {% endif %} +{% if varnish_extra_instance_failover_enable %} + if (!std.healthy(req.backend_hint)) { + set req.backend_hint = app_director.backend(); } +{% else %} + # Extra instance failover is disabled - requests will return 5xx + # in case all of the the extra instances are not healthy. +{% endif %} + +} {% endif %} {% if varnish_vcl_recv_extra %} diff --git a/site.step-45-app-deploy.yml b/site.step-45-app-deploy.yml index 3f6c46c66..0ad3b46f8 100644 --- a/site.step-45-app-deploy.yml +++ b/site.step-45-app-deploy.yml @@ -120,6 +120,9 @@ pio_tasks: app when: mageops_pio_worker_enable + - role: cs.gtm + when: gtm_enabled + tasks: - name: Execute custom post deploy tasks include_tasks: "{{ mageops_extra_tasks_deploy }}" From ff2e830058b3ec7cb4e0e56e5bf698e564dc29c7 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Tue, 3 Mar 2026 13:21:30 +0100 Subject: [PATCH 04/15] Restrict GTM app ingress on ports 8080/8081 to Varnish security group --- roles/cs.aws-security-group/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/cs.aws-security-group/tasks/main.yml b/roles/cs.aws-security-group/tasks/main.yml index a9df3bc8b..98e3e50d1 100644 --- a/roles/cs.aws-security-group/tasks/main.yml +++ b/roles/cs.aws-security-group/tasks/main.yml @@ -93,6 +93,7 @@ ports: - 8080 - 8081 + group_name: "{{ aws_security_group_varnish_name }}" register: aws_security_group_app - name: Create security group for persistant node From 0f93b88346f955897d3f482cb60000f792920b52 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Tue, 3 Mar 2026 13:33:27 +0100 Subject: [PATCH 05/15] Fix GTM vcl_recv branch syntax by using valid VCL 'elsif' --- roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 index 5988ce84d..e9af13db4 100644 --- a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 @@ -31,7 +31,7 @@ if (req.http.Host == "{{ mageops_gtm_domain }}" ) { set req.backend_hint = gtm_director.backend(); return (pass); - } elseif (req.http.Host == "{{ mageops_gtm_preview_domain }}" ) { + } elsif (req.http.Host == "{{ mageops_gtm_preview_domain }}" ) { set req.backend_hint = gtm_preview_director.backend(); return (pass); } else { From ee5113ff66d9d9f1a27fe6e352cb9a08eb2bc626 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Tue, 3 Mar 2026 13:55:08 +0100 Subject: [PATCH 06/15] Run GTM role only with standalone Varnish topology --- site.step-45-app-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site.step-45-app-deploy.yml b/site.step-45-app-deploy.yml index 0ad3b46f8..c00d5dd47 100644 --- a/site.step-45-app-deploy.yml +++ b/site.step-45-app-deploy.yml @@ -121,7 +121,7 @@ when: mageops_pio_worker_enable - role: cs.gtm - when: gtm_enabled + when: gtm_enabled and varnish_standalone tasks: - name: Execute custom post deploy tasks From 0f507292cef4c409afa6975a440e8a712d68faeb Mon Sep 17 00:00:00 2001 From: Piotr Matras Date: Fri, 6 Mar 2026 14:34:19 +0100 Subject: [PATCH 07/15] feat: DEVOPS-621 add search ajax suggest cache type --- roles/cs.magento-configure/defaults/main/app-etc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/cs.magento-configure/defaults/main/app-etc.yml b/roles/cs.magento-configure/defaults/main/app-etc.yml index a73623a25..a4a5b0f83 100644 --- a/roles/cs.magento-configure/defaults/main/app-etc.yml +++ b/roles/cs.magento-configure/defaults/main/app-etc.yml @@ -75,6 +75,7 @@ magento_app_etc_config: vertex: 1 elasticsuite: 1 graphql_query_resolver_result: 1 + search_ajax_suggest: 1 install: date: "Tue, 11 Nov 2016 11:11:00 +0000" From d857551629a6c15228babe89231e86d0d05508e3 Mon Sep 17 00:00:00 2001 From: Piotr Matras Date: Mon, 9 Mar 2026 09:28:48 +0100 Subject: [PATCH 08/15] feat: DEVOPS-623 add realpath cache configuration --- roles/cs.php/defaults/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/cs.php/defaults/main.yml b/roles/cs.php/defaults/main.yml index 35a0534e4..474bc0e81 100644 --- a/roles/cs.php/defaults/main.yml +++ b/roles/cs.php/defaults/main.yml @@ -22,8 +22,9 @@ php_session_gc_maxlifetime: 10800 php_max_input_time: 600 php_max_input_vars: 2000 -php_realpath_cache_size: "4M" -php_realpath_cache_ttl: "3600" +# https://experienceleague.adobe.com/en/docs/commerce-operations/implementation-playbook/best-practices/planning/realpath-cache-size +php_realpath_cache_size: "10M" +php_realpath_cache_ttl: "7200" php_upload_max_filesize: "64M" php_post_max_size: "32M" php_session_name: PHPSESSID From 4473c4ce80ef409e3c4ae8ba40c4910c13624211 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Tue, 10 Mar 2026 10:04:52 +0100 Subject: [PATCH 09/15] Align php realpath cache group vars with updated defaults --- group_vars/all.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/group_vars/all.yml b/group_vars/all.yml index a43301d40..265ac1b68 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1425,8 +1425,8 @@ php_xdebug_remote_port: 9000 php_max_execution_time: "600" php_max_input_time: "600" php_max_input_vars: "2000" -php_realpath_cache_size: "4M" -php_realpath_cache_ttl: "3600" +php_realpath_cache_size: "10M" +php_realpath_cache_ttl: "7200" php_upload_max_filesize: "64M" php_post_max_size: "32M" php_session_name: "PHPSESSID" @@ -2156,4 +2156,4 @@ mageops_new_relic_enabled: no # --------------------- # ----- Google Tag Manger ----- # --------------------- -gtm_enabled: no \ No newline at end of file +gtm_enabled: no From bdf8c090a30c5dd6e0fc4570316baab881dbc933 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Wed, 11 Mar 2026 11:14:37 +0100 Subject: [PATCH 10/15] Implement rclone replacement for s3fs --- group_vars/all.yml | 5 ++ .../tasks/035-clean-old-static-content.yml | 4 +- .../defaults/main.yml | 2 +- .../cs.magento-shared-storage/tasks/main.yml | 8 ++- roles/cs.s3-fuse-rclone/defaults/main.yml | 2 + roles/cs.s3-fuse-rclone/meta/main.yml | 3 + roles/cs.s3-fuse-rclone/tasks/main.yml | 27 ++++++++ .../templates/rclone.conf.j2 | 5 ++ roles/cs.s3-fuse-s3fs/tasks/main.yml | 2 +- roles/cs.s3-mount/defaults/main.yml | 38 +++++++++-- roles/cs.s3-mount/meta/main.yml | 6 +- .../cs.s3-mount/tasks/000-validate-config.yml | 36 +++++++++++ .../tasks/001-prepare-mount-options.yml | 21 ++++++ roles/cs.s3-mount/tasks/002-setup-mounts.yml | 56 +++++++++------- .../cs.s3-mount/tasks/003-setup-watchdog.yml | 17 +++-- roles/cs.s3-mount/tasks/main.yml | 6 +- .../cs.s3-mount/templates/mount-watchdog.bash | 64 ++++++++++++++++--- site.step-40-app-node.yml | 7 +- 18 files changed, 255 insertions(+), 54 deletions(-) create mode 100644 roles/cs.s3-fuse-rclone/defaults/main.yml create mode 100644 roles/cs.s3-fuse-rclone/meta/main.yml create mode 100644 roles/cs.s3-fuse-rclone/tasks/main.yml create mode 100644 roles/cs.s3-fuse-rclone/templates/rclone.conf.j2 create mode 100644 roles/cs.s3-mount/tasks/000-validate-config.yml create mode 100644 roles/cs.s3-mount/tasks/001-prepare-mount-options.yml diff --git a/group_vars/all.yml b/group_vars/all.yml index 265ac1b68..df68f5444 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1245,6 +1245,11 @@ magento_media_storage_strategy: "{{ aws_use | ternary('aws-s3', 'local-fs') }}" # Any extra buckets you want for you app magento_s3fs_buckets_extra: [] +# Which S3 mount backend should be used when `magento_media_storage_strategy` +# or extra mounts require S3-backed storage. +# Possible values: s3fs, goofys, rclone +s3_mount_backend: s3fs + # ------------------------------------------------------------- # -------- Magento Import/Export AWS EFS file shares -------- # ------------------------------------------------------------- diff --git a/roles/cs.magento-configure/tasks/035-clean-old-static-content.yml b/roles/cs.magento-configure/tasks/035-clean-old-static-content.yml index 405bd7075..ecdd145ca 100644 --- a/roles/cs.magento-configure/tasks/035-clean-old-static-content.yml +++ b/roles/cs.magento-configure/tasks/035-clean-old-static-content.yml @@ -1,7 +1,7 @@ ### # Magento does it anyway during setup:upgrade, but lets make sure anyway. # - Use `rm` because ansible cannot easily delete only contents in a -# performant way, especially that this might be an s3fs mount. +# performant way, especially that this might be an S3-backed mount. # - Caution: This will not remove dotfiles, but we don't care. ### - name: Make sure previous Mangeto static cache files are gone @@ -13,4 +13,4 @@ retries: 5 delay: 10 until: not _magento_install_clean_statics is failed - \ No newline at end of file + diff --git a/roles/cs.magento-shared-storage/defaults/main.yml b/roles/cs.magento-shared-storage/defaults/main.yml index cfa3fab06..3e10e0702 100644 --- a/roles/cs.magento-shared-storage/defaults/main.yml +++ b/roles/cs.magento-shared-storage/defaults/main.yml @@ -1,6 +1,6 @@ # What to use for storing Magento media that must be shared across nodes # Possible values: -# - aws-s3 (default if `aws_use`) - store on AWS S3 media bucket and mount it via s3fs +# - aws-s3 (default if `aws_use`) - store on AWS S3 media bucket and mount it via the selected S3 backend # - aws-efs - store on AWS EFS and mount it as NFS share on the node # - local-fs (default if not `aws_use`) - do nothing, just store the files on the local fs # Note: If you set this to `aws-efs` then the media S3 bucket will not be created, but diff --git a/roles/cs.magento-shared-storage/tasks/main.yml b/roles/cs.magento-shared-storage/tasks/main.yml index f50d82698..8cfa8f204 100644 --- a/roles/cs.magento-shared-storage/tasks/main.yml +++ b/roles/cs.magento-shared-storage/tasks/main.yml @@ -17,6 +17,10 @@ ----------- Media storage strategy --------- *** {{ magento_media_storage_strategy }} *** {% if magento_shared_storage_s3_enabled %} + -------------- S3 Backend ---------------- + + {{ s3_mount_backend | default('s3fs') }} + ----------------- S3 Mounts --------------- {{ magento_s3_app_mounts | to_nice_yaml }} @@ -50,8 +54,8 @@ efs_name: "{{ magento_efs_app_node_name }}" efs_tags: "{{ aws_tags_default | combine(aws_tags_role_storage, aws_tags_role_shared_storage) }}" efs_mounts: "{{ magento_efs_app_mounts }}" - # This is a workaround for EFS mounts overlaying the crappy S3FS mounts + # This is a workaround for EFS mounts overlaying the S3-backed mounts. efs_mount_watchdog_cron_enable: "{{ magento_media_storage_strategy == 'aws-s3' }}" - name: Clean up shared Magento asset cache - include: cleanup-static-cache-releases.yml \ No newline at end of file + include: cleanup-static-cache-releases.yml diff --git a/roles/cs.s3-fuse-rclone/defaults/main.yml b/roles/cs.s3-fuse-rclone/defaults/main.yml new file mode 100644 index 000000000..9fa4e48e8 --- /dev/null +++ b/roles/cs.s3-fuse-rclone/defaults/main.yml @@ -0,0 +1,2 @@ +s3_mount_rclone_remote_name: mageops-s3 +s3_mount_rclone_config_path: /etc/rclone.conf diff --git a/roles/cs.s3-fuse-rclone/meta/main.yml b/roles/cs.s3-fuse-rclone/meta/main.yml new file mode 100644 index 000000000..0b9adabf6 --- /dev/null +++ b/roles/cs.s3-fuse-rclone/meta/main.yml @@ -0,0 +1,3 @@ +allow_duplicates: no +dependencies: + - role: cs.repo-epel diff --git a/roles/cs.s3-fuse-rclone/tasks/main.yml b/roles/cs.s3-fuse-rclone/tasks/main.yml new file mode 100644 index 000000000..480eee9e9 --- /dev/null +++ b/roles/cs.s3-fuse-rclone/tasks/main.yml @@ -0,0 +1,27 @@ +- name: Install rclone and FUSE runtime packages + dnf: + name: + - rclone + - fuse3 + state: present + +- name: Add read permission to other users + file: + path: /etc/fuse.conf + state: file + mode: "o=r" + +- name: Allow non-root users to specify the allow_other or allow_root mount options + lineinfile: + dest: /etc/fuse.conf + regexp: "^#\\s*user_allow_other$" + line: "user_allow_other" + state: present + backrefs: yes + +- name: Write rclone config + template: + src: rclone.conf.j2 + dest: "{{ s3_mount_rclone_config_path }}" + owner: root + mode: "0644" diff --git a/roles/cs.s3-fuse-rclone/templates/rclone.conf.j2 b/roles/cs.s3-fuse-rclone/templates/rclone.conf.j2 new file mode 100644 index 000000000..6ed8fae6e --- /dev/null +++ b/roles/cs.s3-fuse-rclone/templates/rclone.conf.j2 @@ -0,0 +1,5 @@ +[{{ s3_mount_rclone_remote_name }}] +type = s3 +provider = AWS +env_auth = true +region = {{ aws_region }} diff --git a/roles/cs.s3-fuse-s3fs/tasks/main.yml b/roles/cs.s3-fuse-s3fs/tasks/main.yml index 77cb5e0e6..068de74a8 100644 --- a/roles/cs.s3-fuse-s3fs/tasks/main.yml +++ b/roles/cs.s3-fuse-s3fs/tasks/main.yml @@ -12,7 +12,7 @@ - name: Allow non-root users to specify the allow_other or allow_root mount options lineinfile: dest: "/etc/fuse.conf" - regexp: "^#user_allow_other$" + regexp: "^#\\s*user_allow_other$" line: "user_allow_other" state: present backrefs: yes diff --git a/roles/cs.s3-mount/defaults/main.yml b/roles/cs.s3-mount/defaults/main.yml index 4916ca737..704b9c759 100644 --- a/roles/cs.s3-mount/defaults/main.yml +++ b/roles/cs.s3-mount/defaults/main.yml @@ -1,13 +1,33 @@ -s3fs_mount_use_goofys: false +s3_mount_backend: s3fs +s3_mount_backends_all: + - s3fs + - goofys + - rclone -s3fs_mount_fstype: "{{ s3fs_mount_use_goofys | ternary('fuse', 'fuse.s3fs') }}" -s3fs_mount_fuse_binary: "{{ s3fs_mount_use_goofys | ternary('goofys', 's3fs') }}" +# These are the helper fstypes written to fstab. +s3_mount_fstype_map: + s3fs: fuse + goofys: fuse + rclone: rclone + +s3_mount_binary_map: + s3fs: s3fs + goofys: goofys + rclone: rclone + +s3_mount_uses_cache_map: + s3fs: true + goofys: false + rclone: true s3fs_cache_dir: /tmp/s3 s3fs_use_cache: yes s3fs_ensure_diskfree_mb: 1024 +s3_mount_rclone_remote_name: mageops-s3 +s3_mount_rclone_config_path: /etc/rclone.conf + s3fs_options_s3fs: - "ahbe_conf=/etc/s3fs/ahbe.conf" - "auto_unmount" @@ -27,6 +47,16 @@ s3fs_options_goofys: - "--uid={{ s3fs_owner_uid }}" - "--gid={{ s3fs_owner_gid }}" -s3fs_mount_opts: "{{ s3fs_mount_use_goofys | ternary(s3fs_options_goofys, s3fs_options_s3fs) }}" +s3_mount_options_rclone: + - "args2env" + - "config={{ s3_mount_rclone_config_path }}" + - "uid={{ s3fs_owner_uid }}" + - "gid={{ s3fs_owner_gid }}" + - "umask=0002" + +s3_mount_options_map: + s3fs: "{{ s3fs_options_s3fs }}" + goofys: "{{ s3fs_options_goofys }}" + rclone: "{{ s3_mount_options_rclone }}" s3fs_premount_bind_mounts: [] diff --git a/roles/cs.s3-mount/meta/main.yml b/roles/cs.s3-mount/meta/main.yml index c6e46e150..f0c89ce15 100644 --- a/roles/cs.s3-mount/meta/main.yml +++ b/roles/cs.s3-mount/meta/main.yml @@ -1,5 +1,7 @@ dependencies: - role: cs.s3-fuse-s3fs - when: not s3fs_mount_use_goofys + when: s3_mount_backend == 's3fs' - role: cs.s3-fuse-goofys - when: s3fs_mount_use_goofys \ No newline at end of file + when: s3_mount_backend == 'goofys' + - role: cs.s3-fuse-rclone + when: s3_mount_backend == 'rclone' diff --git a/roles/cs.s3-mount/tasks/000-validate-config.yml b/roles/cs.s3-mount/tasks/000-validate-config.yml new file mode 100644 index 000000000..d8cc86f85 --- /dev/null +++ b/roles/cs.s3-mount/tasks/000-validate-config.yml @@ -0,0 +1,36 @@ +- name: Validate S3 mount backend selector + assert: + that: + - s3_mount_backend in s3_mount_backends_all + fail_msg: >- + Unsupported S3 mount backend "{{ s3_mount_backend }}". + Use one of: {{ s3_mount_backends_all | join(', ') }} + +- name: Prepare backend-specific S3 mount facts + set_fact: + s3_mount_fstype: "{{ s3_mount_fstype_map[s3_mount_backend] }}" + s3_mount_binary: "{{ s3_mount_binary_map[s3_mount_backend] }}" + s3_mount_opts_base: "{{ s3_mount_options_map[s3_mount_backend] }}" + s3_mount_uses_cache: "{{ s3_mount_uses_cache_map[s3_mount_backend] }}" + s3_mount_buckets_resolved: [] + +- name: Resolve configured S3 mount sources + set_fact: + s3_mount_buckets_resolved: "{{ s3_mount_buckets_resolved + [_s3_mount_bucket_resolved] }}" + vars: + _s3_mount_remote_path: "{{ s3fs_bucket.bucket | regex_replace('^([^:]+):/(.*)$', '\\1/\\2') }}" + _s3_mount_bucket_resolved: >- + {{ + s3fs_bucket | combine({ + 'remote_path': _s3_mount_remote_path, + 'mount_src': ( + s3_mount_backend == 'rclone' + ) | ternary( + s3_mount_rclone_remote_name ~ ':' ~ _s3_mount_remote_path, + s3_mount_binary ~ '#' ~ s3fs_bucket.bucket + ) + }) + }} + loop: "{{ s3fs_buckets }}" + loop_control: + loop_var: s3fs_bucket diff --git a/roles/cs.s3-mount/tasks/001-prepare-mount-options.yml b/roles/cs.s3-mount/tasks/001-prepare-mount-options.yml new file mode 100644 index 000000000..c7d8e47f8 --- /dev/null +++ b/roles/cs.s3-mount/tasks/001-prepare-mount-options.yml @@ -0,0 +1,21 @@ +- name: Initialize S3 mount options + set_fact: + s3_mount_opts: "{{ s3_mount_opts_base }}" + +- name: Enable S3 mount cache + set_fact: + s3_mount_opts: "{{ s3_mount_opts + _s3_mount_cache_opts }}" + vars: + _s3_mount_cache_opts: >- + {{ + (s3_mount_backend == 'rclone') | ternary( + ['vfs_cache_mode=writes', 'cache_dir=' ~ s3fs_cache_dir], + ['use_cache=' ~ s3fs_cache_dir] + ) + }} + when: s3fs_use_cache and s3_mount_uses_cache + +- name: Disable rclone VFS cache when local cache is disabled + set_fact: + s3_mount_opts: "{{ s3_mount_opts + ['vfs_cache_mode=off'] }}" + when: s3_mount_backend == 'rclone' and not s3fs_use_cache diff --git a/roles/cs.s3-mount/tasks/002-setup-mounts.yml b/roles/cs.s3-mount/tasks/002-setup-mounts.yml index 4ac709a82..85c88b006 100644 --- a/roles/cs.s3-mount/tasks/002-setup-mounts.yml +++ b/roles/cs.s3-mount/tasks/002-setup-mounts.yml @@ -1,33 +1,46 @@ -- name: Enable S3FS cache - set_fact: - s3fs_mount_opts: "{{ s3fs_mount_opts + ['use_cache=' ~ s3fs_cache_dir] }}" - when: s3fs_use_cache and not s3fs_mount_use_goofys - - name: Create S3 mount remote source dirs become: no local_action: module: aws_s3 - bucket: "{{ s3fs_bucket.bucket.split(':')[0] }}" - object: "{{ s3fs_bucket.bucket.split(':')[1] }}" + bucket: "{{ s3_mount_bucket.bucket.split(':')[0] }}" + object: "{{ s3_mount_bucket.bucket.split(':')[1] }}" mode: create region: "{{ aws_region }}" - when: s3fs_bucket.bucket is search(':/') - loop: "{{ s3fs_buckets }}" + when: s3_mount_bucket.bucket is search(':/') + loop: "{{ s3_mount_buckets_resolved }}" loop_control: - loop_var: s3fs_bucket + loop_var: s3_mount_bucket # `findmnt` returns 1 if there are not mounts yet - name: Get current S3 mounts - shell: findmnt -l -n -o TARGET -t "{{ s3fs_mount_fstype }}" + shell: >- + findmnt -l -n -o TARGET,SOURCE,FSTYPE | + awk '$2 ~ /^(s3fs|goofys)#/ || $2 ~ /^{{ s3_mount_rclone_remote_name }}:/ { print $1 "\t" $2 "\t" $3 }' register: s3fs_current_mounts_check failed_when: s3fs_current_mounts_check.rc not in [0, 1] changed_when: false -# Use realpath to normalize trailing slashes and other discrepancies -- name: Set current S3 mounts +- name: Initialize current S3 mount entry list + set_fact: + s3_mount_current_entries: [] + +- name: Parse current S3 mount entries set_fact: - s3fs_current_mountpoints: "{{ s3fs_current_mounts_check.stdout_lines | map('realpath') | list }}" - s3fs_target_mountpoints: "{{ s3fs_buckets | map(attribute='mountpoint') | map('realpath') | list }}" + s3_mount_current_entries: "{{ s3_mount_current_entries + [_s3_mount_entry] }}" + vars: + _s3_mount_entry_parts: "{{ s3_mount_entry.split('\t') }}" + _s3_mount_entry: + mountpoint: "{{ _s3_mount_entry_parts[0] | realpath }}" + source: "{{ _s3_mount_entry_parts[1] }}" + fstype: "{{ _s3_mount_entry_parts[2] }}" + loop: "{{ s3fs_current_mounts_check.stdout_lines }}" + loop_control: + loop_var: s3_mount_entry + +- name: Set current S3 mountpoints + set_fact: + s3fs_current_mountpoints: "{{ s3_mount_current_entries | map(attribute='mountpoint') | list }}" + s3fs_target_mountpoints: "{{ s3_mount_buckets_resolved | map(attribute='mountpoint') | map('realpath') | list }}" - name: Remove S3 mounts that should not be present mount: @@ -45,12 +58,11 @@ - name: Create S3 mount mount: - path: "{{ s3fs_bucket.mountpoint }}" - src: "{{ s3fs_mount_fuse_binary }}#{{ s3fs_bucket.bucket }}" - fstype: fuse - opts: "_netdev,allow_other,auto,{{ s3fs_mount_opts | join(',') }}" + path: "{{ s3_mount_bucket.mountpoint }}" + src: "{{ s3_mount_bucket.mount_src }}" + fstype: "{{ s3_mount_fstype }}" + opts: "_netdev,allow_other,auto,{{ s3_mount_opts | join(',') }}" state: mounted - loop: "{{ s3fs_buckets }}" + loop: "{{ s3_mount_buckets_resolved }}" loop_control: - loop_var: s3fs_bucket - + loop_var: s3_mount_bucket diff --git a/roles/cs.s3-mount/tasks/003-setup-watchdog.yml b/roles/cs.s3-mount/tasks/003-setup-watchdog.yml index 31139d789..6902e1462 100644 --- a/roles/cs.s3-mount/tasks/003-setup-watchdog.yml +++ b/roles/cs.s3-mount/tasks/003-setup-watchdog.yml @@ -8,28 +8,27 @@ cron: name: "S3FS - FIX MOUNT #{{ s3fs_mount_id }}" minute: "*/2" - job: "/usr/local/bin/mageops-mount-watchdog fix '{{ s3fs_mount.mountpoint }}'" - loop: "{{ s3fs_buckets }}" + job: "/usr/local/bin/mageops-mount-watchdog fix '{{ s3_mount.mountpoint }}'" + loop: "{{ s3_mount_buckets_resolved }}" loop_control: index_var: s3fs_mount_id - loop_var: s3fs_mount + loop_var: s3_mount - name: Install daily remount cronjob cron: name: "S3FS - REMOUNT #{{ s3fs_mount_id }}" minute: 0 hour: 4 - job: "/usr/bin/systemd-cat --identifier=mageops-mount-watchdog /usr/local/bin/mageops-mount-watchdog remount '{{ s3fs_mount.mountpoint }}'" - loop: "{{ s3fs_buckets }}" + job: "/usr/bin/systemd-cat --identifier=mageops-mount-watchdog /usr/local/bin/mageops-mount-watchdog remount '{{ s3_mount.mountpoint }}'" + loop: "{{ s3_mount_buckets_resolved }}" loop_control: index_var: s3fs_mount_id - loop_var: s3fs_mount + loop_var: s3_mount -- name: Clear S3FS temp daily +- name: Clear S3 mount cache daily cron: name: "S3FS - CLEAN CACHE" hour: 3 minute: 45 job: /usr/bin/bash -c 'rm -rf "{{ s3fs_cache_dir }}"/*' - when: s3fs_use_cache and not s3fs_mount_use_goofys - + when: s3fs_use_cache and s3_mount_uses_cache diff --git a/roles/cs.s3-mount/tasks/main.yml b/roles/cs.s3-mount/tasks/main.yml index 5cfb2486f..8954ff3f7 100644 --- a/roles/cs.s3-mount/tasks/main.yml +++ b/roles/cs.s3-mount/tasks/main.yml @@ -1,10 +1,14 @@ +- import_tasks: 000-validate-config.yml +- import_tasks: 001-prepare-mount-options.yml + - import_tasks: 001-create-premount-backups.yml - name: Setup mounts block: - include_tasks: 002-setup-mounts.yml - include_tasks: 003-setup-watchdog.yml + when: s3_mount_backend == 's3fs' - include_tasks: 004-restore-premount-backups.yml when: s3fs_premount_bind_mounts | length always: - - include_tasks: 005-cleanup.yml \ No newline at end of file + - include_tasks: 005-cleanup.yml diff --git a/roles/cs.s3-mount/templates/mount-watchdog.bash b/roles/cs.s3-mount/templates/mount-watchdog.bash index 50fe6de73..a2e04c8bb 100644 --- a/roles/cs.s3-mount/templates/mount-watchdog.bash +++ b/roles/cs.s3-mount/templates/mount-watchdog.bash @@ -3,8 +3,8 @@ set -eu if [[ $# -ne 2 ]] ; then - echo -e "What: MageOps Mount WatchDog" - echo -e "Why: Detect and attempt to fix a crashed FUSE mount" + echo -e "What: MageOps S3 Mount WatchDog" + echo -e "Why: Detect and attempt to fix a crashed S3 mount" echo -e "How: $0 fix|remount /path/to/mountpoint" exit 1 fi @@ -37,6 +37,51 @@ mount-watchdog::get_source() { findmnt --fstab --first-only --noheadings --output SOURCE "${MOUNTPOINT}" } +mount-watchdog::get_fstype() { + findmnt --fstab --first-only --noheadings --output FSTYPE "${MOUNTPOINT}" +} + +mount-watchdog::get_backend() { + local SOURCE + local FSTYPE + + SOURCE="$(mount-watchdog::get_source)" + FSTYPE="$(mount-watchdog::get_fstype)" + + if [[ "${SOURCE}" == s3fs#* ]] ; then + echo "s3fs" + elif [[ "${SOURCE}" == goofys#* ]] ; then + echo "goofys" + elif [[ "${FSTYPE}" == "rclone" ]] ; then + echo "rclone" + else + return 1 + fi +} + +mount-watchdog::get_kill_pattern() { + local BACKEND + local SOURCE + local SOURCE_PATTERN + + BACKEND="$(mount-watchdog::get_backend)" || return 1 + SOURCE="$(mount-watchdog::get_source)" + + case "${BACKEND}" in + s3fs) + SOURCE_PATTERN="$(printf '%s' "${SOURCE}" | sed 's/#\+/.*/g')" + echo "s3fs.*${SOURCE_PATTERN}.*${MOUNTPOINT}" + ;; + goofys) + SOURCE_PATTERN="$(printf '%s' "${SOURCE}" | sed 's/#\+/.*/g')" + echo "goofys.*${SOURCE_PATTERN}.*${MOUNTPOINT}" + ;; + rclone) + echo "rclone.*${SOURCE}.*${MOUNTPOINT}" + ;; + esac +} + mount-watchdog::unmount() { ( mount-watchdog::log "INFO" "Attempting NORMAL unmount" ; umount "${MOUNTPOINT}" ) || \ ( mount-watchdog::log "INFO" "Attempting FORCED unmount" ; umount -f "${MOUNTPOINT}" ) || \ @@ -54,9 +99,14 @@ mount-watchdog::mount() { } mount-watchdog::kill() { - local PATTERN="$(mount-watchdog::get_source | sed 's/#\+/.*/g').*${MOUNTPOINT}" - - mount-watchdog::log "INFO" "Attempting to kill fuse processes matching '$PATTERN'" + local PATTERN + + PATTERN="$(mount-watchdog::get_kill_pattern)" || { + mount-watchdog::log "WARNING" "Unable to determine backend kill pattern" + return 1 + } + + mount-watchdog::log "INFO" "Attempting to kill mount processes matching '$PATTERN'" if ! pkill -f "$PATTERN" --echo 2>/dev/null ; then mount-watchdog::log "NOTICE" "No process found to be killed" @@ -89,7 +139,7 @@ mount-watchdog::fix() { mount-watchdog::log "WARNING" "Unmounting failed" >&2 if mount-watchdog::kill ; then - mount-watchdog::log "NOTICE" "Attempting one more unmount after successfull kill" >&2 + mount-watchdog::log "NOTICE" "Attempting one more unmount after successful kill" >&2 mount-watchdog::unmount fi fi @@ -109,5 +159,3 @@ fi "mount-watchdog::${SUBCOMMAND}" - - diff --git a/site.step-40-app-node.yml b/site.step-40-app-node.yml index 504a8781c..efdca46b3 100644 --- a/site.step-40-app-node.yml +++ b/site.step-40-app-node.yml @@ -133,10 +133,13 @@ when: blackfire_install - role: cs.s3-fuse-s3fs - when: aws_use and not s3fs_mount_use_goofys | default(false) + when: aws_use and s3_mount_backend | default('s3fs') == 's3fs' - role: cs.s3-fuse-goofys - when: aws_use and s3fs_mount_use_goofys | default(false) + when: aws_use and s3_mount_backend | default('s3fs') == 'goofys' + + - role: cs.s3-fuse-rclone + when: aws_use and s3_mount_backend | default('s3fs') == 'rclone' - role: pinkeen.postfix From 48b2fd5c64dea5b52f6f91fa3d9fbc36dcf6bc33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3=C5=BAwiak?= Date: Mon, 16 Mar 2026 13:30:34 +0100 Subject: [PATCH 11/15] Add slowlog depth trace option to PHP --- group_vars/all.yml | 2 +- roles/cs.php-fpm/defaults/main.yml | 1 + roles/cs.php-fpm/templates/php-fpm.pool.conf | 3 +++ site.step-40-app-node.yml | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/group_vars/all.yml b/group_vars/all.yml index df68f5444..aa7405c4d 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1480,7 +1480,7 @@ php_cli_opcache_file_cache_enable: no php_fpm_pm_type: static php_fpm_pm_max_children: "" php_fpm_pm_max_requests: 100 - +mageops_php_fpm_slowlog_trace_depth: 30 # ----------------------------- # -------- Blackfire -------- # ----------------------------- diff --git a/roles/cs.php-fpm/defaults/main.yml b/roles/cs.php-fpm/defaults/main.yml index 4e69685fe..88ee7c2cd 100644 --- a/roles/cs.php-fpm/defaults/main.yml +++ b/roles/cs.php-fpm/defaults/main.yml @@ -14,6 +14,7 @@ php_fpm_pm_min_spare_servers: 5 php_fpm_pm_max_spare_servers: 5 php_fpm_pm_max_requests: 500 php_fpm_terminate_timeout: 600 +php_fpm_slowlog_trace_depth: 30 # Used for naming files, directories and services php_fpm_pool_name: "app" diff --git a/roles/cs.php-fpm/templates/php-fpm.pool.conf b/roles/cs.php-fpm/templates/php-fpm.pool.conf index 278eb1f12..14022eef1 100644 --- a/roles/cs.php-fpm/templates/php-fpm.pool.conf +++ b/roles/cs.php-fpm/templates/php-fpm.pool.conf @@ -26,6 +26,9 @@ ping.response = "{{ php_fpm_pool_ping_response }}" request_terminate_timeout = {{ php_fpm_terminate_timeout }} request_slowlog_timeout = 5 +request_slowlog_trace_depth = {{ php_fpm_slowlog_trace_depth }} + + slowlog = {{ php_fpm_log_dir_path }}/{{ php_fpm_pool_name }}.slow.log catch_workers_output = yes diff --git a/site.step-40-app-node.yml b/site.step-40-app-node.yml index efdca46b3..98e8ec9e9 100644 --- a/site.step-40-app-node.yml +++ b/site.step-40-app-node.yml @@ -110,6 +110,7 @@ php_fpm_pool_user: "{{ magento_user }}" php_fpm_pool_group: "{{ magento_group }}" php_fpm_terminate_timeout: "{{ mageops_http_pipeline_request_timeout_override }}" + php_fpm_slowlog_trace_depth: "{{ mageops_php_fpm_slowlog_trace_depth }}" - role: cs.php-tideways From 13e3a9a9a8c6a3a5f4283d426f5c4b577883f402 Mon Sep 17 00:00:00 2001 From: Szpadel Date: Tue, 17 Mar 2026 10:37:24 +0100 Subject: [PATCH 12/15] Apply suggestions from code review Co-authored-by: Szpadel --- group_vars/all.yml | 2 +- site.step-40-app-node.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/group_vars/all.yml b/group_vars/all.yml index aa7405c4d..8c5879ab4 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1480,7 +1480,7 @@ php_cli_opcache_file_cache_enable: no php_fpm_pm_type: static php_fpm_pm_max_children: "" php_fpm_pm_max_requests: 100 -mageops_php_fpm_slowlog_trace_depth: 30 +php_fpm_slowlog_trace_depth: 30 # ----------------------------- # -------- Blackfire -------- # ----------------------------- diff --git a/site.step-40-app-node.yml b/site.step-40-app-node.yml index 98e8ec9e9..efdca46b3 100644 --- a/site.step-40-app-node.yml +++ b/site.step-40-app-node.yml @@ -110,7 +110,6 @@ php_fpm_pool_user: "{{ magento_user }}" php_fpm_pool_group: "{{ magento_group }}" php_fpm_terminate_timeout: "{{ mageops_http_pipeline_request_timeout_override }}" - php_fpm_slowlog_trace_depth: "{{ mageops_php_fpm_slowlog_trace_depth }}" - role: cs.php-tideways From 94337ccebd5ab4e63efe6b9b3831dc5628eb6048 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Tue, 17 Mar 2026 10:45:04 +0100 Subject: [PATCH 13/15] Fix plugin reinstall logic Version check was using unpopulated variables for upgrade detection --- roles/cs.opensearch/tasks/main.yml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/roles/cs.opensearch/tasks/main.yml b/roles/cs.opensearch/tasks/main.yml index 4a210154a..36a365226 100644 --- a/roles/cs.opensearch/tasks/main.yml +++ b/roles/cs.opensearch/tasks/main.yml @@ -39,6 +39,25 @@ command: systemd-tmpfiles --create when: tmpfiles_config is changed +- name: Detect installed opensearch version + shell: | + find /usr/share/opensearch/lib/ -type f -regex '.*/opensearch-[0-9]+\..*\.jar' -printf '%P' | sed -E 's/opensearch-(([0-9]+\.)+[0-9]+).*.jar/\1/' + register: opensearch_get_installed_version + failed_when: >- + opensearch_get_installed_version.rc | default(0, true) | int != 0 + or opensearch_get_installed_version.stdout_lines | default([], true) | length == 0 + changed_when: false + +- name: Detect running opensearch version + uri: + url: "http://{{ opensearch_network_host }}:{{ opensearch_http_port }}" + return_content: yes + url_password: "{{ mageops_opensearch_password | default(omit) }}" + url_username: "{{ mageops_opensearch_username | default(omit) }}" + register: opensearch_get_running_version + failed_when: false + changed_when: false + - name: Configure opensearch template: src: "{{ item.key }}" @@ -86,15 +105,11 @@ - name: Force update of opensearch plugins on version change block: - - name: Get list of installed opensearch plugins - command: /usr/share/opensearch/bin/opensearch-plugin list --silent - register: opensearch_plugin_list_command - - name: Remove current opensearch plugins so new version is installed command: "/usr/share/opensearch/bin/opensearch-plugin remove {{ item }} --verbose" args: removes: "/usr/share/opensearch/plugins/{{ item }}" - loop: "{{ opensearch_plugin_list_command.stdout_lines }}" + loop: "{{ opensearch_plugins }}" when: >- opensearch_version_number != opensearch_running_version_number @@ -127,4 +142,3 @@ port: "{{ opensearch_http_port }}" delay: 6 timeout: 60 - From 159142f36a1283df51b27515de70d7ab47a951ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3=C5=BAwiak?= Date: Wed, 18 Mar 2026 08:36:04 +0100 Subject: [PATCH 14/15] Move lazy resize to etc.php --- roles/cs.magento-configure/defaults/main/app-etc.yml | 4 ++++ roles/cs.magento-configure/tasks/action/configure-env.yml | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/roles/cs.magento-configure/defaults/main/app-etc.yml b/roles/cs.magento-configure/defaults/main/app-etc.yml index a4a5b0f83..365d71736 100644 --- a/roles/cs.magento-configure/defaults/main/app-etc.yml +++ b/roles/cs.magento-configure/defaults/main/app-etc.yml @@ -80,6 +80,10 @@ magento_app_etc_config: install: date: "Tue, 11 Nov 2016 11:11:00 +0000" +magento_app_etc_config_lazy_resize: + lazy_resize: + secret : "{{ lazy_resize_secret }}" + magento_mysql_ssl_required: db: connection: diff --git a/roles/cs.magento-configure/tasks/action/configure-env.yml b/roles/cs.magento-configure/tasks/action/configure-env.yml index e4fa1de78..ceaaa0ff3 100644 --- a/roles/cs.magento-configure/tasks/action/configure-env.yml +++ b/roles/cs.magento-configure/tasks/action/configure-env.yml @@ -7,6 +7,13 @@ {{ magento_app_etc_config | combine(magento_app_etc_config_http_cache, recursive=true) }} + - name: Enable lazy resize configuration + when: lazy_resize_secret + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_lazy_resize, recursive=true) }} + - name: Enable ElasticSuite configuration when: elasticsuite_version | default(false) and (mageops_elasticsearch_opensearch_flavor == "elasticsearch") set_fact: From e4dcd66059400e5b341f6f527ce28bf953fd6348 Mon Sep 17 00:00:00 2001 From: Piotr Matras Date: Thu, 19 Mar 2026 16:28:22 +0100 Subject: [PATCH 15/15] feat: DEVOPS-627 allow cache search results --- roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 index e9af13db4..7b35f3200 100644 --- a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 @@ -192,8 +192,8 @@ if (req.http.X-Blackfire-Query) { } } -# Bypass customer, shopping cart, checkout and search requests -if (req.url ~ "/customer" || req.url ~ "/checkout" || req.url ~ "/catalogsearch") { +# Bypass customer, shopping cart, checkout +if (req.url ~ "/customer" || req.url ~ "/checkout") { return (pass); }