Fix logrotate and configure logging

README.md

@@ -191,7 +191,10 @@ config/
   cron/mediawiki-jobs             ← MediaWiki job runner cron
   httpd/mediawiki.conf            ← Apache vhost template
   httpd/security.conf             ← Apache security headers
-  logrotate/httpd-mediawiki       ← Log rotation for Apache
+  logrotate/httpd-mediawiki       ← Daily rotation for /var/log/mediawiki/error.log
+  logrotate/mediawiki-access      ← Hourly rotation for /var/log/mediawiki/access.log
+  logrotate/php-fpm-mediawiki     ← Daily rotation for php-fpm's error/www/slow logs
+  logrotate/mariadb                ← Daily rotation for MariaDB's error/slow-query logs
   logrotate/mediawiki-backup      ← Log rotation for backups
   mariadb/mariadb.repo            ← MariaDB 10.11 yum repo
   mariadb/mediawiki.cnf           ← MariaDB tuning
@@ -200,6 +203,7 @@ config/
   mediawiki/robots.txt            ← robots.txt for the wiki
   mediawiki/assets/               ← Static assets copied to DocumentRoot (logos, favicons)
   php/mediawiki.ini               ← PHP tuning
+  php/www-mediawiki.conf          ← php-fpm pool overrides (timeouts, logging — see 01-php.sh)
   system/limits.conf              ← OS limits
   system/mediawiki-backup.sysconfig ← /etc/sysconfig for backup cron
   system/sysctl.conf              ← Kernel tuning

config/cloudwatch/mediawiki-cwa.json

@@ -8,7 +8,7 @@
       "files": {
         "collect_list": [
           {
-            "file_path": "/var/log/httpd/mediawiki-error.log",
+            "file_path": "/var/log/mediawiki/error.log",
             "log_group_name": "/mediawiki/apache/error",
             "log_stream_name": "{instance_id}",
             "retention_in_days": 7,
@@ -21,13 +21,34 @@
             "retention_in_days": 7,
             "auto_removal": true
           },
+          {
+            "file_path": "/var/log/mariadb/mariadb-error.log",
+            "log_group_name": "/mediawiki/mariadb/error",
+            "log_stream_name": "{instance_id}",
+            "retention_in_days": 7,
+            "auto_removal": true
+          },
           {
             "file_path": "/var/log/mariadb/slow.log",
             "log_group_name": "/mediawiki/mariadb/slow",
             "log_stream_name": "{instance_id}",
             "retention_in_days": 7,
             "auto_removal": true
           },
+          {
+            "file_path": "/var/log/mediawiki/php-fpm-error.log",
+            "log_group_name": "/mediawiki/php-fpm/error",
+            "log_stream_name": "{instance_id}",
+            "retention_in_days": 7,
+            "auto_removal": true
+          },
+          {
+            "file_path": "/var/log/mediawiki/php-fpm-www.log",
+            "log_group_name": "/mediawiki/php-fpm/www-error",
+            "log_stream_name": "{instance_id}",
+            "retention_in_days": 7,
+            "auto_removal": true
+          },
           {
             "file_path": "/var/log/mediawiki-backup.log",
             "log_group_name": "/mediawiki/backup",

config/cron/mediawiki-access-logrotate

@@ -0,0 +1,8 @@
+# mediawiki-access-logrotate — cron.d job to rotate the Apache access log hourly
+# Uploaded to /tmp/ by Packer; installed by 03-httpd.sh.
+# Uses a dedicated state file so this run never conflicts with the system's
+# normal daily logrotate.timer pass over /etc/logrotate.d/.
+SHELL=/bin/bash
+
+5 * * * * root /usr/sbin/logrotate --state /var/lib/logrotate/mediawiki-access.status /etc/logrotate-mediawiki-access.conf
+

config/httpd/mediawiki.conf

@@ -97,6 +97,6 @@
         </FilesMatch>
     </Directory>
 
-    ErrorLog  /var/log/httpd/mediawiki-error.log
-    CustomLog /var/log/httpd/mediawiki-access.log combined
+    ErrorLog  /var/log/mediawiki/error.log
+    CustomLog /var/log/mediawiki/access.log combined
 </VirtualHost>

config/logrotate/httpd-mediawiki

@@ -1,6 +1,7 @@
-# httpd-mediawiki — logrotate config for Apache MediaWiki logs
-# Uploaded to /tmp/ by Packer; installed by 03-httpd.sh.
-/var/log/httpd/mediawiki-*.log {
+# httpd-mediawiki — logrotate config for the MediaWiki vhost's Apache error log
+# Uploaded to /tmp/ by Packer; installed by 03-httpd.sh into /etc/logrotate.d/,
+# where it is picked up by the system's normal daily logrotate.timer run.
+/var/log/mediawiki/error.log {
     daily
     missingok
     rotate 14

config/logrotate/mediawiki-access

@@ -0,0 +1,21 @@
+# mediawiki-access — logrotate config for the high-volume Apache access log
+# Uploaded to /tmp/ by Packer; installed by 03-httpd.sh at
+# /etc/logrotate-mediawiki-access.conf (deliberately OUTSIDE /etc/logrotate.d/
+# so the system's daily logrotate.timer never processes it).
+#
+# Invoked hourly, with its own state file, by cron/mediawiki-access-logrotate.
+/var/log/mediawiki/access.log {
+    hourly
+    missingok
+    rotate 48
+    compress
+    delaycompress
+    notifempty
+    maxsize 100M
+    sharedscripts
+    postrotate
+        /usr/bin/systemctl reload httpd > /dev/null 2>&1 || true
+    endscript
+}
+
+

config/logrotate/mediawiki-backup

@@ -7,5 +7,6 @@
     compress
     delaycompress
     notifempty
+    create 644 root root
 }
 

config/logrotate/php-fpm-mediawiki

@@ -0,0 +1,17 @@
+# php-fpm-mediawiki — logrotate config for php-fpm's own logs
+# Uploaded to /tmp/ by Packer; installed by 01-php.sh into /etc/logrotate.d/.
+/var/log/mediawiki/php-fpm-error.log
+/var/log/mediawiki/php-fpm-www.log
+/var/log/mediawiki/php-fpm-slow.log {
+    daily
+    missingok
+    rotate 14
+    compress
+    delaycompress
+    notifempty
+    sharedscripts
+    postrotate
+        /usr/bin/systemctl kill -s USR1 php-fpm.service > /dev/null 2>&1 || true
+    endscript
+}
+

config/mariadb/mediawiki.cnf

@@ -18,7 +18,9 @@ collation_server     = utf8mb4_unicode_ci
 query_cache_type = 0
 query_cache_size = 0
 
-# Logging — enable slow query log for debugging
+# Without an explicit log_error, MariaDB logs only to journald
+# which still works but isn't covered by CloudWatch.
+log_error            = /var/log/mariadb/mariadb-error.log
 slow_query_log      = 1
 slow_query_log_file = /var/log/mariadb/slow.log
 long_query_time     = 2

config/php/mediawiki.ini

@@ -24,6 +24,10 @@ date.timezone           = UTC
 expose_php              = Off
 display_errors          = Off
 log_errors              = On
+; NOTE: for web requests, php-fpm's zz-mediawiki.conf pool override
+; (php_admin_value[error_log]) takes precedence over this — php_admin_value
+; always wins over php.ini. This error_log is therefore only actually
+; effective for PHP CLI (cron/maintenance script) invocations.
 error_log               = /var/log/php_errors.log
 display_startup_errors  = Off
 

config/php/www-mediawiki.conf

@@ -0,0 +1,31 @@
+; www-mediawiki.conf — pool-level overrides for the default [www] pool
+; Installed at /etc/php-fpm.d/zz-mediawiki.conf by 01-php.sh. The "zz-" prefix
+; makes it load after the package's own www.conf (php-fpm.d/*.conf is read in
+; lexical order); directives repeated for the same pool name are last-wins,
+; so these override www.conf without editing or replacing that package file.
+[www]
+
+; ── Guard against a stuck worker holding a pm.max_children slot forever ──────
+; This is the most common real cause of "server reached pm.max_children"
+; warnings — not genuine undersized capacity. max_execution_time in
+; mediawiki.ini is 60s; terminate a little after that as a hard backstop so a
+; single hung request (e.g. a slow external SSO/API call) can't wedge a slot.
+request_terminate_timeout = 65s
+
+; Recycle workers periodically to bound slow memory growth across requests.
+pm.max_requests = 500
+
+; Diagnostics: if pm.max_children is ever exhausted again, this records which
+; request/backtrace was stuck instead of just a bare warning with no context.
+slowlog = /var/log/mediawiki/php-fpm-slow.log
+request_slowlog_timeout = 10s
+
+; ── Logging ───────────────────────────────────────────────────────────────────
+; php_admin_value always overrides php.ini for FPM-handled requests, so
+; mediawiki.ini's error_log is only actually effective for CLI (cron/
+; maintenance) scripts. Route the web-facing pool log into our own directory,
+; alongside the Apache vhost logs, so it rotates under logrotate/php-fpm-mediawiki
+; instead of the package's /var/log/php-fpm/*.log glob.
+php_admin_value[error_log] = /var/log/mediawiki/php-fpm-www.log
+php_admin_flag[log_errors] = on
+

packer/scripts/01-php.sh

@@ -50,6 +50,30 @@ mkdir -p "${PHP_INI_DIR}"
 
 cp /tmp/config/php/mediawiki.ini "${PHP_INI_DIR}/mediawiki.ini"
 
+# ── php-fpm pool tuning + logging ─────────────────────────────────────────────
+# Shared with the Apache vhost logs — see 03-httpd.sh for the full rationale
+# on keeping MediaWiki's own logs out of package-managed directories/globs.
+mkdir -p /var/log/mediawiki
+chmod 755 /var/log/mediawiki
+
+# Drop-in pool overrides (merged into the package's [www] pool — see the file
+# itself for why this doesn't require editing/replacing www.conf).
+cp /tmp/config/php/www-mediawiki.conf /etc/php-fpm.d/zz-mediawiki.conf
+
+# The global error_log (in php-fpm.conf's [global] section, not per-pool) is
+# where the master process logs "server reached pm.max_children" warnings.
+# Redirect it out of /var/log/php-fpm/ for the same reason as the pool log
+# above. This edits the package's php-fpm.conf in place (never deleted), so
+# a future `dnf update` that ships a new default keeps our line intact and
+# drops its own version alongside as *.rpmnew rather than silently reverting it.
+sed -i -E 's#^;?\s*error_log\s*=.*#error_log = /var/log/mediawiki/php-fpm-error.log#' \
+  /etc/php-fpm.conf
+grep -q '^error_log = /var/log/mediawiki/php-fpm-error.log$' /etc/php-fpm.conf \
+  || { echo "ERROR: Failed to redirect php-fpm's global error_log in /etc/php-fpm.conf"; exit 1; }
+
+# Log rotation for the two paths above (daily, compressed, 14-day retention).
+cp /tmp/config/logrotate/php-fpm-mediawiki /etc/logrotate.d/php-fpm-mediawiki
+
 # ── Verify ────────────────────────────────────────────────────────────────────
 php --version
 php -m | grep -E "mbstring|intl|xml|gd|curl|opcache|mysqlnd"

packer/scripts/02-mariadb.sh

@@ -12,6 +12,19 @@ cp /tmp/config/mariadb/mariadb.repo /etc/yum.repos.d/mariadb.repo
 
 dnf install -y MariaDB-server MariaDB-client
 
+# ── Log directory ─────────────────────────────────────────────────────────────
+# The upstream MariaDB.org repo package (unlike AL2023's own mariadb105
+# packages) relies on a systemd-tmpfiles.d rule to create /var/log/mariadb,
+# which is only applied by systemd-tmpfiles-setup.service very early at boot —
+# it does NOT run as part of `dnf install`. Since this directory doesn't exist
+# yet when mysqld first starts below, slow_query_log_file / log_error in
+# mediawiki.cnf would silently fail to open (slow query logging disabled with
+# no visible error, general error log falling back to journald only). Create
+# it explicitly so both are guaranteed to work regardless of tmpfiles timing.
+mkdir -p /var/log/mariadb
+chown mysql:mysql /var/log/mariadb
+chmod 750 /var/log/mariadb
+
 # ── Start MariaDB for setup ───────────────────────────────────────────────────
 systemctl enable mariadb
 systemctl start mariadb

packer/scripts/03-httpd.sh

@@ -39,13 +39,41 @@ fi
 
 # ── MediaWiki vhost ───────────────────────────────────────────────────────────
 mkdir -p /var/www/mediawiki
-mkdir -p /var/log/httpd
+
+# MediaWiki's own error/access logs live in a dedicated directory, deliberately
+# separate from /var/log/httpd/. The stock RPM's /etc/logrotate.d/httpd
+# (glob "/var/log/httpd/*log") already owns everything httpd itself writes
+# there (ssl_error_log, ssl_request_log, etc.) — that file is package-managed
+# and gets silently restored on the next `dnf update httpd` if we delete or
+# edit it, so we leave it completely alone. Giving our vhost logs their own
+# directory means there's no glob overlap for logrotate to conflict on, no
+# matter what the httpd package does with its own config.
+# MediaWiki's own error/access logs live in a dedicated directory, deliberately
+# separate from /var/log/httpd/. The stock RPM's /etc/logrotate.d/httpd
+# (glob "/var/log/httpd/*log") already owns everything httpd itself writes
+# there (ssl_error_log, ssl_request_log, etc.) — that file is package-managed
+# and gets silently restored on the next `dnf update httpd` if we delete or
+# edit it, so we leave it completely alone. Giving our vhost logs their own
+# directory means there's no glob overlap for logrotate to conflict on, no
+# matter what the httpd package does with its own config.
+# (Already created by 01-php.sh, which needs it earlier for php-fpm's own
+# logs — mkdir -p here is just a defensive no-op if run standalone.)
+mkdir -p /var/log/mediawiki
+chmod 755 /var/log/mediawiki
 
 cp /tmp/config/httpd/mediawiki.conf /etc/httpd/conf.d/mediawiki.conf
 
 # ── Log rotation ──────────────────────────────────────────────────────────────
+# Error log: daily, picked up by the system's normal daily logrotate.timer run.
 cp /tmp/config/logrotate/httpd-mediawiki /etc/logrotate.d/httpd-mediawiki
 
+# Access log: high-volume, needs to rotate far more often than once a day.
+# Installed outside /etc/logrotate.d/ so the daily logrotate.timer never
+# touches it, and rotated hourly by its own cron job with its own state file.
+cp /tmp/config/logrotate/mediawiki-access /etc/logrotate-mediawiki-access.conf
+cp /tmp/config/cron/mediawiki-access-logrotate /etc/cron.d/mediawiki-access-logrotate
+chmod 644 /etc/cron.d/mediawiki-access-logrotate
+
 # ── Enable and start ──────────────────────────────────────────────────────────
 httpd -t   # fail fast with a readable error if config is broken
 systemctl enable httpd

packer/scripts/05-extensions.sh

@@ -106,11 +106,24 @@ echo "Composer update completed successfully"
 # DB_MASTER was renamed to DB_PRIMARY in MW 1.37 and removed in 1.42.
 # The 6.x release that fixed this requires PHP 8.4, which MW 1.43 does not
 # support. Patch the installed source directly; DB_PRIMARY has the same value.
-# TODO: [16-Jul-2026 04:01:55 UTC] PHP Deprecated:  Use of wfGetDB was deprecated in MediaWiki 1.39. [Called from MediaWiki\Extension\DiscourseSsoConsumer\Db::ensureCurrentSchema in /var/www/mediawiki/extensions/DiscourseSsoConsumer/src/Db.php at line 83] in /var/www/mediawiki/includes/debug/MWDebug.php on line 385
 echo "Patching DiscourseSsoConsumer: DB_MASTER → DB_PRIMARY"
 grep -rl --include='*.php' 'DB_MASTER' "${EXT_DIR}/DiscourseSsoConsumer" \
   | xargs sed -i 's/\bDB_MASTER\b/DB_PRIMARY/g'
 
+# wfGetDB() itself (independent of the DB_MASTER/DB_PRIMARY constant above)
+# was deprecated in MW 1.39 in favour of MediaWikiServices::getConnectionProvider().
+# Db::ensureCurrentSchema() calls it on every request, spamming
+# /var/log/mediawiki/php-fpm-www.log with a "PHP Deprecated" notice each time.
+# Patch to the non-deprecated equivalent instead of just tolerating the noise.
+echo "Patching DiscourseSsoConsumer: wfGetDB() → MediaWikiServices::getConnectionProvider()"
+{ grep -rlE --include='*.php' 'wfGetDB\s*\(' "${EXT_DIR}/DiscourseSsoConsumer" || true; } \
+  | while IFS= read -r f; do
+      sed -i -E \
+        -e 's/wfGetDB\(\s*DB_PRIMARY\s*\)/\\MediaWiki\\MediaWikiServices::getInstance()->getConnectionProvider()->getPrimaryDatabase()/g' \
+        -e 's/wfGetDB\(\s*DB_REPLICA\s*\)/\\MediaWiki\\MediaWikiServices::getInstance()->getConnectionProvider()->getReplicaDatabase()/g' \
+        "$f"
+    done
+
 EXPECTED_EXTENSIONS=("TemplateStyles" "DiscourseSsoConsumer" "IFrameTag" "PluggableAuth" "JsonConfig" "WikiCategoryTagCloud")
 for ext in "${EXPECTED_EXTENSIONS[@]}"; do
   if [ ! -d "${EXT_DIR}/${ext}" ]; then

packer/scripts/07-backup-setup.sh

@@ -32,8 +32,12 @@ chmod 644 /etc/cron.d/mediawiki-backup
 cp /tmp/config/logrotate/mediawiki-backup /etc/logrotate.d/mediawiki-backup
 
 # ── Create the log file ───────────────────────────────────────────────────────
+# World-readable: the amazon-cloudwatch-agent service does not run as root
+# (it tails files as its own service user), and this log contains no secrets
+# (DB credentials are never echoed by the backup scripts) — 640 root-owned
+# caused "permission denied" for the agent's [inputs.logfile] tailer.
 touch /var/log/mediawiki-backup.log
-chmod 640 /var/log/mediawiki-backup.log
+chmod 644 /var/log/mediawiki-backup.log
 
 # ── Remove staged config (last phase — no longer needed) ─────────────────────
 rm -rf /tmp/config

scripts/backup/backup-with-retention.sh

@@ -60,6 +60,22 @@ command -v mysqldump >/dev/null || fail "mysqldump not found"
 LSETTINGS="${MW_ROOT}/LocalSettings.php"
 [ -f "${LSETTINGS}" ] || fail "LocalSettings.php not found"
 
+# ── Preflight: enough space in the work dir's filesystem? ───────────────────
+# Images copy compresses ~1:1 in the worst case (already-compressed media);
+# require room for a full uncompressed copy of images/ plus the DB, with a
+# safety margin, instead of failing midway through tar/gzip with a cryptic
+# "No space left on device".
+NEEDED_KB=0
+if [ -d "${MW_ROOT}/images" ]; then
+  NEEDED_KB=$(du -sk "${MW_ROOT}/images" | cut -f1)
+fi
+NEEDED_KB=$(( (NEEDED_KB * 12) / 10 + 262144 ))  # +20% margin, +256MB for the DB dump
+AVAIL_KB=$(df -Pk "${WORK_DIR}" | awk 'NR==2 {print $4}')
+if [ "${AVAIL_KB}" -lt "${NEEDED_KB}" ]; then
+  df -h "${WORK_DIR}" >&2
+  fail "Not enough free space in $(dirname "${WORK_DIR}") (have ${AVAIL_KB}KB, need ~${NEEDED_KB}KB). Check for oversized logs under /var/log (see logrotate.d/httpd-mediawiki, logrotate-mediawiki-access.conf) before retrying."
+fi
+
 # ── Extract DB credentials from LocalSettings.php ────────────────────────────
 extract_setting() {
   # Parse a literal assignment from LocalSettings.php without executing PHP code.

scripts/backup/full-backup.sh

@@ -64,6 +64,18 @@ command -v mysqldump >/dev/null || fail "mysqldump not found"
 command -v tar      >/dev/null || fail "tar not found"
 [ -d "${MW_ROOT}" ] || fail "MediaWiki root not found: ${MW_ROOT}"
 
+# ── Preflight: enough space in the work dir's filesystem? ───────────────────
+NEEDED_KB=0
+if [ -d "${MW_ROOT}/images" ]; then
+  NEEDED_KB=$(du -sk "${MW_ROOT}/images" | cut -f1)
+fi
+NEEDED_KB=$(( (NEEDED_KB * 12) / 10 + 262144 ))  # +20% margin, +256MB for the DB dump
+AVAIL_KB=$(df -Pk "${WORK_DIR}" | awk 'NR==2 {print $4}')
+if [ "${AVAIL_KB}" -lt "${NEEDED_KB}" ]; then
+  df -h "${WORK_DIR}" >&2
+  fail "Not enough free space in $(dirname "${WORK_DIR}") (have ${AVAIL_KB}KB, need ~${NEEDED_KB}KB). Check for oversized logs under /var/log before retrying."
+fi
+
 # ── Auto-detect DB credentials from LocalSettings.php ────────────────────────
 LSETTINGS="${MW_ROOT}/LocalSettings.php"
 [ -f "${LSETTINGS}" ] || fail "LocalSettings.php not found at ${LSETTINGS}"