diff --git a/README.md b/README.md index fb07ecd..87b7af3 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,10 @@ config/ cron/mediawiki-jobs ← MediaWiki job runner cron httpd/mediawiki.conf ← Apache vhost template httpd/security.conf ← Apache security headers - logrotate/httpd-mediawiki ← Log rotation for Apache + logrotate/httpd-mediawiki ← Daily rotation for /var/log/mediawiki/error.log + logrotate/mediawiki-access ← Hourly rotation for /var/log/mediawiki/access.log + logrotate/php-fpm-mediawiki ← Daily rotation for php-fpm's error/www/slow logs + logrotate/mariadb ← Daily rotation for MariaDB's error/slow-query logs logrotate/mediawiki-backup ← Log rotation for backups mariadb/mariadb.repo ← MariaDB 10.11 yum repo mariadb/mediawiki.cnf ← MariaDB tuning @@ -200,6 +203,7 @@ config/ mediawiki/robots.txt ← robots.txt for the wiki mediawiki/assets/ ← Static assets copied to DocumentRoot (logos, favicons) php/mediawiki.ini ← PHP tuning + php/www-mediawiki.conf ← php-fpm pool overrides (timeouts, logging — see 01-php.sh) system/limits.conf ← OS limits system/mediawiki-backup.sysconfig ← /etc/sysconfig for backup cron system/sysctl.conf ← Kernel tuning diff --git a/config/cloudwatch/mediawiki-cwa.json b/config/cloudwatch/mediawiki-cwa.json index aa9e111..403a2d7 100644 --- a/config/cloudwatch/mediawiki-cwa.json +++ b/config/cloudwatch/mediawiki-cwa.json @@ -8,7 +8,7 @@ "files": { "collect_list": [ { - "file_path": "/var/log/httpd/mediawiki-error.log", + "file_path": "/var/log/mediawiki/error.log", "log_group_name": "/mediawiki/apache/error", "log_stream_name": "{instance_id}", "retention_in_days": 7, @@ -21,6 +21,13 @@ "retention_in_days": 7, "auto_removal": true }, + { + "file_path": "/var/log/mariadb/mariadb-error.log", + "log_group_name": "/mediawiki/mariadb/error", + "log_stream_name": "{instance_id}", + "retention_in_days": 7, + "auto_removal": true + }, { "file_path": "/var/log/mariadb/slow.log", "log_group_name": "/mediawiki/mariadb/slow", @@ -28,6 +35,20 @@ "retention_in_days": 7, "auto_removal": true }, + { + "file_path": "/var/log/mediawiki/php-fpm-error.log", + "log_group_name": "/mediawiki/php-fpm/error", + "log_stream_name": "{instance_id}", + "retention_in_days": 7, + "auto_removal": true + }, + { + "file_path": "/var/log/mediawiki/php-fpm-www.log", + "log_group_name": "/mediawiki/php-fpm/www-error", + "log_stream_name": "{instance_id}", + "retention_in_days": 7, + "auto_removal": true + }, { "file_path": "/var/log/mediawiki-backup.log", "log_group_name": "/mediawiki/backup", diff --git a/config/cron/mediawiki-access-logrotate b/config/cron/mediawiki-access-logrotate new file mode 100644 index 0000000..2f35d79 --- /dev/null +++ b/config/cron/mediawiki-access-logrotate @@ -0,0 +1,8 @@ +# mediawiki-access-logrotate — cron.d job to rotate the Apache access log hourly +# Uploaded to /tmp/ by Packer; installed by 03-httpd.sh. +# Uses a dedicated state file so this run never conflicts with the system's +# normal daily logrotate.timer pass over /etc/logrotate.d/. +SHELL=/bin/bash + +5 * * * * root /usr/sbin/logrotate --state /var/lib/logrotate/mediawiki-access.status /etc/logrotate-mediawiki-access.conf + diff --git a/config/httpd/mediawiki.conf b/config/httpd/mediawiki.conf index 0a4771b..076628b 100644 --- a/config/httpd/mediawiki.conf +++ b/config/httpd/mediawiki.conf @@ -97,6 +97,6 @@ - ErrorLog /var/log/httpd/mediawiki-error.log - CustomLog /var/log/httpd/mediawiki-access.log combined + ErrorLog /var/log/mediawiki/error.log + CustomLog /var/log/mediawiki/access.log combined diff --git a/config/logrotate/httpd-mediawiki b/config/logrotate/httpd-mediawiki index 9d8aead..17005b2 100644 --- a/config/logrotate/httpd-mediawiki +++ b/config/logrotate/httpd-mediawiki @@ -1,6 +1,7 @@ -# httpd-mediawiki — logrotate config for Apache MediaWiki logs -# Uploaded to /tmp/ by Packer; installed by 03-httpd.sh. -/var/log/httpd/mediawiki-*.log { +# httpd-mediawiki — logrotate config for the MediaWiki vhost's Apache error log +# Uploaded to /tmp/ by Packer; installed by 03-httpd.sh into /etc/logrotate.d/, +# where it is picked up by the system's normal daily logrotate.timer run. +/var/log/mediawiki/error.log { daily missingok rotate 14 diff --git a/config/logrotate/mediawiki-access b/config/logrotate/mediawiki-access new file mode 100644 index 0000000..f858e6f --- /dev/null +++ b/config/logrotate/mediawiki-access @@ -0,0 +1,21 @@ +# mediawiki-access — logrotate config for the high-volume Apache access log +# Uploaded to /tmp/ by Packer; installed by 03-httpd.sh at +# /etc/logrotate-mediawiki-access.conf (deliberately OUTSIDE /etc/logrotate.d/ +# so the system's daily logrotate.timer never processes it). +# +# Invoked hourly, with its own state file, by cron/mediawiki-access-logrotate. +/var/log/mediawiki/access.log { + hourly + missingok + rotate 48 + compress + delaycompress + notifempty + maxsize 100M + sharedscripts + postrotate + /usr/bin/systemctl reload httpd > /dev/null 2>&1 || true + endscript +} + + diff --git a/config/logrotate/mediawiki-backup b/config/logrotate/mediawiki-backup index 72a6be0..9584e17 100644 --- a/config/logrotate/mediawiki-backup +++ b/config/logrotate/mediawiki-backup @@ -7,5 +7,6 @@ compress delaycompress notifempty + create 644 root root } diff --git a/config/logrotate/php-fpm-mediawiki b/config/logrotate/php-fpm-mediawiki new file mode 100644 index 0000000..df26a2f --- /dev/null +++ b/config/logrotate/php-fpm-mediawiki @@ -0,0 +1,17 @@ +# php-fpm-mediawiki — logrotate config for php-fpm's own logs +# Uploaded to /tmp/ by Packer; installed by 01-php.sh into /etc/logrotate.d/. +/var/log/mediawiki/php-fpm-error.log +/var/log/mediawiki/php-fpm-www.log +/var/log/mediawiki/php-fpm-slow.log { + daily + missingok + rotate 14 + compress + delaycompress + notifempty + sharedscripts + postrotate + /usr/bin/systemctl kill -s USR1 php-fpm.service > /dev/null 2>&1 || true + endscript +} + diff --git a/config/mariadb/mediawiki.cnf b/config/mariadb/mediawiki.cnf index 3b3ef18..21b8ec8 100644 --- a/config/mariadb/mediawiki.cnf +++ b/config/mariadb/mediawiki.cnf @@ -18,7 +18,9 @@ collation_server = utf8mb4_unicode_ci query_cache_type = 0 query_cache_size = 0 -# Logging — enable slow query log for debugging +# Without an explicit log_error, MariaDB logs only to journald +# which still works but isn't covered by CloudWatch. +log_error = /var/log/mariadb/mariadb-error.log slow_query_log = 1 slow_query_log_file = /var/log/mariadb/slow.log long_query_time = 2 diff --git a/config/php/mediawiki.ini b/config/php/mediawiki.ini index 21d0b70..eb29f9a 100644 --- a/config/php/mediawiki.ini +++ b/config/php/mediawiki.ini @@ -24,6 +24,10 @@ date.timezone = UTC expose_php = Off display_errors = Off log_errors = On +; NOTE: for web requests, php-fpm's zz-mediawiki.conf pool override +; (php_admin_value[error_log]) takes precedence over this — php_admin_value +; always wins over php.ini. This error_log is therefore only actually +; effective for PHP CLI (cron/maintenance script) invocations. error_log = /var/log/php_errors.log display_startup_errors = Off diff --git a/config/php/www-mediawiki.conf b/config/php/www-mediawiki.conf new file mode 100644 index 0000000..ddc1d4a --- /dev/null +++ b/config/php/www-mediawiki.conf @@ -0,0 +1,31 @@ +; www-mediawiki.conf — pool-level overrides for the default [www] pool +; Installed at /etc/php-fpm.d/zz-mediawiki.conf by 01-php.sh. The "zz-" prefix +; makes it load after the package's own www.conf (php-fpm.d/*.conf is read in +; lexical order); directives repeated for the same pool name are last-wins, +; so these override www.conf without editing or replacing that package file. +[www] + +; ── Guard against a stuck worker holding a pm.max_children slot forever ────── +; This is the most common real cause of "server reached pm.max_children" +; warnings — not genuine undersized capacity. max_execution_time in +; mediawiki.ini is 60s; terminate a little after that as a hard backstop so a +; single hung request (e.g. a slow external SSO/API call) can't wedge a slot. +request_terminate_timeout = 65s + +; Recycle workers periodically to bound slow memory growth across requests. +pm.max_requests = 500 + +; Diagnostics: if pm.max_children is ever exhausted again, this records which +; request/backtrace was stuck instead of just a bare warning with no context. +slowlog = /var/log/mediawiki/php-fpm-slow.log +request_slowlog_timeout = 10s + +; ── Logging ─────────────────────────────────────────────────────────────────── +; php_admin_value always overrides php.ini for FPM-handled requests, so +; mediawiki.ini's error_log is only actually effective for CLI (cron/ +; maintenance) scripts. Route the web-facing pool log into our own directory, +; alongside the Apache vhost logs, so it rotates under logrotate/php-fpm-mediawiki +; instead of the package's /var/log/php-fpm/*.log glob. +php_admin_value[error_log] = /var/log/mediawiki/php-fpm-www.log +php_admin_flag[log_errors] = on + diff --git a/packer/scripts/01-php.sh b/packer/scripts/01-php.sh index ad43436..bf7d2bb 100644 --- a/packer/scripts/01-php.sh +++ b/packer/scripts/01-php.sh @@ -50,6 +50,30 @@ mkdir -p "${PHP_INI_DIR}" cp /tmp/config/php/mediawiki.ini "${PHP_INI_DIR}/mediawiki.ini" +# ── php-fpm pool tuning + logging ───────────────────────────────────────────── +# Shared with the Apache vhost logs — see 03-httpd.sh for the full rationale +# on keeping MediaWiki's own logs out of package-managed directories/globs. +mkdir -p /var/log/mediawiki +chmod 755 /var/log/mediawiki + +# Drop-in pool overrides (merged into the package's [www] pool — see the file +# itself for why this doesn't require editing/replacing www.conf). +cp /tmp/config/php/www-mediawiki.conf /etc/php-fpm.d/zz-mediawiki.conf + +# The global error_log (in php-fpm.conf's [global] section, not per-pool) is +# where the master process logs "server reached pm.max_children" warnings. +# Redirect it out of /var/log/php-fpm/ for the same reason as the pool log +# above. This edits the package's php-fpm.conf in place (never deleted), so +# a future `dnf update` that ships a new default keeps our line intact and +# drops its own version alongside as *.rpmnew rather than silently reverting it. +sed -i -E 's#^;?\s*error_log\s*=.*#error_log = /var/log/mediawiki/php-fpm-error.log#' \ + /etc/php-fpm.conf +grep -q '^error_log = /var/log/mediawiki/php-fpm-error.log$' /etc/php-fpm.conf \ + || { echo "ERROR: Failed to redirect php-fpm's global error_log in /etc/php-fpm.conf"; exit 1; } + +# Log rotation for the two paths above (daily, compressed, 14-day retention). +cp /tmp/config/logrotate/php-fpm-mediawiki /etc/logrotate.d/php-fpm-mediawiki + # ── Verify ──────────────────────────────────────────────────────────────────── php --version php -m | grep -E "mbstring|intl|xml|gd|curl|opcache|mysqlnd" diff --git a/packer/scripts/02-mariadb.sh b/packer/scripts/02-mariadb.sh index 99b74dc..d5b2074 100644 --- a/packer/scripts/02-mariadb.sh +++ b/packer/scripts/02-mariadb.sh @@ -12,6 +12,19 @@ cp /tmp/config/mariadb/mariadb.repo /etc/yum.repos.d/mariadb.repo dnf install -y MariaDB-server MariaDB-client +# ── Log directory ───────────────────────────────────────────────────────────── +# The upstream MariaDB.org repo package (unlike AL2023's own mariadb105 +# packages) relies on a systemd-tmpfiles.d rule to create /var/log/mariadb, +# which is only applied by systemd-tmpfiles-setup.service very early at boot — +# it does NOT run as part of `dnf install`. Since this directory doesn't exist +# yet when mysqld first starts below, slow_query_log_file / log_error in +# mediawiki.cnf would silently fail to open (slow query logging disabled with +# no visible error, general error log falling back to journald only). Create +# it explicitly so both are guaranteed to work regardless of tmpfiles timing. +mkdir -p /var/log/mariadb +chown mysql:mysql /var/log/mariadb +chmod 750 /var/log/mariadb + # ── Start MariaDB for setup ─────────────────────────────────────────────────── systemctl enable mariadb systemctl start mariadb diff --git a/packer/scripts/03-httpd.sh b/packer/scripts/03-httpd.sh index 745ad87..e645ebd 100644 --- a/packer/scripts/03-httpd.sh +++ b/packer/scripts/03-httpd.sh @@ -39,13 +39,41 @@ fi # ── MediaWiki vhost ─────────────────────────────────────────────────────────── mkdir -p /var/www/mediawiki -mkdir -p /var/log/httpd + +# MediaWiki's own error/access logs live in a dedicated directory, deliberately +# separate from /var/log/httpd/. The stock RPM's /etc/logrotate.d/httpd +# (glob "/var/log/httpd/*log") already owns everything httpd itself writes +# there (ssl_error_log, ssl_request_log, etc.) — that file is package-managed +# and gets silently restored on the next `dnf update httpd` if we delete or +# edit it, so we leave it completely alone. Giving our vhost logs their own +# directory means there's no glob overlap for logrotate to conflict on, no +# matter what the httpd package does with its own config. +# MediaWiki's own error/access logs live in a dedicated directory, deliberately +# separate from /var/log/httpd/. The stock RPM's /etc/logrotate.d/httpd +# (glob "/var/log/httpd/*log") already owns everything httpd itself writes +# there (ssl_error_log, ssl_request_log, etc.) — that file is package-managed +# and gets silently restored on the next `dnf update httpd` if we delete or +# edit it, so we leave it completely alone. Giving our vhost logs their own +# directory means there's no glob overlap for logrotate to conflict on, no +# matter what the httpd package does with its own config. +# (Already created by 01-php.sh, which needs it earlier for php-fpm's own +# logs — mkdir -p here is just a defensive no-op if run standalone.) +mkdir -p /var/log/mediawiki +chmod 755 /var/log/mediawiki cp /tmp/config/httpd/mediawiki.conf /etc/httpd/conf.d/mediawiki.conf # ── Log rotation ────────────────────────────────────────────────────────────── +# Error log: daily, picked up by the system's normal daily logrotate.timer run. cp /tmp/config/logrotate/httpd-mediawiki /etc/logrotate.d/httpd-mediawiki +# Access log: high-volume, needs to rotate far more often than once a day. +# Installed outside /etc/logrotate.d/ so the daily logrotate.timer never +# touches it, and rotated hourly by its own cron job with its own state file. +cp /tmp/config/logrotate/mediawiki-access /etc/logrotate-mediawiki-access.conf +cp /tmp/config/cron/mediawiki-access-logrotate /etc/cron.d/mediawiki-access-logrotate +chmod 644 /etc/cron.d/mediawiki-access-logrotate + # ── Enable and start ────────────────────────────────────────────────────────── httpd -t # fail fast with a readable error if config is broken systemctl enable httpd diff --git a/packer/scripts/05-extensions.sh b/packer/scripts/05-extensions.sh index 674308e..633d663 100644 --- a/packer/scripts/05-extensions.sh +++ b/packer/scripts/05-extensions.sh @@ -106,11 +106,24 @@ echo "Composer update completed successfully" # DB_MASTER was renamed to DB_PRIMARY in MW 1.37 and removed in 1.42. # The 6.x release that fixed this requires PHP 8.4, which MW 1.43 does not # support. Patch the installed source directly; DB_PRIMARY has the same value. -# TODO: [16-Jul-2026 04:01:55 UTC] PHP Deprecated: Use of wfGetDB was deprecated in MediaWiki 1.39. [Called from MediaWiki\Extension\DiscourseSsoConsumer\Db::ensureCurrentSchema in /var/www/mediawiki/extensions/DiscourseSsoConsumer/src/Db.php at line 83] in /var/www/mediawiki/includes/debug/MWDebug.php on line 385 echo "Patching DiscourseSsoConsumer: DB_MASTER → DB_PRIMARY" grep -rl --include='*.php' 'DB_MASTER' "${EXT_DIR}/DiscourseSsoConsumer" \ | xargs sed -i 's/\bDB_MASTER\b/DB_PRIMARY/g' +# wfGetDB() itself (independent of the DB_MASTER/DB_PRIMARY constant above) +# was deprecated in MW 1.39 in favour of MediaWikiServices::getConnectionProvider(). +# Db::ensureCurrentSchema() calls it on every request, spamming +# /var/log/mediawiki/php-fpm-www.log with a "PHP Deprecated" notice each time. +# Patch to the non-deprecated equivalent instead of just tolerating the noise. +echo "Patching DiscourseSsoConsumer: wfGetDB() → MediaWikiServices::getConnectionProvider()" +{ grep -rlE --include='*.php' 'wfGetDB\s*\(' "${EXT_DIR}/DiscourseSsoConsumer" || true; } \ + | while IFS= read -r f; do + sed -i -E \ + -e 's/wfGetDB\(\s*DB_PRIMARY\s*\)/\\MediaWiki\\MediaWikiServices::getInstance()->getConnectionProvider()->getPrimaryDatabase()/g' \ + -e 's/wfGetDB\(\s*DB_REPLICA\s*\)/\\MediaWiki\\MediaWikiServices::getInstance()->getConnectionProvider()->getReplicaDatabase()/g' \ + "$f" + done + EXPECTED_EXTENSIONS=("TemplateStyles" "DiscourseSsoConsumer" "IFrameTag" "PluggableAuth" "JsonConfig" "WikiCategoryTagCloud") for ext in "${EXPECTED_EXTENSIONS[@]}"; do if [ ! -d "${EXT_DIR}/${ext}" ]; then diff --git a/packer/scripts/07-backup-setup.sh b/packer/scripts/07-backup-setup.sh index d330b32..ffdc200 100755 --- a/packer/scripts/07-backup-setup.sh +++ b/packer/scripts/07-backup-setup.sh @@ -32,8 +32,12 @@ chmod 644 /etc/cron.d/mediawiki-backup cp /tmp/config/logrotate/mediawiki-backup /etc/logrotate.d/mediawiki-backup # ── Create the log file ─────────────────────────────────────────────────────── +# World-readable: the amazon-cloudwatch-agent service does not run as root +# (it tails files as its own service user), and this log contains no secrets +# (DB credentials are never echoed by the backup scripts) — 640 root-owned +# caused "permission denied" for the agent's [inputs.logfile] tailer. touch /var/log/mediawiki-backup.log -chmod 640 /var/log/mediawiki-backup.log +chmod 644 /var/log/mediawiki-backup.log # ── Remove staged config (last phase — no longer needed) ───────────────────── rm -rf /tmp/config diff --git a/scripts/backup/backup-with-retention.sh b/scripts/backup/backup-with-retention.sh index e038df1..328e815 100755 --- a/scripts/backup/backup-with-retention.sh +++ b/scripts/backup/backup-with-retention.sh @@ -60,6 +60,22 @@ command -v mysqldump >/dev/null || fail "mysqldump not found" LSETTINGS="${MW_ROOT}/LocalSettings.php" [ -f "${LSETTINGS}" ] || fail "LocalSettings.php not found" +# ── Preflight: enough space in the work dir's filesystem? ─────────────────── +# Images copy compresses ~1:1 in the worst case (already-compressed media); +# require room for a full uncompressed copy of images/ plus the DB, with a +# safety margin, instead of failing midway through tar/gzip with a cryptic +# "No space left on device". +NEEDED_KB=0 +if [ -d "${MW_ROOT}/images" ]; then + NEEDED_KB=$(du -sk "${MW_ROOT}/images" | cut -f1) +fi +NEEDED_KB=$(( (NEEDED_KB * 12) / 10 + 262144 )) # +20% margin, +256MB for the DB dump +AVAIL_KB=$(df -Pk "${WORK_DIR}" | awk 'NR==2 {print $4}') +if [ "${AVAIL_KB}" -lt "${NEEDED_KB}" ]; then + df -h "${WORK_DIR}" >&2 + fail "Not enough free space in $(dirname "${WORK_DIR}") (have ${AVAIL_KB}KB, need ~${NEEDED_KB}KB). Check for oversized logs under /var/log (see logrotate.d/httpd-mediawiki, logrotate-mediawiki-access.conf) before retrying." +fi + # ── Extract DB credentials from LocalSettings.php ──────────────────────────── extract_setting() { # Parse a literal assignment from LocalSettings.php without executing PHP code. diff --git a/scripts/backup/full-backup.sh b/scripts/backup/full-backup.sh index b84505c..f16d0a3 100644 --- a/scripts/backup/full-backup.sh +++ b/scripts/backup/full-backup.sh @@ -64,6 +64,18 @@ command -v mysqldump >/dev/null || fail "mysqldump not found" command -v tar >/dev/null || fail "tar not found" [ -d "${MW_ROOT}" ] || fail "MediaWiki root not found: ${MW_ROOT}" +# ── Preflight: enough space in the work dir's filesystem? ─────────────────── +NEEDED_KB=0 +if [ -d "${MW_ROOT}/images" ]; then + NEEDED_KB=$(du -sk "${MW_ROOT}/images" | cut -f1) +fi +NEEDED_KB=$(( (NEEDED_KB * 12) / 10 + 262144 )) # +20% margin, +256MB for the DB dump +AVAIL_KB=$(df -Pk "${WORK_DIR}" | awk 'NR==2 {print $4}') +if [ "${AVAIL_KB}" -lt "${NEEDED_KB}" ]; then + df -h "${WORK_DIR}" >&2 + fail "Not enough free space in $(dirname "${WORK_DIR}") (have ${AVAIL_KB}KB, need ~${NEEDED_KB}KB). Check for oversized logs under /var/log before retrying." +fi + # ── Auto-detect DB credentials from LocalSettings.php ──────────────────────── LSETTINGS="${MW_ROOT}/LocalSettings.php" [ -f "${LSETTINGS}" ] || fail "LocalSettings.php not found at ${LSETTINGS}"