DPE-5355 Ensure logs flushes (#543)

* ensure flush runs and some refactor

* var names

* text fix

* add parameter to setup call

* version bump

* install file with correct permission

* format refactor

Author: paulomach

config.yaml

@@ -13,7 +13,7 @@ options:
   profile:
     description: |
       profile representing the scope of deployment, and used to be able to enable high-level
-      high-level customisation of sysconfigs, resource checks/allocation, warning levels, etc.
+      customisation of sysconfigs, resource checks/allocation, warning levels, etc.
       Allowed values are: “production” and “testing”.
     type: string
     default: production

lib/charms/mysql/v0/mysql.py

@@ -133,7 +133,7 @@ def wait_until_mysql_connection(self) -> None:
 # Increment this major API version when introducing breaking changes
 LIBAPI = 0
 
-LIBPATCH = 77
+LIBPATCH = 78
 
 UNIT_TEARDOWN_LOCKNAME = "unit-teardown"
 UNIT_ADD_LOCKNAME = "unit-add"
@@ -1004,7 +1004,7 @@ def render_mysqld_configuration(  # noqa: C901
             "log_error": f"{snap_common}/var/log/mysql/error.log",
             "general_log": "ON",
             "general_log_file": f"{snap_common}/var/log/mysql/general.log",
-            "slow_query_log_file": f"{snap_common}/var/log/mysql/slowquery.log",
+            "slow_query_log_file": f"{snap_common}/var/log/mysql/slow.log",
             "binlog_expire_logs_seconds": f"{binlog_retention_seconds}",
             "loose-audit_log_policy": "LOGINS",
             "loose-audit_log_file": f"{snap_common}/var/log/mysql/audit.log",

src/charm.py

@@ -297,6 +297,7 @@ def _on_config_changed(self, _) -> None:
         self._mysql.write_content_to_file(
             path=MYSQLD_CUSTOM_CONFIG_FILE, content=new_config_content
         )
+        self._mysql.setup_logrotate_and_cron(self.text_logs)
 
         if (
             self.mysql_config.keys_requires_restart(changed_config)
@@ -620,6 +621,17 @@ def unit_address(self) -> str:
         """Returns the unit's address."""
         return self.get_unit_address(self.unit)
 
+    @property
+    def text_logs(self) -> list:
+        """Enabled text logs."""
+        # slow logs isn't enabled by default
+        text_logs = ["error", "general"]
+
+        if self.config.plugin_audit_enabled:
+            text_logs.append("audit")
+
+        return text_logs
+
     def install_workload(self) -> bool:
         """Exponential backoff retry to install and configure MySQL.
 
@@ -654,7 +666,7 @@ def workload_initialise(self) -> None:
         self.hostname_resolution.update_etc_hosts(None)
 
         self._mysql.write_mysqld_config()
-        self._mysql.setup_logrotate_and_cron()
+        self._mysql.setup_logrotate_and_cron(self.text_logs)
         self._mysql.reset_root_password_and_start_mysqld()
         self._mysql.configure_mysql_users()
 

src/flush_mysql_logs.py

@@ -60,3 +60,4 @@ def _flush_mysql_logs(self, _) -> None:
             return
 
         self.charm._mysql.flush_mysql_logs(text_logs)
+        logger.debug(f"Flushed {text_logs.lower()}")

src/mysql_vm_helpers.py

@@ -12,7 +12,7 @@
 import subprocess
 import tempfile
 import typing
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, Iterable, List, Optional, Tuple
 
 import jinja2
 from charms.mysql.v0.mysql import (
@@ -280,29 +280,48 @@ def write_mysqld_config(self) -> None:
             content=content_str,
         )
 
-    def setup_logrotate_and_cron(self) -> None:
-        """Create and write the logrotate config file."""
+    def setup_logrotate_and_cron(self, enabled_log_files: Iterable) -> None:
+        """Setup log rotation configuration for text files.
+
+        Args:
+            enabled_log_files: a iterable of enabled text logs
+        """
         logger.debug("Creating logrotate config file")
+        config_path = "/etc/logrotate.d/flush_mysql_logs"
+        script_path = f"{self.charm.charm_dir}/logrotation.sh"
+        cron_path = "/etc/cron.d/flush_mysql_logs"
+        logs_dir = f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql"
 
         with open("templates/logrotate.j2", "r") as file:
             template = jinja2.Template(file.read())
 
-        rendered = template.render(
+        logrotate_conf_content = template.render(
             system_user=MYSQL_SYSTEM_USER,
-            snap_common_directory=CHARMED_MYSQL_COMMON_DIRECTORY,
+            log_dir=logs_dir,
             charm_directory=self.charm.charm_dir,
             unit_name=self.charm.unit.name,
+            enabled_log_files=enabled_log_files,
         )
 
-        with open("/etc/logrotate.d/flush_mysql_logs", "w") as file:
-            file.write(rendered)
+        self.write_content_to_file(
+            config_path, logrotate_conf_content, owner="root", permission=0o644
+        )
 
-        cron = (
-            "* 1-23 * * * root logrotate -f /etc/logrotate.d/flush_mysql_logs\n"
-            "1-59 0 * * * root logrotate -f /etc/logrotate.d/flush_mysql_logs\n"
+        with open("templates/run_log_rotation.sh.j2", "r") as file:
+            template = jinja2.Template(file.read())
+
+        logrotation_script_content = template.render(
+            log_path=f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql",
+            enabled_log_files=enabled_log_files,
+            logrotate_conf=config_path,
+            owner=MYSQL_SYSTEM_USER,
+            group=MYSQL_SYSTEM_USER,
         )
-        with open("/etc/cron.d/flush_mysql_logs", "w") as file:
-            file.write(cron)
+
+        self.write_content_to_file(script_path, logrotation_script_content, permission=0o550)
+
+        cron_content = f"* 1-23 * * * root {script_path}\n1-59 0 * * * root {script_path}\n"
+        self.write_content_to_file(cron_path, cron_content, owner="root")
 
     def reset_root_password_and_start_mysqld(self) -> None:
         """Reset the root user password and start mysqld."""

src/upgrade.py

@@ -199,7 +199,7 @@ def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None:  # noqa: C901
             self.charm._mysql.start_mysqld()
             if self.charm.config.plugin_audit_enabled:
                 self.charm._mysql.install_plugins(["audit_log", "audit_log_filter"])
-            self.charm._mysql.setup_logrotate_and_cron()
+            self.charm._mysql.setup_logrotate_and_cron(self.charm.text_logs)
         except VersionError:
             logger.exception("Failed to upgrade MySQL dependencies")
             self.set_unit_failed()

templates/logrotate.j2

@@ -21,38 +21,14 @@ nomail
 nosharedscripts
 nocopytruncate
 
-{{ snap_common_directory }}/var/log/mysql/error.log {
-    olddir archive_error
+{% for log in enabled_log_files %}
+{{ log_dir }}/{{ log }}.log {
+    olddir archive_{{ log }}
     postrotate
         juju_command=/usr/bin/juju-run
         if command -v /usr/bin/juju-exec; then juju_command=/usr/bin/juju-exec; fi
-        "$juju_command" -u {{ unit_name }} LOGS_TYPE=ERROR JUJU_DISPATCH_PATH=hooks/flush_mysql_logs {{ charm_directory }}/dispatch
+        "$juju_command" -u {{ unit_name }} LOGS_TYPE={{ log|upper }} JUJU_DISPATCH_PATH=hooks/flush_mysql_logs {{ charm_directory }}/dispatch
     endscript
 }
+{% endfor %}
 
-{{ snap_common_directory }}/var/log/mysql/general.log {
-    olddir archive_general
-    postrotate
-        juju_command=/usr/bin/juju-run
-        if command -v /usr/bin/juju-exec; then juju_command=/usr/bin/juju-exec; fi
-        "$juju_command" -u {{ unit_name }} LOGS_TYPE=GENERAL JUJU_DISPATCH_PATH=hooks/flush_mysql_logs {{ charm_directory }}/dispatch
-    endscript
-}
-
-{{ snap_common_directory }}/var/log/mysql/slowquery.log {
-    olddir archive_slowquery
-    postrotate
-        juju_command=/usr/bin/juju-run
-        if command -v /usr/bin/juju-exec; then juju_command=/usr/bin/juju-exec; fi
-        "$juju_command" -u {{ unit_name }} LOGS_TYPE=SLOW JUJU_DISPATCH_PATH=hooks/flush_mysql_logs {{ charm_directory }}/dispatch
-    endscript
-}
-
-{{ snap_common_directory }}/var/log/mysql/audit.log {
-    olddir archive_audit
-    postrotate
-        juju_command=/usr/bin/juju-run
-        if command -v /usr/bin/juju-exec; then juju_command=/usr/bin/juju-exec; fi
-        "$juju_command" -u {{ unit_name }} LOGS_TYPE=AUDIT JUJU_DISPATCH_PATH=hooks/flush_mysql_logs {{ charm_directory }}/dispatch
-    endscript
-}

templates/run_log_rotation.sh.j2

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# install files to ensure flush runs
+{% for log in enabled_log_files %}
+[[ -f "{{ log_path }}/{{ log }}.log" ]] || install -o "{{ owner }}" -g "{{ group }}" -m 640 /dev/null "{{ log_path }}/{{ log }}.log"
+{% endfor %}
+
+logrotate -f {{ logrotate_conf }}
+
+# vim: set filetype=sh
+

tests/integration/high_availability/test_replication.py

@@ -356,13 +356,11 @@ async def test_log_rotation(ops_test: OpsTest, highly_available_cluster) -> None
     app = get_application_name(ops_test, "mysql")
     unit = ops_test.model.applications[app].units[0]
 
-    # Exclude slowquery log files as slowquery logs are not enabled by default
     log_types = ["error", "general", "audit"]
     log_files = ["error.log", "general.log", "audit.log"]
     archive_directories = [
         "archive_error",
         "archive_general",
-        "archive_slowquery",
         "archive_audit",
     ]
 
@@ -418,7 +416,6 @@ async def test_log_rotation(ops_test: OpsTest, highly_available_cluster) -> None
     ), f"❌ unexpected files/directories in log directory: {ls_la_output}"
 
     logger.info("Ensuring log files were rotated")
-    # Exclude checking slowquery log rotation as slowquery logs are disabled by default
     for log in set(log_types):
         file_contents = await read_contents_from_file_in_unit(
             ops_test, unit, f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql/{log}.log"

tests/unit/test_mysql.py

@@ -1882,7 +1882,7 @@ def test_render_mysqld_configuration(self, _get_available_memory):
             "log_error": "/var/log/mysql/error.log",
             "general_log": "ON",
             "general_log_file": "/var/log/mysql/general.log",
-            "slow_query_log_file": "/var/log/mysql/slowquery.log",
+            "slow_query_log_file": "/var/log/mysql/slow.log",
             "binlog_expire_logs_seconds": "604800",
             "loose-audit_log_format": "JSON",
             "loose-audit_log_policy": "LOGINS",

tests/unit/test_mysqlsh_helpers.py

@@ -291,7 +291,7 @@ def test_write_mysqld_config(
             "log_error = /var/snap/charmed-mysql/common/var/log/mysql/error.log",
             "general_log = ON",
             "general_log_file = /var/snap/charmed-mysql/common/var/log/mysql/general.log",
-            "slow_query_log_file = /var/snap/charmed-mysql/common/var/log/mysql/slowquery.log",
+            "slow_query_log_file = /var/snap/charmed-mysql/common/var/log/mysql/slow.log",
             "binlog_expire_logs_seconds = 604800",
             "loose-audit_log_policy = LOGINS",
             "loose-audit_log_file = /var/snap/charmed-mysql/common/var/log/mysql/audit.log",
@@ -334,7 +334,7 @@ def test_write_mysqld_config(
             "log_error = /var/snap/charmed-mysql/common/var/log/mysql/error.log",
             "general_log = ON",
             "general_log_file = /var/snap/charmed-mysql/common/var/log/mysql/general.log",
-            "slow_query_log_file = /var/snap/charmed-mysql/common/var/log/mysql/slowquery.log",
+            "slow_query_log_file = /var/snap/charmed-mysql/common/var/log/mysql/slow.log",
             "binlog_expire_logs_seconds = 604800",
             "loose-audit_log_policy = LOGINS",
             "loose-audit_log_file = /var/snap/charmed-mysql/common/var/log/mysql/audit.log",