[DPE-3705, DPE-3542] Reintroduce fallback keys and fix TLS secrets initialization (#427)

* Reintroduce fallback keys and fix TLS secrets initialization

* Update mysql libpatch

* Fix bug in implementation to avoid overwriting secret value in set_secret

* Update data_interfaces charm lib to v0.31

* Remove secret with fallback key in set_secret

* Fix typo

Author: shayancanonical

lib/charms/data_platform_libs/v0/data_interfaces.py

@@ -295,10 +295,21 @@ def _on_topic_requested(self, event: TopicRequestedEvent):
 import json
 import logging
 from abc import ABC, abstractmethod
-from collections import namedtuple
+from collections import UserDict, namedtuple
 from datetime import datetime
 from enum import Enum
-from typing import Callable, Dict, List, Optional, Set, Tuple, Union
+from typing import (
+    Callable,
+    Dict,
+    ItemsView,
+    KeysView,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+    ValuesView,
+)
 
 from ops import JujuVersion, Model, Secret, SecretInfo, SecretNotFoundError
 from ops.charm import (
@@ -320,7 +331,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 29
+LIBPATCH = 31
 
 PYDEPS = ["ops>=2.0.0"]
 
@@ -612,6 +623,102 @@ def remove(self, label: str) -> None:
 # Base Data
 
 
+class DataDict(UserDict[str, str]):
+    """Python Standard Library 'dict' - like representation of Relation Data."""
+
+    def __init__(self, relation_data: "Data", relation_id: int):
+        self.relation_data = relation_data
+        self.relation_id = relation_id
+
+    @property
+    def data(self) -> Dict[str, str]:
+        """Return the full content of the Abstract Relation Data dictionary."""
+        result = self.relation_data.fetch_my_relation_data([self.relation_id])
+        try:
+            result_remote = self.relation_data.fetch_relation_data([self.relation_id])
+        except NotImplementedError:
+            result_remote = {self.relation_id: {}}
+        if result:
+            result_remote[self.relation_id].update(result[self.relation_id])
+        return result_remote.get(self.relation_id, {})
+
+    def __setitem__(self, key: str, item: str) -> None:
+        """Set an item of the Abstract Relation Data dictionary."""
+        self.relation_data.update_relation_data(self.relation_id, {key: item})
+
+    def __getitem__(self, key: str) -> str:
+        """Get an item of the Abstract Relation Data dictionary."""
+        result = None
+        if not (result := self.relation_data.fetch_my_relation_field(self.relation_id, key)):
+            try:
+                result = self.relation_data.fetch_relation_field(self.relation_id, key)
+            except NotImplementedError:
+                pass
+        if not result:
+            raise KeyError
+        return result
+
+    def __eq__(self, d: dict) -> bool:
+        """Equality."""
+        return self.data == d
+
+    def __repr__(self) -> str:
+        """String representation Abstract Relation Data dictionary."""
+        return repr(self.data)
+
+    def __len__(self) -> int:
+        """Length of the Abstract Relation Data dictionary."""
+        return len(self.data)
+
+    def __delitem__(self, key: str) -> None:
+        """Delete an item of the Abstract Relation Data dictionary."""
+        self.relation_data.delete_relation_data(self.relation_id, [key])
+
+    def has_key(self, key: str) -> bool:
+        """Does the key exist in the Abstract Relation Data dictionary?"""
+        return key in self.data
+
+    def update(self, items: Dict[str, str]):
+        """Update the Abstract Relation Data dictionary."""
+        self.relation_data.update_relation_data(self.relation_id, items)
+
+    def keys(self) -> KeysView[str]:
+        """Keys of the Abstract Relation Data dictionary."""
+        return self.data.keys()
+
+    def values(self) -> ValuesView[str]:
+        """Values of the Abstract Relation Data dictionary."""
+        return self.data.values()
+
+    def items(self) -> ItemsView[str, str]:
+        """Items of the Abstract Relation Data dictionary."""
+        return self.data.items()
+
+    def pop(self, item: str) -> str:
+        """Pop an item of the Abstract Relation Data dictionary."""
+        result = self.relation_data.fetch_my_relation_field(self.relation_id, item)
+        if not result:
+            raise KeyError(f"Item {item} doesn't exist.")
+        self.relation_data.delete_relation_data(self.relation_id, [item])
+        return result
+
+    def __contains__(self, item: str) -> bool:
+        """Does the Abstract Relation Data dictionary contain item?"""
+        return item in self.data.values()
+
+    def __iter__(self):
+        """Iterate through the Abstract Relation Data dictionary."""
+        return iter(self.data)
+
+    def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
+        """Safely get an item of the Abstract Relation Data dictionary."""
+        try:
+            if result := self[key]:
+                return result
+        except KeyError:
+            return default
+
+
 class Data(ABC):
     """Base relation data mainpulation (abstract) class."""
 
@@ -929,6 +1036,10 @@ def _delete_relation_data_without_secrets(
     # Public interface methods
     # Handling Relation Fields seamlessly, regardless if in databag or a Juju Secret
 
+    def as_dict(self, relation_id: int) -> UserDict[str, str]:
+        """Dict behavior representation of the Abstract Data."""
+        return DataDict(self, relation_id)
+
     def get_relation(self, relation_name, relation_id) -> Relation:
         """Safe way of retrieving a relation."""
         relation = self._model.get_relation(relation_name, relation_id)
@@ -1787,6 +1898,14 @@ def __init__(self, unit: Unit, *args, **kwargs):
         self.local_unit = unit
         self.component = unit
 
+    def update_relation_data(self, relation_id: int, data: dict) -> None:
+        """This method makes no sense for a Other Peer Relation."""
+        raise NotImplementedError("It's not possible to update data of another unit.")
+
+    def delete_relation_data(self, relation_id: int, fields: List[str]) -> None:
+        """This method makes no sense for a Other Peer Relation."""
+        raise NotImplementedError("It's not possible to delete data of another unit.")
+
 
 class DataPeerOtherUnitEventHandlers(DataPeerEventHandlers):
     """Requires-side of the relation."""
@@ -1809,18 +1928,18 @@ def __init__(
         additional_secret_fields: Optional[List[str]] = [],
         secret_field_name: Optional[str] = None,
         deleted_label: Optional[str] = None,
-        unique_key: str = "",
     ):
-        DataPeerData.__init__(
+        DataPeerOtherUnitData.__init__(
             self,
+            unit,
             charm.model,
             relation_name,
             extra_user_roles,
             additional_secret_fields,
             secret_field_name,
             deleted_label,
         )
-        DataPeerEventHandlers.__init__(self, charm, self, unique_key)
+        DataPeerOtherUnitEventHandlers.__init__(self, charm, self)
 
 
 # General events

lib/charms/mysql/v0/mysql.py

@@ -99,6 +99,7 @@ def wait_until_mysql_connection(self) -> None:
     PEER,
     ROOT_PASSWORD_KEY,
     ROOT_USERNAME,
+    SECRET_KEY_FALLBACKS,
     SERVER_CONFIG_PASSWORD_KEY,
     SERVER_CONFIG_USERNAME,
 )
@@ -114,7 +115,7 @@ def wait_until_mysql_connection(self) -> None:
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 56
+LIBPATCH = 57
 
 UNIT_TEARDOWN_LOCKNAME = "unit-teardown"
 UNIT_ADD_LOCKNAME = "unit-add"
@@ -411,8 +412,8 @@ def __init__(self, *args):
             additional_secret_fields=[
                 "key",
                 "csr",
-                "cert",
-                "cauth",
+                "certificate",
+                "certificate-authority",
                 "chain",
             ],
             secret_field_name=SECRET_INTERNAL_LABEL,
@@ -580,6 +581,13 @@ def has_cos_relation(self) -> bool:
 
         return len(active_cos_relations) > 0
 
+    def peer_relation_data(self, scope: Scopes) -> DataPeer:
+        """Returns the peer relation data per scope."""
+        if scope == APP_SCOPE:
+            return self.peer_relation_app
+        elif scope == UNIT_SCOPE:
+            return self.peer_relation_unit
+
     def get_secret(
         self,
         scope: Scopes,
@@ -591,11 +599,15 @@ def get_secret(
         Else retrieve from peer databag. This is to account for cases where secrets are stored in
         peer databag but the charm is then refreshed to a newer revision.
         """
+        if scope not in get_args(Scopes):
+            raise ValueError("Unknown secret scope")
+
         peers = self.model.get_relation(PEER)
-        if scope == APP_SCOPE:
-            value = self.peer_relation_app.fetch_my_relation_field(peers.id, key)
-        else:
-            value = self.peer_relation_unit.fetch_my_relation_field(peers.id, key)
+        if not (value := self.peer_relation_data(scope).fetch_my_relation_field(peers.id, key)):
+            if key in SECRET_KEY_FALLBACKS:
+                value = self.peer_relation_data(scope).fetch_my_relation_field(
+                    peers.id, SECRET_KEY_FALLBACKS[key]
+                )
         return value
 
     def set_secret(self, scope: Scopes, key: str, value: Optional[str]) -> None:
@@ -607,24 +619,30 @@ def set_secret(self, scope: Scopes, key: str, value: Optional[str]) -> None:
             raise MySQLSecretError("Can only set app secrets on the leader unit")
 
         if not value:
-            return self.remove_secret(scope, key)
+            if key in SECRET_KEY_FALLBACKS:
+                self.remove_secret(scope, SECRET_KEY_FALLBACKS[key])
+            self.remove_secret(scope, key)
+            return
 
         peers = self.model.get_relation(PEER)
-        if scope == APP_SCOPE:
-            self.peer_relation_app.update_relation_data(peers.id, {key: value})
-        elif scope == UNIT_SCOPE:
-            self.peer_relation_unit.update_relation_data(peers.id, {key: value})
+
+        fallback_key_to_secret_key = {v: k for k, v in SECRET_KEY_FALLBACKS.items()}
+        if key in fallback_key_to_secret_key:
+            if self.peer_relation_data(scope).fetch_my_relation_field(peers.id, key):
+                self.remove_secret(scope, key)
+            self.peer_relation_data(scope).update_relation_data(
+                peers.id, {fallback_key_to_secret_key[key]: value}
+            )
+        else:
+            self.peer_relation_data(scope).update_relation_data(peers.id, {key: value})
 
     def remove_secret(self, scope: Scopes, key: str) -> None:
         """Removing a secret."""
         if scope not in get_args(Scopes):
             raise RuntimeError("Unknown secret scope.")
 
         peers = self.model.get_relation(PEER)
-        if scope == APP_SCOPE:
-            self.peer_relation_app.delete_relation_data(peers.id, [key])
-        else:
-            self.peer_relation_unit.delete_relation_data(peers.id, [key])
+        self.peer_relation_data(scope).delete_relation_data(peers.id, [key])
 
 
 class MySQLMemberState(str, enum.Enum):

src/constants.py

@@ -47,3 +47,12 @@
 GR_MAX_MEMBERS = 9
 HOSTNAME_DETAILS = "hostname-details"
 COS_AGENT_RELATION_NAME = "cos-agent"
+SECRET_KEY_FALLBACKS = {
+    "root-password": "root_password",
+    "server-config-password": "server_config_password",
+    "cluster-admin-password": "cluster_admin_password",
+    "monitoring-password": "monitoring_password",
+    "backups-password": "backups_password",
+    "certificate": "cert",
+    "certificate-authority": "ca",
+}