Merge pull request #66 from homeylab/61-notification

61 notification

Author: pchang388

README.md

@@ -17,6 +17,8 @@ Table of Contents
     - [Modify Markdown Files](#modify-markdown-files)
   - [Object Storage](#object-storage)
     - [Minio Backups](#minio-backups)
+  - [Notifications](#notifications)
+    - [apprise](#apprise)
   - [Potential Breaking Upgrades](#potential-breaking-upgrades)
   - [Future Items](#future-items)
 
@@ -251,6 +253,15 @@ assets:
   export_meta: false
 keep_last: 5
 run_interval: 0
+notifications:
+  apprise:
+    service_urls:
+      - "json://localhost:8080/notify"
+    config_path: ""
+    plugin_paths: []
+    storage_path: ""
+    custom_title: ""
+    custom_attachment_path: ""
 ```
 
 #### Options and Descriptions
@@ -480,6 +491,25 @@ minio:
 | `path` | `str` | `false` | Optional, path of the backup to use. Will use root bucket path if not set. `<bucket_name>:/<path>/bookstack-<timestamp>.tgz` |
 | `keep_last` | `int` | `false` | Optional (default: `0`), if exporter can delete older archives in minio.<br>- set to `1+` if you want to retain a certain number of archives<br>-  `0` will result in no action done |
 
+## Notifications
+It is possible to send notifications when an export run fails. Currently, the only supported notification service is [apprise](https://github.com/caronc/apprise). Apprise is a general purpose notification service and has a variety of integrations and includes generic HTTP POST.
+
+### apprise
+The apprise configuration is a part of the configuration yaml file and can be modified under `notifications.apprise`.
+
+| Item | Type | Description |
+| ---- | ---- | ----------- |
+| `apprise.service_urls` | `List<str>` | Provide the apprise urls for apprise to send notifications to. Can also be provided as environment variable: `APPRISE_URLS`, see example further below. |
+| `apprise.config_path` | `str` | If specified, overrides `apprise.service_urls`. Can specify the path to an apprise configuration file |
+| `apprise.plugin_paths` | `List<str>` | Provide the plugin paths for apprise to use |
+| `apprise.storage_path` | `str` | For persistent storage, specify a path for apprise to use |
+| `apprise.custom_title` | `str` | Replace the default message title for apprise notifications |
+| `apprise.custom_attachment_path` | `str` | To include a custom attachment to the apprise notification, specify the path to a file | 
+
+`apprise.service_urls` can contain sensitive information and can be specified as an environment variable instead as a string list, example: `export APPRISE_URLS='["json://localhost:8080/notify"]'`.
+
+**If using apprise for notifications, one of `apprise.service_urls` or `apprise.config_path` should be specified.**
+
 ## Potential Breaking Upgrades
 Below are versions that have major changes to the way configuration or exporter runs.
 

bookstack_file_exporter/archiver/minio_archiver.py

@@ -80,11 +80,9 @@ def _scan_objects(self, file_extension: str) -> List[MinioObject]:
         # get all objects in archive path/directory
         full_list: List[MinioObject] = self._client.list_objects(self.bucket, prefix=path_prefix)
         # validate and filter out non managed objects
-        if full_list:
-            return [object for object in full_list
-                    if object.object_name.endswith(file_extension)
-                        and filter_str in object.object_name]
-        return []
+        return [object for object in full_list
+                if object.object_name.endswith(file_extension)
+                    and filter_str in object.object_name]
 
     def _get_stale_objects(self, file_extension: str) -> List[MinioObject]:
         minio_objects = self._scan_objects(file_extension)

bookstack_file_exporter/common/util.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Dict
+import os
+from typing import Dict, Union
 import urllib3
 # pylint: disable=import-error
 import requests
@@ -70,3 +71,27 @@ def should_verify(url: str) -> str:
         if url.startswith("https"):
             return "https://"
         return "http://"
+
+def check_var(env_key: str, default_val: Union[list[str],str], can_error: bool = False) -> str:
+    """
+    :param: env_key = the environment variable to check
+    :param: default_val = the default value if any to set if env variable not set
+    :param: can_error = whether or not missing both env_key and default_val should
+            trigger an exception
+    
+    :return: env_key if present or default_val if not
+    :throws: ValueError if both parameters are empty.
+    """
+    env_value = os.environ.get(env_key, "")
+    # env value takes precedence
+    if env_value:
+        log.debug("""env key: %s specified.
+                    Will override configuration file value if set.""", env_key)
+        return env_value
+    # check for optional inputs, if env and input is missing
+    if not can_error:
+        if not env_value and not default_val:
+            raise ValueError(f"""{env_key} is not specified in env and is
+                                missing from configuration - at least one should be set""")
+    # fall back to configuration file value if present
+    return default_val

bookstack_file_exporter/config_helper/config_helper.py

@@ -5,6 +5,7 @@
 # pylint: disable=import-error
 import yaml
 
+from bookstack_file_exporter.common.util import check_var
 from bookstack_file_exporter.config_helper import models
 from bookstack_file_exporter.config_helper.remote import StorageProviderConfig
 
@@ -83,17 +84,17 @@ def _generate_credentials(self) -> Tuple[str, str]:
         token_secret = self.user_inputs.credentials.token_secret
 
         # check to see if env var is specified, if so, it takes precedence
-        token_id = self._check_var(_BOOKSTACK_TOKEN_FIELD, token_id)
-        token_secret = self._check_var(_BOOKSTACK_TOKEN_SECRET_FIELD, token_secret)
+        token_id = check_var(_BOOKSTACK_TOKEN_FIELD, token_id)
+        token_secret = check_var(_BOOKSTACK_TOKEN_SECRET_FIELD, token_secret)
         return token_id, token_secret
 
     def _generate_remote_config(self) -> Dict[str, StorageProviderConfig]:
         object_config = {}
         # check for optional minio credentials if configuration is set in yaml configuration file
         if self.user_inputs.minio:
-            minio_access_key = self._check_var(_MINIO_ACCESS_KEY_FIELD,
+            minio_access_key = check_var(_MINIO_ACCESS_KEY_FIELD,
                                                self.user_inputs.minio.access_key)
-            minio_secret_key = self._check_var(_MINIO_SECRET_KEY_FIELD,
+            minio_secret_key = check_var(_MINIO_SECRET_KEY_FIELD,
                                                self.user_inputs.minio.secret_key)
 
             object_config["minio"] = StorageProviderConfig(minio_access_key,
@@ -177,24 +178,24 @@ def object_storage_config(self) -> Dict[str, StorageProviderConfig]:
         """return remote storage configuration"""
         return self._object_storage_config
 
-    @staticmethod
-    def _check_var(env_key: str, default_val: str) -> str:
-        """
-        :param: env_key = the environment variable to check
-        :param: default_val = the default value if any to set if env variable not set
-        
-        :return: env_key if present or default_val if not
-        :throws: ValueError if both parameters are empty.
-        """
-        env_value = os.environ.get(env_key, "")
-        # env value takes precedence
-        if env_value:
-            log.debug("""env key: %s specified.
-                       Will override configuration file value if set.""", env_key)
-            return env_value
-        # check for optional inputs, if env and input is missing
-        if not env_value and not default_val:
-            raise ValueError(f"""{env_key} is not specified in env and is
-                              missing from configuration - at least one should be set""")
-        # fall back to configuration file value if present
-        return default_val
+    # @staticmethod
+    # def _check_var(env_key: str, default_val: str) -> str:
+    #     """
+    #     :param: env_key = the environment variable to check
+    #     :param: default_val = the default value if any to set if env variable not set
+
+    #     :return: env_key if present or default_val if not
+    #     :throws: ValueError if both parameters are empty.
+    #     """
+    #     env_value = os.environ.get(env_key, "")
+    #     # env value takes precedence
+    #     if env_value:
+    #         log.debug("""env key: %s specified.
+    #                    Will override configuration file value if set.""", env_key)
+    #         return env_value
+    #     # check for optional inputs, if env and input is missing
+    #     if not env_value and not default_val:
+    #         raise ValueError(f"""{env_key} is not specified in env and is
+    #                           missing from configuration - at least one should be set""")
+    #     # fall back to configuration file value if present
+    #     return default_val

bookstack_file_exporter/config_helper/models.py

@@ -36,6 +36,19 @@ class HttpConfig(BaseModel):
     retry_count: Optional[int] = 5
     additional_headers: Optional[Dict[str, str]] = {}
 
+class AppRiseNotifyConfig(BaseModel):
+    """YAML schema for user provided app rise settings"""
+    service_urls: Optional[List[str]] = []
+    config_path: Optional[str] = ""
+    plugin_paths: Optional[List[str]] = []
+    storage_path: Optional[str] = ""
+    custom_title: Optional[str] = ""
+    custom_attachment_path: Optional[str] = ""
+
+class Notifications(BaseModel):
+    """YAML schema for user provided notification settings"""
+    apprise: Optional[AppRiseNotifyConfig] = None
+
 # pylint: disable=too-few-public-methods
 class UserInput(BaseModel):
     """YAML schema for user provided configuration file"""
@@ -48,3 +61,4 @@ class UserInput(BaseModel):
     keep_last: Optional[int] = 0
     run_interval: Optional[int] = 0
     http_config: Optional[HttpConfig] = HttpConfig()
+    notifications: Optional[Notifications] = None

bookstack_file_exporter/config_helper/notifications.py

@@ -0,0 +1,62 @@
+import json
+import logging
+import os
+from typing import Union
+
+from bookstack_file_exporter.config_helper import models
+from bookstack_file_exporter.common.util import check_var
+
+log = logging.getLogger(__name__)
+
+_APPRISE_FIELDS = {
+    "urls": "APPRISE_URLS",
+    # "config_path": "APPRISE_CONFIG_PATH",
+    # "plugin_paths": "APPRISE_PLUGIN_PATHS",
+    # "storage_path": "APPRISE_STORAGE_PATH"
+}
+
+_DEFAULT_TITLE = "Bookstack File Exporter Failed"
+
+# pylint: disable=too-few-public-methods
+class AppRiseNotifyConfig:
+    """
+    Convenience class to hold apprise notification configuration
+    
+    Args:
+        :config: <models.AppRiseNotifyConfig> = user input configuration
+    
+    Returns:
+        AppRiseNotifyConfig instance for holding configuration
+    """
+    def __init__(self, config: models.AppRiseNotifyConfig):
+        self.service_urls: Union[str, list] = check_var(_APPRISE_FIELDS["urls"],
+                                  config.service_urls, can_error=True)
+        self.config_path = config.config_path
+        self.plugin_paths = config.plugin_paths
+        self.storage_path = config.storage_path
+        self.custom_title = config.custom_title
+        self.custom_attachment = config.custom_attachment_path
+
+    def validate(self) -> None:
+        """validate apprise configuration"""
+        if not self.config_path and not self.service_urls:
+            raise ValueError("""apprise config_path and service_urls are
+                              missing from configuration - at least one should be set""")
+
+        # if not config path/file given, then we use service_urls
+        if not self.config_path:
+            # if not config path style, we try service_urls
+            # if service_urls defined in env, override main config file value
+            if os.environ.get(_APPRISE_FIELDS["urls"]) is not None:
+                try:
+                    new_urls = json.loads(self.service_urls)
+                    self.service_urls = new_urls
+                # json errors can be hard to debug, add helpful log message
+                except json.decoder.JSONDecodeError as url_err:
+                    log.Error("Failed to parse env var for apprise urls. \
+                            Ensure proper json string format")
+                    raise url_err
+
+        # set default custom_title if not provided
+        if not self.custom_title:
+            self.custom_title = _DEFAULT_TITLE

bookstack_file_exporter/config_helper/remote.py

@@ -25,7 +25,7 @@ def __init__(self, access_key: str, secret_key: str, config: ObjectStorageConfig
         self.config = config
         self._access_key = access_key
         self._secret_key = secret_key
-        self._valid_checker = {'minio': self._is_minio_valid()}
+        self._valid_checker = {'minio': self._is_minio_valid}
 
     @property
     def access_key(self) -> str:
@@ -40,7 +40,7 @@ def secret_key(self) -> str:
     def is_valid(self, storage_type: str) -> bool:
         """check if object storage config is valid"""
         return self._valid_checker[storage_type]
-    
+
     def _is_minio_valid(self) -> bool:
         """check if minio config is valid"""
         # required values - keys and bucket already checked so skip

bookstack_file_exporter/notify/__init__.py

@@ -0,0 +1,48 @@
+import logging
+
+from bookstack_file_exporter.config_helper import models, notifications
+from bookstack_file_exporter.notify import notifiers
+
+
+log = logging.getLogger(__name__)
+
+# pylint: disable=too-few-public-methods
+class NotifyHandler:
+    """
+    NotifyHandler helps push out notifications for failed export runs
+
+    Args:
+        :config: <models.Notifications> = User input configuration for notification handlers
+    
+    Returns:
+        NotifyHandler instance to help handle notification integrations.
+    """
+    def __init__(self, config: models.Notifications):
+        self.targets = self._get_targets(config)
+        self._supported_notifiers={
+            "apprise": self._handle_apprise
+        }
+
+    def _get_targets(self, config: models.Notifications):
+        targets = {}
+
+        if config.apprise:
+            targets["apprise"] = config.apprise
+
+        return targets
+
+    def do_notify(self, excep: Exception) -> None:
+        """handle notification sending for all configured targets"""
+        if len(self.targets) == 0:
+            log.debug("No notification targets found")
+            return
+        for target, config in self.targets.items():
+            log.debug("Starting notification handling for: %s", target)
+            self._supported_notifiers[target](config, excep)
+
+
+    def _handle_apprise(self, config: models.AppRiseNotifyConfig, excep: Exception):
+        a_config = notifications.AppRiseNotifyConfig(config)
+        a_config.validate()
+        apprise = notifiers.AppRiseNotify(a_config)
+        apprise.notify(excep)

bookstack_file_exporter/notify/handler.py

@@ -0,0 +1,66 @@
+from datetime import datetime
+from apprise import Apprise, AppriseAsset, AppriseConfig
+
+from bookstack_file_exporter.config_helper import notifications
+
+# pylint: disable=too-few-public-methods
+class AppRiseNotify:
+    """
+    AppRiseNotify helps send notifications via apprise for failed export runs
+    
+    Args:
+        :config: <notifications.AppRiseNotifyConfig> = Configuration with user inputs and 
+                                                       general options
+    
+    Returns:
+        AppRiseNotify instance to help handle apprise notification integration.
+    """
+    def __init__(self, config: notifications.AppRiseNotifyConfig):
+        self.config = config
+        self._client = self._create_client()
+
+    def _create_client(self):
+        client = Apprise()
+        asset = AppriseAsset()
+
+        if self.config.storage_path:
+            asset.storage_path=self.config.storage_path
+
+        if self.config.plugin_paths:
+            asset.plugin_paths = self.config.plugin_paths
+
+        if self.config.config_path:
+            app_config = AppriseConfig()
+            app_config.add(self.config.config_path)
+            client.add(app_config)
+        else:
+            client.add(self.config.service_urls)
+
+        client.asset=asset
+        return client
+
+    def _get_message_body(self, error_msg: str) -> str:
+        timestamp = datetime.today().strftime('%Y-%m-%d %H:%M:%S')
+        body = f"""
+        Bookstack File Exporter encountered an unrecoverable error.
+        
+        Occurred At: {timestamp}
+        
+        Error message: {error_msg}
+        """
+        return body
+
+    def notify(self, excep: Exception):
+        """send notification with exception message"""
+        custom_body = self._get_message_body(str(excep))
+        if self.config.custom_attachment:
+            self._client.notify(
+                title=self.config.custom_title,
+                body=custom_body,
+                attach=self.config.custom_attachment
+            )
+        else:
+            self._client.notify(
+                title=self.config.custom_title,
+                body=custom_body
+            )