feat: Dynamic templating (#165)

Reviewed-on: https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api/pulls/165

nixos/module.nix

@@ -5,6 +5,19 @@ let
   config-id = "default";
   nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
   nix = "${config.nix.package.out}/bin/nix";
+  sp-fetch-remote-module = pkgs.writeShellApplication {
+    name = "sp-fetch-remote-module";
+    runtimeInputs = [ config.nix.package.out ];
+    text = ''
+      if [ "$#" -ne 1 ]; then
+        echo "Usage: $0 <URL>"
+        exit 1
+      fi
+
+      URL="$1"
+      nix eval --file /etc/sp-fetch-remote-module.nix --raw --apply "f: f { flakeURL = \"$URL\"; }"
+    '';
+  };
 in
 {
   options.services.selfprivacy-api = {
@@ -46,11 +59,14 @@ in
         pkgs.util-linux
         pkgs.e2fsprogs
         pkgs.iproute2
+        pkgs.postgresql_16.out
+        sp-fetch-remote-module
       ];
       after = [ "network-online.target" ];
       wants = [ "network-online.target" ];
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
+        # Do not forget to edit Postgres identMap if you change the user!
         User = "root";
         ExecStart = "${selfprivacy-graphql-api}/bin/app.py";
         Restart = "always";
@@ -81,11 +97,14 @@ in
         pkgs.util-linux
         pkgs.e2fsprogs
         pkgs.iproute2
+        pkgs.postgresql_16.out
+        sp-fetch-remote-module
       ];
       after = [ "network-online.target" ];
       wants = [ "network-online.target" ];
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
+        # Do not forget to edit Postgres identMap if you change the user!
         User = "root";
         ExecStart = "${pkgs.python312Packages.huey}/bin/huey_consumer.py selfprivacy_api.task_registry.huey";
         Restart = "always";

selfprivacy_api/backup/__init__.py

@@ -246,9 +246,10 @@ def back_up(
         try:
             if service.can_be_backed_up() is False:
                 raise ValueError("cannot backup a non-backuppable service")
-            folders = service.get_folders()
+            folders = service.get_folders_to_back_up()
             service_name = service.get_id()
-            service.pre_backup()
+            service.pre_backup(job=job)
+            Jobs.update(job, status=JobStatus.RUNNING, status_text="Uploading backup")
             snapshot = Backups.provider().backupper.start_backup(
                 folders,
                 service_name,
@@ -258,12 +259,12 @@ def back_up(
             Backups._on_new_snapshot_created(service_name, snapshot)
             if reason == BackupReason.AUTO:
                 Backups._prune_auto_snaps(service)
-            service.post_backup()
+            service.post_backup(job=job)
         except Exception as error:
             Jobs.update(job, status=JobStatus.ERROR, error=str(error))
             raise error
 
-        Jobs.update(job, status=JobStatus.FINISHED)
+        Jobs.update(job, status=JobStatus.FINISHED, result="Backup finished")
         if reason in [BackupReason.AUTO, BackupReason.PRE_RESTORE]:
             Jobs.set_expiration(job, AUTOBACKUP_JOB_EXPIRATION_SECONDS)
         return Backups.sync_date_from_cache(snapshot)
@@ -452,6 +453,7 @@ def restore_snapshot(
             with StoppedService(service):
                 if not service.is_always_active():
                     Backups.assert_dead(service)
+                service.pre_restore(job=job)
                 if strategy == RestoreStrategy.INPLACE:
                     Backups._inplace_restore(service, snapshot, job)
                 else:  # verify_before_download is our default
@@ -464,7 +466,7 @@ def restore_snapshot(
                         service, snapshot.id, verify=True
                     )
 
-                service.post_restore()
+                service.post_restore(job=job)
                 Jobs.update(
                     job,
                     status=JobStatus.RUNNING,
@@ -515,7 +517,7 @@ def _restore_service_from_snapshot(
         snapshot_id: str,
         verify=True,
     ) -> None:
-        folders = service.get_folders()
+        folders = service.get_folders_to_back_up()
 
         Backups.provider().backupper.restore_from_backup(
             snapshot_id,
@@ -715,7 +717,7 @@ def space_usable_for_service(service: Service) -> int:
         Returns the amount of space available on the volume the given
         service is located on.
         """
-        folders = service.get_folders()
+        folders = service.get_folders_to_back_up()
         if folders == []:
             raise ValueError("unallocated service", service.get_id())
 

selfprivacy_api/backup/backuppers/restic_backupper.py

@@ -4,12 +4,13 @@
 import json
 import datetime
 import tempfile
+import logging
+import os
 
 from typing import List, Optional, TypeVar, Callable
 from collections.abc import Iterable
 from json.decoder import JSONDecodeError
-from os.path import exists, join
-from os import mkdir
+from os.path import exists, join, isfile, islink, isdir
 from shutil import rmtree
 from selfprivacy_api.utils.waitloop import wait_until_success
 
@@ -28,6 +29,8 @@
 
 T = TypeVar("T", bound=Callable)
 
+logger = logging.getLogger(__name__)
+
 
 def unlocked_repo(func: T) -> T:
     """unlock repo and retry if it appears to be locked"""
@@ -363,7 +366,27 @@ def restored_size(self, snapshot_id: str) -> int:
                 parsed_output = ResticBackupper.parse_json_output(output)
                 return parsed_output["total_size"]
             except ValueError as error:
-                raise ValueError("cannot restore a snapshot: " + output) from error
+                raise ValueError("Cannot restore a snapshot: " + output) from error
+
+    def _rm_all_folder_contents(self, folder: str) -> None:
+        """
+        Remove all contents of a folder, including subfolders.
+
+        Raises:
+            ValueError: If it encounters an error while removing contents.
+        """
+        try:
+            for filename in os.listdir(folder):
+                path = join(folder, filename)
+                try:
+                    if isfile(path) or islink(path):
+                        os.unlink(path)
+                    elif isdir(path):
+                        rmtree(path)
+                except Exception as error:
+                    raise ValueError("Cannot remove folder contents: ", path) from error
+        except OSError as error:
+            raise ValueError("Cannot access folder: ", folder) from error
 
     @unlocked_repo
     def restore_from_backup(
@@ -376,7 +399,7 @@ def restore_from_backup(
         Restore from backup with restic
         """
         if folders is None or folders == []:
-            raise ValueError("cannot restore without knowing where to!")
+            raise ValueError("Cannot restore without knowing where to!")
 
         with tempfile.TemporaryDirectory() as temp_dir:
             if verify:
@@ -394,9 +417,9 @@ def restore_from_backup(
             else:  # attempting inplace restore
                 for folder in folders:
                     wait_until_success(
-                        lambda: rmtree(folder), timeout_sec=FILESYSTEM_TIMEOUT_SEC
+                        lambda: self._rm_all_folder_contents(folder),
+                        timeout_sec=FILESYSTEM_TIMEOUT_SEC,
                     )
-                    mkdir(folder)
                 self._raw_verified_restore(snapshot_id, target="/")
                 return
 

selfprivacy_api/dependencies.py

@@ -27,4 +27,4 @@ async def get_token_header(
 
 def get_api_version() -> str:
     """Get API version"""
-    return "3.4.0"
+    return "3.5.0"

selfprivacy_api/graphql/common_types/service.py

@@ -6,6 +6,7 @@
 from selfprivacy_api.graphql.common_types.backup import BackupReason
 from selfprivacy_api.graphql.common_types.dns import DnsRecord
 
+from selfprivacy_api.models.services import License
 from selfprivacy_api.services import ServiceManager
 from selfprivacy_api.services import Service as ServiceInterface
 from selfprivacy_api.services import ServiceDnsRecord
@@ -71,6 +72,28 @@ class ServiceStatusEnum(Enum):
     OFF = "OFF"
 
 
+@strawberry.enum
+class SupportLevelEnum(Enum):
+    """Enum representing the support level of a service."""
+
+    NORMAL = "normal"
+    EXPERIMENTAL = "experimental"
+    DEPRECATED = "deprecated"
+    COMMUNITY = "community"
+    UNKNOWN = "unknown"
+
+
+@strawberry.experimental.pydantic.type(model=License)
+class LicenseType:
+    free: strawberry.auto
+    full_name: strawberry.auto
+    redistributable: strawberry.auto
+    short_name: strawberry.auto
+    spdx_id: strawberry.auto
+    url: strawberry.auto
+    deprecated: strawberry.auto
+
+
 def get_storage_usage(root: "Service") -> ServiceStorageUsage:
     """Get storage usage for a service"""
     service = ServiceManager.get_service_by_id(root.id)
@@ -176,10 +199,15 @@ class Service:
     is_required: bool
     is_enabled: bool
     is_installed: bool
+    is_system_service: bool
     can_be_backed_up: bool
     backup_description: str
     status: ServiceStatusEnum
     url: Optional[str]
+    license: List[LicenseType]
+    homepage: Optional[str]
+    source_page: Optional[str]
+    support_level: SupportLevelEnum
 
     @strawberry.field
     def dns_records(self) -> Optional[List[DnsRecord]]:
@@ -207,7 +235,10 @@ def configuration(self) -> Optional[List[ConfigItem]]:
         if not config_items:
             return None
         # By the "type" field convert every dict into a ConfigItem. In the future there will be more types.
-        return [config_item_to_graphql(config_items[item]) for item in config_items]
+        unsorted_config_items = [config_items[item] for item in config_items]
+        # Sort the items by their weight. If there is no weight, implicitly set it to 50.
+        config_items = sorted(unsorted_config_items, key=lambda x: x.get("weight", 50))
+        return [config_item_to_graphql(item) for item in config_items]
 
     # TODO: fill this
     @strawberry.field
@@ -238,6 +269,13 @@ def service_to_graphql_service(service: ServiceInterface) -> Service:
         backup_description=service.get_backup_description(),
         status=ServiceStatusEnum(service.get_status().value),
         url=service.get_url(),
+        is_system_service=service.is_system_service(),
+        license=[
+            LicenseType.from_pydantic(license) for license in service.get_license()
+        ],
+        homepage=service.get_homepage(),
+        source_page=service.get_source_page(),
+        support_level=SupportLevelEnum(service.get_support_level().value),
     )
 
 

selfprivacy_api/graphql/queries/backup.py

@@ -13,6 +13,7 @@
     Service,
     ServiceStatusEnum,
     SnapshotInfo,
+    SupportLevelEnum,
     service_to_graphql_service,
 )
 from selfprivacy_api.graphql.common_types.backup import AutobackupQuotas
@@ -53,6 +54,11 @@ def tombstone_service(service_id: str) -> Service:
         can_be_backed_up=False,
         backup_description="",
         is_installed=False,
+        homepage=None,
+        source_page=None,
+        license=[],
+        is_system_service=False,
+        support_level=SupportLevelEnum.UNKNOWN,
     )
 
 

selfprivacy_api/graphql/queries/services.py

@@ -16,5 +16,8 @@
 class Services:
     @strawberry.field
     def all_services(self) -> typing.List[Service]:
-        services = ServiceManager.get_all_services()
-        return [service_to_graphql_service(service) for service in services]
+        services = [
+            service_to_graphql_service(service)
+            for service in ServiceManager.get_all_services()
+        ]
+        return sorted(services, key=lambda service: service.display_name)