Add multiple mirror sync with extensive include/exclude pattern matching

docs/source/using/mirroring.rst

@@ -61,6 +61,18 @@ Creating a mirror channel is similar to creating proxy channels except that you
      "mirror_mode": "mirror"
    }
 
+To mirror packages from multiple source channels, provide a list of URLs in the ``mirror_channel_url`` attribute:
+
+.. code:: json
+
+   {
+     "name": "mirror-channel",
+     "private": false,
+     "mirror_channel_url": ["https://conda.anaconda.org/btel", "https://conda.anaconda.org/conda-forge"],
+     "mirror_mode": "mirror"
+   }
+
+Note that setting multiple mirror channel urls will work for mirror channels only. Proxy channels can only mirror a single channel and will therefore use only the first url in the list.
 
 
 .. code:: bash
@@ -80,7 +92,7 @@ Mirror channels are read only (you can not add or change packages in these chann
 
    curl ${QUETZ_HOST}/api/channels/mirror-channel/packages
 
-You can also postpone the synchronising the channel by adding ``{"actions": []}`` to the request:
+You can also postpone the synchronisation of the channel by adding ``{"actions": []}`` to the request:
 
 .. code:: bash
 
@@ -94,6 +106,8 @@ You can also postpone the synchronising the channel by adding ``{"actions": []}`
             "mirror_mode":"mirror",
             "actions": []}'
 
+Otherwise, this will be done automatically after the channel is created.
+
 
 Synchronising mirror channel
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -118,8 +132,8 @@ If you don't want to mirror all packages for a channel or can't mirror packages
 :excludelist: Don't download packages in list.
 :proxylist: Parse package metadata, but redirect downloads to upstream server for packages in list.
 
-
-It's possible to change metadata after creating a channel using the PATCH ``/api​/channels​/{channel_name}`` endpoint:
+After creating a mirror channel you can only modify some attributes, e.g. ``private``, ``size_limit``, ``metadata`` and ``ttl``.
+The ``metadata`` attribute` holds the ``includelist``, ``excludelist`` or ``proxylist`` attributes, which can therefore be changed after creating a channel using the PATCH ``/api​/channels​/{channel_name}`` endpoint:
 
 .. code:: json
 

pyproject.toml

@@ -20,7 +20,7 @@ check-imports = ["quetz"]
 ignore = ["W004"]
 
 [tool.tbump.version]
-current = "0.9.2"
+current = "0.10.0"
 regex = '''
   (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
   ((?P<channel>a|b|rc|.dev)(?P<release>\d+))?

quetz/_version.py

@@ -1,2 +1,2 @@
-version_info = (0, 9, 2, "", "")
+version_info = (0, 10, 0, "", "")
 __version__ = '.'.join(filter(lambda s: len(s) > 0, map(str, version_info)))

quetz/dao.py

@@ -311,10 +311,12 @@ def create_channel(
                 "only ASCII characters should be used in channel name"
             )
 
+        # note: due to backwards compatibility, the rest model still calls it
+        # 'mirror_channel_url' but the db model calls it 'mirror_channel_urls'
         channel = Channel(
             name=data.name,
             description=data.description,
-            mirror_channel_url=data.mirror_channel_url,
+            mirror_channel_urls=data.mirror_channel_url,
             mirror_mode=data.mirror_mode,
             private=data.private,
             ttl=data.ttl,
@@ -332,6 +334,19 @@ def create_channel(
 
         return channel
 
+    def remove_package(self, package_name: str, channel_name: Optional[str] = None):
+        """
+        Remove package from database but only from the given channel if specified.
+        Due to cascading behaviour, this will also remove all package versions and
+        package members.
+        """
+        query = self.db.query(Package).filter(Package.name == package_name)
+        if channel_name:
+            query = query.filter(Package.channel_name == channel_name)
+
+        query.delete()
+        self.db.commit()
+
     def cleanup_channel_db(
         self,
         channel_name: str,
@@ -344,7 +359,7 @@ def cleanup_channel_db(
             Package.channel_name == channel_name
         )
         if package_name:
-            all_packages = all_packages.filter(
+            all_packages = all_packages.join(PackageVersion).filter(
                 PackageVersion.package_name == package_name
             )
         for each_package in all_packages:
@@ -379,7 +394,7 @@ def cleanup_channel_db(
             Package.channel_name == channel_name
         )
         if package_name:
-            all_packages = all_packages.filter(
+            all_packages = all_packages.join(PackageVersion).filter(
                 PackageVersion.package_name == package_name
             )
         for each_package in all_packages:
@@ -412,7 +427,7 @@ def cleanup_channel_db(
             Package.channel_name == channel_name
         )
         if package_name:
-            all_packages = all_packages.filter(
+            all_packages = all_packages.join(PackageVersion).filter(
                 PackageVersion.package_name == package_name
             )
         for x, each_package in enumerate(all_packages):
@@ -1009,6 +1024,11 @@ def get_channel_datas(self, channel_name: str):
         )
 
     def assert_size_limits(self, channel_name: str, size: int):
+        """
+        validate that adding a package of size `size` to channel `channel_name`
+        does not exceed the channel size limit.
+        raises: QuotaError
+        """
         channel_size, channel_size_limit = (
             self.db.query(Channel.size, Channel.size_limit)
             .filter(Channel.name == channel_name)

quetz/db_models.py

@@ -173,6 +173,7 @@ class Package(Base):
         ),
         viewonly=True,
         lazy="select",
+        cascade="all,delete-orphan",
     )
 
     @property
@@ -200,14 +201,32 @@ class Channel(Base):
     )
     description = Column(String)
     private = Column(Boolean, default=False)
-    mirror_channel_url = Column(String)
+    mirror_channel_url = Column("mirror_channel_url", String)
     mirror_mode = Column(String)
     channel_metadata = Column(String, server_default='{}', nullable=False)
     timestamp_mirror_sync = Column(Integer, default=0)
     size = Column(BigInteger, default=0)
     size_limit = Column(BigInteger, default=None)
     ttl = Column(Integer, server_default=f'{60 * 60 * 10}', nullable=False)  # 10 hours
 
+    @property
+    def mirror_channel_urls(self):
+        if self.mirror_channel_url is None:
+            return []
+        elif ";" in self.mirror_channel_url:
+            return self.mirror_channel_url.split(";")
+        else:
+            return [self.mirror_channel_url]
+
+    @mirror_channel_urls.setter
+    def mirror_channel_urls(self, value):
+        if isinstance(value, str):
+            self.mirror_channel_url = value
+        elif isinstance(value, list):
+            self.mirror_channel_url = ";".join(value)
+        else:
+            self.mirror_channel_url = None  # type: ignore
+
     packages = relationship(
         'Package', back_populates='channel', cascade="all,delete", uselist=True
     )

quetz/deps.py

@@ -125,7 +125,7 @@ def __call__(
 
         auth.assert_channel_read(channel)
 
-        mirror_url = channel.mirror_channel_url
+        mirror_url = channel.mirror_channel_urls
 
         is_proxy = mirror_url and channel.mirror_mode == "proxy"
         is_mirror = mirror_url and channel.mirror_mode == "mirror"

quetz/main.py

@@ -704,9 +704,15 @@ def post_channel(
     if not new_channel.mirror_channel_url:
         auth.assert_create_channel()
 
-    is_mirror = new_channel.mirror_channel_url and new_channel.mirror_mode == "mirror"
+    is_mirror = (
+        new_channel.mirror_channel_url
+        and new_channel.mirror_mode == rest_models.MirrorMode.mirror
+    )
 
-    is_proxy = new_channel.mirror_channel_url and new_channel.mirror_mode == "proxy"
+    is_proxy = (
+        new_channel.mirror_channel_url
+        and new_channel.mirror_mode == rest_models.MirrorMode.proxy
+    )
 
     if is_mirror:
         auth.assert_create_mirror_channel()
@@ -742,29 +748,32 @@ def post_channel(
         else:
             size_limit = None
 
+    # create database model (channel) from rest model (new_channel)
     channel = dao.create_channel(new_channel, user_id, authorization.OWNER, size_limit)
     pkgstore.create_channel(new_channel.name)
     if not is_proxy:
         indexing.update_indexes(dao, pkgstore, new_channel.name)
 
     # register mirror
     if is_mirror and register_mirror:
-        mirror_url = str(new_channel.mirror_channel_url)
-        mirror_url = mirror_url.replace("get", "api/channels")
-        headers = {"x-api-key": mirror_api_key} if mirror_api_key else {}
-        api_endpoint = str(request.url.replace(query=None)) + '/' + new_channel.name
-        request.url
-        response = session.post(
-            mirror_url + '/mirrors',
-            json={
-                "url": api_endpoint.replace("api/channels", "get"),
-                "api_endpoint": api_endpoint,
-                "metrics_endpoint": api_endpoint.replace("api", "metrics"),
-            },
-            headers=headers,
-        )
-        if response.status_code != 201:
-            logger.warning(f"could not register mirror due to error {response.text}")
+        for mirror_url in channel.mirror_channel_urls:
+            mirror_url = mirror_url.replace("get", "api/channels")
+            headers = {"x-api-key": mirror_api_key} if mirror_api_key else {}
+            api_endpoint = str(request.url.replace(query=None)) + '/' + new_channel.name
+            request.url
+            response = session.post(
+                mirror_url + '/mirrors',
+                json={
+                    "url": api_endpoint.replace("api/channels", "get"),
+                    "api_endpoint": api_endpoint,
+                    "metrics_endpoint": api_endpoint.replace("api", "metrics"),
+                },
+                headers=headers,
+            )
+            if response.status_code != 201:
+                logger.warning(
+                    f"could not register mirror due to error {response.text}"
+                )
 
     for action in actions:
         task.execute_channel_action(
@@ -1479,6 +1488,25 @@ def _assert_filename_package_name_consistent(file_name: str, package_name: str):
     after=after_log(logger, logging.WARNING),
 )
 def _extract_and_upload_package(file, channel_name, channel_proxylist):
+    """
+    Extracts information from a conda package and uploads it to a package store.
+
+    This is automatically reried on error (indicating an unsuccessful upload)
+    The wait time between retries is exponentially increasing.
+
+    Parameters:
+        file (object): The conda package file object to be uploaded.
+        channel_name (str): The name of the channel where the package will be uploaded.
+        channel_proxylist (list): A list of package names that are proxied.
+        If the package is in this list, it will not be uploaded.
+
+    Returns:
+        conda_info: contains extracted information from the conda package.
+
+    Raises:
+        Exception: If there is an error in extracting conda info from the package
+        or in uploading the package to the store.
+    """
     try:
         conda_info = CondaInfo(file.file, file.filename)
     except Exception as e:
@@ -1798,14 +1826,22 @@ def serve_path(
         except ValueError:
             pass
 
-    if is_package_request and channel.mirror_channel_url:
+    if is_package_request and channel.mirror_channel_urls:
         # if we exclude the package from syncing, redirect to original URL
+        # use the first mirror url entry for the redirect
         channel_proxylist = json.loads(channel.channel_metadata).get('proxylist', [])
         if channel_proxylist and package_name and package_name in channel_proxylist:
-            return RedirectResponse(f"{channel.mirror_channel_url}/{path}")
+            # proxying packages only works for the first url in the list
+            redirect_url = channel.mirror_channel_urls[0]
+            return RedirectResponse(f"{redirect_url}/{path}")
 
-    if channel.mirror_channel_url and channel.mirror_mode == "proxy":
-        repository = RemoteRepository(channel.mirror_channel_url, session)
+    # note: proxy mode only works with one mirror url (checked on channel creation)
+    if (
+        channel.mirror_channel_urls
+        and channel.mirror_mode == rest_models.MirrorMode.proxy
+    ):
+        proxy_url = channel.mirror_channel_urls[0]
+        repository = RemoteRepository(proxy_url, session)
         if not pkgstore.file_exists(channel.name, path):
             download_remote_file(repository, pkgstore, channel.name, path)
         elif path.endswith(".json"):

quetz/pkgstores.py

@@ -67,6 +67,10 @@ def support_redirect(self) -> bool:
 
     @abc.abstractmethod
     def create_channel(self, name):
+        """
+        create backend resources for a channel with a given name,
+        e.g. create a directory or bucket.
+        """
         pass
 
     @abc.abstractmethod