Merge pull request #493 from ddalcino/retry_checksums

[Security] Use SHA-256 checksums from trusted mirrors only

Author: miurahr

aqt/archives.py

@@ -19,15 +19,15 @@
 # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
+import binascii
 import posixpath
 import xml.etree.ElementTree as ElementTree
 from dataclasses import dataclass, field
 from logging import getLogger
 from typing import Dict, Iterable, List, Optional, Tuple
 
-from aqt.exceptions import ArchiveDownloadError, ArchiveListError, NoPackageFound
-from aqt.helper import Settings, getUrl, ssplit
+from aqt.exceptions import ArchiveDownloadError, ArchiveListError, ChecksumDownloadFailure, NoPackageFound
+from aqt.helper import Settings, get_hash, getUrl, ssplit
 from aqt.metadata import QtRepoProperty, Version
 
 
@@ -45,10 +45,10 @@ def __post_init__(self):
 @dataclass
 class QtPackage:
     name: str
-    archive_url: str
+    base_url: str
+    archive_path: str
     archive: str
     package_desc: str
-    hashurl: str
     pkg_update_name: str
     version: Optional[Version] = field(default=None)
 
@@ -59,9 +59,9 @@ def __repr__(self):
     def __str__(self):
         v_info = f", version={self.version}" if self.version else ""
         return (
-            f"QtPackage(name={self.name}, url={self.archive_url}, "
+            f"QtPackage(name={self.name}, url={self.archive_path}, "
             f"archive={self.archive}, desc={self.package_desc}"
-            f"hashurl={self.hashurl}{v_info})"
+            f"{v_info})"
         )
 
 
@@ -140,7 +140,7 @@ def __init__(
         self.os_name: str = os_name
         self.all_extra: bool = all_extra
         self.arch_list: List[str] = [item.get("arch") for item in Settings.qt_combinations]
-        self.base: str = posixpath.join(base, "online/qtsdkrepository")
+        self.base: str = base
         self.logger = getLogger("aqt.archives")
         self.archives: List[QtPackage] = []
         self.subarchives: Optional[Iterable[str]] = subarchives
@@ -217,22 +217,26 @@ def _target_packages(self) -> ModuleToPackage:
     def _get_archives(self):
         # Get packages index
 
-        # archive_path: windows_x86/desktop/qt5_59_src_doc_examples
-        archive_path = posixpath.join(
+        # os_target_folder: online/qtsdkrepository/windows_x86/desktop/qt5_59_src_doc_examples
+        os_target_folder = posixpath.join(
+            "online/qtsdkrepository",
             self.os_name + ("_x86" if self.os_name == "windows" else "_x64"),
             self.target,
             f"qt{self.version.major}_{self._version_str()}{self._arch_ext()}",
         )
-        update_xml_url = posixpath.join(self.base, archive_path, "Updates.xml")
-        archive_url = posixpath.join(self.base, archive_path)
-        self._download_update_xml(update_xml_url)
-        self._parse_update_xml(archive_url, self._target_packages())
+        update_xml_path = posixpath.join(os_target_folder, "Updates.xml")
+        self._download_update_xml(update_xml_path)
+        self._parse_update_xml(os_target_folder, self._target_packages())
 
-    def _download_update_xml(self, update_xml_url):
+    def _download_update_xml(self, update_xml_path):
         """Hook for unit test."""
-        self.update_xml_text = getUrl(update_xml_url, self.timeout)
+        xml_hash = binascii.unhexlify(get_hash(update_xml_path, "sha256", self.timeout))
+        if xml_hash == "":
+            raise ChecksumDownloadFailure(f"Checksum for '{update_xml_path}' is empty")
+        update_xml_text = getUrl(posixpath.join(self.base, update_xml_path), self.timeout, xml_hash)
+        self.update_xml_text = update_xml_text
 
-    def _parse_update_xml(self, archive_url, target_packages: Optional[ModuleToPackage]):
+    def _parse_update_xml(self, os_target_folder, target_packages: Optional[ModuleToPackage]):
         if not target_packages:
             target_packages = ModuleToPackage({})
         try:
@@ -270,22 +274,21 @@ def _parse_update_xml(self, archive_url, target_packages: Optional[ModuleToPacka
                 archive_name = archive.split("-", maxsplit=1)[0]
                 if should_filter_archives and archive_name not in self.subarchives:
                     continue
-                package_url = posixpath.join(
-                    # https://download.qt.io/online/qtsdkrepository/linux_x64/desktop/qt5_5150/
-                    archive_url,
+                archive_path = posixpath.join(
+                    # online/qtsdkrepository/linux_x64/desktop/qt5_5150/
+                    os_target_folder,
                     # qt.qt5.5150.gcc_64/
                     pkg_name,
                     # 5.15.0-0-202005140804qtbase-Linux-RHEL_7_6-GCC-Linux-RHEL_7_6-X86_64.7z
                     full_version + archive,
                 )
-                hashurl = package_url + ".sha1"
                 self.archives.append(
                     QtPackage(
                         name=archive_name,
-                        archive_url=package_url,
+                        base_url=self.base,
+                        archive_path=archive_path,
                         archive=archive,
                         package_desc=package_desc,
-                        hashurl=hashurl,
                         pkg_update_name=pkg_name,  # For testing purposes
                     )
                 )
@@ -431,25 +434,20 @@ def handle_missing_updates_xml(self, e: ArchiveDownloadError):
         raise ArchiveListError(msg, suggested_action=[help_msg]) from e
 
     def _get_archives(self):
-        _a = "_x64"
-        if self.os_name == "windows":
-            _a = "_x86"
-
-        archive_url = posixpath.join(
-            # https://download.qt.io/online/qtsdkrepository/
-            self.base,
+        os_target_folder = posixpath.join(
+            "online/qtsdkrepository",
             # linux_x64/
-            self.os_name + _a,
+            self.os_name + ("_x86" if self.os_name == "windows" else "_x64"),
             # desktop/
             self.target,
             # tools_ifw/
             self.tool_name,
         )
-        update_xml_url = posixpath.join(archive_url, "Updates.xml")
+        update_xml_url = posixpath.join(os_target_folder, "Updates.xml")
         self._download_update_xml(update_xml_url)  # call super method.
-        self._parse_update_xml(archive_url, None)
+        self._parse_update_xml(os_target_folder, None)
 
-    def _parse_update_xml(self, archive_url, *ignored):
+    def _parse_update_xml(self, os_target_folder, *ignored):
         try:
             self.update_xml = ElementTree.fromstring(self.update_xml_text)
         except ElementTree.ParseError as perror:
@@ -472,22 +470,21 @@ def _parse_update_xml(self, archive_url, *ignored):
             message = f"The package '{self.arch}' contains no downloadable archives!"
             raise NoPackageFound(message)
         for archive in ssplit(downloadable_archives):
-            package_url = posixpath.join(
-                # https://download.qt.io/online/qtsdkrepository/linux_x64/desktop/tools_ifw/
-                archive_url,
+            archive_path = posixpath.join(
+                # online/qtsdkrepository/linux_x64/desktop/tools_ifw/
+                os_target_folder,
                 # qt.tools.ifw.41/
                 name,
                 #  4.1.1-202105261130ifw-linux-x64.7z
                 f"{named_version}{archive}",
             )
-            hashurl = package_url + ".sha1"
             self.archives.append(
                 QtPackage(
                     name=name,
-                    archive_url=package_url,
+                    base_url=self.base,
+                    archive_path=archive_path,
                     archive=archive,
                     package_desc=package_desc,
-                    hashurl=hashurl,
                     pkg_update_name=name,  # Redundant
                 )
             )

aqt/exceptions.py

@@ -20,6 +20,8 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 from typing import Iterable
 
+DOCS_CONFIG = "https://aqtinstall.readthedocs.io/en/stable/configuration.html#configuration"
+
 
 class AqtException(Exception):
     def __init__(self, *args, **kwargs):
@@ -47,6 +49,19 @@ class ArchiveChecksumError(ArchiveDownloadError):
     pass
 
 
+class ChecksumDownloadFailure(ArchiveDownloadError):
+    def __init__(self, *args, **kwargs):
+        kwargs["suggested_action"] = kwargs.pop("suggested_action", []).extend(
+            [
+                "Check your internet connection",
+                "Consider modifying `requests.max_retries_to_retrieve_hash` in settings.ini",
+                f"Consider modifying `mirrors.trusted_mirrors` in settings.ini (see {DOCS_CONFIG})",
+            ]
+        )
+        kwargs["should_show_help"] = True
+        super(ChecksumDownloadFailure, self).__init__(*args, **kwargs)
+
+
 class ArchiveConnectionError(AqtException):
     pass
 

aqt/helper.py

@@ -24,18 +24,26 @@
 import json
 import logging.config
 import os
+import posixpath
+import secrets
 import sys
 import xml.etree.ElementTree as ElementTree
 from logging import getLogger
 from logging.handlers import QueueListener
 from pathlib import Path
-from typing import Callable, Dict, List, Tuple
+from typing import Callable, Dict, Generator, List, Optional, Tuple
 from urllib.parse import urlparse
 
 import requests
 import requests.adapters
 
-from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError, ArchiveListError
+from aqt.exceptions import (
+    ArchiveChecksumError,
+    ArchiveConnectionError,
+    ArchiveDownloadError,
+    ArchiveListError,
+    ChecksumDownloadFailure,
+)
 
 
 def _get_meta(url: str):
@@ -47,7 +55,13 @@ def _check_content_type(ct: str) -> bool:
     return any(ct.startswith(t) for t in candidate)
 
 
-def getUrl(url: str, timeout) -> str:
+def getUrl(url: str, timeout, expected_hash: Optional[bytes] = None) -> str:
+    """
+    Gets a file from `url` via HTTP GET.
+
+    No caller should call this function without providing an expected_hash, unless
+    the caller is `get_hash`, which cannot know what the expected hash should be.
+    """
     logger = getLogger("aqt.helper")
     with requests.sessions.Session() as session:
         retries = requests.adapters.Retry(
@@ -76,6 +90,14 @@ def getUrl(url: str, timeout) -> str:
                 msg = f"Failed to retrieve file at {url}\nServer response code: {r.status_code}, reason: {r.reason}"
                 raise ArchiveDownloadError(msg)
         result = r.text
+        filename = url.split("/")[-1]
+        actual_hash = hashlib.sha256(bytes(result, "utf-8")).digest()
+        if expected_hash is not None and expected_hash != actual_hash:
+            raise ArchiveChecksumError(
+                f"Downloaded file {filename} is corrupted! Detect checksum error.\n"
+                f"Expect {expected_hash.hex()}: {url}\n"
+                f"Actual {actual_hash.hex()}: {filename}"
+            )
     return result
 
 
@@ -134,6 +156,42 @@ def retry_on_errors(action: Callable[[], any], acceptable_errors: Tuple, num_ret
             raise e from e
 
 
+def retry_on_bad_connection(function: Callable[[str], any], base_url: str):
+    logger = getLogger("aqt.helper")
+    fallback_url = secrets.choice(Settings.fallbacks)
+    try:
+        return function(base_url)
+    except ArchiveConnectionError:
+        logger.warning(f"Connection to '{base_url}' failed. Retrying with fallback '{fallback_url}'.")
+        return function(fallback_url)
+
+
+def iter_list_reps(_list: List, num_reps: int) -> Generator:
+    list_index = 0
+    for i in range(num_reps):
+        yield _list[list_index]
+        list_index += 1
+        if list_index >= len(_list):
+            list_index = 0
+
+
+def get_hash(archive_path: str, algorithm: str, timeout) -> str:
+    logger = getLogger("aqt.helper")
+    for base_url in iter_list_reps(Settings.trusted_mirrors, Settings.max_retries_to_retrieve_hash):
+        url = posixpath.join(base_url, f"{archive_path}.{algorithm}")
+        logger.debug(f"Attempt to download checksum at {url}")
+        try:
+            r = getUrl(url, timeout)
+            # sha256 & md5 files are: "some_hash archive_filename"
+            return r.split(" ")[0]
+        except (ArchiveConnectionError, ArchiveDownloadError):
+            pass
+    filename = archive_path.split("/")[-1]
+    raise ChecksumDownloadFailure(
+        f"Failed to download checksum for the file '{filename}' from mirrors '{Settings.trusted_mirrors}"
+    )
+
+
 def altlink(url: str, alt: str):
     """
     Blacklisting redirected(alt) location based on Settings.blacklist configuration.
@@ -225,7 +283,9 @@ def xml_to_modules(
 
 class MyConfigParser(configparser.ConfigParser):
     def getlist(self, section: str, option: str, fallback=[]) -> List[str]:
-        value = self.get(section, option)
+        value = self.get(section, option, fallback=None)
+        if value is None:
+            return fallback
         try:
             result = list(filter(None, (x.strip() for x in value.splitlines())))
         except Exception:
@@ -361,10 +421,18 @@ def max_retries_on_connection_error(self):
     def max_retries_on_checksum_error(self):
         return self.config.getint("requests", "max_retries_on_checksum_error", fallback=int(self.max_retries))
 
+    @property
+    def max_retries_to_retrieve_hash(self):
+        return self.config.getint("requests", "max_retries_to_retrieve_hash", fallback=int(self.max_retries))
+
     @property
     def backoff_factor(self):
         return self.config.getfloat("requests", "retry_backoff", fallback=0.1)
 
+    @property
+    def trusted_mirrors(self):
+        return self.config.getlist("mirrors", "trusted_mirrors", fallback=[self.baseurl])
+
     @property
     def fallbacks(self):
         return self.config.getlist("mirrors", "fallbacks", fallback=[])