fetch: add pypi backend

acbs/fetch.py

@@ -1,7 +1,10 @@
+import http.client
 import logging
 import os
 import shutil
 import subprocess
+import json
+
 from typing import Callable, Dict, List, Optional, Tuple
 
 from acbs.base import ACBSPackageInfo, ACBSSourceInfo
@@ -67,34 +70,46 @@ def tarball_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Opti
         if not info.chksum[1] and not generate_mode:
             raise ValueError('No checksum found. Please specify the checksum!')
         full_path = os.path.join(source_location, filename)
-        flag_path = os.path.join(source_location, f'{filename}.dl')
-        if os.path.exists(full_path) and not os.path.exists(flag_path):
-            info.source_location = full_path
-            return info
         try:
-            # `touch ${flag_path}`, some servers may not support Range, so this is to ensure
-            # if the download has finished successfully, we don't overwrite the downloaded file
-            with open(flag_path, 'wb') as f:
-                f.write(b'')
-            subprocess.check_call(
-                ['wget', '-c', info.url, '-O', full_path])
+            wget_download(info.url, full_path)
             info.source_location = full_path
-            os.unlink(flag_path)  # delete the flag
             return info
         except Exception:
             raise AssertionError('Failed to fetch source with Wget!')
         return None
     return None
 
 
+def wget_download(url: str, full_path: str):
+    flag_path = full_path + ".dl"
+    if os.path.exists(full_path) and not os.path.exists(flag_path):
+        return
+    try:
+        # `touch ${flag_path}`, some servers may not support Range, so this is to ensure
+        # if the download has finished successfully, we don't overwrite the downloaded file
+        with open(flag_path, 'wb') as f:
+            f.write(b'')
+        subprocess.check_call(
+            ['wget', '-c', url, '-O', full_path])
+        os.unlink(flag_path)  # delete the flag
+        return
+    except Exception:
+        raise AssertionError('Failed to fetch source with Wget!')
+
 def tarball_processor_innner(package: ACBSPackageInfo, index: int, source_name: str, decompress=True) -> None:
     info = package.source_uri[index]
     if not info.source_location:
         raise ValueError('Where is the source file?')
     logging.info('Computing %s checksum for %s...' % (info.chksum, info.source_location))
     check_hash_hashlib(info.chksum, info.source_location)
+
     server_filename = os.path.basename(info.url)
     extension = guess_extension_name(server_filename)
+    if len(extension) == 0:
+        # also guess from downloaded file name
+        # pypi (maybe other fetcher) use tarball processor, but has no file extension in info.url
+        extension = guess_extension_name(info.source_location)
+
     # this name is used in the build directory (will be seen by the build scripts)
     # the name will be, e.g. 'acbs-0.1.0.tar.gz'
     facade_name = info.source_name or '{name}-{version}{index}{extension}'.format(
@@ -115,6 +130,38 @@ def tarball_processor(package: ACBSPackageInfo, index: int, source_name: str) ->
     return tarball_processor_innner(package, index, source_name)
 
 
+def pypi_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Optional[ACBSSourceInfo]:
+    # https://warehouse.pypa.io/api-reference/json.html#release
+    api = f"/pypi/{info.url}/{info.revision}/json"
+    logging.info(f"Querying PyPI API endpoint for source URL...")
+    conn = http.client.HTTPSConnection("pypi.org")
+    conn.request("GET", api)
+    response = conn.getresponse()
+    if response.status != 200:
+        logging.error(f"Got response {response.status}")
+        raise RuntimeError("Failed to query PyPI API endpoint")
+    result = json.load(response)
+
+    actual_url = ""
+    for r in result["urls"]:
+        if r["packagetype"] == "sdist":
+            actual_url = r["url"]
+            break
+    if actual_url == "":
+        raise RuntimeError("Can't find source URL")
+    logging.info(f"Source URL is {actual_url}")
+
+    ext = guess_extension_name(actual_url)
+    full_path = os.path.join(source_location, name + ext)
+    try:
+        wget_download(actual_url, full_path)
+        info.source_location = full_path
+        return info
+    except Exception:
+        raise AssertionError('Failed to fetch source with Wget!')
+    return None
+
+
 def blob_processor(package: ACBSPackageInfo, index: int, source_name: str) -> None:
     return tarball_processor_innner(package, index, source_name, False)
 
@@ -303,5 +350,6 @@ def fossil_processor(package: ACBSPackageInfo, index: int, source_name: str) ->
     'FOSSIL': (fossil_fetch, fossil_processor),
     'TARBALL': (tarball_fetch, tarball_processor),
     'FILE': (tarball_fetch, blob_processor),
+    'PYPI': (pypi_fetch, tarball_processor),
     'NONE': (dummy_fetch, dummy_processor),
 }

acbs/parser.py

@@ -72,6 +72,8 @@ def parse_fetch_options(options: str, acbs_source_info: ACBSSourceInfo):
             acbs_source_info.use_url_name = v.strip() == 'true'
         elif k == 'commit':
             acbs_source_info.revision = v.strip()
+        elif k == 'version':
+            acbs_source_info.revision = v.strip()
         elif k == 'copy-repo':
             acbs_source_info.copy_repo = v.strip() == 'true'
         elif k == 'submodule':