CI: make package lint more robust

Includes some fixes in obtain_manifest().

.github/workflows/package-lint.yml

@@ -58,7 +58,7 @@ jobs:
             echo >> /tmp/lint-report.md
 
             ipkfile=/tmp/$(sha256sum "${changed_file}" | cut -d ' ' -f 1).ipk
-            python3 -m repogen.downloadipk -i "${changed_file}" -o "${ipkfile}"
+            python3 -m repogen.downloadipk -i "${changed_file}" -o "${ipkfile}" >> /tmp/lint-report.md || { export lint_retcode=1; continue; }
 
             echo '### Compatibility Check Results' >> /tmp/lint-report.md
 

repogen/downloadipk.py

@@ -1,8 +1,12 @@
+from sys import stderr, exit
 from pathlib import Path
+import json.decoder
 
 import requests
+import requests.exceptions
 
 from repogen import pkg_info
+from jsonschema.exceptions import ValidationError
 
 if __name__ == '__main__':
     import argparse
@@ -12,8 +16,42 @@
     parser.add_argument('-o', '--output', required=True)
     args = parser.parse_args()
 
-    pkginfo = pkg_info.from_package_info_file(Path(args.info))
-    with requests.get(pkginfo['manifest']['ipkUrl'], allow_redirects=True) as resp:
-        with open(args.output, 'wb') as f:
-            f.write(resp.content)
-            print(f'IPK file downloaded: {args.output}')
+    try:
+        pkginfo = pkg_info.from_package_info_file(Path(args.info))
+    except (requests.exceptions.JSONDecodeError, json.decoder.JSONDecodeError) as e:
+        print(f'Could not parse manifest: {e}')
+        exit(2)
+    except (IOError) as e:
+        print(f'Could not open package info file: {e.strerror}')
+        exit(3)
+    except ValidationError as e:
+        print(f'Could not parse package info: {e.message}')
+        exit(4)
+
+    if pkginfo is None:
+        print(f'Failed to get manifest for unknown reason')
+        exit(5)
+
+    try:
+        ipk_url = pkginfo['manifest']['ipkUrl']
+    except KeyError as e:
+        print(f'Invalid package info: missing key {e}')
+        exit(6)
+
+    try:
+        with requests.get(ipk_url, allow_redirects=True) as resp:
+            try:
+                with open(args.output, 'wb') as f:
+                    try:
+                        f.write(resp.content)
+                    except IOError as e:
+                        print(f'Could not write to output IPK file: {e.strerror}')
+                        exit(9)
+                    else:
+                        print(f'IPK file downloaded: {args.output}', file=stderr)
+            except IOError as e:
+                print(f'Could not open output IPK file: {e.strerror}')
+                exit(8)
+    except requests.exceptions.RequestException as e:
+        print(f'Could not download IPK: {e}')
+        exit(7)

repogen/pkg_info.py

@@ -32,7 +32,7 @@ class PackageInfo(TypedDict):
     lastmodified_str: str
 
 
-def load_registry(info_path: Path) -> (str, PackageRegistry):
+def load_registry(info_path: Path) -> tuple[str, PackageRegistry]:
     extension = info_path.suffix
     content: PackageRegistry
     if extension == '.yml':

repogen/pkg_manifest.py

@@ -3,8 +3,7 @@
 import urllib.parse
 from datetime import datetime
 from email.utils import parsedate_to_datetime
-from json import JSONDecodeError
-from typing import Tuple, TypedDict, Optional, NotRequired, Literal
+from typing import TypedDict, Optional, NotRequired, Literal
 from urllib.parse import urljoin
 from urllib.request import url2pathname
 
@@ -33,15 +32,17 @@ class PackageManifest(TypedDict):
     installedSize: NotRequired[int]
 
 
-def obtain_manifest(pkgid: str, channel: str, uri: str, offline: bool = False) -> Tuple[PackageManifest, datetime]:
+def obtain_manifest(pkgid: str, channel: str, uri: str, offline: bool = False) -> tuple[PackageManifest, datetime]:
     parsed = urllib.parse.urlparse(uri)
     if parsed.scheme == 'file':
         manifest_path = url2pathname(parsed.path)
         try:
             with open(manifest_path, encoding='utf-8') as f:
                 return json.load(f), datetime.fromtimestamp(os.stat(manifest_path).st_mtime)
-        except IOError or JSONDecodeError:
-            os.unlink(manifest_path)
+        except (IOError, json.decoder.JSONDecodeError) as e:
+            # XXX: not sure this should be happening; it was broken before
+            #os.unlink(manifest_path)
+            raise e
     else:
         cache_name = f'manifest_{pkgid}_{channel}.json'
         cache_file = cache.path(cache_name)
@@ -61,11 +62,14 @@ def obtain_manifest(pkgid: str, channel: str, uri: str, offline: bool = False) -
                     resp.headers['last-modified'])
                 os.utime(cache_file, (last_modified.timestamp(), last_modified.timestamp()))
             return manifest, last_modified
+        except (requests.exceptions.JSONDecodeError, requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema) as e:
+            # XXX: fall back to cache?
+            raise e
         except requests.exceptions.RequestException as e:
             if cache_file.exists():
                 try:
                     with cache.open_file(cache_name, encoding='utf-8') as f:
                         return json.load(f), datetime.fromtimestamp(cache_file.stat().st_mtime)
-                except IOError or JSONDecodeError:
+                except (IOError, json.decoder.JSONDecodeError):
                     cache_file.unlink()
             raise e