From 1e793d301df30daaf38a2377b2f881ebbb6df593 Mon Sep 17 00:00:00 2001 From: Mariotaku Date: Mon, 15 Apr 2024 15:50:01 +0900 Subject: [PATCH] host selected packages (hbchannel & updater) on site --- pelicanconf.py | 6 ++++++ repogen/apidata.py | 37 +++++++++++++++++++++++++++++-------- repogen/downloadipk.py | 2 +- repogen/plugin.py | 8 +++++--- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/pelicanconf.py b/pelicanconf.py index 4348827..71e4052 100644 --- a/pelicanconf.py +++ b/pelicanconf.py @@ -84,6 +84,12 @@ ('utility', 'Utilities'), ] +# Following packages will have their IPKs downloaded and hosted on the site +HOST_PACKAGES: set[str] = { + 'org.webosbrew.hbchannel', + 'org.webosbrew.safeupdate' +} + DEFAULT_PAGINATION = 20 # Uncomment following line if you want document-relative URLs when developing diff --git a/repogen/apidata.py b/repogen/apidata.py index 5714158..0af8958 100755 --- a/repogen/apidata.py +++ b/repogen/apidata.py @@ -4,9 +4,10 @@ import os import urllib.parse from pathlib import Path -from typing import List +from typing import List, Set import more_itertools +import requests from markdown import Markdown from repogen import pkg_info @@ -31,10 +32,27 @@ def fix_manifest_url(item: PackageInfo, app_dir: Path): item['manifestUrl'] = manifest_url[manifest_url.find('/api/apps'):] -def generate(packages: List[PackageInfo], outdir: Path): +def save_ipk(item: PackageInfo, apps_dir: Path, site_url: str): + """ + Save the ipk file to the apps directory, and modify PackageInfo to point to the local file. + """ + manifest = item['manifest'] + app_ipk = apps_dir.joinpath(item["id"], 'releases', f'{manifest["ipkHash"]["sha256"]}.ipk') + if not app_ipk.parent.exists(): + app_ipk.parent.mkdir(parents=True, exist_ok=True) + # Not likely to happen, but just in case someday we host some ipk files by ourselves directly. + if site_url and manifest['ipkUrl'].startswith(site_url): + return + with requests.get(manifest['ipkUrl'], allow_redirects=True) as resp: + with ensure_open(app_ipk, 'wb') as f: + f.write(resp.content) + manifest['ipkUrl'] = f'{site_url.removesuffix("/")}/{"/".join(app_ipk.parts[-4:])}' + + +def generate(packages: List[PackageInfo], api_dir: Path, apps_dir: Path = None, host_packages: Set[str] = None): markdown = Markdown() - appsdir: Path = outdir.joinpath('apps') + appsdir: Path = api_dir.joinpath('apps') site_url = siteurl() def package_item(p_info: PackageInfo, in_apps_dir: bool, is_details: bool) -> PackageInfo: @@ -56,7 +74,7 @@ def package_item(p_info: PackageInfo, in_apps_dir: bool, is_details: bool) -> Pa max_page = math.ceil(packages_length / ITEMS_PER_PAGE) def save_page(page: int, items: [PackageInfo]): - json_file = appsdir.joinpath('%d.json' % page) if page > 1 else outdir.joinpath('apps.json') + json_file = appsdir.joinpath('%d.json' % page) if page > 1 else api_dir.joinpath('apps.json') with ensure_open(json_file, 'w', encoding='utf-8') as pf: json.dump({ 'paging': { @@ -71,13 +89,16 @@ def save_page(page: int, items: [PackageInfo]): chunks = more_itertools.chunked(packages, ITEMS_PER_PAGE) if packages else [[]] for index, chunk in enumerate(chunks): for item in chunk: - app_dir = appsdir.joinpath(item['id']) - releases_dir = app_dir.joinpath('releases') - fix_manifest_url(item, app_dir) + api_app_dir = appsdir.joinpath(item['id']) + releases_dir = api_app_dir.joinpath('releases') + if host_packages and item['id'] in host_packages: + save_ipk(item, apps_dir, site_url) + fix_manifest_url(item, api_app_dir) + # This will be used by dev-manager-desktop app_info = releases_dir.joinpath('latest.json') with ensure_open(app_info, 'w', encoding='utf-8') as f: json.dump(package_item(item, True, True), f) - desc_html = app_dir.joinpath('full_description.html') + desc_html = api_app_dir.joinpath('full_description.html') with ensure_open(desc_html, 'w', encoding='utf-8') as f: f.write(markdown.convert(item['description'])) save_page(index + 1, chunk) diff --git a/repogen/downloadipk.py b/repogen/downloadipk.py index 4fede05..bd3a526 100644 --- a/repogen/downloadipk.py +++ b/repogen/downloadipk.py @@ -20,7 +20,7 @@ except (requests.exceptions.JSONDecodeError, json.decoder.JSONDecodeError) as e: print(f'Could not parse manifest: {e}') exit(2) - except (IOError) as e: + except IOError as e: print(f'Could not open package info file: {e.strerror}') exit(3) except ValidationError as e: diff --git a/repogen/plugin.py b/repogen/plugin.py index f146807..e3e8bed 100644 --- a/repogen/plugin.py +++ b/repogen/plugin.py @@ -30,7 +30,7 @@ def read(self, filename: str): info['manifest']['iconUri'] = info['iconUri'] metadata = { 'title': info['title'], - 'override_save_as': f'apps/{info["id"]}.html', + 'override_save_as': f'apps/{info["id"]}/index.html', 'template': 'app', 'status': 'hidden', 'modified': info['lastmodified'], @@ -93,12 +93,14 @@ def apps_list_href(page): def add_app_api_data(generator: StaticGenerator): packages = generator.settings['PACKAGES'].values() + output_path = generator.settings['OUTPUT_PATH'] + host_packages = generator.settings.get('HOST_PACKAGES', None) def pool_list(pool: str): return list(sorted(filter(lambda pkg: pkg['pool'] == pool, packages), key=lambda pkg: pkg['title'].lower())) - apidata.generate(pool_list('main'), Path(generator.settings['OUTPUT_PATH'], 'api')) - apidata.generate(pool_list('non-free'), Path(generator.settings['OUTPUT_PATH'], 'api', 'non-free')) + apidata.generate(pool_list('main'), Path(output_path, 'api'), Path(output_path, 'apps'), host_packages) + apidata.generate(pool_list('non-free'), Path(output_path, 'api', 'non-free')) def register():