fixed site generation issue

repogen/__main__.py

@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-
 import argparse
 from os import path
+from pathlib import Path
+
 from repogen import apppage, apidata, pkg_info
 
 parser = argparse.ArgumentParser()
@@ -19,6 +21,6 @@
 packages = pkg_info.list_packages(args.input_dir)
 
 if args.gen_api:
-    apidata.generate(packages, path.join(args.output_dir, 'api'))
+    apidata.generate(packages, Path(args.output_dir, 'api'))
 
-apppage.generate(packages, path.join(args.output_dir, 'apps'), gen_details=args.gen_details, gen_list=args.gen_list)
+apppage.generate(packages, Path(args.output_dir, 'apps'), gen_details=args.gen_details, gen_list=args.gen_list)

repogen/apidata.py

@@ -2,44 +2,35 @@
 import json
 import math
 import urllib.parse
-from os import makedirs
-from os.path import exists, join
 from pathlib import Path
 from typing import List
 
 import more_itertools
 from markdown import Markdown
 
 from repogen import pkg_info
-from repogen.common import ITEMS_PER_PAGE
+from repogen.common import ITEMS_PER_PAGE, ensure_open
 from repogen.pkg_info import PackageInfo
 
 MANIFEST_KEYS = ('id', 'title', 'iconUri', 'manifestUrl', 'manifest', 'manifestUrlBeta', 'manifestBeta', 'pool')
 
 
-def fix_manifest_url(item: PackageInfo, app_dir: str):
-    manifest_url = urllib.parse.urlparse(item['manifestUrl'])
-    if manifest_url.scheme != 'file':
+def fix_manifest_url(item: PackageInfo, app_dir: Path):
+    if urllib.parse.urlparse(item['manifestUrl']).scheme != 'file':
         return
-
-    manifests_dir = Path(app_dir).joinpath('manifests')
-    if not manifests_dir.exists():
-        manifests_dir.mkdir()
     manifest = item["manifest"]
-    manifest_path = manifests_dir.joinpath(f'{manifest["version"]}.json')
-    with manifest_path.open(mode='w') as mf:
+    manifest_path = app_dir.joinpath('manifests', f'{manifest["version"]}.json')
+
+    with ensure_open(manifest_path, mode='w') as mf:
         json.dump(manifest, mf)
-    item['manifestUrl'] = manifest_path.as_posix().removeprefix('content')
+    manifest_url = manifest_path.as_posix()
+    item['manifestUrl'] = manifest_url[manifest_url.find('/api/apps'):]
 
 
-def generate(packages: List[PackageInfo], outdir: str):
+def generate(packages: List[PackageInfo], outdir: Path):
     markdown = Markdown()
 
-    if not exists(outdir):
-        makedirs(outdir)
-    appsdir = join(outdir, 'apps')
-    if not exists(appsdir):
-        makedirs(appsdir)
+    appsdir: Path = outdir.joinpath('apps')
 
     def package_item(p_info: PackageInfo, in_apps_dir: bool):
         package = {k: p_info[k] for k in MANIFEST_KEYS if k in p_info}
@@ -55,8 +46,8 @@ def package_item(p_info: PackageInfo, in_apps_dir: bool):
     max_page = math.ceil(packages_length / ITEMS_PER_PAGE)
 
     def save_page(page: int, items: [PackageInfo]):
-        json_file = join(appsdir, '%d.json' % page) if page > 1 else join(outdir, 'apps.json')
-        with open(json_file, 'w', encoding='utf-8') as pf:
+        json_file = appsdir.joinpath('%d.json' % page) if page > 1 else outdir.joinpath('apps.json')
+        with ensure_open(json_file, 'w', encoding='utf-8') as pf:
             json.dump({
                 'paging': {
                     'page': page,
@@ -70,16 +61,14 @@ def save_page(page: int, items: [PackageInfo]):
     chunks = more_itertools.chunked(packages, ITEMS_PER_PAGE) if packages else [[]]
     for index, chunk in enumerate(chunks):
         for item in chunk:
-            app_dir = join(appsdir, item['id'])
+            app_dir = appsdir.joinpath(item['id'])
+            releases_dir = app_dir.joinpath('releases')
             fix_manifest_url(item, app_dir)
-            releases_dir = join(app_dir, 'releases')
-            if not exists(releases_dir):
-                makedirs(releases_dir)
-            app_info = join(releases_dir, 'latest.json')
-            with open(app_info, 'w', encoding='utf-8') as f:
+            app_info = releases_dir.joinpath('latest.json')
+            with ensure_open(app_info, 'w', encoding='utf-8') as f:
                 json.dump(package_item(item, True), f)
-            desc_html = join(app_dir, 'full_description.html')
-            with open(desc_html, 'w', encoding='utf-8') as f:
+            desc_html = app_dir.joinpath('full_description.html')
+            with ensure_open(desc_html, 'w', encoding='utf-8') as f:
                 f.write(markdown.convert(item['description']))
         save_page(index + 1, chunk)
     print('Generated json data for %d packages.' % len(packages))

repogen/apppage.py

@@ -1,18 +1,20 @@
 # -*- coding: utf-8 -*-
 import math
-from os import makedirs
-from os.path import dirname, exists, join
+from os.path import dirname, join
+from pathlib import Path
+from typing import List
 
 import more_itertools
 import pystache
 
 from repogen import pkg_info
 from repogen.common import ITEMS_PER_PAGE
+from repogen.pkg_info import PackageInfo
 
 
 class AppListingGenerator:
 
-    def __init__(self, packages):
+    def __init__(self, packages: List[PackageInfo]):
         with open(join(dirname(__file__), 'templates', 'apps', 'detail.md'), encoding='utf-8') as f:
             self.details_template = f.read()
         with open(join(dirname(__file__), 'templates', 'apps', 'list.html'), encoding='utf-8') as f:
@@ -84,11 +86,11 @@ def gen_list(self, outdir):
             self._gen_page(outdir, items, pagination)
 
 
-def generate(packages, outdir, gen_details=True, gen_list=True):
+def generate(packages: List[PackageInfo], outdir: Path, gen_details=True, gen_list=True):
     generator = AppListingGenerator(packages)
 
-    if not exists(outdir):
-        makedirs(outdir)
+    if not outdir.exists():
+        outdir.mkdir(parents=True)
 
     if gen_details:
         generator.gen_details(outdir)
@@ -107,4 +109,4 @@ def generate(packages, outdir, gen_details=True, gen_list=True):
     parser.add_argument('-o', '--output-dir', required=True)
     args = parser.parse_args()
 
-    generate(pkg_info.list_packages(args.input_dir), args.output_dir)
+    generate(pkg_info.list_packages(args.input_dir), Path(args.output_dir))

repogen/cache.py

@@ -1,19 +1,15 @@
 import os
 from pathlib import Path
-from typing import IO, TypeVar, Callable
+from typing import IO
+
+from repogen.common import copy_signature
 
 _approot = Path(__file__).parent.parent
 
 assert _approot.samefile(os.getcwd())
 
 _cachepath = _approot.joinpath('cache')
 
-F = TypeVar("F", bound=Callable)
-
-
-def copy_signature(_: F) -> Callable[..., F]:
-    return lambda f: f
-
 
 def path(name: str) -> Path:
     return _cachepath.joinpath(name)

repogen/common.py

@@ -1,12 +1,20 @@
 # -*- coding: utf-8 -*-
 import urllib
-from os import path
+from os import path, PathLike
+from pathlib import Path
+from typing import Callable, TypeVar
 from urllib.parse import urljoin
 
 import requests
 
 ITEMS_PER_PAGE: int = 30
 
+F = TypeVar("F", bound=Callable)
+
+
+def copy_signature(_: F) -> Callable[..., F]:
+    return lambda f: f
+
 
 def url_fixup(u: str) -> str:
     parsed = urllib.parse.urlparse(u)
@@ -25,3 +33,11 @@ def url_size(u):
     if not content_length:
         return 0
     return int(content_length)
+
+
+@copy_signature(open)
+def ensure_open(file: str | PathLike[str], *args, **kwargs):
+    if not isinstance(file, Path):
+        file = Path(file)
+    file.parent.mkdir(parents=True, exist_ok=True)
+    return open(file, *args, **kwargs)

repogen/lintpkg.py

@@ -92,6 +92,7 @@ def _validate_manifest_url(url: str, key: str, e: [str]):
     args = parser.parse_args()
 
     lint_pkginfo = pkg_info.from_package_info_file(args.file)
+    print(lint_pkginfo)
     if lint_pkginfo is None:
         raise ValueError('No package info')
 

repogen/pkg_info.py

@@ -5,7 +5,7 @@
 from os.path import isfile, join
 from typing import TypedDict, Optional, List, NotRequired
 
-import bleach
+import nh3
 
 from repogen.common import url_fixup
 from repogen.pkg_manifest import obtain_manifest, PackageManifest
@@ -55,7 +55,7 @@ def from_package_info(pkgid: str, content: PackageRegistry, offline=False):
         'iconUri': content['iconUri'],
         'manifestUrl': manifest_url,
         'category': content['category'],
-        'description': bleach.clean(content.get('description', '')),
+        'description': nh3.clean(content.get('description', ''), attributes={'*': {'align'}}, link_rel=None),
     }
     if 'detailIconUri' in content:
         pkginfo['detailIconUri'] = content['detailIconUri']

repogen/plugin.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from pathlib import Path
 
 from markdown import Markdown
 from more_itertools import chunked
@@ -74,8 +75,8 @@ def add_app_api_data(generator: StaticGenerator):
     def pool_list(pool: str):
         return list(sorted(filter(lambda pkg: pkg['pool'] == pool, packages), key=lambda pkg: pkg['title'].lower()))
 
-    apidata.generate(pool_list('main'), os.path.join(generator.settings['OUTPUT_PATH'], 'api'))
-    apidata.generate(pool_list('non-free'), os.path.join(generator.settings['OUTPUT_PATH'], 'api', 'non-free'))
+    apidata.generate(pool_list('main'), Path(generator.settings['OUTPUT_PATH'], 'api'))
+    apidata.generate(pool_list('non-free'), Path(generator.settings['OUTPUT_PATH'], 'api', 'non-free'))
 
 
 def register():

requirements.txt

@@ -2,15 +2,15 @@ pelican~=4.8.0
 pelican-webassets~=2.0.0
 markdown~=3.4.1
 pyyaml~=6.0
-pystache
+pystache~=0.6.0
 more_itertools~=9.0
-bleach~=6.0
+nh3~=0.2.3
 requests~=2.28.0
 urllib3~=1.26.0
 webassets~=2.0
-pyscss~=1.4.0
 cssmin~=0.2.0
 livereload~=2.6.3
 invoke~=2.0.0
 ar~=0.3.2
+git+https://github.com/Kronuz/pyScss.git@60414f5d573315a8458b5fbcdf69e5c648c44a9a#egg=pyscss~=1.4.0
 git+https://github.com/webosbrew/[email protected]#egg=pelican-theme-webosbrew

theme/templates/app.html

@@ -5,7 +5,7 @@
 
 {% block head %}
   {{ super() }}
-  {% assets filters="pyscss", output="styles/app.css", "styles/app.scss" %}
+  {% assets filters="pyscss,cssmin", output="styles/app.css", "styles/app.scss" %}
   <link href="{{ SITEURL }}/{{ ASSET_URL }}" type="text/css" rel="stylesheet">
   {% endassets %}
 

theme/templates/apps.html

@@ -5,7 +5,7 @@
 
 {% block head %}
   {{ super() }}
-  {% assets filters="pyscss", output="styles/apps.css", "styles/apps.scss" %}
+  {% assets filters="pyscss,cssmin", output="styles/apps.css", "styles/apps.scss" %}
   <link href="{{ SITEURL }}/{{ ASSET_URL }}" type="text/css" rel="stylesheet">
   {% endassets %}