From 63ba2578a974fb4373fd8f3b1fa26384c5dd2c37 Mon Sep 17 00:00:00 2001 From: Mariotaku Date: Wed, 5 Apr 2023 16:37:40 +0900 Subject: [PATCH] fixed site generation issue --- repogen/__main__.py | 6 +++-- repogen/apidata.py | 47 +++++++++++++++------------------------ repogen/apppage.py | 16 +++++++------ repogen/cache.py | 10 +++------ repogen/common.py | 18 ++++++++++++++- repogen/lintpkg.py | 1 + repogen/pkg_info.py | 4 ++-- repogen/plugin.py | 5 +++-- requirements.txt | 6 ++--- theme/templates/app.html | 2 +- theme/templates/apps.html | 2 +- 11 files changed, 62 insertions(+), 55 deletions(-) diff --git a/repogen/__main__.py b/repogen/__main__.py index 49c4446..f0b0dc8 100644 --- a/repogen/__main__.py +++ b/repogen/__main__.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- import argparse from os import path +from pathlib import Path + from repogen import apppage, apidata, pkg_info parser = argparse.ArgumentParser() @@ -19,6 +21,6 @@ packages = pkg_info.list_packages(args.input_dir) if args.gen_api: - apidata.generate(packages, path.join(args.output_dir, 'api')) + apidata.generate(packages, Path(args.output_dir, 'api')) -apppage.generate(packages, path.join(args.output_dir, 'apps'), gen_details=args.gen_details, gen_list=args.gen_list) +apppage.generate(packages, Path(args.output_dir, 'apps'), gen_details=args.gen_details, gen_list=args.gen_list) diff --git a/repogen/apidata.py b/repogen/apidata.py index 9fd1046..c2e57a2 100755 --- a/repogen/apidata.py +++ b/repogen/apidata.py @@ -2,8 +2,6 @@ import json import math import urllib.parse -from os import makedirs -from os.path import exists, join from pathlib import Path from typing import List @@ -11,35 +9,28 @@ from markdown import Markdown from repogen import pkg_info -from repogen.common import ITEMS_PER_PAGE +from repogen.common import ITEMS_PER_PAGE, ensure_open from repogen.pkg_info import PackageInfo MANIFEST_KEYS = ('id', 'title', 'iconUri', 'manifestUrl', 'manifest', 'manifestUrlBeta', 'manifestBeta', 'pool') -def fix_manifest_url(item: PackageInfo, app_dir: str): - manifest_url = urllib.parse.urlparse(item['manifestUrl']) - if manifest_url.scheme != 'file': +def fix_manifest_url(item: PackageInfo, app_dir: Path): + if urllib.parse.urlparse(item['manifestUrl']).scheme != 'file': return - - manifests_dir = Path(app_dir).joinpath('manifests') - if not manifests_dir.exists(): - manifests_dir.mkdir() manifest = item["manifest"] - manifest_path = manifests_dir.joinpath(f'{manifest["version"]}.json') - with manifest_path.open(mode='w') as mf: + manifest_path = app_dir.joinpath('manifests', f'{manifest["version"]}.json') + + with ensure_open(manifest_path, mode='w') as mf: json.dump(manifest, mf) - item['manifestUrl'] = manifest_path.as_posix().removeprefix('content') + manifest_url = manifest_path.as_posix() + item['manifestUrl'] = manifest_url[manifest_url.find('/api/apps'):] -def generate(packages: List[PackageInfo], outdir: str): +def generate(packages: List[PackageInfo], outdir: Path): markdown = Markdown() - if not exists(outdir): - makedirs(outdir) - appsdir = join(outdir, 'apps') - if not exists(appsdir): - makedirs(appsdir) + appsdir: Path = outdir.joinpath('apps') def package_item(p_info: PackageInfo, in_apps_dir: bool): package = {k: p_info[k] for k in MANIFEST_KEYS if k in p_info} @@ -55,8 +46,8 @@ def package_item(p_info: PackageInfo, in_apps_dir: bool): max_page = math.ceil(packages_length / ITEMS_PER_PAGE) def save_page(page: int, items: [PackageInfo]): - json_file = join(appsdir, '%d.json' % page) if page > 1 else join(outdir, 'apps.json') - with open(json_file, 'w', encoding='utf-8') as pf: + json_file = appsdir.joinpath('%d.json' % page) if page > 1 else outdir.joinpath('apps.json') + with ensure_open(json_file, 'w', encoding='utf-8') as pf: json.dump({ 'paging': { 'page': page, @@ -70,16 +61,14 @@ def save_page(page: int, items: [PackageInfo]): chunks = more_itertools.chunked(packages, ITEMS_PER_PAGE) if packages else [[]] for index, chunk in enumerate(chunks): for item in chunk: - app_dir = join(appsdir, item['id']) + app_dir = appsdir.joinpath(item['id']) + releases_dir = app_dir.joinpath('releases') fix_manifest_url(item, app_dir) - releases_dir = join(app_dir, 'releases') - if not exists(releases_dir): - makedirs(releases_dir) - app_info = join(releases_dir, 'latest.json') - with open(app_info, 'w', encoding='utf-8') as f: + app_info = releases_dir.joinpath('latest.json') + with ensure_open(app_info, 'w', encoding='utf-8') as f: json.dump(package_item(item, True), f) - desc_html = join(app_dir, 'full_description.html') - with open(desc_html, 'w', encoding='utf-8') as f: + desc_html = app_dir.joinpath('full_description.html') + with ensure_open(desc_html, 'w', encoding='utf-8') as f: f.write(markdown.convert(item['description'])) save_page(index + 1, chunk) print('Generated json data for %d packages.' % len(packages)) diff --git a/repogen/apppage.py b/repogen/apppage.py index 8ffa1bf..5df68ae 100644 --- a/repogen/apppage.py +++ b/repogen/apppage.py @@ -1,18 +1,20 @@ # -*- coding: utf-8 -*- import math -from os import makedirs -from os.path import dirname, exists, join +from os.path import dirname, join +from pathlib import Path +from typing import List import more_itertools import pystache from repogen import pkg_info from repogen.common import ITEMS_PER_PAGE +from repogen.pkg_info import PackageInfo class AppListingGenerator: - def __init__(self, packages): + def __init__(self, packages: List[PackageInfo]): with open(join(dirname(__file__), 'templates', 'apps', 'detail.md'), encoding='utf-8') as f: self.details_template = f.read() with open(join(dirname(__file__), 'templates', 'apps', 'list.html'), encoding='utf-8') as f: @@ -84,11 +86,11 @@ def gen_list(self, outdir): self._gen_page(outdir, items, pagination) -def generate(packages, outdir, gen_details=True, gen_list=True): +def generate(packages: List[PackageInfo], outdir: Path, gen_details=True, gen_list=True): generator = AppListingGenerator(packages) - if not exists(outdir): - makedirs(outdir) + if not outdir.exists(): + outdir.mkdir(parents=True) if gen_details: generator.gen_details(outdir) @@ -107,4 +109,4 @@ def generate(packages, outdir, gen_details=True, gen_list=True): parser.add_argument('-o', '--output-dir', required=True) args = parser.parse_args() - generate(pkg_info.list_packages(args.input_dir), args.output_dir) + generate(pkg_info.list_packages(args.input_dir), Path(args.output_dir)) diff --git a/repogen/cache.py b/repogen/cache.py index 2c33d64..870c027 100644 --- a/repogen/cache.py +++ b/repogen/cache.py @@ -1,6 +1,8 @@ import os from pathlib import Path -from typing import IO, TypeVar, Callable +from typing import IO + +from repogen.common import copy_signature _approot = Path(__file__).parent.parent @@ -8,12 +10,6 @@ _cachepath = _approot.joinpath('cache') -F = TypeVar("F", bound=Callable) - - -def copy_signature(_: F) -> Callable[..., F]: - return lambda f: f - def path(name: str) -> Path: return _cachepath.joinpath(name) diff --git a/repogen/common.py b/repogen/common.py index ef8e652..2ff2e84 100644 --- a/repogen/common.py +++ b/repogen/common.py @@ -1,12 +1,20 @@ # -*- coding: utf-8 -*- import urllib -from os import path +from os import path, PathLike +from pathlib import Path +from typing import Callable, TypeVar from urllib.parse import urljoin import requests ITEMS_PER_PAGE: int = 30 +F = TypeVar("F", bound=Callable) + + +def copy_signature(_: F) -> Callable[..., F]: + return lambda f: f + def url_fixup(u: str) -> str: parsed = urllib.parse.urlparse(u) @@ -25,3 +33,11 @@ def url_size(u): if not content_length: return 0 return int(content_length) + + +@copy_signature(open) +def ensure_open(file: str | PathLike[str], *args, **kwargs): + if not isinstance(file, Path): + file = Path(file) + file.parent.mkdir(parents=True, exist_ok=True) + return open(file, *args, **kwargs) diff --git a/repogen/lintpkg.py b/repogen/lintpkg.py index 443abd5..e3abe9f 100644 --- a/repogen/lintpkg.py +++ b/repogen/lintpkg.py @@ -92,6 +92,7 @@ def _validate_manifest_url(url: str, key: str, e: [str]): args = parser.parse_args() lint_pkginfo = pkg_info.from_package_info_file(args.file) + print(lint_pkginfo) if lint_pkginfo is None: raise ValueError('No package info') diff --git a/repogen/pkg_info.py b/repogen/pkg_info.py index edb52d4..67751e6 100644 --- a/repogen/pkg_info.py +++ b/repogen/pkg_info.py @@ -5,7 +5,7 @@ from os.path import isfile, join from typing import TypedDict, Optional, List, NotRequired -import bleach +import nh3 from repogen.common import url_fixup from repogen.pkg_manifest import obtain_manifest, PackageManifest @@ -55,7 +55,7 @@ def from_package_info(pkgid: str, content: PackageRegistry, offline=False): 'iconUri': content['iconUri'], 'manifestUrl': manifest_url, 'category': content['category'], - 'description': bleach.clean(content.get('description', '')), + 'description': nh3.clean(content.get('description', ''), attributes={'*': {'align'}}, link_rel=None), } if 'detailIconUri' in content: pkginfo['detailIconUri'] = content['detailIconUri'] diff --git a/repogen/plugin.py b/repogen/plugin.py index 625c3e6..2fca10a 100644 --- a/repogen/plugin.py +++ b/repogen/plugin.py @@ -1,5 +1,6 @@ import logging import os +from pathlib import Path from markdown import Markdown from more_itertools import chunked @@ -74,8 +75,8 @@ def add_app_api_data(generator: StaticGenerator): def pool_list(pool: str): return list(sorted(filter(lambda pkg: pkg['pool'] == pool, packages), key=lambda pkg: pkg['title'].lower())) - apidata.generate(pool_list('main'), os.path.join(generator.settings['OUTPUT_PATH'], 'api')) - apidata.generate(pool_list('non-free'), os.path.join(generator.settings['OUTPUT_PATH'], 'api', 'non-free')) + apidata.generate(pool_list('main'), Path(generator.settings['OUTPUT_PATH'], 'api')) + apidata.generate(pool_list('non-free'), Path(generator.settings['OUTPUT_PATH'], 'api', 'non-free')) def register(): diff --git a/requirements.txt b/requirements.txt index 6ed9631..2c6ba2d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,15 +2,15 @@ pelican~=4.8.0 pelican-webassets~=2.0.0 markdown~=3.4.1 pyyaml~=6.0 -pystache +pystache~=0.6.0 more_itertools~=9.0 -bleach~=6.0 +nh3~=0.2.3 requests~=2.28.0 urllib3~=1.26.0 webassets~=2.0 -pyscss~=1.4.0 cssmin~=0.2.0 livereload~=2.6.3 invoke~=2.0.0 ar~=0.3.2 +git+https://github.com/Kronuz/pyScss.git@60414f5d573315a8458b5fbcdf69e5c648c44a9a#egg=pyscss~=1.4.0 git+https://github.com/webosbrew/pelican-theme-webosbrew.git@v1.0.4#egg=pelican-theme-webosbrew \ No newline at end of file diff --git a/theme/templates/app.html b/theme/templates/app.html index cae5f69..436ae1b 100644 --- a/theme/templates/app.html +++ b/theme/templates/app.html @@ -5,7 +5,7 @@ {% block head %} {{ super() }} - {% assets filters="pyscss", output="styles/app.css", "styles/app.scss" %} + {% assets filters="pyscss,cssmin", output="styles/app.css", "styles/app.scss" %} {% endassets %} diff --git a/theme/templates/apps.html b/theme/templates/apps.html index 7a05d7e..5fa143d 100644 --- a/theme/templates/apps.html +++ b/theme/templates/apps.html @@ -5,7 +5,7 @@ {% block head %} {{ super() }} - {% assets filters="pyscss", output="styles/apps.css", "styles/apps.scss" %} + {% assets filters="pyscss,cssmin", output="styles/apps.css", "styles/apps.scss" %} {% endassets %}