Skip to content

Commit a41c5f7

Browse files
gershnikgershnik
committed
Adding command line script
1 parent 44efdec commit a41c5f7

11 files changed

+509
-1
lines changed

‎CHANGELOG.md

+6
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
55

66
## Unreleased
77

8+
### Added
9+
- Command line script to build repositories. Invoke with `repopulator` (or `python3 -m repopulator`)
10+
11+
### Changed
12+
- AptRepo.add_distribution informational fields arguments are now optional rather than required.
13+
814
## [1.0] - 2024-06-08
915

1016
### Added

‎pyproject.toml

+3
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,6 @@ Homepage = 'https://github.com/gershnik/repopulator'
6262
Documentation = 'https://gershnik.github.io/repopulator'
6363
Issues = 'https://github.com/gershnik/repopulator/issues'
6464
Changelog = 'https://github.com/gershnik/repopulator/blob/master/CHANGELOG.md'
65+
66+
[project.scripts]
67+
repopulator = 'repopulator.__main__:main'

‎src/repopulator/__main__.py

+340
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,340 @@
1+
# SPDX-License-Identifier: BSD-3-Clause
2+
# Copyright (c) 2024, Eugene Gershnik
3+
# Use of this source code is governed by a BSD-style
4+
# license that can be found in the LICENSE.txt file or at
5+
# https://opensource.org/licenses/BSD-3-Clause
6+
7+
# pylint: disable=missing-function-docstring
8+
9+
"""Command line utility"""
10+
11+
from __future__ import annotations
12+
13+
import sys
14+
import argparse
15+
16+
from abc import ABCMeta, abstractmethod
17+
from pathlib import Path
18+
from typing import Any, Sequence, Tuple
19+
20+
from repopulator.alpine import AlpineRepo
21+
from repopulator.apt import AptPackage, AptRepo
22+
from repopulator.freebsd import FreeBSDRepo
23+
from repopulator.pacman import PacmanRepo
24+
from repopulator.rpm import RpmRepo
25+
from repopulator.pgp_signer import PgpSigner
26+
from repopulator.pki_signer import PkiSigner
27+
28+
29+
class _Handler(metaclass=ABCMeta):
30+
@abstractmethod
31+
def add_parser(self, key: str, subparsers: argparse._SubParsersAction[argparse.ArgumentParser]):
32+
...
33+
34+
@abstractmethod
35+
def handle(self, args: argparse.Namespace) -> int:
36+
...
37+
38+
class _AlpineHandler(_Handler):
39+
def add_parser(self, key: str, subparsers: argparse._SubParsersAction[argparse.ArgumentParser]):
40+
parser: argparse.ArgumentParser = subparsers.add_parser(key, description='Create Alpine apk repo')
41+
42+
parser.add_argument('-o', '--output', dest='dest', type=Path, metavar='DEST',
43+
help='output path to export the repository to')
44+
45+
parser.add_argument('-d', '--desc', type=str, dest='desc', required=True,
46+
help='repository description')
47+
parser.add_argument('-k', '--key', type=Path, dest='key_path', metavar='PATH', required=True,
48+
help='path of the private key for signing. If -s/--signer option is not supplied '
49+
'the stem of the private key filename is used as the name. '
50+
'So for example a key [email protected] will result in [email protected] '
51+
'being used as a signer name.')
52+
parser.add_argument('-w', '--password', type=str, dest='key_password', metavar='PASSWORD',
53+
help='private key password')
54+
parser.add_argument('-s', '--signer', type=str, dest='signer',
55+
help='name of the signer. This can be used to override name deduced from the key filename')
56+
57+
parser.add_argument('-p', '--packages', nargs='+', metavar='PACKAGE',
58+
help='.apk file(s) to add to repository. To override apk architecture use filename:arch format. '
59+
'For example foo-doc-1.1-r0.apk:x86_64')
60+
61+
62+
def handle(self, args: argparse.Namespace):
63+
desc: str = args.desc
64+
packages: Sequence[str] = args.packages
65+
key_path: Path = args.key_path
66+
key_password: str | None = args.key_password
67+
signer_name: str | None = args.signer
68+
dest: Path = args.dest
69+
70+
if signer_name is None:
71+
last_dot_idx = key_path.name.rfind('.')
72+
if last_dot_idx == 0:
73+
print('unable to determine signer name from the key, please use --signer option', file=sys.stderr)
74+
return 1
75+
signer_name = key_path.name[0:last_dot_idx]
76+
77+
print(f'Signing as {signer_name}')
78+
79+
repo = AlpineRepo(desc)
80+
for p in packages:
81+
parts = p.split(':', 2)
82+
if len(parts) == 2:
83+
print(f'Adding {parts[0]} with architecture {parts[1]}')
84+
repo.add_package(parts[0], force_arch=parts[1])
85+
else:
86+
print(f'Adding {parts[0]}')
87+
repo.add_package(parts[0])
88+
signer = PkiSigner(key_path, key_password)
89+
90+
repo.export(dest, signer, signer_name)
91+
92+
return 0
93+
94+
95+
class _FreeBSDHandler(_Handler):
96+
def add_parser(self, key: str, subparsers: argparse._SubParsersAction[argparse.ArgumentParser]):
97+
parser: argparse.ArgumentParser = subparsers.add_parser(key, description='Create FreeBSD pkg repo')
98+
99+
parser.add_argument('-o', '--output', dest='dest', type=Path, metavar='DEST',
100+
help='output path to export the repository to')
101+
parser.add_argument('-k', '--key', type=Path, dest='key_path', metavar='PATH', required=True,
102+
help='path of the private key for signing.')
103+
parser.add_argument('-w', '--password', type=str, dest='key_password', metavar='PASSWORD',
104+
help='private key password')
105+
parser.add_argument('-p', '--packages', nargs='+', metavar='PACKAGE',
106+
help='.pkg file(s) to add to repository.')
107+
108+
def handle(self, args: argparse.Namespace):
109+
packages: Sequence[str] = args.packages
110+
key_path: Path = args.key_path
111+
key_password: str | None = args.key_password
112+
dest: Path = args.dest
113+
114+
repo = FreeBSDRepo()
115+
116+
for p in packages:
117+
print(f'Adding {p}')
118+
repo.add_package(p)
119+
120+
signer = PkiSigner(key_path, key_password)
121+
122+
repo.export(dest, signer)
123+
124+
return 0
125+
126+
class _RpmHandler(_Handler):
127+
def add_parser(self, key: str, subparsers: argparse._SubParsersAction[argparse.ArgumentParser]):
128+
parser: argparse.ArgumentParser = subparsers.add_parser(key, description='Create RPM repo')
129+
130+
parser.add_argument('-o', '--output', dest='dest', type=Path, metavar='DEST',
131+
help='output path to export the repository to')
132+
parser.add_argument('-k', '--key', type=Path, dest='key_name', metavar='NAME', required=True,
133+
help='Name or ID of the GPG key for signing')
134+
parser.add_argument('-w', '--password', type=str, dest='key_password', metavar='PASSWORD', required=True,
135+
help='GPG key password')
136+
parser.add_argument('-p', '--packages', nargs='+', metavar='PACKAGE',
137+
help='.rpm file(s) to add to repository.')
138+
139+
140+
def handle(self, args: argparse.Namespace):
141+
packages: Sequence[str] = args.packages
142+
key_name: str = args.key_name
143+
key_password: str = args.key_password
144+
dest: Path = args.dest
145+
146+
repo = RpmRepo()
147+
148+
for package in packages:
149+
print(f'Adding {package}')
150+
repo.add_package(package)
151+
152+
signer = PgpSigner(key_name=key_name, key_pwd=key_password)
153+
154+
repo.export(dest, signer)
155+
156+
return 0
157+
158+
class _PacmanHandler(_Handler):
159+
def add_parser(self, key: str, subparsers: argparse._SubParsersAction[argparse.ArgumentParser]):
160+
parser: argparse.ArgumentParser = subparsers.add_parser(key, description='Create Pacman repo')
161+
162+
parser.add_argument('-o', '--output', dest='dest', type=Path, metavar='DEST',
163+
help='output path to export the repository to')
164+
165+
parser.add_argument('-n', '--name', type=str, dest='name', required=True,
166+
help='repository name')
167+
parser.add_argument('-k', '--key', type=Path, dest='key_name', metavar='NAME', required=True,
168+
help='Name or ID of the GPG key for signing')
169+
parser.add_argument('-w', '--password', type=str, dest='key_password', metavar='PASSWORD', required=True,
170+
help='GPG key password')
171+
parser.add_argument('-p', '--packages', nargs='+', metavar='PACKAGE',
172+
help='.zst file to add to repository. If a .sig file with the same name exists next to it, '
173+
' it will be automatically used to supply the package signature')
174+
175+
176+
def handle(self, args: argparse.Namespace):
177+
name: str = args.name
178+
packages: Sequence[str] = args.packages
179+
key_name: str = args.key_name
180+
key_password: str = args.key_password
181+
dest: Path = args.dest
182+
183+
repo = PacmanRepo(name)
184+
185+
for p in packages:
186+
print(f'Adding {p}')
187+
repo.add_package(p)
188+
189+
signer = PgpSigner(key_name=key_name, key_pwd=key_password)
190+
191+
repo.export(dest, signer)
192+
193+
return 0
194+
195+
class _AptDistroAction(argparse.Action):
196+
def __call__(self,
197+
parser: argparse.ArgumentParser,
198+
namespace: argparse.Namespace,
199+
values: str | Sequence[Any] | None,
200+
option_string: str | None = None) -> None:
201+
if not isinstance(values, str):
202+
raise argparse.ArgumentError(self, 'distribution option must have a single value')
203+
if not hasattr(namespace, 'distros'):
204+
namespace.distros = {}
205+
distro = namespace.distros.get(values)
206+
if distro is None:
207+
distro = argparse.Namespace()
208+
distro.origin = None
209+
distro.label = None
210+
distro.suite = None
211+
distro.codename = None
212+
distro.version = None
213+
distro.desc = None
214+
distro.packages = []
215+
namespace.distros[values] = distro
216+
namespace.current_distro = distro
217+
218+
219+
class _AptStoreAction(argparse._StoreAction):
220+
def __call__(self,
221+
parser: argparse.ArgumentParser,
222+
namespace: argparse.Namespace,
223+
values: str | Sequence[Any] | None,
224+
option_string: str | None = None) -> None:
225+
if not hasattr(namespace, 'distros'):
226+
name = argparse._get_action_name(self)
227+
raise argparse.ArgumentError(self, f'you must use --distro before {name}')
228+
super().__call__(parser, namespace.current_distro, values, option_string)
229+
230+
class _AptHandler(_Handler):
231+
def add_parser(self, key: str, subparsers: argparse._SubParsersAction[argparse.ArgumentParser]):
232+
parser: argparse.ArgumentParser = subparsers.add_parser(key, description='Create APT repo')
233+
234+
parser.add_argument('-o', '--output', dest='dest', type=Path, metavar='DEST',
235+
help='output path to export the repository to')
236+
parser.add_argument('-k', '--key', type=Path, dest='key_name', metavar='NAME', required=True,
237+
help='Name or ID of the GPG key for signing')
238+
parser.add_argument('-w', '--password', type=str, dest='key_password', metavar='PASSWORD', required=True,
239+
help='GPG key password')
240+
241+
parser.add_argument('-d', '--distro', type=str, dest='distro', metavar='DISTRO', required=True, action=_AptDistroAction,
242+
help='Distribution name. This can be a relative path like `stable/updates`. All subsequent '
243+
'per-distribution options apply to this distribution '
244+
'Conversely this option is required to precede all per-distribution options. Multiple '
245+
'distributions may be specified on the same command line')
246+
247+
parser.add_argument('-g', '--origin', type=str, dest='origin', metavar='STRING', required=False, action=_AptStoreAction,
248+
help='current distribution origin')
249+
parser.add_argument('-l', '--label', type=str, dest='label', metavar='STRING', required=False, action=_AptStoreAction,
250+
help='current distribution label')
251+
parser.add_argument('-s', '--suite', type=str, dest='suite', metavar='STRING', required=False, action=_AptStoreAction,
252+
help='current distribution suite')
253+
parser.add_argument('-c', '--codename', type=str, dest='codename', metavar='STRING', required=False, action=_AptStoreAction,
254+
help='current distribution codename')
255+
parser.add_argument('--dist-version', type=str, dest='version', metavar='STRING', required=False, action=_AptStoreAction,
256+
help='current distribution version')
257+
parser.add_argument('--desc', type=str, dest='desc', metavar='STRING', required=False, action=_AptStoreAction,
258+
help='current distribution description')
259+
parser.add_argument('-p', '--packages', nargs='+', metavar='PACKAGE', action=_AptStoreAction,
260+
help='.deb file(s) to add to the current distribution. To specify a component for each package '
261+
'use `filename:component` format. For example `foo-1.2.3_amd64.deb:contrib` will assign '
262+
'foo-1.2.3_amd64.deb to contrib component. '
263+
'If no component is specified `main` is assumed.')
264+
265+
def handle(self, args: argparse.Namespace):
266+
distros: dict[str, argparse.Namespace] = args.distros
267+
key_name: str = args.key_name
268+
key_password: str = args.key_password
269+
dest: Path = args.dest
270+
271+
repo = AptRepo()
272+
273+
all_packages: dict[str, AptPackage] = {}
274+
for distro_path, distro_args in distros.items():
275+
# normalize the list of packages to set((name, component))
276+
normalized: set[Tuple[str, str]] = set()
277+
if distro_args.packages is not None:
278+
for package in distro_args.packages:
279+
name, _, component = package.partition(':')
280+
if len(component) == 0:
281+
component = 'main'
282+
normalized.add((name, component))
283+
284+
print(f'Adding distribution: {distro_path}')
285+
distro = repo.add_distribution(distro_path,
286+
origin=distro_args.origin,
287+
label=distro_args.label,
288+
suite=distro_args.suite,
289+
codename=distro_args.codename,
290+
version=distro_args.version,
291+
description=distro_args.desc)
292+
293+
for name, component in normalized:
294+
repo_object = all_packages.get(name)
295+
if repo_object is None:
296+
print(f'Adding new package: {name}')
297+
repo_object = repo.add_package(name)
298+
all_packages[name] = repo_object
299+
print(f'Assigning package: {name} to component {component}')
300+
repo.assign_package(repo_object, distro, component)
301+
302+
303+
signer = PgpSigner(key_name=key_name, key_pwd=key_password)
304+
305+
repo.export(dest, signer)
306+
307+
return 0
308+
309+
310+
def main():
311+
"""script entry point"""
312+
313+
repo_types: dict[str, _Handler] = {
314+
'alpine': _AlpineHandler(),
315+
'apt': _AptHandler(),
316+
'freebsd': _FreeBSDHandler(),
317+
'pacman': _PacmanHandler(),
318+
'rpm': _RpmHandler(),
319+
}
320+
321+
322+
parser = argparse.ArgumentParser(
323+
prog='repopulator',
324+
description='Populates software repositories',
325+
epilog="Use repopulator TYPE -h to get more help for each type's options"
326+
)
327+
subparsers = parser.add_subparsers(
328+
help='type of repository to create, one of: ' + ', '.join(repo_types),
329+
metavar='TYPE',
330+
dest='repo_key',
331+
required=True
332+
)
333+
for repo_key, handler in repo_types.items():
334+
handler.add_parser(repo_key, subparsers)
335+
336+
args = parser.parse_args()
337+
return repo_types[args.repo_key].handle(args)
338+
339+
if __name__ == '__main__':
340+
sys.exit(main())

‎tests/conftest.py

+18
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
# SPDX-License-Identifier: BSD-3-Clause
2+
# Copyright (c) 2024, Eugene Gershnik
3+
# Use of this source code is governed by a BSD-style
4+
# license that can be found in the LICENSE.txt file or at
5+
# https://opensource.org/licenses/BSD-3-Clause
6+
17
# pylint: skip-file
28

39
import os
@@ -90,10 +96,22 @@ def pgp_signer():
9096
key_pwd = os.environ['PGP_KEY_PASSWD'],
9197
homedir=os.environ.get('GNUPGHOME'))
9298

99+
@pytest.fixture
100+
def pgp_cmd():
101+
return ['-k', os.environ['PGP_KEY_NAME'], '-w', os.environ['PGP_KEY_PASSWD']]
102+
103+
93104
@pytest.fixture
94105
def pki_signer():
95106
return PkiSigner((Path(os.environ['BSD_KEY'])), os.environ.get('BSD_KEY_PASSWD'))
96107

108+
@pytest.fixture
109+
def pki_cmd():
110+
ret = ['-k', os.environ['BSD_KEY']]
111+
if (pwd := os.environ.get('BSD_KEY_PASSWD')) is not None:
112+
ret += ['-w', pwd]
113+
return ret
114+
97115

98116
@pytest.fixture(scope='session')
99117
def fixed_datetime():

0 commit comments

Comments
 (0)