From d6bdb03fc1ea49cdd4b615cb110873f8e09ec5d6 Mon Sep 17 00:00:00 2001 From: Gleb Popov <6yearold@gmail.com> Date: Mon, 13 Mar 2023 15:02:38 +0300 Subject: [PATCH 1/2] Implement the FreeBSD backend. The configuration file for the FreeBSD backend is expected to point to the Poudriere-built repository root. The ArchiveRoot property should be the path to Poudriere "data/packages" directory and Suite names correspond to its subdirectories. Here is an sample config: { "ProjectName": "FreeBSD", "MediaBaseUrl": "http://pkg.FreeBSD.org/FreeBSD:13:amd64/latest/media", "ArchiveRoot": "/usr/local/poudriere/data/packages/", "Backend": "freebsd", "Suites": { "131amd64-default": { "sections": ["default"], "architectures": ["amd64"] } } } Sponsored by: Serenity Cybersecurity, LLC --- src/asgen/backends/freebsd/fbsdpkg.d | 102 ++++++++++++++++++++++ src/asgen/backends/freebsd/fbsdpkgindex.d | 93 ++++++++++++++++++++ src/asgen/backends/freebsd/package.d | 24 +++++ src/asgen/config.d | 8 +- src/asgen/engine.d | 4 + src/asgen/meson.build | 4 + 6 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 src/asgen/backends/freebsd/fbsdpkg.d create mode 100644 src/asgen/backends/freebsd/fbsdpkgindex.d create mode 100644 src/asgen/backends/freebsd/package.d diff --git a/src/asgen/backends/freebsd/fbsdpkg.d b/src/asgen/backends/freebsd/fbsdpkg.d new file mode 100644 index 0000000..4381766 --- /dev/null +++ b/src/asgen/backends/freebsd/fbsdpkg.d @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2023 Serenity Cyber Security, LLC + * Author: Gleb Popov + * + * Licensed under the GNU Lesser General Public License Version 3 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the license, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this software. If not, see . + */ + +module asgen.backends.freebsd.fbsdpkg; + +import std.stdio; +import std.json; +import std.path; +import std.string; +import std.array : empty; +import asgen.backends.interfaces; +import asgen.logging; +import asgen.zarchive; + + +final class FreeBSDPackage : Package +{ +private: + JSONValue pkgjson; + + string pkgFname; + PackageKind _kind; + +public: + this (string pkgRoot, JSONValue[string] j) + { + pkgjson = j; + pkgFname = buildPath (pkgRoot, pkgjson["repopath"].str()); + _kind = PackageKind.PHYSICAL; + } + + @property override string name () const { return pkgjson["name"].str(); } + @property override string ver () const { return pkgjson["version"].str(); } + @property override string arch () const { return pkgjson["arch"].str(); } + @property override string maintainer () const { return pkgjson["maintainer"].str(); } + @property override string getFilename () const { return pkgFname; } + + + @property override const(string[string]) summary () const + { + string[string] sums; + sums["en"] = pkgjson["comment"].str(); + return sums; + } + + @property override const(string[string]) description () const + { + string[string] descs; + descs["en"] = pkgjson["desc"].str(); + return descs; + } + + override + const(ubyte[]) getFileData (string fname) + { + ArchiveDecompressor ad; + ad.open (pkgFname); + + return ad.readData(fname); + } + + @property override + string[] contents () + { + ArchiveDecompressor ad; + ad.open (pkgFname); + + auto c = ad.readContents(); + + //throw new Exception(join(c, "\n")); + + return c; + } + + override + void finish () + { + } + + @property override + PackageKind kind () @safe pure + { + return this._kind; + } +} diff --git a/src/asgen/backends/freebsd/fbsdpkgindex.d b/src/asgen/backends/freebsd/fbsdpkgindex.d new file mode 100644 index 0000000..6e28037 --- /dev/null +++ b/src/asgen/backends/freebsd/fbsdpkgindex.d @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2023 Serenity Cyber Security, LLC + * Author: Gleb Popov + * + * Licensed under the GNU Lesser General Public License Version 3 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the license, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this software. If not, see . + */ + +module asgen.backends.freebsd.fbsdpkgindex; + +import std.json; +import std.stdio; +import std.path; +import std.string; +import std.conv : to; +import std.array : appender; +import std.algorithm : remove; +static import std.file; + +import asgen.logging; +import asgen.zarchive; +import asgen.backends.interfaces; +import asgen.backends.freebsd.fbsdpkg; + + +final class FreeBSDPackageIndex : PackageIndex +{ + +private: + string rootDir; + Package[][string] pkgCache; + +public: + + this (string dir) + { + this.rootDir = dir; + if (!std.file.exists (dir)) + throw new Exception ("Directory '%s' does not exist.", dir); + } + + void release () + { + pkgCache = null; + } + + Package[] packagesFor (string suite, string section, string arch, bool withLongDescs = true) + { + auto pkgRoot = buildPath (rootDir, suite); + auto listsTarFname = buildPath (pkgRoot, "packagesite.pkg"); + if (!std.file.exists (listsTarFname)) { + logWarning ("Package lists file '%s' does not exist.", listsTarFname); + return []; + } + + ArchiveDecompressor ad; + ad.open (listsTarFname); + logDebug ("Opened: %s", listsTarFname); + + auto d = ad.readData("packagesite.yaml"); + auto pkgs = appender!(Package[]); + + foreach(entry; d.split('\n')) { + auto j = parseJSON(assumeUTF(entry)); + if (j.type == JSONType.object) + pkgs ~= to!Package(new FreeBSDPackage(pkgRoot, j.object)); + } + + return pkgs.data; + } + + Package packageForFile (string fname, string suite = null, string section = null) + { + return null; // FIXME: not implemented + } + + bool hasChanges (DataStore dstore, string suite, string section, string arch) + { + return true; + } +} diff --git a/src/asgen/backends/freebsd/package.d b/src/asgen/backends/freebsd/package.d new file mode 100644 index 0000000..e7961ee --- /dev/null +++ b/src/asgen/backends/freebsd/package.d @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2023 Serenity Cyber Security, LLC + * Author: Gleb Popov + * + * Licensed under the GNU Lesser General Public License Version 3 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the license, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this software. If not, see . + */ + +module asgen.backends.freebsd; + +public import asgen.backends.freebsd.fbsdpkg; +public import asgen.backends.freebsd.fbsdpkgindex; diff --git a/src/asgen/config.d b/src/asgen/config.d index b2cbd3e..6f95a22 100644 --- a/src/asgen/config.d +++ b/src/asgen/config.d @@ -68,7 +68,8 @@ enum Backend { Ubuntu, Archlinux, RpmMd, - Alpinelinux + Alpinelinux, + FreeBSD } /** @@ -408,6 +409,11 @@ public: this.backend = Backend.Alpinelinux; this.metadataType = DataType.XML; break; + case "freebsd": + this.backendName = "FreeBSD"; + this.backend = Backend.FreeBSD; + this.metadataType = DataType.XML; + break; default: break; } diff --git a/src/asgen/engine.d b/src/asgen/engine.d index 9e7f216..67526d1 100644 --- a/src/asgen/engine.d +++ b/src/asgen/engine.d @@ -49,6 +49,7 @@ import asgen.backends.debian; import asgen.backends.ubuntu; import asgen.backends.archlinux; import asgen.backends.alpinelinux; +import asgen.backends.freebsd; static if (HAVE_RPMMD) import asgen.backends.rpmmd; @@ -99,6 +100,9 @@ public: case Backend.Alpinelinux: pkgIndex = new AlpinePackageIndex(conf.archiveRoot); break; + case Backend.FreeBSD: + pkgIndex = new FreeBSDPackageIndex (conf.archiveRoot); + break; default: throw new Exception("No backend specified, can not continue!"); } diff --git a/src/asgen/meson.build b/src/asgen/meson.build index 5252f14..1d69af1 100644 --- a/src/asgen/meson.build +++ b/src/asgen/meson.build @@ -64,6 +64,10 @@ backend_sources = [ 'backends/ubuntu/package.d', 'backends/ubuntu/ubupkg.d', 'backends/ubuntu/ubupkgindex.d', + + 'backends/freebsd/package.d', + 'backends/freebsd/fbsdpkg.d', + 'backends/freebsd/fbsdpkgindex.d', ] if get_option('rpmmd') From 51b18f91fbff45623ecd1188d53a7b25fafa1383 Mon Sep 17 00:00:00 2001 From: Gleb Popov <6yearold@gmail.com> Date: Tue, 14 Mar 2023 08:23:26 +0300 Subject: [PATCH 2/2] Add I/O caching for the FreeBSD backend. This reduces the running time considerably when processing large packages like LLVM and Plasma Workspace. The implementation follows ideas from other backends. Sponsored by: Serenity Cybersecurity, LLC --- src/asgen/backends/freebsd/fbsdpkg.d | 21 ++++++++++++--------- src/asgen/backends/freebsd/fbsdpkgindex.d | 13 ++++++++++++- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/asgen/backends/freebsd/fbsdpkg.d b/src/asgen/backends/freebsd/fbsdpkg.d index 4381766..a5552e1 100644 --- a/src/asgen/backends/freebsd/fbsdpkg.d +++ b/src/asgen/backends/freebsd/fbsdpkg.d @@ -38,6 +38,9 @@ private: string pkgFname; PackageKind _kind; + ArchiveDecompressor pkgArchive; + string[] contentsL = null; + public: this (string pkgRoot, JSONValue[string] j) { @@ -70,23 +73,23 @@ public: override const(ubyte[]) getFileData (string fname) { - ArchiveDecompressor ad; - ad.open (pkgFname); + if (!pkgArchive.isOpen) + pkgArchive.open (this.getFilename); - return ad.readData(fname); + return pkgArchive.readData(fname); } @property override string[] contents () { - ArchiveDecompressor ad; - ad.open (pkgFname); - - auto c = ad.readContents(); + if (!this.contentsL.empty) + return this.contentsL; - //throw new Exception(join(c, "\n")); + if (!pkgArchive.isOpen) + pkgArchive.open (this.getFilename); - return c; + this.contentsL = pkgArchive.readContents (); + return this.contentsL; } override diff --git a/src/asgen/backends/freebsd/fbsdpkgindex.d b/src/asgen/backends/freebsd/fbsdpkgindex.d index 6e28037..03f49b5 100644 --- a/src/asgen/backends/freebsd/fbsdpkgindex.d +++ b/src/asgen/backends/freebsd/fbsdpkgindex.d @@ -56,7 +56,7 @@ public: pkgCache = null; } - Package[] packagesFor (string suite, string section, string arch, bool withLongDescs = true) + private Package[] loadPackages (string suite, string section, string arch) { auto pkgRoot = buildPath (rootDir, suite); auto listsTarFname = buildPath (pkgRoot, "packagesite.pkg"); @@ -81,6 +81,17 @@ public: return pkgs.data; } + Package[] packagesFor (string suite, string section, string arch, bool withLongDescs = true) + { + immutable id = "%s-%s-%s".format (suite, section, arch); + if (id !in pkgCache) { + auto pkgs = loadPackages (suite, section, arch); + synchronized (this) pkgCache[id] = pkgs; + } + + return pkgCache[id]; + } + Package packageForFile (string fname, string suite = null, string section = null) { return null; // FIXME: not implemented