Skip to content

Commit f9f2c5c

Browse files
committed
zerocopy nonsense via cista::offset
1 parent a4f49e9 commit f9f2c5c

34 files changed

Lines changed: 11336 additions & 557 deletions

extra/bash-completion

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,9 @@ _pkgfile() {
2020
local shortopts=(-l -s -u -b -C -D -g -i -q -R -r -h -V -v -w -z -0)
2121
local longopts=(--list --search --update --binaries --glob --ignorecase
2222
--quiet --regex --help --version --verbose --raw --null)
23-
local longoptsarg=(--compress --config --cachedir --repo)
23+
local longoptsarg=(--config --cachedir --repo)
2424
local allopts=("${shortopts[@]}" "${longopts[@]}" "${longoptsarg[@]}")
2525

26-
local compressopts=(none gzip bzip2 lz4 lzma lzop xz zstd)
27-
2826
# maybe mangle the arguments in case we're looking at a --longopt=$val
2927
[[ $cur = '=' ]] && cur=
3028
if [[ $prev = '=' ]] && __inarray "$prevprev" "${allopts[@]}"; then
@@ -42,10 +40,6 @@ _pkgfile() {
4240
compopt -o filenames
4341
return 0
4442
;;
45-
-z|--compress)
46-
COMPREPLY=($(compgen -W '${compressopts[*]}' -- "$cur"))
47-
return 0
48-
;;
4943
-R|--repo)
5044
local repos=$(sed '/^\[\(.*\)\]$/!d;s//\1/g;/options/d' /etc/pacman.conf)
5145
COMPREPLY=($(compgen -W '$repos' -- "$cur"))

extra/zsh-completion

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,6 @@ _repos(){
88
compadd "$@" -a repositories
99
}
1010

11-
_compression(){
12-
local -a cmd _comps
13-
_comps=('none' 'gzip' 'bzip2' 'lzma' 'lzop' 'lz4' 'xz' 'zstd')
14-
typeset -U _comps
15-
compadd "$@" -a _comps
16-
}
17-
1811
_action_none(){
1912
_arguments \
2013
"$_shortopts[@]" \
@@ -37,7 +30,6 @@ _longopts=(
3730
'--verbose[output more]'
3831
'--raw[disable output justification]'
3932
'--null[null terminate output]'
40-
'--compress=[compress downloaded repos]: :_compression'
4133
'--config=[use an alternate pacman config]: :_files'
4234
'--cachedir=[use an alternate cache directory]: :_files -/'
4335
)
@@ -58,7 +50,6 @@ _shortopts=(
5850
'*-v[output more]'
5951
'*-w[disable output justification]'
6052
'*-0[null terminate output]'
61-
'*-z[compress downloaded repos]: :_compression'
6253
'*-C[use an alternate pacman config]: :_files'
6354
'*-D[use an alternate cache directory]: :_files -/'
6455
)

man/pkgfile.pod

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -97,15 +97,6 @@ Avoid justification of 2 column output.
9797

9898
=over 4
9999

100-
=item B<-z>, B<--compress>[B<=>I<COMPRESSION>]
101-
102-
Repack downloaded repos with the optionally supplied compression method, which
103-
may be one of B<none>, B<gzip>, B<bzip2>, B<lzop>, B<lz4>, B<lzma>, B<xz>, or
104-
B<zstd>. If this flag is passed without a compression method, this defaults to
105-
B<gzip>. If this flag is not passed at all, no compression will be applied.
106-
Applying any form of compression will decrease performance, but may be
107-
desirable for disk space concerns.
108-
109100
=back
110101

111102
=head1 GENERAL OPTIONS

man/pkgfiled.pod

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,6 @@ On the initial sync, ignore timestamp comparisons and rewrite all found database
3333

3434
Exit after pkgfiled's initial sync, rather than continuing to listen for changes.
3535

36-
=item B<-z>, B<--compress>[B<=>I<COMPRESSION>]
37-
38-
Repack repos with the optionally supplied compression method, which may be one
39-
of B<none>, B<gzip>, B<bzip2>, B<lzop>, B<lz4>, B<lzma>, B<xz>, or B<zstd>. If
40-
this flag is passed without a compression method, this defaults to B<gzip>. If
41-
this flag is not passed at all, no compression will be applied. Applying any
42-
form of compression will decrease performance, but may be desirable for disk
43-
space concerns.
44-
4536
=item B<-h>, B<--help>
4637

4738
Print help and exit.

meson.build

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,7 @@ endif
2121

2222
configure_file(output: 'config.hh', configuration: conf)
2323

24-
add_project_arguments(
25-
'-include',
26-
'config.hh',
27-
'-fno-exceptions',
28-
language: 'cpp',
29-
)
24+
add_project_arguments('-include', 'config.hh', language: 'cpp')
3025

3126
libpcre = dependency('libpcre', version: '>= 8.30')
3227
libarchive = dependency('libarchive', version: '>= 3.2.0')
@@ -36,6 +31,7 @@ pthreads = dependency('threads')
3631
stdcppfs = cpp.find_library('stdc++fs')
3732
gtest = dependency('gtest', required: false)
3833
gmock = dependency('gmock', required: false)
34+
cista = declare_dependency(include_directories: 'third_party/cista')
3935

4036
pod2man = find_program('pod2man')
4137
pkgconfig = find_program('pkg-config')
@@ -48,15 +44,15 @@ libcommon = static_library(
4844
src/archive_converter.cc src/archive_converter.hh
4945
src/archive_io.cc src/archive_io.hh
5046
src/archive_reader.cc src/archive_reader.hh
51-
src/compress.cc src/compress.hh
47+
src/db.cc src/db.hh
5248
src/filter.cc src/filter.hh
5349
src/repo.cc src/repo.hh
5450
src/result.cc src/result.hh
5551
src/update.cc src/update.hh
5652
src/queue.hh
5753
'''.split(),
5854
),
59-
dependencies: [libpcre, libarchive, libcurl, pthreads, stdcppfs],
55+
dependencies: [cista, libpcre, libarchive, libcurl, pthreads, stdcppfs],
6056
install: false,
6157
)
6258

@@ -75,14 +71,15 @@ executable(
7571
'''.split(),
7672
),
7773
link_with: [libcommon],
74+
dependencies: [cista],
7875
install: true,
7976
)
8077

8178
executable(
8279
'pkgfiled',
8380
'src/pkgfiled.cc',
8481
link_with: [libcommon],
85-
dependencies: [libsystemd, stdcppfs],
82+
dependencies: [cista, libsystemd, stdcppfs],
8683
install: true,
8784
)
8885

@@ -190,7 +187,7 @@ if gtest.found() and gmock.found()
190187
'''.split(),
191188
),
192189
link_with: [libcommon, gtest_main],
193-
dependencies: [gmock, gtest, libpcre],
190+
dependencies: [cista, gmock, gtest, libpcre],
194191
),
195192
protocol: 'gtest',
196193
)
@@ -203,15 +200,22 @@ py3 = python.find_installation('python3')
203200

204201
python_requirement = '>=3.7'
205202
if py3.found() and py3.language_version().version_compare(python_requirement)
206-
foreach input : ['tests/list.py', 'tests/search.py', 'tests/update.py']
203+
integration_tests = [
204+
'tests/database.py',
205+
'tests/list.py',
206+
'tests/search.py',
207+
'tests/update.py'
208+
]
209+
210+
foreach input : integration_tests
207211
basename = input.split('/')[-1].split('.')[0]
208212

209213
test(
210214
'pkgfile_@0@_integration_test'.format(basename),
211215
py3,
212216
args: [join_paths(meson.project_source_root(), input)],
213217
env: ['PYTHONDONTWRITEBYTECODE=1'],
214-
suite: 'integration'
218+
suite: 'integration',
215219
)
216220
endforeach
217221
else

src/archive_converter.cc

Lines changed: 48 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -5,139 +5,102 @@
55

66
#include <filesystem>
77
#include <format>
8+
#include <fstream>
89
#include <iostream>
910

1011
namespace fs = std::filesystem;
1112

1213
namespace pkgfile {
1314

14-
// static
15-
std::unique_ptr<ArchiveConverter> ArchiveConverter::New(
16-
const std::string& reponame, int fd_in, std::string base_filename_out,
17-
int compress, int repo_chunk_bytes) {
18-
const char* error;
19-
20-
auto reader = ReadArchive::New(fd_in, &error);
21-
if (reader == nullptr) {
22-
std::cerr << std::format(
23-
"error: failed to create archive reader for {}: {}\n", reponame, error);
24-
return nullptr;
15+
namespace {
16+
17+
std::pair<std::string_view, std::string_view> ParsePkgname(
18+
std::string_view entryname) {
19+
const auto pkgrel = entryname.rfind('-');
20+
if (pkgrel == entryname.npos) {
21+
return {};
2522
}
2623

27-
auto writer = WriteArchive::New(
28-
MakeArchiveChunkFilename(base_filename_out, 0, true), compress, &error);
29-
if (writer == nullptr) {
30-
std::cerr << std::format("error: failed to open file for writing: {}: {}\n",
31-
base_filename_out, error);
32-
return nullptr;
24+
const auto pkgver = entryname.substr(0, pkgrel).rfind('-');
25+
if (pkgver == entryname.npos) {
26+
return {};
3327
}
3428

35-
return std::make_unique<ArchiveConverter>(
36-
reponame, std::move(base_filename_out), compress, repo_chunk_bytes,
37-
std::move(reader), std::move(writer));
29+
return {entryname.substr(0, pkgver), entryname.substr(pkgver + 1)};
3830
}
3931

32+
} // namespace
33+
4034
std::string ArchiveConverter::MakeArchiveChunkFilename(
4135
const std::string& base_filename, int chunk_number, bool tempfile) {
4236
return std::format("{}.{:03d}{}", base_filename, chunk_number,
4337
tempfile ? "~" : "");
4438
}
4539

4640
bool ArchiveConverter::NextArchiveChunk() {
47-
if (!out_->Close()) {
48-
return false;
49-
}
50-
51-
const char* error;
52-
53-
auto writer = WriteArchive::New(
54-
MakeArchiveChunkFilename(base_filename_out_, ++chunk_number_, true),
55-
compress_, &error);
56-
if (writer == nullptr) {
57-
std::cerr << std::format("error: failed to open file for writing: {}: {}\n",
58-
base_filename_out_, error);
59-
return false;
60-
}
41+
std::string chunk_name =
42+
MakeArchiveChunkFilename(base_filename_out_, chunk_number_++, true);
43+
cista::buf mmap{cista::mmap{chunk_name.c_str()}};
44+
cista::serialize<cista::mode::NONE>(mmap, data_);
6145

62-
out_ = std::move(writer);
46+
data_.clear();
6347

6448
return true;
6549
}
6650

67-
int ArchiveConverter::WriteCpioEntry(archive_entry* ae,
68-
const fs::path& entryname) {
69-
pkgfile::ArchiveReader reader(in_->read_archive());
51+
int ArchiveConverter::WriteMetaEntry(const fs::path& entryname) {
52+
ArchiveReader reader(in_->read_archive());
7053
std::string_view line;
7154

7255
// discard the first line
7356
reader.GetLine(&line);
7457

75-
std::string entry;
76-
while (reader.GetLine(&line) == ARCHIVE_OK) {
77-
// do the copy, with a slash prepended
78-
std::format_to(std::back_inserter(entry), "/{}\n", line);
58+
auto [name, version] = ParsePkgname(entryname.c_str());
59+
if (name.empty()) {
60+
return 0;
7961
}
8062

81-
// adjust the entry size for removing the first line and adding slashes
82-
archive_entry_set_size(ae, entry.size());
83-
84-
// inodes in cpio archives are dumb.
85-
archive_entry_set_ino64(ae, 0);
86-
87-
// store the metadata as simply $pkgname-$pkgver-$pkgrel
88-
archive_entry_update_pathname_utf8(ae, entryname.parent_path().c_str());
89-
90-
if (archive_write_header(out_->write_archive(), ae) != ARCHIVE_OK) {
91-
std::cerr << std::format("error: failed to write entry header: {}/{}: {}\n",
92-
reponame_, archive_entry_pathname(ae),
93-
strerror(errno));
94-
return -errno;
95-
}
63+
auto& pkg = data_[name];
64+
pkg.version = version;
9665

97-
if (archive_write_data(out_->write_archive(), entry.c_str(), entry.size()) !=
98-
static_cast<ssize_t>(entry.size())) {
99-
std::cerr << std::format("error: failed to write entry: {}/{}: {}\n",
100-
reponame_, archive_entry_pathname(ae),
101-
strerror(errno));
102-
return -errno;
66+
int bytesize = 0;
67+
while (reader.GetLine(&line) == ARCHIVE_OK) {
68+
// do the copy, with a slash prepended
69+
bytesize += pkg.files.emplace_back(std::format("/{}", line)).size();
10370
}
10471

105-
return entry.size();
72+
return bytesize;
10673
}
10774

10875
bool ArchiveConverter::Finalize() {
109-
in_->Close();
110-
111-
if (!out_->Close()) {
112-
return false;
113-
}
76+
NextArchiveChunk();
11477

11578
struct stat st;
116-
fstat(in_->fd(), &st);
79+
in_->Stat(&st);
80+
in_->Close();
11781

11882
const struct timeval times[] = {
11983
{st.st_atim.tv_sec, 0},
12084
{st.st_mtim.tv_sec, 0},
12185
};
12286

123-
for (int i = 0; i <= chunk_number_; ++i) {
87+
for (int i = 0; i < chunk_number_; ++i) {
12488
std::string path = MakeArchiveChunkFilename(base_filename_out_, i, true);
89+
std::string dest = MakeArchiveChunkFilename(base_filename_out_, i, false);
12590

12691
if (utimes(path.c_str(), times) < 0) {
12792
std::cerr << std::format("warning: failed to set filetimes on {}: {}\n",
128-
out_->path(), strerror(errno));
93+
path, strerror(errno));
12994
}
13095

131-
const fs::path dest = path.substr(0, path.size() - 1);
132-
13396
std::error_code ec;
13497
if (fs::rename(path, dest, ec); ec.value() != 0) {
13598
std::cerr << std::format("error: renaming tmpfile to {} failed: {}\n",
136-
dest.string(), ec.message());
99+
dest, ec.message());
137100
}
138101
}
139102

140-
for (int i = chunk_number_ + 1;; ++i) {
103+
for (int i = chunk_number_;; ++i) {
141104
std::string path = MakeArchiveChunkFilename(base_filename_out_, i, false);
142105

143106
std::error_code ec;
@@ -162,17 +125,19 @@ bool ArchiveConverter::RewriteArchive() {
162125
chunk_size = 0;
163126
}
164127

165-
fs::path entryname = archive_entry_pathname(ae);
128+
const fs::path entryname = archive_entry_pathname(ae);
166129

167130
// ignore everything but the /files metadata
168-
if (entryname.filename() == "files") {
169-
const int bytes_written = WriteCpioEntry(ae, entryname);
170-
if (bytes_written < 0) {
171-
return false;
172-
}
131+
if (entryname.filename() != "files") {
132+
continue;
133+
}
173134

174-
chunk_size += bytes_written;
135+
const int bytes_written = WriteMetaEntry(entryname.parent_path());
136+
if (bytes_written < 0) {
137+
return false;
175138
}
139+
140+
chunk_size += bytes_written;
176141
}
177142

178143
return Finalize();

0 commit comments

Comments
 (0)