Skip to content

Commit 1aa5d9f

Browse files
Geod24Geod24
committed
feat: Implement support for Github-based index, bypassing the registry
This implement a package 'index' similar to that found for Homebrew, Nix, Cargo, etc... It allows us to remove a SPOF in our critical infrastructure, as a Github outage would always cause a registry being unusable anyway. There are multiple steps to having a useful index: - For transition purpose, we add a hidden command to Dub that export an `index.yaml`; - In the future, users should register their packages by adding an entry to `index.yaml`, the index definition file of the registry. This is used as the source of all packages; - `dub` now has a hidden `index-build` command to allow it to build the index based on an index definition file (`index.yaml`). Using this, it queries the various APIs to generate JSON index files that are stored under a pre-defined hierarchy. - Finally, a `PackageSupplier` is added to make use of this new feature; In the future, the registration process needs to be moved from the registry to Github to make this migration complete. This *can* be done by exposing a user-friendly interface on `code.dlang.org`, if making an MR to the index is deemed too complicated.
1 parent b0578bc commit 1aa5d9f

File tree

12 files changed

+1904
-2
lines changed

12 files changed

+1904
-2
lines changed

‎build-files.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ source/dub/generators/sublimetext.d
1818
source/dub/generators/targetdescription.d
1919
source/dub/generators/visuald.d
2020
source/dub/init.d
21+
source/dub/index/bitbucket.d
22+
source/dub/index/client.d
23+
source/dub/index/data.d
24+
source/dub/index/github.d
25+
source/dub/index/gitlab.d
26+
source/dub/index/utils.d
2127
source/dub/internal/configy/attributes.d
2228
source/dub/internal/configy/backend/node.d
2329
source/dub/internal/configy/backend/yaml.d

‎source/dub/commandline.d

Lines changed: 253 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,10 @@ CommandGroup[] getCommands() @safe pure nothrow
7373
new ListOverridesCommand,
7474
new CleanCachesCommand,
7575
new ConvertCommand,
76-
)
76+
// This is index management but those commands are hidden
77+
new IndexBuildCommand,
78+
new IndexFromRegistryCommand,
79+
),
7780
];
7881
}
7982

@@ -2983,6 +2986,255 @@ class ConvertCommand : Command {
29832986
}
29842987

29852988

2989+
/******************************************************************************/
2990+
/* Index management
2991+
/******************************************************************************/
2992+
2993+
public class IndexBuildCommand : Command {
2994+
import dub.index.bitbucket;
2995+
import dub.index.data;
2996+
import dub.index.github;
2997+
import dub.index.gitlab;
2998+
import dub.index.utils;
2999+
import std.random;
3000+
import std.range;
3001+
3002+
/// Index file to use
3003+
private string index = "index.yaml";
3004+
/// Filename to write to
3005+
private string output = "index-build-result";
3006+
/// Packages to filter in - assume all if empty
3007+
private string[] include;
3008+
/// Packages to filter out
3009+
private string[] exclude;
3010+
/// Bearer token to use to authenticate requests
3011+
private string githubToken, gitlabToken, bitbucketToken;
3012+
/// Kind of packages to include (default: all kinds)
3013+
private string[] kind;
3014+
/// Whether to force the iteration of tags or not
3015+
/// This needs to be used if some tags need to be reprocessed
3016+
private bool force_tags;
3017+
/// Force the package to be entirely reprocessed. Imply `--force-tags`.
3018+
private bool force;
3019+
/// Whether to use a randomized sample of packages
3020+
private bool random;
3021+
/// The number of packages to update
3022+
private uint maxUpdates = uint.max;
3023+
/// Source and target index to use, mutually exclusive with `random`
3024+
private uint fromIdx = 0, toIdx = uint.max;
3025+
3026+
this() @safe pure nothrow
3027+
{
3028+
this.name = "index-build";
3029+
this.description = "Generate the rich index from the index.yaml file";
3030+
this.helpText = [ "This command is for internal use only. Do not use it." ];
3031+
this.hidden = true;
3032+
}
3033+
3034+
override void prepare(scope CommandArgs args) {
3035+
args.getopt("bitbucket-token", &this.bitbucketToken, ["Bearer token to use when issuing Bitbucket requests"]);
3036+
args.getopt("github-token", &this.githubToken, ["Bearer token to use when issuing Github requests"]);
3037+
args.getopt("gitlab-token", &this.gitlabToken, ["Bearer token to use when issuing GitLab requests"]);
3038+
args.getopt("output", &this.output, ["Where to output the data (path to a folder)"]);
3039+
args.getopt("index", &this.index, ["Index file to use - default to 'index.yaml'"]);
3040+
args.getopt("include", &this.include, ["Which packages to filter in - if not, assume all"]);
3041+
args.getopt("exclude", &this.exclude, ["Which packages to filter out - if not, assume none"]);
3042+
args.getopt("kind", &this.kind, ["Kind of packages to include (github, gitlab, bitbucket). Default: all"]);
3043+
args.getopt("force", &this.force, ["Force Dub to reprocess packages even if it has cache informations"]);
3044+
args.getopt("force-tags", &this.force_tags,
3045+
["Force Dub to re-list tags, but do not reload the recipe if the commit hasn't changed."]);
3046+
args.getopt("random", &this.random, ["Randomize the order in which packages are processed"]);
3047+
args.getopt("max-updates", &this.maxUpdates, ["Maximum number of packages to process"]);
3048+
args.getopt("from", &this.fromIdx, ["Index to seek to before iterating the list of packages (default: 0)"]);
3049+
args.getopt("to", &this.toIdx, ["Index to stop at when iterating the list of packages (default: end of list)"]);
3050+
3051+
enforce(this.fromIdx <= this.toIdx, "Cannot have source index (`--from`) be past end index (`--to`)");
3052+
enforce(this.fromIdx == 0 || !this.random,
3053+
"Cannot specify source index (`--from`) for random sampling (`--random`)");
3054+
enforce(this.toIdx == uint.max || !this.random,
3055+
"Cannot specify end index (`--to`) for random sampling (`--random`)");
3056+
}
3057+
3058+
override int execute(Dub dub, string[] free_args, string[] app_args)
3059+
{
3060+
import dub.index.client : RepositoryClient;
3061+
import dub.index.data;
3062+
import dub.internal.configy.easy;
3063+
static import std.file;
3064+
import std.typecons;
3065+
3066+
enforceUsage(free_args.length == 0, "Expected no free argument.");
3067+
enforceUsage(app_args.length == 0, "Expected zero application arguments.");
3068+
3069+
const isUpdate = std.file.exists(this.output);
3070+
if (isUpdate)
3071+
enforce(std.file.isDir(this.output), this.output ~ ": is not a directory");
3072+
else
3073+
std.file.mkdirRecurse(this.output);
3074+
3075+
auto indexN = parseConfigFileSimple!PackageList(this.index);
3076+
if (indexN.isNull()) return 1;
3077+
auto indexC = indexN.get();
3078+
logInfoNoTag("Found %s packages in the index file", indexC.packages.length);
3079+
3080+
const NativePath outputPath = NativePath(this.output);
3081+
size_t processed, updated, notsupported;
3082+
string[] included, excluded, errored;
3083+
scope gh = new GithubClient(this.githubToken);
3084+
scope gl = new GitLabClient(this.gitlabToken);
3085+
scope bb = new BitbucketClient(this.bitbucketToken);
3086+
3087+
void update (scope RepositoryClient client, in PackageEntry pkg) {
3088+
const target = getPackageDescriptionPath(outputPath, PackageName(pkg.name.value));
3089+
ensureDirectory(target.parentPath());
3090+
const targetStr = target.toNativeString();
3091+
auto previous = !this.force && std.file.exists(targetStr) ?
3092+
parseConfigFileSimple!(IndexedPackage!0)(targetStr, StrictMode.Ignore) :
3093+
Nullable!(IndexedPackage!0).init;
3094+
if (this.force_tags && !previous.isNull())
3095+
previous.get().cache = CacheInfo.init;
3096+
auto res = updateDescription(client, pkg, previous);
3097+
if (previous.isNull() || previous.get() != res) {
3098+
std.file.write(targetStr, res.serializeToJsonString());
3099+
++updated;
3100+
}
3101+
}
3102+
3103+
// Update a single package - this code is in its own function as it
3104+
// is wrapped in a try-catch in the `foreach` to process as many packages
3105+
// as possible
3106+
void updatePackageIndex (in PackageEntry pkg) {
3107+
logInfo("[%s] Processing included package", pkg.name.value);
3108+
switch (pkg.source.kind) {
3109+
case `github`:
3110+
scope client = gh.new Repository(pkg.source.owner, pkg.source.project);
3111+
update(client, pkg);
3112+
break;
3113+
case `gitlab`:
3114+
scope client = gl.new Project(pkg.source.owner, pkg.source.project);
3115+
update(client, pkg);
3116+
break;
3117+
case `bitbucket`:
3118+
scope client = bb.new Repository(pkg.source.owner, pkg.source.project);
3119+
update(client, pkg);
3120+
break;
3121+
default:
3122+
throw new Exception("Package kind not supported: " ~ pkg.source.kind);
3123+
}
3124+
}
3125+
3126+
int processEntry (size_t idx, ref PackageEntry pkg) {
3127+
if (updated >= this.maxUpdates) return 1;
3128+
if (this.include.length && !this.include.canFind(pkg.name.value))
3129+
return 0;
3130+
if (this.exclude.canFind(pkg.name.value)) {
3131+
excluded ~= pkg.name.value;
3132+
return 0;
3133+
}
3134+
if (this.kind.length && !this.kind.canFind(pkg.source.kind))
3135+
return 0;
3136+
3137+
++processed;
3138+
try
3139+
updatePackageIndex(pkg);
3140+
catch (Exception exc) {
3141+
errored ~= pkg.name.value;
3142+
// If we get a 404 here, it might be a dead package
3143+
logError("[%s] Could not build index for package: %s",
3144+
pkg.name.value, exc.message());
3145+
}
3146+
3147+
if (this.include.length) {
3148+
included ~= pkg.name.value;
3149+
if (included.length % 10 == 0) {
3150+
const rl = gh.getRateLimit();
3151+
logDebug("Requests still available: %s/%s", rl.remaining, rl.limit);
3152+
}
3153+
}
3154+
else if (idx % 10 == 0) {
3155+
const rl = gh.getRateLimit();
3156+
logDebug("Requests still available: %s/%s", rl.remaining, rl.limit);
3157+
}
3158+
return 0;
3159+
}
3160+
3161+
if (this.random) { // Can't use `std.random : choose` because bugs
3162+
foreach (idx, pkg; indexC.packages.randomCover().enumerate)
3163+
if (processEntry(idx, pkg))
3164+
break;
3165+
} else {
3166+
const startIdx = min(this.fromIdx, indexC.packages.length);
3167+
const endIdx = min(this.toIdx, indexC.packages.length);
3168+
foreach (idx, pkg; indexC.packages[startIdx .. endIdx])
3169+
if (processEntry(idx, pkg))
3170+
break;
3171+
}
3172+
3173+
logInfoNoTag("Updated %s packages out of %s processed (%s excluded, %s errors, %s not supported)",
3174+
updated, processed, excluded.length, errored.length, notsupported);
3175+
if (this.include.length && included != this.include)
3176+
logWarn("Not all explicitly-included packages have been processed!");
3177+
if (errored.length)
3178+
logWarn("The following packages errored out:\n%(\t- %s\n%)", errored);
3179+
if (!this.kind.length || this.kind.canFind(`github`)) {
3180+
const rl = gh.getRateLimit();
3181+
logInfoNoTag("Github requests still available: %s/%s", rl.remaining, rl.limit);
3182+
}
3183+
return 0;
3184+
}
3185+
}
3186+
3187+
public class IndexFromRegistryCommand : Command {
3188+
/// Filename to write to
3189+
private string output = "index.yaml";
3190+
/// Bypass cache, always query the registry
3191+
private bool force;
3192+
3193+
this() @safe pure nothrow
3194+
{
3195+
this.name = "index-fromregistry";
3196+
this.description = "Generate the index.yaml file from the remote registry";
3197+
this.helpText = [ "This command is for internal use only. Do not use it." ];
3198+
this.hidden = true;
3199+
}
3200+
3201+
override void prepare(scope CommandArgs args) {
3202+
args.getopt("O", &this.output, ["Where to output the data ('-' is supported)"]);
3203+
args.getopt("f", &this.force, ["Bypass the cache and always query the registry"]);
3204+
}
3205+
3206+
override int execute(Dub dub, string[] free_args, string[] app_args)
3207+
{
3208+
import dub.internal.vibecompat.inet.url;
3209+
import dub.packagesuppliers.registry;
3210+
import std.format;
3211+
3212+
enforceUsage(free_args.length == 0, "Expected zero arguments.");
3213+
enforceUsage(app_args.length == 0, "Expected zero application arguments.");
3214+
3215+
scope registry = new RegistryPackageSupplier(URL(defaultRegistryURLs[1]));
3216+
scope allPkgs = registry.getPackageDump(this.force);
3217+
writeln("Found ", allPkgs.array.length, " packages");
3218+
scope output = this.output == "-" ? stdout : File(this.output, "w+");
3219+
scope writer = output.lockingTextWriter();
3220+
writer.formattedWrite("packages:\n");
3221+
foreach (pkg; allPkgs.array) {
3222+
writer.formattedWrite(` %s:
3223+
source:
3224+
kind: %s
3225+
owner: %s
3226+
project: %s
3227+
`,
3228+
pkg["name"].opt!string, pkg["repository"]["kind"].opt!string,
3229+
pkg["repository"]["owner"].opt!string,
3230+
pkg["repository"]["project"].opt!string);
3231+
}
3232+
3233+
return 0;
3234+
}
3235+
}
3236+
3237+
29863238
/******************************************************************************/
29873239
/* HELP */
29883240
/******************************************************************************/

‎source/dub/dub.d

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ deprecated("use defaultRegistryURLs") enum defaultRegistryURL = defaultRegistryU
4747

4848
/// The URL to the official package registry and it's default fallback registries.
4949
static immutable string[] defaultRegistryURLs = [
50+
"dub+index+https://github.com/dlang/dub-index.git",
5051
"https://code.dlang.org/",
5152
"https://codemirror.dlang.org/"
5253
];
@@ -413,6 +414,12 @@ class Dub {
413414
switch (url.startsWith("dub+", "mvn+", "file://"))
414415
{
415416
case 1:
417+
// `startsWith` takes the shortest match so if we provide `dub+`
418+
// `and `dub+index+` it will always match the former...
419+
if (url.startsWith(`dub+index+`)) {
420+
return new IndexPackageSupplier(URL(url["dub+index+".length .. $]),
421+
this.m_dirs.userPackages ~ "index");
422+
}
416423
return new RegistryPackageSupplier(URL(url[4..$]));
417424
case 2:
418425
return new MavenRegistryPackageSupplier(URL(url[4..$]));
@@ -1291,7 +1298,7 @@ class Dub {
12911298
try
12921299
results ~= tuple(ps.description, ps.searchPackages(query));
12931300
catch (Exception e) {
1294-
logWarn("Searching %s for '%s' failed: %s", ps.description, query, e.msg);
1301+
logWarn("Searching %s for '%s' failed: %s", ps.description, query, e);
12951302
}
12961303
}
12971304
return results.filter!(tup => tup[1].length);

0 commit comments

Comments
 (0)