From 116b67a4078e1fc861fea00cb89ac63d3c4e377e Mon Sep 17 00:00:00 2001 From: Matthias Klumpp Date: Sun, 24 Feb 2019 17:44:16 +0100 Subject: [PATCH] Speed up locale search by moving it to its own database cache --- src/asgen/contentsstore.d | 78 ++++++++++++++++++++++++------ src/asgen/handlers/localehandler.d | 22 ++++----- 2 files changed, 73 insertions(+), 27 deletions(-) diff --git a/src/asgen/contentsstore.d b/src/asgen/contentsstore.d index b4281dde..58e50bd4 100644 --- a/src/asgen/contentsstore.d +++ b/src/asgen/contentsstore.d @@ -43,6 +43,7 @@ private: MDB_envp dbEnv; MDB_dbi dbContents; MDB_dbi dbIcons; + MDB_dbi dbLocale; bool opened; @@ -84,9 +85,9 @@ public: scope (failure) dbEnv.mdb_env_close (); checkError (rc, "mdb_env_create"); - // We are going to use at max 2 sub-databases: - // contents and icons - rc = dbEnv.mdb_env_set_maxdbs (2); + // We are going to use at max 3 sub-databases: + // contents, icons and locale + rc = dbEnv.mdb_env_set_maxdbs (3); checkError (rc, "mdb_env_set_maxdbs"); // set a huge map size to be futureproof. @@ -118,6 +119,10 @@ public: rc = txn.mdb_dbi_open ("icondata", MDB_CREATE, &dbIcons); checkError (rc, "open icon-info database"); + // contains list of locale files and related data + rc = txn.mdb_dbi_open ("localedata", MDB_CREATE, &dbLocale); + checkError (rc, "open locale-info database"); + rc = txn.mdb_txn_commit (); checkError (rc, "mdb_txn_commit"); } @@ -181,6 +186,10 @@ public: res = txn.mdb_del (dbIcons, &key, null); if (res != MDB_NOTFOUND) checkError (res, "mdb_del (icons)"); + + res = txn.mdb_del (dbLocale, &key, null); + if (res != MDB_NOTFOUND) + checkError (res, "mdb_del (locale)"); } bool packageExists (string pkid) @@ -206,21 +215,28 @@ public: void addContents (string pkid, string[] contents) { - MDB_val key; - MDB_val contentsVal, iconsVal; - - // filter out icon filenames and filenames of icon-related stuff (e.g. theme.index) + // filter out icon filenames and filenames of icon-related stuff (e.g. theme.index), + // as well as locale information auto iconInfo = appender!(string[]); - foreach (ref c; contents) { - if ((c.startsWith ("/usr/share/icons/")) || - (c.startsWith ("/usr/share/pixmaps/"))) { - iconInfo ~= c; + auto localeInfo = appender!(string[]); + foreach (ref f; contents) { + if ((f.startsWith ("/usr/share/icons/")) || + (f.startsWith ("/usr/share/pixmaps/"))) { + iconInfo ~= f; + continue; } + + if (f.startsWith ("/usr/share/locale/")) { + localeInfo ~= f; + continue; + } } immutable contentsStr = contents.join ("\n"); synchronized (this) { + MDB_val key, contentsVal; + key = makeDbValue (pkid); contentsVal = makeDbValue (contentsStr); @@ -231,19 +247,34 @@ public: auto res = txn.mdb_put (dbContents, &key, &contentsVal, 0); checkError (res, "mdb_put"); + // if we have icon information, store that too if (!iconInfo.data.empty) { - // we have icon information, store it too + MDB_val iconsVal; + immutable iconsStr = iconInfo.data.join ("\n"); iconsVal = makeDbValue (iconsStr); res = txn.mdb_put (dbIcons, &key, &iconsVal, 0); checkError (res, "mdb_put (icons)"); } + + // store locale + if (!localeInfo.data.empty) { + MDB_val localeVal; + + immutable localeStr = localeInfo.data.join ("\n"); + localeVal = makeDbValue (localeStr); + + res = txn.mdb_put (dbLocale, &key, &localeVal, 0); + checkError (res, "mdb_put (locale)"); + } } } - private HashMap!(string, string) getFilesMap (string[] pkids, MDB_dbi dbi) + private HashMap!(string, string) getFilesMap (string[] pkids, MDB_dbi dbi, bool useBaseName = false) { + import std.path : baseName; + MDB_cursorp cur; auto txn = newTransaction (MDB_RDONLY); @@ -267,7 +298,10 @@ public: auto contents = to!string (data); foreach (ref c; contents.split ("\n")) { - pkgCMap[c] = pkid; + if (useBaseName) + pkgCMap[c.baseName] = pkid; + else + pkgCMap[c] = pkid; } } @@ -284,6 +318,14 @@ public: return getFilesMap (pkids, dbIcons); } + auto getLocaleMap (string[] pkids) + { + // we make the assumption here that all locale for a given domain are in one package. + // otherwise this global search will get even more insane. + // (that's why useBaseName is set to "true" - this could maybe change in future though + return getFilesMap (pkids, dbLocale, true); + } + private string[] getContentsList (string pkid, MDB_dbi dbi) { MDB_val pkey, cval; @@ -319,6 +361,11 @@ public: return getContentsList (pkid, dbIcons); } + string[] getLocaleFiles (string pkid) + { + return getContentsList (pkid, dbLocale); + } + HashSet!(immutable string) getPackageIdSet () { MDB_cursorp cur; @@ -356,6 +403,9 @@ public: res = txn.mdb_del (dbIcons, &key, null); if (res != MDB_NOTFOUND) checkError (res, "mdb_del (icons)"); + res = txn.mdb_del (dbLocale, &key, null); + if (res != MDB_NOTFOUND) + checkError (res, "mdb_del (locale)"); } } diff --git a/src/asgen/handlers/localehandler.d b/src/asgen/handlers/localehandler.d index f52faaf0..16eca19a 100644 --- a/src/asgen/handlers/localehandler.d +++ b/src/asgen/handlers/localehandler.d @@ -136,29 +136,25 @@ private: // we make the assumption here that all locale for a given domain are in one package. // otherwise this global search will get even more insane. - foreach (info; parallel (ccache.getContentsMap (pkgMap.keys).byKeyValue, 100)) { - immutable fname = info.key; + // the key of the map returned by getLocaleMap will therefore contain only the locale + // file basename instead of a full path + auto dbLocaleMap = ccache.getLocaleMap (pkgMap.keys); + foreach (info; dbLocaleMap.byKeyValue) { + immutable id = info.key; immutable pkgid = info.value; - if (!fname.startsWith ("/usr/share/locale/")) - continue; - immutable id = fname.baseName; - // check if we already have a package - lookups in this HashMap are faster // due to its smaller size and (most of the time) outweight the following additional // lookup for the right package entity. - synchronized (this) { - if (localeIdPkgMap.get (id, null) !is null) - continue; - } + if (localeIdPkgMap.get (id, null) !is null) + continue; Package pkg; if (pkgid !is null) pkg = pkgMap.get (pkgid, null); - if (pkg !is null) { - synchronized (this) localeIdPkgMap[id] = pkg; - } + if (pkg !is null) + localeIdPkgMap[id] = pkg; } logDebug ("Created new LocaleHandler.");