Skip to content

Commit

Permalink
Use compound index when getting search candidates
Browse files Browse the repository at this point in the history
  • Loading branch information
@bengl
bengl authored and Loïc Hermann committed Nov 27, 2020
1 parent 730fb10 commit 4e2021c
Show file tree
Hide file tree
Showing 4 changed files with 253 additions and 6 deletions.
29 changes: 23 additions & 6 deletions lib/datastore.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,27 +255,44 @@ Datastore.prototype.updateIndexes = function (oldDoc, newDoc) {
Datastore.prototype.getCandidates = function (query, dontExpireStaleDocs, callback) {
var indexNames = Object.keys(this.indexes)
, self = this
, usableQueryKeys;
, usableQueryKeys
, basicQueryKeys
, compoundQueryKeys;

if (typeof dontExpireStaleDocs === 'function') {
callback = dontExpireStaleDocs;
dontExpireStaleDocs = false;
}


async.waterfall([
// STEP 1: get candidates list by checking indexes from most to least frequent usecase
function (cb) {
// For a basic match

usableQueryKeys = [];
Object.keys(query).forEach(function (k) {
if (typeof query[k] === 'string' || typeof query[k] === 'number' || typeof query[k] === 'boolean' || util.isDate(query[k]) || query[k] === null) {
usableQueryKeys.push(k);
}
});
usableQueryKeys = _.intersection(usableQueryKeys, indexNames);
if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]]));

// For a basic match
basicQueryKeys = _.intersection(usableQueryKeys, indexNames);
if (basicQueryKeys.length > 0) {
return cb(null, self.indexes[basicQueryKeys[0]].getMatching(query[basicQueryKeys[0]]));
}

// For a compound match
compoundQueryKeys = [];
indexNames.forEach(function(indexName){
if (indexName.indexOf(',') === -1) return;
var subIndexNames = indexName.split(',');
if (_.intersection(subIndexNames, usableQueryKeys).length === subIndexNames.length) {
compoundQueryKeys.push(subIndexNames);
}
});

if (compoundQueryKeys.length > 0) {
return cb(null, self.indexes[compoundQueryKeys[0]].getMatching(_.pick(query,compoundQueryKeys[0])));
}

// For a $in match
Expand Down
22 changes: 22 additions & 0 deletions test/db.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,28 @@ describe('Database', function () {
});
});

it('Can use a compound index to get docs with a basic match', function (done) {
d.ensureIndex({ fieldName: ['tf', 'tg'] }, function (err) {
d.insert({ tf: 4, tg: 0, foo: 1 }, function () {
d.insert({ tf: 6, tg: 0, foo: 2 }, function () {
d.insert({ tf: 4, tg: 1, foo: 3 }, function (err, _doc1) {
d.insert({ tf: 6, tg: 1, foo: 4 }, function () {
d.getCandidates({ tf: 4, tg: 1 }, function (err, data) {
var doc1 = _.find(data, function (d) { return d._id === _doc1._id; })
;

data.length.should.equal(1);
assert.deepEqual(doc1, { _id: doc1._id, tf: 4, tg: 1, foo: 3 });

done();
});
});
});
});
});
});
});

it('Can use an index to get docs with a $in match', function (done) {
d.ensureIndex({ fieldName: 'tf' }, function (err) {
d.insert({ tf: 4 }, function (err) {
Expand Down
36 changes: 36 additions & 0 deletions test/indexes.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,32 @@ describe('Indexes', function () {
doc3.a.should.equal(42);
});

it('Can insert pointers to documents in the index correctly when they have compound fields', function () {
var idx = new Index({ fieldName: ['tf', 'tg'] })
, doc1 = { a: 5, tf: 'hello', tg: 'world' }
, doc2 = { a: 8, tf: 'hello', tg: 'bloup' }
, doc3 = { a: 2, tf: 'bloup', tg: 'bloup' }
;

idx.insert(doc1);
idx.insert(doc2);
idx.insert(doc3);


// The underlying BST now has 3 nodes which contain the docs where it's expected
idx.tree.getNumberOfKeys().should.equal(3);
console.log(idx.tree);
assert.deepEqual(idx.tree.search({tf: 'hello', tg: 'world'}), [{ a: 5, tf: 'hello', tg: 'world' }]);
assert.deepEqual(idx.tree.search({tf: 'hello', tg: 'bloup'}), [{ a: 8, tf: 'hello', tg: 'bloup' }]);
assert.deepEqual(idx.tree.search({tf: 'bloup', tg: 'bloup'}), [{ a: 2, tf: 'bloup', tg: 'bloup' }]);

// The nodes contain pointers to the actual documents
idx.tree.search({tf: 'hello', tg: 'bloup'})[0].should.equal(doc2);
idx.tree.search({tf: 'bloup', tg: 'bloup'})[0].a = 42;
doc3.a.should.equal(42);

});

it('Inserting twice for the same fieldName in a unique index will result in an error thrown', function () {
var idx = new Index({ fieldName: 'tf', unique: true })
, doc1 = { a: 5, tf: 'hello' }
Expand All @@ -44,6 +70,16 @@ describe('Indexes', function () {
(function () { idx.insert(doc1); }).should.throw();
});

it('Inserting twice for the same compound fieldName in a unique index will result in an error thrown', function () {
var idx = new Index({ fieldName: ['tf', 'tg'], unique: true })
, doc1 = { a: 5, tf: 'hello', tg: 'world' }
;

idx.insert(doc1);
idx.tree.getNumberOfKeys().should.equal(1);
(function () { idx.insert(doc1); }).should.throw();
});

it('Inserting twice for a fieldName the docs dont have with a unique index results in an error thrown', function () {
var idx = new Index({ fieldName: 'nope', unique: true })
, doc1 = { a: 5, tf: 'hello' }
Expand Down
172 changes: 172 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1


assertion-error@^1.0.1:
version "1.1.0"
resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b"
integrity sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==

[email protected]:
version "0.2.10"
resolved "https://registry.yarnpkg.com/async/-/async-0.2.10.tgz#b6bbe0b0674b9d719708ca38de8c237cb526c3d1"
integrity sha1-trvgsGdLnXGXCMo43owjfLUmw9E=

[email protected]:
version "0.2.5"
resolved "https://registry.yarnpkg.com/binary-search-tree/-/binary-search-tree-0.2.5.tgz#7dbb3b210fdca082450dad2334c304af39bdc784"
integrity sha1-fbs7IQ/coIJFDa0jNMMErzm9x4Q=
dependencies:
underscore "~1.4.4"

chai@^3.2.0:
version "3.5.0"
resolved "https://registry.yarnpkg.com/chai/-/chai-3.5.0.tgz#4d02637b067fe958bdbfdd3a40ec56fef7373247"
integrity sha1-TQJjewZ/6Vi9v906QOxW/vc3Mkc=
dependencies:
assertion-error "^1.0.1"
deep-eql "^0.1.3"
type-detect "^1.0.0"

[email protected]:
version "0.6.1"
resolved "https://registry.yarnpkg.com/commander/-/commander-0.6.1.tgz#fa68a14f6a945d54dbbe50d8cdb3320e9e3b1a06"
integrity sha1-+mihT2qUXVTbvlDYzbMyDp47GgY=

[email protected]:
version "1.1.1"
resolved "https://registry.yarnpkg.com/commander/-/commander-1.1.1.tgz#50d1651868ae60eccff0a2d9f34595376bc6b041"
integrity sha1-UNFlGGiuYOzP8KLZ80WVN2vGsEE=
dependencies:
keypress "0.1.x"

debug@*:
version "4.3.1"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee"
integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==
dependencies:
ms "2.1.2"

deep-eql@^0.1.3:
version "0.1.3"
resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2"
integrity sha1-71WKyrjeJSBs1xOQbXTlaTDrafI=
dependencies:
type-detect "0.1.1"

[email protected]:
version "1.0.2"
resolved "https://registry.yarnpkg.com/diff/-/diff-1.0.2.tgz#4ae73f1aee8d6fcf484f1a1ce77ce651d9b7f0c9"
integrity sha1-Suc/Gu6Nb89ITxoc53zmUdm38Mk=

[email protected]:
version "0.0.2"
resolved "https://registry.yarnpkg.com/exec-time/-/exec-time-0.0.2.tgz#6331dc860cc5aa97a63d956e0fd847df8b15708c"
integrity sha1-YzHchgzFqpemPZVuD9hH34sVcIw=

[email protected]:
version "1.5.1"
resolved "https://registry.yarnpkg.com/growl/-/growl-1.5.1.tgz#1decd1f22a4b30dae7d363799ec624cf40cc0070"
integrity sha1-HezR8ipLMNrn02N5nsYkz0DMAHA=

immediate@~3.0.5:
version "3.0.6"
resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=

[email protected]:
version "0.26.3"
resolved "https://registry.yarnpkg.com/jade/-/jade-0.26.3.tgz#8f10d7977d8d79f2f6ff862a81b0513ccb25686c"
integrity sha1-jxDXl32NefL2/4YqgbBRPMslaGw=
dependencies:
commander "0.6.1"
mkdirp "0.3.0"

[email protected]:
version "0.1.0"
resolved "https://registry.yarnpkg.com/keypress/-/keypress-0.1.0.tgz#4a3188d4291b66b4f65edb99f806aa9ae293592a"
integrity sha1-SjGI1CkbZrT2XtuZ+AaqmuKTWSo=

[email protected]:
version "3.1.1"
resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=
dependencies:
immediate "~3.0.5"

localforage@^1.3.0:
version "1.9.0"
resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.9.0.tgz#f3e4d32a8300b362b4634cc4e066d9d00d2f09d1"
integrity sha512-rR1oyNrKulpe+VM9cYmcFn6tsHuokyVHFaCM3+osEmxaHTbEk8oQu6eGDfS6DQLWi/N67XRmB8ECG37OES368g==
dependencies:
lie "3.1.1"

minimist@^1.2.5:
version "1.2.5"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==

[email protected]:
version "0.3.0"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.0.tgz#1bbf5ab1ba827af23575143490426455f481fe1e"
integrity sha1-G79asbqCevI1dRQ0kEJkVfSB/h4=

[email protected]:
version "0.3.3"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.3.tgz#595e251c1370c3a68bab2136d0e348b8105adf13"
integrity sha1-WV4lHBNww6aLqyE20ONIuBBa3xM=

mkdirp@~0.5.1:
version "0.5.5"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"
integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==
dependencies:
minimist "^1.2.5"

[email protected]:
version "1.4.3"
resolved "https://registry.yarnpkg.com/mocha/-/mocha-1.4.3.tgz#f36630651e0686fe283d1bcac820b52acc091407"
integrity sha1-82YwZR4Ghv4oPRvKyCC1KswJFAc=
dependencies:
commander "0.6.1"
debug "*"
diff "1.0.2"
growl "1.5.x"
jade "0.26.3"
mkdirp "0.3.3"
ms "0.3.0"

[email protected]:
version "0.3.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-0.3.0.tgz#03edc348d613e66a56486cfdac53bcbe899cbd61"
integrity sha1-A+3DSNYT5mpWSGz9rFO8vomcvWE=

[email protected]:
version "2.1.2"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==

[email protected]:
version "2.9.203"
resolved "https://registry.yarnpkg.com/request/-/request-2.9.203.tgz#6c1711a5407fb94a114219563e44145bcbf4723a"
integrity sha1-bBcRpUB/uUoRQhlWPkQUW8v0cjo=

[email protected]:
version "1.3.4"
resolved "https://registry.yarnpkg.com/sinon/-/sinon-1.3.4.tgz#9fe5d746acb5e078f26f6598a18b13cfa066c4ab"
integrity sha1-n+XXRqy14Hjyb2WYoYsTz6BmxKs=

[email protected]:
version "0.1.1"
resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-0.1.1.tgz#0ba5ec2a885640e470ea4e8505971900dac58822"
integrity sha1-C6XsKohWQORw6k6FBZcZANrFiCI=

type-detect@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2"
integrity sha1-diIXzAbbJY7EiQihKY6LlRIejqI=

underscore@~1.4.4:
version "1.4.4"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604"
integrity sha1-YaajIBBiKvoHljvzJSA88SI51gQ=

0 comments on commit 4e2021c

Please sign in to comment.