Skip to content

Commit 8f45ca5

Browse files
rkistnerrkistner
committed
Add tests; fix parameter query lookup issue.
1 parent 65889dd commit 8f45ca5

File tree

4 files changed

+414
-15
lines changed

4 files changed

+414
-15
lines changed

‎modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts

+9-7
Original file line numberDiff line numberDiff line change
@@ -877,6 +877,7 @@ export class MongoSyncBucketStorage
877877
private async getDataBucketChanges(
878878
options: GetCheckpointChangesOptions
879879
): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {
880+
const limit = 1000;
880881
const bucketStateUpdates = await this.db.bucket_state
881882
.find(
882883
{
@@ -888,15 +889,15 @@ export class MongoSyncBucketStorage
888889
projection: {
889890
'_id.b': 1
890891
},
891-
limit: 1001,
892-
batchSize: 1001,
892+
limit: limit + 1,
893+
batchSize: limit + 1,
893894
singleBatch: true
894895
}
895896
)
896897
.toArray();
897898

898899
const buckets = bucketStateUpdates.map((doc) => doc._id.b);
899-
const invalidateDataBuckets = buckets.length > 1000;
900+
const invalidateDataBuckets = buckets.length > limit;
900901

901902
return {
902903
invalidateDataBuckets: invalidateDataBuckets,
@@ -907,23 +908,24 @@ export class MongoSyncBucketStorage
907908
private async getParameterBucketChanges(
908909
options: GetCheckpointChangesOptions
909910
): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
911+
const limit = 1000;
910912
const parameterUpdates = await this.db.bucket_parameters
911913
.find(
912914
{
913-
_id: { $gt: BigInt(options.lastCheckpoint), $lt: BigInt(options.nextCheckpoint) },
915+
_id: { $gt: BigInt(options.lastCheckpoint), $lte: BigInt(options.nextCheckpoint) },
914916
'key.g': this.group_id
915917
},
916918
{
917919
projection: {
918920
lookup: 1
919921
},
920-
limit: 1001,
921-
batchSize: 1001,
922+
limit: limit + 1,
923+
batchSize: limit + 1,
922924
singleBatch: true
923925
}
924926
)
925927
.toArray();
926-
const invalidateParameterUpdates = parameterUpdates.length > 1000;
928+
const invalidateParameterUpdates = parameterUpdates.length > limit;
927929

928930
return {
929931
invalidateParameterBuckets: invalidateParameterUpdates,

‎modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap

+171
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,74 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
357357
]
358358
`;
359359

360+
exports[`sync - mongodb > sync updates to data query only 1`] = `
361+
[
362+
{
363+
"checkpoint": {
364+
"buckets": [
365+
{
366+
"bucket": "by_user["user1"]",
367+
"checksum": 0,
368+
"count": 0,
369+
"priority": 3,
370+
},
371+
],
372+
"last_op_id": "1",
373+
"write_checkpoint": undefined,
374+
},
375+
},
376+
{
377+
"checkpoint_complete": {
378+
"last_op_id": "1",
379+
},
380+
},
381+
]
382+
`;
383+
384+
exports[`sync - mongodb > sync updates to data query only 2`] = `
385+
[
386+
{
387+
"checkpoint_diff": {
388+
"last_op_id": "2",
389+
"removed_buckets": [],
390+
"updated_buckets": [
391+
{
392+
"bucket": "by_user["user1"]",
393+
"checksum": 1418351250,
394+
"count": 1,
395+
"priority": 3,
396+
},
397+
],
398+
"write_checkpoint": undefined,
399+
},
400+
},
401+
{
402+
"data": {
403+
"after": "0",
404+
"bucket": "by_user["user1"]",
405+
"data": [
406+
{
407+
"checksum": 1418351250n,
408+
"data": "{"id":"list1","user_id":"user1","name":"User 1"}",
409+
"object_id": "list1",
410+
"object_type": "lists",
411+
"op": "PUT",
412+
"op_id": "2",
413+
"subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8",
414+
},
415+
],
416+
"has_more": false,
417+
"next_after": "2",
418+
},
419+
},
420+
{
421+
"checkpoint_complete": {
422+
"last_op_id": "2",
423+
},
424+
},
425+
]
426+
`;
427+
360428
exports[`sync - mongodb > sync updates to global data 1`] = `
361429
[
362430
{
@@ -468,3 +536,106 @@ exports[`sync - mongodb > sync updates to global data 3`] = `
468536
},
469537
]
470538
`;
539+
540+
exports[`sync - mongodb > sync updates to parameter query + data 1`] = `
541+
[
542+
{
543+
"checkpoint": {
544+
"buckets": [],
545+
"last_op_id": "0",
546+
"write_checkpoint": undefined,
547+
},
548+
},
549+
{
550+
"checkpoint_complete": {
551+
"last_op_id": "0",
552+
},
553+
},
554+
]
555+
`;
556+
557+
exports[`sync - mongodb > sync updates to parameter query + data 2`] = `
558+
[
559+
{
560+
"checkpoint_diff": {
561+
"last_op_id": "2",
562+
"removed_buckets": [],
563+
"updated_buckets": [
564+
{
565+
"bucket": "by_user["user1"]",
566+
"checksum": 1418351250,
567+
"count": 1,
568+
"priority": 3,
569+
},
570+
],
571+
"write_checkpoint": undefined,
572+
},
573+
},
574+
{
575+
"data": {
576+
"after": "0",
577+
"bucket": "by_user["user1"]",
578+
"data": [
579+
{
580+
"checksum": 1418351250n,
581+
"data": "{"id":"list1","user_id":"user1","name":"User 1"}",
582+
"object_id": "list1",
583+
"object_type": "lists",
584+
"op": "PUT",
585+
"op_id": "1",
586+
"subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8",
587+
},
588+
],
589+
"has_more": false,
590+
"next_after": "1",
591+
},
592+
},
593+
{
594+
"checkpoint_complete": {
595+
"last_op_id": "2",
596+
},
597+
},
598+
]
599+
`;
600+
601+
exports[`sync - mongodb > sync updates to parameter query only 1`] = `
602+
[
603+
{
604+
"checkpoint": {
605+
"buckets": [],
606+
"last_op_id": "0",
607+
"write_checkpoint": undefined,
608+
},
609+
},
610+
{
611+
"checkpoint_complete": {
612+
"last_op_id": "0",
613+
},
614+
},
615+
]
616+
`;
617+
618+
exports[`sync - mongodb > sync updates to parameter query only 2`] = `
619+
[
620+
{
621+
"checkpoint_diff": {
622+
"last_op_id": "1",
623+
"removed_buckets": [],
624+
"updated_buckets": [
625+
{
626+
"bucket": "by_user["user1"]",
627+
"checksum": 0,
628+
"count": 0,
629+
"priority": 3,
630+
},
631+
],
632+
"write_checkpoint": undefined,
633+
},
634+
},
635+
{
636+
"checkpoint_complete": {
637+
"last_op_id": "1",
638+
},
639+
},
640+
]
641+
`;

‎packages/service-core-tests/src/test-utils/general-utils.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,16 @@ export function testRules(content: string): storage.PersistedSyncRulesContent {
3232
};
3333
}
3434

35-
export function makeTestTable(name: string, columns?: string[] | undefined) {
36-
const relId = utils.hashData('table', name, (columns ?? ['id']).join(','));
35+
export function makeTestTable(name: string, replicaIdColumns?: string[] | undefined) {
36+
const relId = utils.hashData('table', name, (replicaIdColumns ?? ['id']).join(','));
3737
const id = new bson.ObjectId('6544e3899293153fa7b38331');
3838
return new storage.SourceTable(
3939
id,
4040
storage.SourceTable.DEFAULT_TAG,
4141
relId,
4242
'public',
4343
name,
44-
(columns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })),
44+
(replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })),
4545
true
4646
);
4747
}

0 commit comments

Comments
 (0)