Skip to content

Commit 87606bf

Browse files
pubkeyclaude
andauthored
fix: RxPipeline.awaitIdle() resolved before pre-existing source documents were processed (#9538)
Claude-Session: https://claude.ai/code/session_015P6w6hnnyVPXwotts6YQ2F Co-authored-by: Claude <noreply@anthropic.com>
1 parent 00d44aa commit 87606bf

3 files changed

Lines changed: 142 additions & 5 deletions

File tree

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"$schema": "./schema.json",
3+
"text": "FIX `RxPipeline.awaitIdle()` and reads on the destination collection resolved before the source documents that existed when the pipeline was created were processed, when the leader election had not finished yet or the pipeline ran on a non-leader instance",
4+
"links": [],
5+
"priority": 5
6+
}

‎src/plugins/pipeline/rx-pipeline.ts‎

Lines changed: 56 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,15 @@ export class RxPipeline<RxDocType> {
4848
lastProcessedDocTime = new BehaviorSubject(0);
4949
somethingChanged = new Subject();
5050

51+
/**
52+
* Is false until it is known that the source collection
53+
* contains no documents that are newer than the stored checkpoint.
54+
* Source documents that have been written before the pipeline
55+
* was created, emit no event to this instance, so the storage
56+
* must be asked for them.
57+
*/
58+
initialStateKnown = false;
59+
changeCount = 0;
5160

5261

5362
waitBeforeWriteFn = async () => {
@@ -98,6 +107,9 @@ export class RxPipeline<RxDocType> {
98107
public readonly batchSize = 100
99108
) {
100109
this.checkpointId = 'rx-pipeline-' + identifier;
110+
this.subs.push(
111+
this.somethingChanged.subscribe(() => this.changeCount = this.changeCount + 1)
112+
);
101113
this.source.onClose.push(() => this.close());
102114
/**
103115
* The checkpoint belongs to the derived data in the destination.
@@ -234,13 +246,32 @@ export class RxPipeline<RxDocType> {
234246
if (this.error) {
235247
throw this.error;
236248
}
237-
if (
238-
this.stopped ||
239-
this.lastProcessedDocTime.getValue() >= this.lastSourceDocTime.getValue()
249+
if (this.stopped) {
250+
/**
251+
* A stopped pipeline never processes the remaining documents.
252+
*/
253+
done = true;
254+
} else if (this.lastProcessedDocTime.getValue() < this.lastSourceDocTime.getValue()) {
255+
await firstValueFrom(this.somethingChanged);
256+
} else if (
257+
this.initialStateKnown ||
258+
this.source.closed ||
259+
this.destination.closed
240260
) {
241261
done = true;
242262
} else {
243-
await firstValueFrom(this.somethingChanged);
263+
const changeCountBefore = this.changeCount;
264+
const hasUnprocessed = await hasUnprocessedDocuments(this);
265+
if (this.changeCount !== changeCountBefore) {
266+
/**
267+
* The pipeline state has changed while the storage was asked,
268+
* so the result might be outdated and must be fetched again.
269+
*/
270+
} else if (hasUnprocessed) {
271+
await firstValueFrom(this.somethingChanged);
272+
} else {
273+
this.initialStateKnown = true;
274+
}
244275
}
245276
}
246277
}
@@ -249,12 +280,15 @@ export class RxPipeline<RxDocType> {
249280
await this.processQueue;
250281
this.stopped = true;
251282
this.destination.awaitBeforeReads.delete(this.waitBeforeWriteFn);
252-
this.subs.forEach(s => s.unsubscribe());
253283
/**
254284
* A stopped pipeline never processes the remaining documents,
255285
* so pending awaitIdle() calls must be woken up.
286+
* Must be emitted before the subscriptions are closed, so that
287+
* an awaitIdle() call that is asking the storage at this moment,
288+
* detects the change and checks the stopped state again.
256289
*/
257290
this.somethingChanged.next({});
291+
this.subs.forEach(s => s.unsubscribe());
258292
await this.processQueue;
259293
}
260294

@@ -307,6 +341,22 @@ export async function getCheckpointDoc<RxDocType>(
307341
}
308342
}
309343

344+
/**
345+
* Returns true when the source collection contains documents
346+
* that are newer than the stored checkpoint of the pipeline.
347+
*/
348+
export async function hasUnprocessedDocuments<RxDocType>(
349+
pipeline: RxPipeline<RxDocType>
350+
): Promise<boolean> {
351+
const checkpointDoc = await getCheckpointDoc(pipeline);
352+
const docsSinceResult = await getChangedDocumentsSince(
353+
pipeline.source.storageInstance,
354+
1,
355+
checkpointDoc ? checkpointDoc.data.checkpoint : undefined
356+
);
357+
return docsSinceResult.documents.length > 0;
358+
}
359+
310360
export async function setCheckpointDoc<RxDocType>(
311361
pipeline: RxPipeline<RxDocType>,
312362
newCheckpoint: CheckpointDocData,
@@ -354,6 +404,7 @@ export async function addPipeline<RxDocType>(
354404
const startPromise = waitForLeadership ? this.database.waitForLeadership() : PROMISE_RESOLVE_VOID;
355405
startPromise.then(() => {
356406
pipeline.trigger();
407+
pipeline.somethingChanged.next({});
357408
pipeline.subs.push(
358409
this.eventBulks$.pipe(
359410
filter((bulk: RxChangeEventBulk<RxDocType>) => {

‎test/unit/rx-pipeline.test.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,86 @@ describe('rx-pipeline.test.js', () => {
156156
});
157157
});
158158
describe('.awaitIdle()', () => {
159+
it('should not resolve before the documents that existed at creation are processed', async () => {
160+
if (!config.storage.hasMultiInstance) {
161+
return;
162+
}
163+
const c1 = await humansCollection.createMultiInstance(randomToken(10), 3);
164+
const c2 = await humansCollection.create(0);
165+
const pipeline = await c1.addPipeline({
166+
destination: c2,
167+
handler: async (docs) => {
168+
for (const doc of docs) {
169+
await c2.insert(schemaObjects.humanData(doc.passportId));
170+
}
171+
},
172+
identifier: randomToken(10)
173+
});
174+
await pipeline.awaitIdle();
175+
const destinationDocs = await c2.find().exec();
176+
assert.strictEqual(destinationDocs.length, 3);
177+
178+
await c1.database.close();
179+
await c2.database.close();
180+
});
181+
it('should block reads on the destination until the documents that existed at creation are processed', async () => {
182+
if (!config.storage.hasMultiInstance) {
183+
return;
184+
}
185+
const c1 = await humansCollection.createMultiInstance(randomToken(10), 3);
186+
const c2 = await humansCollection.create(0);
187+
await c1.addPipeline({
188+
destination: c2,
189+
handler: async (docs) => {
190+
for (const doc of docs) {
191+
await c2.insert(schemaObjects.humanData(doc.passportId));
192+
}
193+
},
194+
identifier: randomToken(10)
195+
});
196+
const destinationDocs = await c2.find().exec();
197+
assert.strictEqual(destinationDocs.length, 3);
198+
199+
await c1.database.close();
200+
await c2.database.close();
201+
});
202+
it('should resolve on a non-leader instance when the leader has processed all documents', async () => {
203+
if (!config.storage.hasMultiInstance) {
204+
return;
205+
}
206+
const identifier = randomToken(10);
207+
const name = randomToken(10);
208+
const c1 = await humansCollection.createMultiInstance(name, 5);
209+
await c1.database.waitForLeadership();
210+
const c2 = await humansCollection.createMultiInstance(name);
211+
const destination1 = await humansCollection.createMultiInstance(name + 'dest');
212+
const destination2 = await humansCollection.createMultiInstance(name + 'dest');
213+
const p2 = await c2.addPipeline({
214+
destination: destination2,
215+
handler: () => { },
216+
batchSize: 2,
217+
identifier
218+
});
219+
await c1.addPipeline({
220+
destination: destination1,
221+
handler: async (docs) => {
222+
await promiseWait(20);
223+
for (const doc of docs) {
224+
await destination1.insert(schemaObjects.humanData(doc.passportId));
225+
}
226+
},
227+
batchSize: 2,
228+
identifier
229+
});
230+
await p2.awaitIdle();
231+
const destinationDocs = await destination2.find().exec();
232+
assert.strictEqual(destinationDocs.length, 5);
233+
234+
await c1.database.close();
235+
await c2.database.close();
236+
await destination1.database.close();
237+
await destination2.database.close();
238+
});
159239
it('should have updated its internal timestamps', async () => {
160240
const c1 = await humansCollection.create(0);
161241
await c1.database.waitForLeadership();

0 commit comments

Comments
 (0)