@@ -48,6 +48,15 @@ export class RxPipeline<RxDocType> {
4848 lastProcessedDocTime = new BehaviorSubject ( 0 ) ;
4949 somethingChanged = new Subject ( ) ;
5050
51+ /**
52+ * Is false until it is known that the source collection
53+ * contains no documents that are newer than the stored checkpoint.
54+ * Source documents that have been written before the pipeline
55+ * was created, emit no event to this instance, so the storage
56+ * must be asked for them.
57+ */
58+ initialStateKnown = false ;
59+ changeCount = 0 ;
5160
5261
5362 waitBeforeWriteFn = async ( ) => {
@@ -98,6 +107,9 @@ export class RxPipeline<RxDocType> {
98107 public readonly batchSize = 100
99108 ) {
100109 this . checkpointId = 'rx-pipeline-' + identifier ;
110+ this . subs . push (
111+ this . somethingChanged . subscribe ( ( ) => this . changeCount = this . changeCount + 1 )
112+ ) ;
101113 this . source . onClose . push ( ( ) => this . close ( ) ) ;
102114 /**
103115 * The checkpoint belongs to the derived data in the destination.
@@ -234,13 +246,32 @@ export class RxPipeline<RxDocType> {
234246 if ( this . error ) {
235247 throw this . error ;
236248 }
237- if (
238- this . stopped ||
239- this . lastProcessedDocTime . getValue ( ) >= this . lastSourceDocTime . getValue ( )
249+ if ( this . stopped ) {
250+ /**
251+ * A stopped pipeline never processes the remaining documents.
252+ */
253+ done = true ;
254+ } else if ( this . lastProcessedDocTime . getValue ( ) < this . lastSourceDocTime . getValue ( ) ) {
255+ await firstValueFrom ( this . somethingChanged ) ;
256+ } else if (
257+ this . initialStateKnown ||
258+ this . source . closed ||
259+ this . destination . closed
240260 ) {
241261 done = true ;
242262 } else {
243- await firstValueFrom ( this . somethingChanged ) ;
263+ const changeCountBefore = this . changeCount ;
264+ const hasUnprocessed = await hasUnprocessedDocuments ( this ) ;
265+ if ( this . changeCount !== changeCountBefore ) {
266+ /**
267+ * The pipeline state has changed while the storage was asked,
268+ * so the result might be outdated and must be fetched again.
269+ */
270+ } else if ( hasUnprocessed ) {
271+ await firstValueFrom ( this . somethingChanged ) ;
272+ } else {
273+ this . initialStateKnown = true ;
274+ }
244275 }
245276 }
246277 }
@@ -249,12 +280,15 @@ export class RxPipeline<RxDocType> {
249280 await this . processQueue ;
250281 this . stopped = true ;
251282 this . destination . awaitBeforeReads . delete ( this . waitBeforeWriteFn ) ;
252- this . subs . forEach ( s => s . unsubscribe ( ) ) ;
253283 /**
254284 * A stopped pipeline never processes the remaining documents,
255285 * so pending awaitIdle() calls must be woken up.
286+ * Must be emitted before the subscriptions are closed, so that
287+ * an awaitIdle() call that is asking the storage at this moment,
288+ * detects the change and checks the stopped state again.
256289 */
257290 this . somethingChanged . next ( { } ) ;
291+ this . subs . forEach ( s => s . unsubscribe ( ) ) ;
258292 await this . processQueue ;
259293 }
260294
@@ -307,6 +341,22 @@ export async function getCheckpointDoc<RxDocType>(
307341 }
308342}
309343
344+ /**
345+ * Returns true when the source collection contains documents
346+ * that are newer than the stored checkpoint of the pipeline.
347+ */
348+ export async function hasUnprocessedDocuments < RxDocType > (
349+ pipeline : RxPipeline < RxDocType >
350+ ) : Promise < boolean > {
351+ const checkpointDoc = await getCheckpointDoc ( pipeline ) ;
352+ const docsSinceResult = await getChangedDocumentsSince (
353+ pipeline . source . storageInstance ,
354+ 1 ,
355+ checkpointDoc ? checkpointDoc . data . checkpoint : undefined
356+ ) ;
357+ return docsSinceResult . documents . length > 0 ;
358+ }
359+
310360export async function setCheckpointDoc < RxDocType > (
311361 pipeline : RxPipeline < RxDocType > ,
312362 newCheckpoint : CheckpointDocData ,
@@ -354,6 +404,7 @@ export async function addPipeline<RxDocType>(
354404 const startPromise = waitForLeadership ? this . database . waitForLeadership ( ) : PROMISE_RESOLVE_VOID ;
355405 startPromise . then ( ( ) => {
356406 pipeline . trigger ( ) ;
407+ pipeline . somethingChanged . next ( { } ) ;
357408 pipeline . subs . push (
358409 this . eventBulks$ . pipe (
359410 filter ( ( bulk : RxChangeEventBulk < RxDocType > ) => {
0 commit comments