From 2bf65ddb4a79c88383ba4be8050066e2f3357bdc Mon Sep 17 00:00:00 2001 From: Nathaniel Tucker Date: Thu, 12 Jun 2025 11:53:23 +0100 Subject: [PATCH 1/3] feat: Normalize into immutablejs state --- .../src/normalize/NormalizeDelegate.ts | 42 ++++++++++++++----- packages/normalizr/src/normalize/normalize.ts | 16 ++++--- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/packages/normalizr/src/normalize/NormalizeDelegate.ts b/packages/normalizr/src/normalize/NormalizeDelegate.ts index a4a1ad685300..b211aa2dcfbb 100644 --- a/packages/normalizr/src/normalize/NormalizeDelegate.ts +++ b/packages/normalizr/src/normalize/NormalizeDelegate.ts @@ -1,4 +1,4 @@ -import { +import type { EntityTable, NormalizedIndex, INormalizeDelegate, @@ -7,12 +7,22 @@ import { import { getCheckLoop } from './getCheckLoop.js'; import { POJODelegate } from '../delegate/Delegate.js'; import { INVALID } from '../denormalize/symbol.js'; +import { NormalizedSchema } from '../types.js'; /** Full normalize() logic for POJO state */ export class NormalizeDelegate extends POJODelegate - implements INormalizeDelegate + implements INormalizeDelegate, NormalizedSchema { + // declare readonly normalized: NormalizedSchema; + declare result: any; + declare readonly entities: EntityTable; + declare readonly indexes: { + [entityKey: string]: { + [indexName: string]: { [lookup: string]: string }; + }; + }; + declare readonly entitiesMeta: { [entityKey: string]: { [pk: string]: { @@ -26,8 +36,10 @@ export class NormalizeDelegate declare readonly meta: { fetchedAt: number; date: number; expiresAt: number }; declare checkLoop: (entityKey: string, pk: string, input: object) => boolean; - protected newEntities = new Map>(); - protected newIndexes = new Map>(); + protected new = { + entities: new Map>(), + indexes: new Map>(), + }; constructor( state: { @@ -46,7 +58,15 @@ export class NormalizeDelegate actionMeta: { fetchedAt: number; date: number; expiresAt: number }, ) { super(state); - this.entitiesMeta = state.entitiesMeta; + // this.normalized = NormalizedSchema = { + // result: '' as any, + // entities: { ...state.entities }, + // indexes: { ...state.indexes }, + // entitiesMeta: { ...state.entitiesMeta }, + // }; + this.entities = { ...state.entities }; + this.indexes = { ...state.indexes }; + this.entitiesMeta = { ...state.entitiesMeta }; this.meta = actionMeta; this.checkLoop = getCheckLoop(); } @@ -57,8 +77,8 @@ export class NormalizeDelegate protected getNewEntities(key: string): Map { // first time we come across this type of entity - if (!this.newEntities.has(key)) { - this.newEntities.set(key, new Map()); + if (!this.new.entities.has(key)) { + this.new.entities.set(key, new Map()); // we will be editing these, so we need to clone them first this.entities[key] = { ...this.entities[key], @@ -68,15 +88,15 @@ export class NormalizeDelegate }; } - return this.newEntities.get(key) as Map; + return this.new.entities.get(key) as Map; } protected getNewIndexes(key: string): Map { - if (!this.newIndexes.has(key)) { - this.newIndexes.set(key, new Map()); + if (!this.new.indexes.has(key)) { + this.new.indexes.set(key, new Map()); this.indexes[key] = { ...this.indexes[key] }; } - return this.newIndexes.get(key) as Map; + return this.new.indexes.get(key) as Map; } /** Updates an entity using merge lifecycles when it has previously been set */ diff --git a/packages/normalizr/src/normalize/normalize.ts b/packages/normalizr/src/normalize/normalize.ts index 6568d29c3538..e84a16fcb111 100644 --- a/packages/normalizr/src/normalize/normalize.ts +++ b/packages/normalizr/src/normalize/normalize.ts @@ -79,15 +79,13 @@ See https://dataclient.io/rest/api/RestEndpoint#parseResponse for more informati } } - const ret: NormalizedSchema = { - result: '' as any, - entities: { ...entities }, - indexes: { ...indexes }, - entitiesMeta: { ...entitiesMeta }, - }; - const visit = getVisit(new NormalizeDelegate(ret, meta)); - ret.result = visit(schema, input, input, undefined, args); - return ret; + const delegate = new NormalizeDelegate( + { entities, indexes, entitiesMeta }, + meta, + ); + const visit = getVisit(delegate); + delegate.result = visit(schema, input, input, undefined, args); + return delegate as any; }; function expectedSchemaType(schema: Schema) { From ea0a4dc2bda8813aa39051b08180c7e2502e401d Mon Sep 17 00:00:00 2001 From: Nathaniel Tucker Date: Tue, 22 Jul 2025 11:55:37 -0400 Subject: [PATCH 2/3] enhance: Hoist state to delegate --- .../normalizr/src/delegate/BaseDelegate.ts | 8 - packages/normalizr/src/delegate/Delegate.ts | 23 +- .../src/normalize/NormalizeDelegate.imm.ts | 197 ++++++++++++++++++ 3 files changed, 209 insertions(+), 19 deletions(-) create mode 100644 packages/normalizr/src/normalize/NormalizeDelegate.imm.ts diff --git a/packages/normalizr/src/delegate/BaseDelegate.ts b/packages/normalizr/src/delegate/BaseDelegate.ts index 769c4638af05..8c93f221e26c 100644 --- a/packages/normalizr/src/delegate/BaseDelegate.ts +++ b/packages/normalizr/src/delegate/BaseDelegate.ts @@ -9,14 +9,6 @@ import type { Dep } from '../memo/WeakDependencyMap.js'; /** Basic state interfaces for normalize side */ export abstract class BaseDelegate { - declare entities: any; - declare indexes: any; - - constructor({ entities, indexes }: { entities: any; indexes: any }) { - this.entities = entities; - this.indexes = indexes; - } - abstract getEntities(key: string): EntitiesInterface | undefined; abstract getEntity(key: string, pk: string): object | undefined; abstract getIndex(...path: IndexPath): object | undefined; diff --git a/packages/normalizr/src/delegate/Delegate.ts b/packages/normalizr/src/delegate/Delegate.ts index b72081d98db3..71317633e2f0 100644 --- a/packages/normalizr/src/delegate/Delegate.ts +++ b/packages/normalizr/src/delegate/Delegate.ts @@ -5,27 +5,28 @@ import type { } from '../interface.js'; import { BaseDelegate } from './BaseDelegate.js'; -/** Basic POJO state interfaces for normalize side */ +/** Basic POJO state interfaces for normalize side + * Used directly as QueryDelegate, and inherited by NormalizeDelegate + */ export class POJODelegate extends BaseDelegate { - declare entities: EntityTable; - declare indexes: { - [entityKey: string]: { - [indexName: string]: { [lookup: string]: string }; - }; + declare state: { + entities: EntityTable; + indexes: NormalizedIndex; }; constructor(state: { entities: EntityTable; indexes: NormalizedIndex }) { - super(state); + super(); + this.state = state; } // we must expose the entities object to track in our WeakDependencyMap // however, this should not be part of the public API protected getEntitiesObject(key: string): object | undefined { - return this.entities[key]; + return this.state.entities[key]; } getEntities(key: string): EntitiesInterface | undefined { - const entities = this.entities[key]; + const entities = this.state.entities[key]; if (entities === undefined) return undefined; return { keys(): IterableIterator { @@ -38,12 +39,12 @@ export class POJODelegate extends BaseDelegate { } getEntity(key: string, pk: string): any { - return this.entities[key]?.[pk]; + return this.state.entities[key]?.[pk]; } // this is different return value than QuerySnapshot getIndex(key: string, field: string): object | undefined { - return this.indexes[key]?.[field]; + return this.state.indexes[key]?.[field]; } getIndexEnd(entity: object | undefined, value: string) { diff --git a/packages/normalizr/src/normalize/NormalizeDelegate.imm.ts b/packages/normalizr/src/normalize/NormalizeDelegate.imm.ts new file mode 100644 index 000000000000..e1544422f8e2 --- /dev/null +++ b/packages/normalizr/src/normalize/NormalizeDelegate.imm.ts @@ -0,0 +1,197 @@ +import type { + INormalizeDelegate, + Mergeable, + EntitiesInterface, +} from '../interface.js'; +import { getCheckLoop } from './getCheckLoop.js'; +import { ImmDelegate } from '../delegate/Delegate.imm.js'; +import { INVALID } from '../denormalize/symbol.js'; + +export type ImmutableJSEntityTable = { + get(key: string): EntitiesInterface | undefined; + getIn(k: [key: string, pk: string]): { toJS(): any } | undefined; + setIn(k: [key: string, pk: string], value: any); +}; +type ImmutableJSMeta = { + getIn(k: [key: string, pk: string]): + | { + date: number; + expiresAt: number; + fetchedAt: number; + } + | undefined; + setIn( + k: [key: string, pk: string], + value: { + date: number; + expiresAt: number; + fetchedAt: number; + }, + ); +}; + +/** Full normalize() logic for ImmutableJS state */ +export class NormalizeDelegate + extends ImmDelegate + implements INormalizeDelegate +{ + declare readonly entitiesMeta: ImmutableJSMeta; + + declare readonly meta: { fetchedAt: number; date: number; expiresAt: number }; + declare checkLoop: (entityKey: string, pk: string, input: object) => boolean; + + protected newEntities = new Map>(); + protected newIndexes = new Map>(); + + constructor( + state: { + entities: ImmutableJSEntityTable; + indexes: ImmutableJSEntityTable; + entitiesMeta: ImmutableJSMeta; + }, + actionMeta: { fetchedAt: number; date: number; expiresAt: number }, + ) { + super(state); + this.entitiesMeta = state.entitiesMeta; + this.meta = actionMeta; + this.checkLoop = getCheckLoop(); + } + + protected getNewEntity(key: string, pk: string) { + return this.getNewEntities(key).get(pk); + } + + protected getNewEntities(key: string): Map { + // first time we come across this type of entity + if (!this.newEntities.has(key)) { + this.newEntities.set(key, new Map()); + } + + return this.newEntities.get(key) as Map; + } + + protected getNewIndexes(key: string): Map { + if (!this.newIndexes.has(key)) { + this.newIndexes.set(key, new Map()); + } + return this.newIndexes.get(key) as Map; + } + + /** Updates an entity using merge lifecycles when it has previously been set */ + mergeEntity( + schema: Mergeable & { indexes?: any }, + pk: string, + incomingEntity: any, + ) { + const key = schema.key; + + // default when this is completely new entity + let nextEntity = incomingEntity; + let nextMeta = this.meta; + + // if we already processed this entity during this normalization (in another nested place) + let entity = this.getNewEntity(key, pk); + if (entity) { + nextEntity = schema.merge(entity, incomingEntity); + } else { + // if we find it in the store + entity = this.getEntity(key, pk); + if (entity) { + const meta = this.getMeta(key, pk); + nextEntity = schema.mergeWithStore( + meta, + nextMeta, + entity, + incomingEntity, + ); + nextMeta = schema.mergeMetaWithStore( + meta, + nextMeta, + entity, + incomingEntity, + ); + } + } + + // once we have computed the merged values, set them + this.setEntity(schema, pk, nextEntity, nextMeta); + } + + /** Sets an entity overwriting any previously set values */ + setEntity( + schema: { key: string; indexes?: any }, + pk: string, + entity: any, + meta: { fetchedAt: number; date: number; expiresAt: number } = this.meta, + ) { + const key = schema.key; + const newEntities = this.getNewEntities(key); + const updateMeta = !newEntities.has(pk); + newEntities.set(pk, entity); + + // update index + if (schema.indexes) { + // typescript should know indexes is defined now + this.handleIndexes(schema as any, pk, entity); + } + + // set this after index updates so we know what indexes to remove from + this._setEntity(key, pk, entity); + + if (updateMeta) this._setMeta(key, pk, meta); + } + + handleIndexes( + schema: { key: string; indexes: any }, + pk: string, + entity: any, + ) { + const { key } = schema; + const newIndexes = this.getNewIndexes(key); + const storeEntity = this.entities.getIn([key, pk]); + for (const index of schema.indexes) { + if (!newIndexes.has(index)) { + newIndexes.set(index, this.indexes.getIn([key, index]) ?? {}); + } + const indexMap = newIndexes.get(index); + if (storeEntity) { + delete indexMap[storeEntity[index]]; + } + // entity already in cache but the index changed + if (storeEntity && storeEntity[index] !== entity[index]) { + indexMap[storeEntity[index]] = INVALID; + } + if (index in entity) { + indexMap[entity[index]] = pk; + } /* istanbul ignore next */ else if ( + process.env.NODE_ENV !== 'production' + ) { + console.warn(`Index not found in entity. Indexes must be top-level members of your entity. +Index: ${index} +Entity: ${JSON.stringify(entity, undefined, 2)}`); + } + } + } + + /** Invalidates an entity, potentially triggering suspense */ + invalidate(schema: { key: string; indexes?: any }, pk: string) { + // set directly: any queued updates are meaningless with delete + this.setEntity(schema, pk, INVALID); + } + + protected _setEntity(key: string, pk: string, entity: any) { + this.entities.setIn([key, pk], entity); + } + + protected _setMeta( + key: string, + pk: string, + meta: { fetchedAt: number; date: number; expiresAt: number }, + ) { + this.entitiesMeta.setIn([key, pk], meta); + } + + getMeta(key: string, pk: string) { + return this.entitiesMeta.getIn([key, pk]); + } +} From 047331f8c677f2ff748ef5469fb09e69e7bdcccb Mon Sep 17 00:00:00 2001 From: Nathaniel Tucker Date: Tue, 22 Jul 2025 11:56:21 -0400 Subject: [PATCH 3/3] enhance(in-progress): Store result in own object --- .../src/normalize/NormalizeDelegate.ts | 181 +++++++++++------- packages/normalizr/src/normalize/normalize.ts | 7 +- 2 files changed, 115 insertions(+), 73 deletions(-) diff --git a/packages/normalizr/src/normalize/NormalizeDelegate.ts b/packages/normalizr/src/normalize/NormalizeDelegate.ts index b211aa2dcfbb..8768f66e7b92 100644 --- a/packages/normalizr/src/normalize/NormalizeDelegate.ts +++ b/packages/normalizr/src/normalize/NormalizeDelegate.ts @@ -7,39 +7,39 @@ import type { import { getCheckLoop } from './getCheckLoop.js'; import { POJODelegate } from '../delegate/Delegate.js'; import { INVALID } from '../denormalize/symbol.js'; -import { NormalizedSchema } from '../types.js'; +import type { NormalizedSchema } from '../types.js'; /** Full normalize() logic for POJO state */ export class NormalizeDelegate extends POJODelegate - implements INormalizeDelegate, NormalizedSchema + implements INormalizeDelegate { - // declare readonly normalized: NormalizedSchema; - declare result: any; - declare readonly entities: EntityTable; - declare readonly indexes: { - [entityKey: string]: { - [indexName: string]: { [lookup: string]: string }; - }; - }; + declare readonly result: Normalized; + // declare result: any; + // declare readonly entities: EntityTable; + // declare readonly indexes: { + // [entityKey: string]: { + // [indexName: string]: { [lookup: string]: string }; + // }; + // }; - declare readonly entitiesMeta: { - [entityKey: string]: { - [pk: string]: { - date: number; - expiresAt: number; - fetchedAt: number; - }; - }; - }; + // declare readonly entitiesMeta: { + // [entityKey: string]: { + // [pk: string]: { + // date: number; + // expiresAt: number; + // fetchedAt: number; + // }; + // }; + // }; declare readonly meta: { fetchedAt: number; date: number; expiresAt: number }; declare checkLoop: (entityKey: string, pk: string, input: object) => boolean; - protected new = { - entities: new Map>(), - indexes: new Map>(), - }; + // protected new = { + // entities: new Map>(), + // indexes: new Map>(), + // }; constructor( state: { @@ -58,47 +58,11 @@ export class NormalizeDelegate actionMeta: { fetchedAt: number; date: number; expiresAt: number }, ) { super(state); - // this.normalized = NormalizedSchema = { - // result: '' as any, - // entities: { ...state.entities }, - // indexes: { ...state.indexes }, - // entitiesMeta: { ...state.entitiesMeta }, - // }; - this.entities = { ...state.entities }; - this.indexes = { ...state.indexes }; - this.entitiesMeta = { ...state.entitiesMeta }; + this.result = new Normalized(state); this.meta = actionMeta; this.checkLoop = getCheckLoop(); } - protected getNewEntity(key: string, pk: string) { - return this.getNewEntities(key).get(pk); - } - - protected getNewEntities(key: string): Map { - // first time we come across this type of entity - if (!this.new.entities.has(key)) { - this.new.entities.set(key, new Map()); - // we will be editing these, so we need to clone them first - this.entities[key] = { - ...this.entities[key], - }; - this.entitiesMeta[key] = { - ...this.entitiesMeta[key], - }; - } - - return this.new.entities.get(key) as Map; - } - - protected getNewIndexes(key: string): Map { - if (!this.new.indexes.has(key)) { - this.new.indexes.set(key, new Map()); - this.indexes[key] = { ...this.indexes[key] }; - } - return this.new.indexes.get(key) as Map; - } - /** Updates an entity using merge lifecycles when it has previously been set */ mergeEntity( schema: Mergeable & { indexes?: any }, @@ -112,7 +76,7 @@ export class NormalizeDelegate let nextMeta = this.meta; // if we already processed this entity during this normalization (in another nested place) - let entity = this.getNewEntity(key, pk); + let entity = this.result.getEntity(key, pk); if (entity) { nextEntity = schema.merge(entity, incomingEntity); } else { @@ -147,7 +111,7 @@ export class NormalizeDelegate meta: { fetchedAt: number; date: number; expiresAt: number } = this.meta, ) { const key = schema.key; - const newEntities = this.getNewEntities(key); + const newEntities = this.result.getEntities(key); const updateMeta = !newEntities.has(pk); newEntities.set(pk, entity); @@ -156,17 +120,17 @@ export class NormalizeDelegate handleIndexes( pk, schema.indexes, - this.getNewIndexes(key), - this.indexes[key], + this.result.getIndexes(key), + this.clone.indexes[key], entity, - this.entities[key] as any, + this.clone.entities[key] as any, ); } // set this after index updates so we know what indexes to remove from - this._setEntity(key, pk, entity); + this.result.setEntity(key, pk, entity); - if (updateMeta) this._setMeta(key, pk, meta); + if (updateMeta) this.result.setMeta(key, pk, meta); } /** Invalidates an entity, potentially triggering suspense */ @@ -175,11 +139,62 @@ export class NormalizeDelegate this.setEntity(schema, pk, INVALID); } - protected _setEntity(key: string, pk: string, entity: any) { + getMeta(key: string, pk: string) { + return this.result.entitiesMeta[key][pk]; + } +} + +class Normalized implements NormalizedSchema { + result: any = ''; + declare readonly entities: EntityTable; + declare readonly indexes: { + [entityKey: string]: { + [indexName: string]: { [lookup: string]: string }; + }; + }; + + declare readonly entitiesMeta: { + [entityKey: string]: { + [pk: string]: { + date: number; + expiresAt: number; + fetchedAt: number; + }; + }; + }; + + protected new = { + entities: new Map>(), + indexes: new Map>(), + }; + + constructor({ + entities, + indexes, + entitiesMeta, + }: { + entities: EntityTable; + indexes: NormalizedIndex; + entitiesMeta: { + [entityKey: string]: { + [pk: string]: { + date: number; + expiresAt: number; + fetchedAt: number; + }; + }; + }; + }) { + this.entities = { ...entities }; + this.indexes = { ...indexes }; + this.entitiesMeta = { ...entitiesMeta }; + } + + setEntity(key: string, pk: string, entity: any) { (this.entities[key] as any)[pk] = entity; } - protected _setMeta( + setMeta( key: string, pk: string, meta: { fetchedAt: number; date: number; expiresAt: number }, @@ -187,8 +202,32 @@ export class NormalizeDelegate this.entitiesMeta[key][pk] = meta; } - getMeta(key: string, pk: string) { - return this.entitiesMeta[key][pk]; + getEntity(key: string, pk: string) { + return this.getEntities(key).get(pk); + } + + getEntities(key: string): Map { + // first time we come across this type of entity + if (!this.new.entities.has(key)) { + this.new.entities.set(key, new Map()); + // we will be editing these, so we need to clone them first + this.entities[key] = { + ...this.entities[key], + }; + this.entitiesMeta[key] = { + ...this.entitiesMeta[key], + }; + } + + return this.new.entities.get(key) as Map; + } + + getIndexes(key: string): Map { + if (!this.new.indexes.has(key)) { + this.new.indexes.set(key, new Map()); + this.indexes[key] = { ...this.indexes[key] }; + } + return this.new.indexes.get(key) as Map; } } diff --git a/packages/normalizr/src/normalize/normalize.ts b/packages/normalizr/src/normalize/normalize.ts index e84a16fcb111..e8e7dde7ae8c 100644 --- a/packages/normalizr/src/normalize/normalize.ts +++ b/packages/normalizr/src/normalize/normalize.ts @@ -79,13 +79,16 @@ See https://dataclient.io/rest/api/RestEndpoint#parseResponse for more informati } } + // using cloned or original should not matter as merge checks against a Set to determine whether to merge or not + // however, we should still write a test to specifially test merge detection + // TODO: we need a test to validate that getEntities uses the old version of state and not the cloned one const delegate = new NormalizeDelegate( { entities, indexes, entitiesMeta }, meta, ); const visit = getVisit(delegate); - delegate.result = visit(schema, input, input, undefined, args); - return delegate as any; + delegate.result.result = visit(schema, input, input, undefined, args); + return delegate.result as any as NormalizedSchema; }; function expectedSchemaType(schema: Schema) {