Skip to content

Commit 9bc088d

Browse files
stevensJourneystevensJourney
committed
CRUD improvements
1 parent 9933c84 commit 9bc088d

File tree

7 files changed

+3938
-121
lines changed

7 files changed

+3938
-121
lines changed

‎.changeset/many-hairs-scream.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@journeyapps/powersync-sdk-common': patch
3+
---
4+
5+
Improved connector CRUD uploads to be triggered whenever an internal CRUD operation change is triggered. Improved CRUD upload debouncing to rather use a throttled approach - executing multiple continous write/CRUD operations will now trigger a connector upload at most (every) 1 second (by default).

‎apps/supabase-todolist

‎packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts

Lines changed: 58 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,12 @@ import { UploadQueueStats } from '../db/crud/UploadQueueStatus';
88
import { PowerSyncBackendConnector } from './connection/PowerSyncBackendConnector';
99
import {
1010
AbstractStreamingSyncImplementation,
11+
DEFAULT_CRUD_UPLOAD_THROTTLE_MS,
1112
StreamingSyncImplementationListener
1213
} from './sync/stream/AbstractStreamingSyncImplementation';
1314
import { CrudBatch } from './sync/bucket/CrudBatch';
1415
import { CrudTransaction } from './sync/bucket/CrudTransaction';
15-
import { BucketStorageAdapter } from './sync/bucket/BucketStorageAdapter';
16+
import { BucketStorageAdapter, PSInternalTable } from './sync/bucket/BucketStorageAdapter';
1617
import { CrudEntry } from './sync/bucket/CrudEntry';
1718
import { mutexRunExclusive } from '../utils/mutex';
1819
import { BaseObserver } from '../utils/BaseObserver';
@@ -22,13 +23,19 @@ export interface PowerSyncDatabaseOptions {
2223
schema: Schema;
2324
database: DBAdapter;
2425
retryDelay?: number;
26+
crudUploadThrottleMs?: number;
2527
logger?: ILogger;
2628
}
2729

2830
export interface SQLWatchOptions {
2931
signal?: AbortSignal;
3032
tables?: string[];
3133
throttleMs?: number;
34+
/**
35+
* Allows for watching any SQL table
36+
* by not removing PowerSync table name prefixes
37+
*/
38+
rawTableNames?: boolean;
3239
}
3340

3441
export interface WatchOnChangeEvent {
@@ -45,7 +52,8 @@ export const DEFAULT_WATCH_THROTTLE_MS = 30;
4552

4653
export const DEFAULT_POWERSYNC_DB_OPTIONS = {
4754
retryDelay: 5000,
48-
logger: Logger.get('PowerSyncDatabase')
55+
logger: Logger.get('PowerSyncDatabase'),
56+
crudUploadThrottleMs: DEFAULT_CRUD_UPLOAD_THROTTLE_MS
4957
};
5058

5159
/**
@@ -133,6 +141,21 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
133141
this.sdkVersion = version.rows?.item(0)['powersync_rs_version()'] ?? '';
134142
this.ready = true;
135143
this.iterateListeners((cb) => cb.initialized?.());
144+
this.watchCrudUploads();
145+
}
146+
147+
/**
148+
* Queues a CRUD upload when internal CRUD tables have been updated
149+
*/
150+
protected async watchCrudUploads() {
151+
for await (const event of this.onChange({
152+
tables: [PSInternalTable.CRUD],
153+
rawTableNames: true
154+
})) {
155+
if (this.connected) {
156+
this.syncStreamImplementation?.triggerCrudUpload();
157+
}
158+
}
136159
}
137160

138161
/**
@@ -182,9 +205,9 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
182205

183206
// TODO DB name, verify this is necessary with extension
184207
await this.database.writeTransaction(async (tx) => {
185-
await tx.execute('DELETE FROM ps_oplog WHERE 1');
186-
await tx.execute('DELETE FROM ps_crud WHERE 1');
187-
await tx.execute('DELETE FROM ps_buckets WHERE 1');
208+
await tx.execute(`DELETE FROM ${PSInternalTable.OPLOG} WHERE 1`);
209+
await tx.execute(`DELETE FROM ${PSInternalTable.CRUD} WHERE 1`);
210+
await tx.execute(`DELETE FROM ${PSInternalTable.BUCKETS} WHERE 1`);
188211

189212
const existingTableRows = await tx.execute(
190213
"SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'"
@@ -220,12 +243,14 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
220243
async getUploadQueueStats(includeSize?: boolean): Promise<UploadQueueStats> {
221244
return this.readTransaction(async (tx) => {
222245
if (includeSize) {
223-
const result = await tx.execute('SELECT SUM(cast(data as blob) + 20) as size, count(*) as count FROM ps_crud');
246+
const result = await tx.execute(
247+
`SELECT SUM(cast(data as blob) + 20) as size, count(*) as count FROM ${PSInternalTable.CRUD}`
248+
);
224249

225250
const row = result.rows.item(0);
226251
return new UploadQueueStats(row?.count ?? 0, row?.size ?? 0);
227252
} else {
228-
const result = await tx.execute('SELECT count(*) as count FROM ps_crud');
253+
const result = await tx.execute(`SELECT count(*) as count FROM ${PSInternalTable.CRUD}`);
229254
const row = result.rows.item(0);
230255
return new UploadQueueStats(row?.count ?? 0);
231256
}
@@ -250,9 +275,10 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
250275
* and a single transaction may be split over multiple batches.
251276
*/
252277
async getCrudBatch(limit: number): Promise<CrudBatch | null> {
253-
const result = await this.database.execute('SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT ?', [
254-
limit + 1
255-
]);
278+
const result = await this.database.execute(
279+
`SELECT id, tx_id, data FROM ${PSInternalTable.CRUD} ORDER BY id ASC LIMIT ?`,
280+
[limit + 1]
281+
);
256282

257283
const all: CrudEntry[] = result.rows?._array?.map((row) => CrudEntry.fromRow(row)) ?? [];
258284

@@ -268,11 +294,13 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
268294
const last = all[all.length - 1];
269295
return new CrudBatch(all, haveMore, async (writeCheckpoint?: string) => {
270296
await this.writeTransaction(async (tx) => {
271-
await tx.execute('DELETE FROM ps_crud WHERE id <= ?', [last.clientId]);
272-
if (writeCheckpoint != null && (await tx.execute('SELECT 1 FROM ps_crud LIMIT 1')) == null) {
273-
await tx.execute("UPDATE ps_buckets SET target_op = ? WHERE name='$local'", [writeCheckpoint]);
297+
await tx.execute(`DELETE FROM ${PSInternalTable.CRUD} WHERE id <= ?`, [last.clientId]);
298+
if (writeCheckpoint != null && (await tx.execute(`SELECT 1 FROM ${PSInternalTable.CRUD} LIMIT 1`)) == null) {
299+
await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = ? WHERE name='$local'`, [
300+
writeCheckpoint
301+
]);
274302
} else {
275-
await tx.execute("UPDATE ps_buckets SET target_op = ? WHERE name='$local'", [
303+
await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = ? WHERE name='$local'`, [
276304
this.bucketStorageAdapter.getMaxOpId()
277305
]);
278306
}
@@ -295,7 +323,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
295323
*/
296324
async getNextCrudTransaction(): Promise<CrudTransaction> {
297325
return await this.readTransaction(async (tx) => {
298-
const first = await tx.execute('SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT 1');
326+
const first = await tx.execute(`SELECT id, tx_id, data FROM ${PSInternalTable.CRUD} ORDER BY id ASC LIMIT 1`);
299327

300328
if (!first.rows.length) {
301329
return null;
@@ -306,7 +334,10 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
306334
if (!txId) {
307335
all = [CrudEntry.fromRow(first.rows.item(0))];
308336
} else {
309-
const result = await tx.execute('SELECT id, tx_id, data FROM ps_crud WHERE tx_id = ? ORDER BY id ASC', [txId]);
337+
const result = await tx.execute(
338+
`SELECT id, tx_id, data FROM ${PSInternalTable.CRUD} WHERE tx_id = ? ORDER BY id ASC`,
339+
[txId]
340+
);
310341
all = result.rows._array.map((row) => CrudEntry.fromRow(row));
311342
}
312343

@@ -316,14 +347,16 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
316347
all,
317348
async (writeCheckpoint?: string) => {
318349
await this.writeTransaction(async (tx) => {
319-
await tx.execute('DELETE FROM ps_crud WHERE id <= ?', [last.clientId]);
350+
await tx.execute(`DELETE FROM ${PSInternalTable.CRUD} WHERE id <= ?`, [last.clientId]);
320351
if (writeCheckpoint) {
321-
const check = await tx.execute('SELECT 1 FROM ps_crud LIMIT 1');
352+
const check = await tx.execute(`SELECT 1 FROM ${PSInternalTable.CRUD} LIMIT 1`);
322353
if (!check.rows?.length) {
323-
await tx.execute("UPDATE ps_buckets SET target_op = ? WHERE name='$local'", [writeCheckpoint]);
354+
await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = ? WHERE name='$local'`, [
355+
writeCheckpoint
356+
]);
324357
}
325358
} else {
326-
await tx.execute("UPDATE ps_buckets SET target_op = ? WHERE name='$local'", [
359+
await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = ? WHERE name='$local'`, [
327360
this.bucketStorageAdapter.getMaxOpId()
328361
]);
329362
}
@@ -340,7 +373,6 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
340373
async execute(sql: string, parameters?: any[]) {
341374
await this.waitForReady();
342375
const result = await this.database.execute(sql, parameters);
343-
_.defer(() => this.syncStreamImplementation?.triggerCrudUpload());
344376
return result;
345377
}
346378

@@ -386,7 +418,6 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
386418
await this.waitForReady();
387419
return mutexRunExclusive(AbstractPowerSyncDatabase.transactionMutex, async () => {
388420
const res = await callback(this.database);
389-
_.defer(() => this.syncStreamImplementation?.triggerCrudUpload());
390421
return res;
391422
});
392423
}
@@ -415,7 +446,6 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
415446
async (tx) => {
416447
const res = await callback(tx);
417448
await tx.commit();
418-
_.defer(() => this.syncStreamImplementation?.triggerCrudUpload());
419449
return res;
420450
},
421451
{ timeoutMs: lockTimeout }
@@ -475,10 +505,13 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
475505
const dispose = this.database.registerListener({
476506
tablesUpdated: async (update) => {
477507
const { table } = update;
478-
if (!table.match(POWERSYNC_TABLE_MATCH)) {
508+
const { rawTableNames } = options;
509+
510+
if (!rawTableNames && !table.match(POWERSYNC_TABLE_MATCH)) {
479511
return;
480512
}
481-
const tableName = table.replace(POWERSYNC_TABLE_MATCH, '');
513+
514+
const tableName = rawTableNames ? table : table.replace(POWERSYNC_TABLE_MATCH, '');
482515
throttledTableUpdates.push(tableName);
483516

484517
flushTableUpdates();

‎packages/powersync-sdk-common/src/client/sync/bucket/BucketStorageAdapter.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,13 @@ export interface BucketChecksum {
3737
count: number;
3838
}
3939

40+
export enum PSInternalTable {
41+
DATA = 'ps_data',
42+
CRUD = 'ps_crud',
43+
BUCKETS = 'ps_buckets',
44+
OPLOG = 'ps_oplog'
45+
}
46+
4047
export interface BucketStorageAdapter {
4148
init(): Promise<void>;
4249
saveSyncData(batch: SyncDataBatch): Promise<void>;

‎packages/powersync-sdk-common/src/client/sync/stream/AbstractStreamingSyncImplementation.ts

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -37,24 +37,27 @@ export interface AbstractStreamingSyncImplementationOptions {
3737
uploadCrud: () => Promise<void>;
3838
logger?: ILogger;
3939
retryDelayMs?: number;
40+
crudUploadThrottleMs?: number;
4041
}
4142

4243
export interface StreamingSyncImplementationListener extends BaseListener {
4344
statusChanged?: (status: SyncStatus) => void;
4445
}
4546

47+
export const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
48+
4649
export const DEFAULT_STREAMING_SYNC_OPTIONS = {
4750
retryDelayMs: 5000,
48-
logger: Logger.get('PowerSyncStream')
51+
logger: Logger.get('PowerSyncStream'),
52+
crudUploadThrottleMs: DEFAULT_CRUD_UPLOAD_THROTTLE_MS
4953
};
5054

51-
const CRUD_UPLOAD_DEBOUNCE_MS = 1000;
52-
5355
export abstract class AbstractStreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener> {
5456
protected _lastSyncedAt: Date | null;
5557
protected options: AbstractStreamingSyncImplementationOptions;
5658

5759
syncStatus: SyncStatus;
60+
triggerCrudUpload: () => void;
5861

5962
constructor(options: AbstractStreamingSyncImplementationOptions) {
6063
super();
@@ -67,6 +70,17 @@ export abstract class AbstractStreamingSyncImplementation extends BaseObserver<S
6770
downloading: false
6871
}
6972
});
73+
74+
this.triggerCrudUpload = _.throttle(
75+
() => {
76+
if (!this.syncStatus.connected || this.syncStatus.dataFlowStatus.uploading) {
77+
return;
78+
}
79+
this._uploadAllCrud();
80+
},
81+
this.options.crudUploadThrottleMs,
82+
{ trailing: true }
83+
);
7084
}
7185

7286
get lastSyncedAt() {
@@ -88,17 +102,6 @@ export abstract class AbstractStreamingSyncImplementation extends BaseObserver<S
88102
return this.options.adapter.hasCompletedSync();
89103
}
90104

91-
triggerCrudUpload = _.debounce(
92-
() => {
93-
if (!this.syncStatus.connected || this.syncStatus.dataFlowStatus.uploading) {
94-
return;
95-
}
96-
this._uploadAllCrud();
97-
},
98-
CRUD_UPLOAD_DEBOUNCE_MS,
99-
{ trailing: true }
100-
);
101-
102105
protected async _uploadAllCrud(): Promise<void> {
103106
return this.obtainLock({
104107
type: LockType.CRUD,

‎packages/powersync-sdk-react-native/src/db/PowerSyncDatabase.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ export class PowerSyncDatabase extends AbstractPowerSyncDatabase {
2727
await this.waitForReady();
2828
await connector.uploadData(this);
2929
},
30-
retryDelayMs: this.options.retryDelay
30+
retryDelayMs: this.options.retryDelay,
31+
crudUploadThrottleMs: this.options.crudUploadThrottleMs
3132
});
3233
}
3334
}

0 commit comments

Comments
 (0)