From 9a219345297c8edcc122ba12b863df7232df57ee Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Tue, 17 Feb 2026 12:18:46 +0100 Subject: [PATCH 1/7] node internal ID: added DB migrations --- ...20260217085512-alter-table-node-add-iid.js | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/db/dbMigrator/migration/survey/migrations/20260217085512-alter-table-node-add-iid.js diff --git a/src/db/dbMigrator/migration/survey/migrations/20260217085512-alter-table-node-add-iid.js b/src/db/dbMigrator/migration/survey/migrations/20260217085512-alter-table-node-add-iid.js new file mode 100644 index 0000000..a899509 --- /dev/null +++ b/src/db/dbMigrator/migration/survey/migrations/20260217085512-alter-table-node-add-iid.js @@ -0,0 +1,51 @@ +'use strict' + +var dbm +var type +var seed +var fs = require('fs') +var path = require('path') +var Promise + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate + type = dbm.dataType + seed = seedLink + Promise = options.Promise +} + +exports.up = function (db) { + var filePath = path.join(__dirname, 'sqls', '20260217085512-alter-table-node-add-iid-up.sql') + return new Promise(function (resolve, reject) { + fs.readFile(filePath, { encoding: 'utf-8' }, function (err, data) { + if (err) return reject(err) + console.log('received data: ' + data) + + resolve(data) + }) + }).then(function (data) { + return db.runSql(data) + }) +} + +exports.down = function (db) { + var filePath = path.join(__dirname, 'sqls', '20260217085512-alter-table-node-add-iid-down.sql') + return new Promise(function (resolve, reject) { + fs.readFile(filePath, { encoding: 'utf-8' }, function (err, data) { + if (err) return reject(err) + console.log('received data: ' + data) + + resolve(data) + }) + }).then(function (data) { + return db.runSql(data) + }) +} + +exports._meta = { + version: 1, +} From 7f1a9613de44b79862d5794395f4c7c921405fcd Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Tue, 17 Feb 2026 12:18:59 +0100 Subject: [PATCH 2/7] node internal ID: added DB migrations --- ...17085512-alter-table-node-add-iid-down.sql | 1 + ...0217085512-alter-table-node-add-iid-up.sql | 123 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-down.sql create mode 100644 src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql diff --git a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-down.sql b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-down.sql new file mode 100644 index 0000000..44f074e --- /dev/null +++ b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-down.sql @@ -0,0 +1 @@ +/* Replace with your SQL commands */ \ No newline at end of file diff --git a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql new file mode 100644 index 0000000..2ed7e3e --- /dev/null +++ b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql @@ -0,0 +1,123 @@ +ALTER TABLE node + ADD COLUMN i_id INTEGER; + +ALTER TABLE node + ADD COLUMN p_i_id INTEGER; + +-- Populate the i_id column with the row number for each record_uuid +WITH ranked AS ( + SELECT id, + ROW_NUMBER() OVER (PARTITION BY record_uuid ORDER BY id) AS i_id + FROM node +) +UPDATE node +SET i_id = ranked.i_id +FROM ranked +WHERE node.id = ranked.id; + +ALTER TABLE node + ALTER COLUMN i_id SET NOT NULL; + +-- Populate the p_i_id column by joining the node table with itself based on the parent-child relationship +UPDATE node AS child +SET p_i_id = parent.i_id +FROM node AS parent +WHERE child.parent_uuid = parent.uuid + AND child.record_uuid = parent.record_uuid; + +-- Replace meta.h UUID array with i_id array, preserving order +UPDATE node AS n +SET meta = jsonb_set( + meta, + '{h}', + COALESCE( + ( + SELECT jsonb_agg(p.i_id ORDER BY elems.ordinality) + FROM jsonb_array_elements_text(n.meta->'h') WITH ORDINALITY AS elems(uuid_text, ordinality) + JOIN node AS p ON p.uuid = elems.uuid_text::uuid + ), + '[]'::jsonb + ) +) +WHERE n.meta ? 'h'; + +-- Update activity_log content to use i_id/p_i_id and drop meta.h +UPDATE activity_log AS al +SET content = updated.content +FROM ( + SELECT al.id, + jsonb_set( + jsonb_set( + jsonb_set( + ( + CASE + WHEN al.content ? 'meta' + THEN jsonb_set(al.content, '{meta}', (al.content->'meta') - 'h') + ELSE al.content + END + ) - 'uuid' - 'parentUuid', + '{iId}', + to_jsonb(n.i_id), + true + ), + '{pIId}', + to_jsonb(n.p_i_id), + true + ), + '{recordUuid}', + to_jsonb(n.record_uuid), + true + ) AS content + FROM activity_log AS al + JOIN node AS n ON n.uuid = (al.content->>'uuid')::uuid + WHERE al.type IN ('nodeCreate', 'nodeDelete', 'nodeValueUpdate') +) AS updated +WHERE al.id = updated.id; + +DROP VIEW IF EXISTS activity_log_user_aggregate; +CREATE VIEW activity_log_user_aggregate AS +SELECT + DISTINCT ON ( + date_created::date, + user_uuid, + type, + content_uuid, + content_key, + content_record_uuid, + content_i_id + ) + id, + date_created, + user_uuid, + type, + (content->>'uuid')::uuid as content_uuid, + content->>'key' as content_key, + -- node related fields + (content->>'recordUuid')::uuid as content_record_uuid, + (content->>'iId')::integer as content_i_id, + content +FROM + activity_log +WHERE + NOT system +ORDER BY + date_created::date DESC, + user_uuid, + type, + content_uuid, + content_key, + content_record_uuid, + content_i_id, + date_created DESC; + +-- Add a unique constraint on the combination of record_uuid and i_id +ALTER TABLE node + DROP CONSTRAINT IF EXISTS node_pkey; + +-- Add a composite primary key on record_uuid and i_id +ALTER TABLE node + ADD PRIMARY KEY (record_uuid, i_id); + +-- Create an index on the combination of record_uuid and i_id for faster lookups +CREATE INDEX IF NOT EXISTS node_record_uuid_i_id_idx + ON node (record_uuid, i_id); From cd67494a5522cdfc3c2a7f58d05b13f92c99d12d Mon Sep 17 00:00:00 2001 From: Stefano Ricci Date: Wed, 18 Feb 2026 22:39:44 +0100 Subject: [PATCH 3/7] updated migrations --- ...0217085512-alter-table-node-add-iid-up.sql | 79 ++++++++++++------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql index 2ed7e3e..da6f210 100644 --- a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql +++ b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql @@ -4,16 +4,38 @@ ALTER TABLE node ALTER TABLE node ADD COLUMN p_i_id INTEGER; --- Populate the i_id column with the row number for each record_uuid -WITH ranked AS ( - SELECT id, - ROW_NUMBER() OVER (PARTITION BY record_uuid ORDER BY id) AS i_id - FROM node -) -UPDATE node -SET i_id = ranked.i_id -FROM ranked -WHERE node.id = ranked.id; +CREATE INDEX IF NOT EXISTS node_uuid_idx + ON node (uuid); + +CREATE INDEX IF NOT EXISTS node_id_idx + ON node (id); + +CREATE INDEX IF NOT EXISTS node_record_uuid_id_idx + ON node (record_uuid, id); + +DO $$ +DECLARE + row_count INT; +BEGIN + LOOP + -- Update 50,000 rows at a time + UPDATE node + SET i_id = ranked.i_id + FROM ( + SELECT id, ROW_NUMBER() OVER (PARTITION BY record_uuid ORDER BY id) as i_id + FROM node + WHERE i_id IS NULL -- Only target rows not yet processed + LIMIT 500000 + ) AS ranked + WHERE node.id = ranked.id; + + GET DIAGNOSTICS row_count = ROW_COUNT; + EXIT WHEN row_count = 0; + + COMMIT; -- Finalize this chunk so disk space can be reclaimed + RAISE NOTICE 'Updated 500,000 rows...'; + END LOOP; +END $$; ALTER TABLE node ALTER COLUMN i_id SET NOT NULL; @@ -22,8 +44,8 @@ ALTER TABLE node UPDATE node AS child SET p_i_id = parent.i_id FROM node AS parent -WHERE child.parent_uuid = parent.uuid - AND child.record_uuid = parent.record_uuid; +WHERE child.parent_uuid IS NOT NULL + AND child.parent_uuid = parent.uuid; -- Replace meta.h UUID array with i_id array, preserving order UPDATE node AS n @@ -46,27 +68,30 @@ UPDATE activity_log AS al SET content = updated.content FROM ( SELECT al.id, - jsonb_set( + COALESCE( jsonb_set( jsonb_set( - ( - CASE - WHEN al.content ? 'meta' - THEN jsonb_set(al.content, '{meta}', (al.content->'meta') - 'h') - ELSE al.content - END - ) - 'uuid' - 'parentUuid', - '{iId}', - to_jsonb(n.i_id), + jsonb_set( + ( + CASE + WHEN al.content ? 'meta' + THEN jsonb_set(al.content, '{meta}', (al.content->'meta') - 'h') + ELSE al.content + END + ) - 'parentUuid', + '{iId}', + to_jsonb(n.i_id), + true + ), + '{pIId}', + to_jsonb(n.p_i_id), true ), - '{pIId}', - to_jsonb(n.p_i_id), + '{recordUuid}', + to_jsonb(n.record_uuid), true ), - '{recordUuid}', - to_jsonb(n.record_uuid), - true + al.content -- Fallback to original content if anything fails ) AS content FROM activity_log AS al JOIN node AS n ON n.uuid = (al.content->>'uuid')::uuid From 53b9e8d1cc3f35c8167325ef58cd9425ac06e7c8 Mon Sep 17 00:00:00 2001 From: Stefano Ricci Date: Sun, 22 Feb 2026 22:28:39 +0100 Subject: [PATCH 4/7] improved node i_id update --- ...0217085512-alter-table-node-add-iid-up.sql | 119 ++++-------------- 1 file changed, 24 insertions(+), 95 deletions(-) diff --git a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql index da6f210..876b35e 100644 --- a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql +++ b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql @@ -13,29 +13,30 @@ CREATE INDEX IF NOT EXISTS node_id_idx CREATE INDEX IF NOT EXISTS node_record_uuid_id_idx ON node (record_uuid, id); -DO $$ -DECLARE - row_count INT; -BEGIN - LOOP - -- Update 50,000 rows at a time - UPDATE node - SET i_id = ranked.i_id - FROM ( - SELECT id, ROW_NUMBER() OVER (PARTITION BY record_uuid ORDER BY id) as i_id - FROM node - WHERE i_id IS NULL -- Only target rows not yet processed - LIMIT 500000 - ) AS ranked - WHERE node.id = ranked.id; - - GET DIAGNOSTICS row_count = ROW_COUNT; - EXIT WHEN row_count = 0; - - COMMIT; -- Finalize this chunk so disk space can be reclaimed - RAISE NOTICE 'Updated 500,000 rows...'; - END LOOP; -END $$; +-- Update node i_id START + +CREATE UNLOGGED TABLE temp_node_ranking AS +SELECT + id, + ROW_NUMBER() OVER (PARTITION BY record_uuid ORDER BY id) AS new_i_id +FROM node +ORDER BY id; + +CREATE INDEX idx_temp_node_id ON temp_node_ranking (id); +ANALYZE temp_node_ranking; + +SET LOCAL session_replication_role = 'replica'; + +SET LOCAL work_mem = '512MB'; + +UPDATE node +SET i_id = t.new_i_id +FROM temp_node_ranking t +WHERE node.id = t.id; + +DROP TABLE temp_node_ranking; + +-- Update node i_id END ALTER TABLE node ALTER COLUMN i_id SET NOT NULL; @@ -63,78 +64,6 @@ SET meta = jsonb_set( ) WHERE n.meta ? 'h'; --- Update activity_log content to use i_id/p_i_id and drop meta.h -UPDATE activity_log AS al -SET content = updated.content -FROM ( - SELECT al.id, - COALESCE( - jsonb_set( - jsonb_set( - jsonb_set( - ( - CASE - WHEN al.content ? 'meta' - THEN jsonb_set(al.content, '{meta}', (al.content->'meta') - 'h') - ELSE al.content - END - ) - 'parentUuid', - '{iId}', - to_jsonb(n.i_id), - true - ), - '{pIId}', - to_jsonb(n.p_i_id), - true - ), - '{recordUuid}', - to_jsonb(n.record_uuid), - true - ), - al.content -- Fallback to original content if anything fails - ) AS content - FROM activity_log AS al - JOIN node AS n ON n.uuid = (al.content->>'uuid')::uuid - WHERE al.type IN ('nodeCreate', 'nodeDelete', 'nodeValueUpdate') -) AS updated -WHERE al.id = updated.id; - -DROP VIEW IF EXISTS activity_log_user_aggregate; -CREATE VIEW activity_log_user_aggregate AS -SELECT - DISTINCT ON ( - date_created::date, - user_uuid, - type, - content_uuid, - content_key, - content_record_uuid, - content_i_id - ) - id, - date_created, - user_uuid, - type, - (content->>'uuid')::uuid as content_uuid, - content->>'key' as content_key, - -- node related fields - (content->>'recordUuid')::uuid as content_record_uuid, - (content->>'iId')::integer as content_i_id, - content -FROM - activity_log -WHERE - NOT system -ORDER BY - date_created::date DESC, - user_uuid, - type, - content_uuid, - content_key, - content_record_uuid, - content_i_id, - date_created DESC; - -- Add a unique constraint on the combination of record_uuid and i_id ALTER TABLE node DROP CONSTRAINT IF EXISTS node_pkey; From d308b83da9d080f357f6cfd78ae0ef9e8e91f5cf Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Mon, 23 Feb 2026 14:13:03 +0100 Subject: [PATCH 5/7] fixed constraints update --- .../20260217085512-alter-table-node-add-iid-up.sql | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql index 876b35e..3ebcca9 100644 --- a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql +++ b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql @@ -64,13 +64,12 @@ SET meta = jsonb_set( ) WHERE n.meta ? 'h'; --- Add a unique constraint on the combination of record_uuid and i_id ALTER TABLE node - DROP CONSTRAINT IF EXISTS node_pkey; - --- Add a composite primary key on record_uuid and i_id -ALTER TABLE node - ADD PRIMARY KEY (record_uuid, i_id); + DROP CONSTRAINT IF EXISTS node_parent_fk, + DROP CONSTRAINT IF EXISTS node_pkey, + -- Add a composite primary key on record_uuid and i_id + ADD PRIMARY KEY (record_uuid, i_id), + ADD CONSTRAINT node_parent_fk FOREIGN KEY (record_uuid, p_i_id) REFERENCES "node" (record_uuid, i_id) ON DELETE CASCADE; -- Create an index on the combination of record_uuid and i_id for faster lookups CREATE INDEX IF NOT EXISTS node_record_uuid_i_id_idx From ac81c23d46cb9565fe7ef5c83506a15fb2a32e6b Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Tue, 24 Feb 2026 12:23:15 +0100 Subject: [PATCH 6/7] db migrations: added filter by record_uuid --- .../sqls/20260217085512-alter-table-node-add-iid-up.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql index 3ebcca9..a900e76 100644 --- a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql +++ b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql @@ -57,7 +57,9 @@ SET meta = jsonb_set( ( SELECT jsonb_agg(p.i_id ORDER BY elems.ordinality) FROM jsonb_array_elements_text(n.meta->'h') WITH ORDINALITY AS elems(uuid_text, ordinality) - JOIN node AS p ON p.uuid = elems.uuid_text::uuid + JOIN node AS p + ON p.record_uuid = n.record_uuid + AND p.uuid = elems.uuid_text::uuid ), '[]'::jsonb ) From d7b6149fcc12e1c24682e82f54e0f96bc228937f Mon Sep 17 00:00:00 2001 From: Stefano Ricci Date: Tue, 10 Mar 2026 22:45:23 +0100 Subject: [PATCH 7/7] added record validation migration --- ...0217085512-alter-table-node-add-iid-up.sql | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql index a900e76..59472e5 100644 --- a/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql +++ b/src/db/dbMigrator/migration/survey/migrations/sqls/20260217085512-alter-table-node-add-iid-up.sql @@ -66,6 +66,49 @@ SET meta = jsonb_set( ) WHERE n.meta ? 'h'; +-- Update record validation fields to replace UUID references with i_id references + +UPDATE record r +SET validation = jsonb_set( + r.validation, + '{fields}', + updated_fields.new_fields +) +FROM ( + SELECT + r_inner.id, + jsonb_object_agg( + CASE + -- Case 1: childrenCount_NODEUUID_NODEDEFUUID + WHEN fields.key LIKE 'childrenCount_%' THEN + 'childrenCount_' || COALESCE(n.i_id::text, split_part(fields.key, '_', 2)) || '_' || split_part(fields.key, '_', 3) + + -- Case 2: Plain NODEUUID + ELSE + COALESCE(n.i_id::text, fields.key) + END, + fields.value + ) as new_fields + FROM record r_inner + CROSS JOIN LATERAL jsonb_each(r_inner.validation->'fields') AS fields(key, value) + -- Join logic: if prefixed, get 2nd part; if not, get 1st part + LEFT JOIN node n ON ( + CASE + WHEN fields.key LIKE 'childrenCount_%' THEN split_part(fields.key, '_', 2) + ELSE fields.key + END + )::uuid = n.uuid + GROUP BY r_inner.id +) AS updated_fields +WHERE r.id = updated_fields.id; + +-- Create an index on the combination of record_uuid and i_id for faster lookups + +CREATE INDEX IF NOT EXISTS node_record_uuid_i_id_idx + ON node (record_uuid, i_id); + +-- Finally, update the node table constraints to reflect the new primary key and foreign key relationships + ALTER TABLE node DROP CONSTRAINT IF EXISTS node_parent_fk, DROP CONSTRAINT IF EXISTS node_pkey, @@ -73,6 +116,3 @@ ALTER TABLE node ADD PRIMARY KEY (record_uuid, i_id), ADD CONSTRAINT node_parent_fk FOREIGN KEY (record_uuid, p_i_id) REFERENCES "node" (record_uuid, i_id) ON DELETE CASCADE; --- Create an index on the combination of record_uuid and i_id for faster lookups -CREATE INDEX IF NOT EXISTS node_record_uuid_i_id_idx - ON node (record_uuid, i_id);