[WIP ]feat: update user and organization metadata

rfcs/inactive_users/user_and_organization_meta_update.md

@@ -0,0 +1,77 @@
+# User/Organization metadata update rework
+## Overview and Motivation
+When user or organization metadata needs to be updated, the Service uses the Redis pipeline javascript code.
+For each assigned meta hash always exists a single `audience`, but there is no list of `audiences` assigned to the user or company.
+To achieve easier audience tracking and a combined metadata update, I advise using a Lua based script.
+
+## Audience lists
+Audiences stored in sets formed from `USERS_AUDIENCE` or `ORGANISATION_AUDIENCE` constants and `Id`
+(eg: `{ms-users}10110110111!audiences`). Both keys contain `audience` names that are currently have assigned values.
+
+## utils/updateMetadata.js
+Almost all logic in this file removed and ported into LUA Script.
+This Function checks the consistency of the provided `opts`. If `opts.metadata` and `opts.audiences` are objects, script transforming them to an array containing these objects. Checks count of meta operations and audiences to equal each other.
+Organization meta update request `utils/setOrganizationMetadata.js` uses the same functionality, so the same changes applied to it.
+
+After commands execution result returned from the script, decoded from JSON string.
+
+## script/updateMetadata.lua
+Script repeats all logic including custom scripts support.
+
+### Script parameters:
+1. KEYS[1] Audiences key template.
+2. KEYS[2] used as metadata key template, eg: "{ms-users}{id}!metadata!{audience}".
+3. ARGV[1] Id - organization or user-id.
+4. ARGV[2] JSON encoded opts parameter opts.{script, metadata, audiences}.
+
+### Depending on metadata or script set:
+If `opt.metadata` set:
+ * Script starts iterating audiences.
+ * On each audience, creates metadata key from provided template.
+ * Iterates operations from `opt.metadata`, based on index of `opts.audiences`.
+    ```javascript
+     const opts = {
+       audiences: ['first', 'second'],
+       metadata: [{
+         // first audience commands
+       }, {
+         // second audience commands
+       }], 
+     }
+    ```
+   Commands execute in order: `audiences[0]` => `metadata[0]`,`audiences[1]` => `metadata[1]`, 
+
+If `opt.script` set:
+* Script iterates `audiences` and creates metadata keys from provided template
+ * Iterates `opt.script`:
+    * EVAL's script from `script.lua` and executes with params generated from: metadata keys(look to the previous step)
+     and passed `script.argv`.
+    * If script evaluation fails, script returns redis.error witch description.
+
+When operations/scripts processed, the script forms JSON object like
+```javascript
+const metaResponse = [
+  //forEach audience
+  {
+    '$incr': {
+      field: 'result', // result returned from HINCRBY command
+    },
+    '$remove': intCount, // count of deleted fields
+    '$set': "OK", // or cmd hset result.
+  },
+];
+
+const scriptResponse = {
+  'scriptName': [
+    // values returned from script
+  ],
+};
+```
+
+### Audience list update
+When all update operations succeeded:
+* Script get's current list of user's or organization's audiences from HSET `KEYS[1]`,
+unions them with `opts.audiences` and generates full list metadata keys.
+* Iterates over them to check whether some data exists.
+* If no data exists, the script deletes the corresponding audience from HSET `KEYS[1]`.
+

scripts/updateMetadata.lua

@@ -0,0 +1,141 @@
+local audienceKeyTemplate = KEYS[1]
+local metaDataTemplate = KEYS[2]
+local Id = ARGV[1]
+local updateOptsJson = ARGV[2]
+
+redis.replicate_commands()
+
+local updateOpts = cjson.decode(updateOptsJson)
+
+local function loadScript(code, environment)
+  if setfenv and loadstring then
+    local f = assert(loadstring(code))
+    setfenv(f,environment)
+    return f
+  else
+    return assert(load(code, nil,"t",environment))
+  end
+end
+
+local function tablesUniqueItems(...)
+  local args = {...}
+  local tableWithUniqueItems = {}
+  for _, passedTable in pairs(args) do
+    for __, keyName in pairs(passedTable) do
+      tableWithUniqueItems[keyName] = keyName
+    end
+  end
+  return tableWithUniqueItems
+end
+
+local function makeKey (template, id, audience)
+  local str = template:gsub('{id}', id, 1)
+  if audience ~= nil then
+    str = str:gsub('{audience}', audience, 1)
+  end
+  return str
+end
+
+--
+-- available ops definition
+--
+local function opSet(metaKey, args)
+  local setArgs = {}
+  local result = {}
+
+  for field, value in pairs(args) do
+    table.insert(setArgs, field)
+    table.insert(setArgs, value)
+  end
+
+  local callResult = redis.call("HMSET", metaKey, unpack(setArgs))
+  result[1] = callResult.ok
+  return result
+end
+
+local function opRemove(metaKey, args)
+  local result = 0;
+  for i, field in pairs(args) do
+    result = result + redis.call("HDEL", metaKey, field)
+  end
+  return result
+end
+
+local function opIncr(metaKey, args)
+  local result = {}
+  for field, incrVal in pairs(args) do
+    result[field] = redis.call("HINCRBY", metaKey, field, incrVal)
+  end
+  return result
+end
+
+-- operations index
+local metaOps = {
+  ['$set'] = opSet,
+  ['$remove'] = opRemove,
+  ['$incr'] = opIncr
+}
+
+--
+-- Script body
+--
+local scriptResult = {}
+
+local keysToProcess = {};
+for i, audience in ipairs(updateOpts.audiences) do
+  local key = makeKey(metaDataTemplate, Id, audience)
+  table.insert(keysToProcess, i, key);
+end
+
+if updateOpts.metaOps then
+  for i, op in ipairs(updateOpts.metaOps) do
+    local targetOpKey = keysToProcess[i]
+    local metaProcessResult = {};
+
+    for opName, opArg in pairs(op) do
+      local processFn = metaOps[opName];
+
+      if processFn == nil then
+        return redis.error_reply("Unsupported command:" .. opName)
+      end
+      if type(opArg) ~= "table" then
+        return redis.error_reply("Args for ".. opName .." must be and array")
+      end
+
+      metaProcessResult[opName] = processFn(targetOpKey, opArg)
+    end
+    table.insert(scriptResult, metaProcessResult)
+  end
+
+elseif updateOpts.scripts then
+  local env = {};
+  -- allow read access to this script scope
+  setmetatable(env,{__index=_G})
+
+  for i, script in pairs(updateOpts.scripts) do
+    env.ARGV = script.argv
+    env.KEYS = keysToProcess
+    local fn = loadScript(script.lua, env)
+    scriptResult[script.name] = fn()
+  end
+
+end
+
+local audienceKey = makeKey(audienceKeyTemplate, Id)
+local audiences = redis.call("SMEMBERS", audienceKey)
+local processedAudiences = updateOpts.audiences
+local uniqueAudiences = tablesUniqueItems(audiences, processedAudiences)
+
+for _, audience in pairs(uniqueAudiences) do
+  local metaKey = makeKey(metaDataTemplate, Id, audience)
+  local dataLen = redis.call("HLEN", metaKey)
+
+  if (dataLen > 0) then
+    redis.call("SADD", audienceKey, audience)
+  else
+    redis.call("SREM", audienceKey, audience)
+  end
+end
+
+
+return cjson.encode(scriptResult)

src/constants.js

@@ -18,6 +18,7 @@ module.exports = exports = {
   // hashes
   USERS_DATA: 'data',
   USERS_METADATA: 'metadata',
+  USERS_AUDIENCE: 'users-audiences',
   USERS_TOKENS: 'tokens',
   USERS_API_TOKENS: 'api-tokens',
   USERS_API_TOKENS_ZSET: 'api-tokens-set',
@@ -26,6 +27,7 @@ module.exports = exports = {
   USERS_ORGANIZATIONS: 'user-organizations',
   ORGANIZATIONS_DATA: 'data',
   ORGANIZATIONS_METADATA: 'metadata',
+  ORGANIZATIONS_AUDIENCE: 'organization-audiences',
   ORGANIZATIONS_MEMBERS: 'members',
 
   // standard JWT with TTL

src/utils/setOrganizationMetadata.js

@@ -3,9 +3,18 @@ const Promise = require('bluebird');
 const is = require('is');
 const { HttpStatusError } = require('common-errors');
 const redisKey = require('../utils/key.js');
-const handlePipeline = require('../utils/pipelineError.js');
-const { handleAudience } = require('../utils/updateMetadata.js');
-const { ORGANIZATIONS_METADATA } = require('../constants.js');
+const { prepareOps } = require('./updateMetadata');
+const { ORGANIZATIONS_METADATA, ORGANIZATIONS_AUDIENCE } = require('../constants.js');
+
+const JSONStringify = (data) => JSON.stringify(data);
+
+function callUpdateMetadataScript(redis, id, ops) {
+  const audienceKeyTemplate = redisKey('{id}', ORGANIZATIONS_AUDIENCE);
+  const metaDataTemplate = redisKey('{id}', ORGANIZATIONS_METADATA, '{audience}');
+
+  return redis
+    .updateMetadata(2, audienceKeyTemplate, metaDataTemplate, id, JSONStringify(ops));
+}
 
 /**
  * Updates metadata on a organization object
@@ -19,20 +28,17 @@ async function setOrganizationMetadata(opts) {
   } = opts;
   const audiences = is.array(audience) ? audience : [audience];
 
-  // keys
-  const keys = audiences.map((aud) => redisKey(organizationId, ORGANIZATIONS_METADATA, aud));
-
   // if we have meta, then we can
   if (metadata) {
-    const pipe = redis.pipeline();
-    const metaOps = is.array(metadata) ? metadata : [metadata];
-
-    if (metaOps.length !== audiences.length) {
+    const rawMetaOps = is.array(metadata) ? metadata : [metadata];
+    if (rawMetaOps.length !== audiences.length) {
       return Promise.reject(new HttpStatusError(400, 'audiences must match metadata entries'));
     }
 
-    metaOps.forEach((meta, idx) => handleAudience(pipe, keys[idx], meta));
-    return pipe.exec().then(handlePipeline);
+    const metaOps = rawMetaOps.map((opBlock) => prepareOps(opBlock));
+
+    const scriptOpts = { metaOps, audiences };
+    return callUpdateMetadataScript(redis, organizationId, scriptOpts);
   }
 
   return true;