Users grouping improvements

package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-foris-arena",
-  "version": "2.5.9",
+  "version": "2.5.10",
   "license": "MIT",
   "engines": {
     "node": "^24.11.1"
@@ -237,4 +237,4 @@
     "{common,core,server,test,webapp}/**/*.{js,jsx,ts,tsx}": "eslint --cache --fix"
   },
   "packageManager": "yarn@4.17.1"
-}
+}

server/modules/dataExport/api/dataExportApi.js

@@ -7,6 +7,7 @@ import * as Request from '@server/utils/request'
 
 import * as AuthMiddleware from '@server/modules/auth/authApiMiddleware'
 import { sendTempFileToResponse } from '@server/modules/fileDownload/api/fileDownloadApi'
+import { requireRecordsMatchUserGroupQualifiers } from '@server/modules/record/api/recordQualifierMiddleware'
 import * as SurveyService from '@server/modules/survey/service/surveyService'
 
 import * as DataExportService from '../service/dataExportService'
@@ -18,25 +19,30 @@ const checkExportUuid = (exportUuid) => {
 }
 
 export const init = (app) => {
-  app.post('/survey/:surveyId/data-export', AuthMiddleware.requireRecordsExportPermission, async (req, res, next) => {
-    try {
-      const { surveyId, cycle, recordUuids, search, options } = Request.getParams(req)
-
-      const user = Request.getUser(req)
-
-      const job = DataExportService.startCsvDataExportJob({
-        user,
-        surveyId,
-        cycle,
-        recordUuids,
-        search,
-        options,
-      })
-      res.json({ job: JobUtils.jobToJSON(job) })
-    } catch (error) {
-      next(error)
+  app.post(
+    '/survey/:surveyId/data-export',
+    AuthMiddleware.requireRecordsExportPermission,
+    requireRecordsMatchUserGroupQualifiers,
+    async (req, res, next) => {
+      try {
+        const { surveyId, cycle, recordUuids, search, options } = Request.getParams(req)
+
+        const user = Request.getUser(req)
+
+        const job = DataExportService.startCsvDataExportJob({
+          user,
+          surveyId,
+          cycle,
+          recordUuids,
+          search,
+          options,
+        })
+        res.json({ job: JobUtils.jobToJSON(job) })
+      } catch (error) {
+        next(error)
+      }
     }
-  })
+  )
 
   app.post(
     '/survey/:surveyId/data-summary-export',

server/modules/record/api/recordApi.js

@@ -239,8 +239,9 @@ export const init = (app) => {
   app.get('/survey/:surveyId/records/summary/export', requireRecordListExportPermission, async (req, res, next) => {
     try {
       const { surveyId, cycle, fileFormat = FileFormats.xlsx } = Request.getParams(req)
+      const user = Request.getUser(req)
 
-      await RecordService.exportRecordsSummary({ res, surveyId, cycle, fileFormat })
+      await RecordService.exportRecordsSummary({ res, surveyId, cycle, fileFormat, user })
     } catch (error) {
       next(error)
     }

server/modules/record/api/recordQualifierMiddleware.js

@@ -38,14 +38,13 @@ const _recordMatchesUserGroupQualifiers = async ({ user, surveyId, recordUuid, p
 
   const survey = await fetchSurveyByCycle(Record.getCycle(record))
 
-  const qualifierFilters = await RecordManager.fetchUserQualifierFilters({ user, survey })
+  const qualifierFilters = await SurveyManager.fetchUserQualifierFilters({ user, survey })
 
-  // apply the node value carried by the request (if any) before checking, so an edit that sets
-  // a qualifier attribute to a value outside the user's group qualifiers is rejected too, not just
-  // records that already mismatch before the edit is applied
-  const recordWithPendingNode = pendingNode ? Record.assocNode(pendingNode)(record) : record
-
-  return RecordManager.recordMatchesQualifierFilters({ survey, record: recordWithPendingNode, qualifierFilters })
+  // pendingNode is passed through rather than merged into the record via Record.assocNode: the record
+  // was fetched with fetchForUpdate: false, so its _nodesIndex hasn't been built, and assocNode would
+  // initialize one containing only pendingNode, shadowing every other already-persisted node (see
+  // recordMatchesQualifierFilters' jsdoc for details)
+  return RecordManager.recordMatchesQualifierFilters({ survey, record, qualifierFilters, pendingNode })
 }
 
 /**

server/modules/record/manager/_recordManager/recordQualifierMatcher.js

@@ -1,96 +1,57 @@
-import { Objects, ServiceRegistry } from '@openforis/arena-core'
-import { ServerServiceType } from '@openforis/arena-server'
+import { Objects } from '@openforis/arena-core'
 
-import * as User from '@core/user/user'
-import * as UserGroup from '@core/user/userGroup/userGroup'
-import * as UserGroupQualifier from '@core/user/userGroup/userGroupQualifier'
-import * as Survey from '@core/survey/survey'
 import * as NodeDef from '@core/survey/nodeDef'
-import * as CategoryItem from '@core/survey/categoryItem'
 import * as Record from '@core/record/record'
 import * as Node from '@core/record/node'
 import { NodeValues } from '@core/record/nodeValues'
 
-import { db } from '@server/db/db'
-import { CategoryItemProviderDefault } from '@server/modules/category/manager/categoryItemProviderDefault'
-
-/**
- * Determines the qualifier attribute/value pairs that the given user's group restricts them to, if any.
- * Mirrors the lookup used to auto-fill qualifier attributes when a new record is created
- * (see _applyGroupQualifierValues in recordUpdateManager.js), so write-time and read-time
- * restrictions stay consistent: only the user's first UserGroup in the survey is considered.
- * For code attributes, the qualifier's code is resolved to its category item via a direct DB lookup
- * (works regardless of category size, unlike the survey's category items ref data index, which
- * excludes "big" categories), so the returned value can be compared against a record's node value by
- * item uuid rather than relying on the ref data index being loaded.
- * @param {object} params - The function parameters.
- * @param {object} params.user - The current user.
- * @param {object} params.survey - The survey, with node defs loaded.
- * @param {pgPromise.IDatabase} [client] - The db client.
- * @returns {Promise<Array<{nodeDef: object, value: *}>>} - An empty array when the user is
- * unrestricted (no group, or group with no matching qualifiers), otherwise the list of qualifier
- * filters to apply.
- */
-export const fetchUserQualifierFilters = async ({ user, survey }, client = db) => {
-  const qualifierNodeDefs = Survey.getQualifierNodeDefs(survey)
-  if (qualifierNodeDefs.length === 0) return []
-
-  const userGroupService = ServiceRegistry.getInstance().getService(ServerServiceType.userGroup)
-  const userGroups = await userGroupService.getManyByUser(
-    { userUuid: User.getUuid(user), surveyUuid: Survey.getUuid(survey) },
-    client
-  )
-  const userGroup = userGroups[0]
-  if (!userGroup) return []
-
-  const qualifiers = UserGroup.getQualifiers(userGroup)
-  if (qualifiers.length === 0) return []
-
-  const filters = []
-  for (const nodeDef of qualifierNodeDefs) {
-    const qualifier = qualifiers.find((q) => UserGroupQualifier.getName(q) === NodeDef.getName(nodeDef))
-    const qualifierValue = qualifier ? UserGroupQualifier.getValue(qualifier) : null
-    if (Objects.isEmpty(qualifierValue)) continue
-
-    if (NodeDef.isCode(nodeDef)) {
-      const categoryUuid = NodeDef.getCategoryUuid(nodeDef)
-      const item = await CategoryItemProviderDefault.getItemByCode(
-        { survey, categoryUuid, code: qualifierValue },
-        client
-      )
-      if (!item) continue
-      filters.push({
-        nodeDef,
-        value: Node.newNodeValueCode({ itemUuid: CategoryItem.getUuid(item), code: qualifierValue }),
-      })
-    } else {
-      filters.push({ nodeDef, value: qualifierValue })
-    }
-  }
-  return filters
-}
-
 /**
  * Checks whether a record's qualifier attribute values match the given qualifier filters.
  * @param {object} params - The function parameters.
  * @param {object} params.survey - The survey, with node defs and categories loaded.
  * @param {object} params.record - The record (with nodes loaded) to check.
- * @param {Array<{nodeDef: object, value: string}>} params.qualifierFilters - The qualifier filters,
- * as returned by fetchUserQualifierFilters.
+ * @param {Array<{nodeDef: object, value: *}>} params.qualifierFilters - The qualifier filters,
+ * as returned by SurveyManager.fetchUserQualifierFilters.
+ * @param {object} [params.pendingNode] - A node carried by the current request but not yet persisted
+ * (e.g. a create/update of some attribute), applied on top of the record's own nodes when checking, so
+ * an edit that sets a qualifier attribute to a value outside the user's group qualifiers is rejected
+ * too. Deliberately not merged into `record` itself via `Record.assocNode`: doing so on a record whose
+ * `_nodesIndex` hasn't been built (see `fetchForUpdate` in `fetchRecordAndNodesByUuid`) makes
+ * `assocNode` initialize a new index containing only this one node, which then shadows the full node
+ * list for every other lookup (`Record.getNodeChildrenByDefUuid` and friends trust a present index
+ * instead of falling back to a full scan), making every other already-persisted node - including the
+ * qualifier attribute - invisible.
  * @returns {boolean} - True if qualifierFilters is empty (unrestricted), if the record has not been
- * initialized yet (no nodes: checkin will auto-fill qualifier attributes to match, see
- * _applyGroupQualifierValues in recordUpdateManager.js), or every filter matches the record's
- * corresponding attribute value; false otherwise.
+ * initialized yet (no nodes), if a qualifier attribute node is missing or not yet given a value
+ * (record creation will auto-fill qualifier attributes to match, see _applyGroupQualifierValues in
+ * recordUpdateManager.js, but that happens as one of the last steps of record creation, so a
+ * just-created record may briefly have the node without its value yet), or every filter matches the
+ * record's corresponding attribute value; false when a qualifier attribute has an actual value that
+ * differs from the expected one, or when `pendingNode` explicitly targets a qualifier attribute with
+ * an empty value: the UI never lets a user edit an already-applied qualifier node (see
+ * `isQualifierValueApplied` in nodeDefSwitch.js), so a request clearing it can only be a deliberate
+ * attempt to drop the restriction, not an in-progress record still being initialized.
  */
-export const recordMatchesQualifierFilters = ({ survey, record, qualifierFilters }) => {
+export const recordMatchesQualifierFilters = ({ survey, record, qualifierFilters, pendingNode = null }) => {
   if (qualifierFilters.length === 0) return true
   if (Record.getNodesArray(record).length === 0) return true
 
   const rootNode = Record.getRootNode(record)
 
   return qualifierFilters.every(({ nodeDef, value }) => {
-    const node = Record.getNodeChildrenByDefUuid(rootNode, NodeDef.getUuid(nodeDef))(record)[0]
-    if (!node) return false
+    const nodeDefUuid = NodeDef.getUuid(nodeDef)
+    const isPendingNode = pendingNode && Node.getNodeDefUuid(pendingNode) === nodeDefUuid
+    const node = isPendingNode ? pendingNode : Record.getNodeChildByDefUuid(rootNode, nodeDefUuid)(record)
+
+    if (!node) return true
+
+    if (Objects.isEmpty(Node.getValue(node))) {
+      // a persisted node without a value yet: record is still being initialized (the qualifier
+      // auto-fill is one of the last steps of record creation), not (yet) evidence of a mismatch;
+      // but pendingNode explicitly clearing the value is a deliberate attempt to bypass the
+      // restriction, so it must be rejected rather than given the same benefit of the doubt
+      return !isPendingNode
+    }
     return NodeValues.isValueEqual({ survey, nodeDef, value: Node.getValue(node), valueSearch: value })
   })
 }

server/modules/record/manager/_recordManager/recordUpdateManager.js

@@ -20,8 +20,8 @@ import * as RecordFileManager from '@server/modules/record/manager/recordFileMan
 import * as NodeDefRepository from '@server/modules/nodeDef/repository/nodeDefRepository'
 import * as DataTableUpdateRepository from '@server/modules/surveyRdb/repository/dataTableUpdateRepository'
 import * as DataTableReadRepository from '@server/modules/surveyRdb/repository/dataTableReadRepository'
+import * as SurveyManager from '@server/modules/survey/manager/surveyManager'
 
-import * as RecordQualifierMatcher from './recordQualifierMatcher'
 import * as RecordValidationManager from './recordValidationManager'
 import * as NodeCreationManager from './nodeCreationManager'
 import * as NodeUpdateManager from './nodeUpdateManager'
@@ -79,7 +79,7 @@ const _applyGroupQualifierValues = async (
   { user, survey, record, timezoneOffset, nodesUpdateListener, nodesValidationListener },
   client
 ) => {
-  const qualifierFilters = await RecordQualifierMatcher.fetchUserQualifierFilters({ user, survey }, client)
+  const qualifierFilters = await SurveyManager.fetchUserQualifierFilters({ user, survey }, client)
   if (qualifierFilters.length === 0) return record
 
   const rootNode = Record.getRootNode(record)

server/modules/record/manager/recordManager.js

@@ -27,7 +27,6 @@ import * as UserManager from '@server/modules/user/manager/userManager'
 import * as RecordRepository from '../repository/recordRepository'
 import * as FileRepository from '../repository/fileRepository'
 import * as NodeRepository from '../repository/nodeRepository'
-import * as RecordQualifierMatcher from './_recordManager/recordQualifierMatcher'
 import * as RecordUpdateManager from './_recordManager/recordUpdateManager'
 import { NodeRdbManager } from './_recordManager/nodeRDBManager'
 
@@ -78,7 +77,7 @@ export const fetchRecordsSummaryBySurveyId = async (
     summaryDefs = Survey.getRootSummaryDefs({ cycle })(survey)
     nodeDefKeys = Survey.getNodeDefRootKeysSorted({ cycle })(survey)
     if (user) {
-      qualifierNodeDefFilters = await RecordQualifierMatcher.fetchUserQualifierFilters({ user, survey }, client)
+      qualifierNodeDefFilters = await SurveyManager.fetchUserQualifierFilters({ user, survey }, client)
     }
   }
 
@@ -160,9 +159,7 @@ export const countRecordsBySurveyId = async (
   const survey = await SurveyManager.fetchSurveyAndNodeDefsBySurveyId({ surveyId, cycle, draft: nodeDefsDraft }, client)
   const nodeDefKeys = Survey.getNodeDefRootKeys(survey)
   const summaryDefs = Survey.getRootSummaryDefs({ cycle })(survey)
-  const qualifierNodeDefFilters = user
-    ? await RecordQualifierMatcher.fetchUserQualifierFilters({ user, survey }, client)
-    : []
+  const qualifierNodeDefFilters = user ? await SurveyManager.fetchUserQualifierFilters({ user, survey }, client) : []
 
   return RecordRepository.countRecordsBySurveyId(
     { surveyId, cycle, search, nodeDefKeys, summaryDefs, nodeDefRoot, ownerUuid, qualifierNodeDefFilters },
@@ -262,7 +259,7 @@ export const fetchRecordAndNodesByUuid = async (
 
 export { fetchNodeByUuid, fetchChildNodesByNodeDefUuids } from '../repository/nodeRepository'
 
-export { fetchUserQualifierFilters, recordMatchesQualifierFilters } from './_recordManager/recordQualifierMatcher'
+export { recordMatchesQualifierFilters } from './_recordManager/recordQualifierMatcher'
 
 const fetchNodeRefData = async ({ survey, node, isCode }, client) => {
   const surveyId = Survey.getId(survey)

server/modules/record/repository/recordRepository.js

@@ -341,7 +341,8 @@ export const fetchRecordsSummaryBySurveyId = async (
     const paramName = `qualifierValue${index}`
     const colName = NodeDefTable.getColumnName(nodeDef)
     recordsSelectWhereConditions.push(`"${colName}" = $/${paramName}/`)
-    qualifierFilterParams[paramName] = value
+    const columnValue = NodeDef.isCode(nodeDef) ? value?.code : value
+    qualifierFilterParams[paramName] = columnValue
   })
 
   const whereConditionsJoint = recordsSelectWhereConditions.map((condition) => `(${condition})`).join(' AND ')

server/modules/record/service/recordService.js

@@ -85,11 +85,12 @@ export const {
   updateRecordOwner,
 } = RecordManager
 
-export const exportRecordsSummary = async ({ res, surveyId, cycle, fileFormat }) => {
+export const exportRecordsSummary = async ({ res, surveyId, cycle, fileFormat, user }) => {
   const { list, nodeDefKeys } = await RecordManager.fetchRecordsSummaryBySurveyId({
     surveyId,
     cycle,
     includeCounts: true,
+    user,
   })
 
   const valueFormattersByType = {

server/modules/survey/manager/surveyManager.js

@@ -599,3 +599,5 @@ export const deleteAllActivityLog = async ({ surveyId }, client = db) =>
   ActivityLogRepository.deleteAll({ surveyId }, client)
 
 export const { dropSurveySchema } = SurveyRepository
+
+export { fetchUserQualifierFilters } from './surveyUserGroupQualifierFilters'

server/modules/survey/manager/surveyUserGroupQualifierFilters.js

@@ -0,0 +1,69 @@
+import { Objects, ServiceRegistry } from '@openforis/arena-core'
+import { ServerServiceType } from '@openforis/arena-server'
+
+import * as User from '@core/user/user'
+import * as UserGroup from '@core/user/userGroup/userGroup'
+import * as UserGroupQualifier from '@core/user/userGroup/userGroupQualifier'
+import * as Survey from '@core/survey/survey'
+import * as NodeDef from '@core/survey/nodeDef'
+import * as CategoryItem from '@core/survey/categoryItem'
+import * as Node from '@core/record/node'
+
+import { db } from '@server/db/db'
+import { CategoryItemProviderDefault } from '@server/modules/category/manager/categoryItemProviderDefault'
+
+/**
+ * Determines the qualifier attribute/value pairs that the given user's group restricts them to, if any.
+ * Mirrors the lookup used to auto-fill qualifier attributes when a new record is created
+ * (see _applyGroupQualifierValues in recordUpdateManager.js), so write-time and read-time
+ * restrictions stay consistent: only the user's first UserGroup in the survey is considered.
+ * For code attributes, the qualifier's code is resolved to its category item via a direct DB lookup
+ * (works regardless of category size, unlike the survey's category items ref data index, which
+ * excludes "big" categories), so the returned value can be compared against a record's node value by
+ * item uuid rather than relying on the ref data index being loaded.
+ * @param {object} params - The function parameters.
+ * @param {object} params.user - The current user.
+ * @param {object} params.survey - The survey, with node defs loaded.
+ * @param {pgPromise.IDatabase} [client] - The db client.
+ * @returns {Promise<Array<{nodeDef: object, value: *}>>} - An empty array when the user is
+ * unrestricted (no group, or group with no matching qualifiers), otherwise the list of qualifier
+ * filters to apply.
+ */
+export const fetchUserQualifierFilters = async ({ user, survey }, client = db) => {
+  const qualifierNodeDefs = Survey.getQualifierNodeDefs(survey)
+  if (qualifierNodeDefs.length === 0) return []
+
+  const userGroupService = ServiceRegistry.getInstance().getService(ServerServiceType.userGroup)
+  const userGroups = await userGroupService.getManyByUser(
+    { userUuid: User.getUuid(user), surveyUuid: Survey.getUuid(survey) },
+    client
+  )
+  const userGroup = userGroups[0]
+  if (!userGroup) return []
+
+  const qualifiers = UserGroup.getQualifiers(userGroup)
+  if (qualifiers.length === 0) return []
+
+  const filters = []
+  for (const nodeDef of qualifierNodeDefs) {
+    const qualifier = qualifiers.find((q) => UserGroupQualifier.getName(q) === NodeDef.getName(nodeDef))
+    const qualifierValue = qualifier ? UserGroupQualifier.getValue(qualifier) : null
+    if (Objects.isEmpty(qualifierValue)) continue
+
+    if (NodeDef.isCode(nodeDef)) {
+      const categoryUuid = NodeDef.getCategoryUuid(nodeDef)
+      const item = await CategoryItemProviderDefault.getItemByCode(
+        { survey, categoryUuid, code: qualifierValue },
+        client
+      )
+      if (!item) continue
+      filters.push({
+        nodeDef,
+        value: Node.newNodeValueCode({ itemUuid: CategoryItem.getUuid(item), code: qualifierValue }),
+      })
+    } else {
+      filters.push({ nodeDef, value: qualifierValue })
+    }
+  }
+  return filters
+}