feat: New style RBAC framework (#8766)

This commit introduces a new style Role Based Access Control framework
for cubes. User can now define `accessPolicies` on Cubes and Views which
will be evaluated into `queryRewrite` and visibility rules.

This commit introduces a new config: `contextToRoles(context):
string[]`. It should return a list of user role names based on the
request context. Access Policies are defined per Cube x Role name like

```
access_policy:
  - role: "manager"
    conditions:
      - if "{ security_context.isNotSuspended }"
    row_level:
      filters:
        - member: `access_level`
          operator: lt
          values: [2]
    member_level:
      includes: "*"
      excludes: [`top_secret_field`]
```

Each policy can define a `row_level` and `member_level` rules.
Row level rules can be defined as a list of filters or `allow_all: true`
Member level rules should specify a list of "included" members that the
user with a given role is allowed to see.

When evaluating Cube and View level policies:
- row level filters are joined via AND (least permissive policy wins)
- member level policy at the view always wins (you can expose a hidden
  member of a Cube on a View)

Policies can reference `security_context` (lowercase) when evaluating
policy conditions and filter values.

Author: bsod90

.gitignore

@@ -23,4 +23,4 @@ testings/
 rust/cubesql/profile.json
 .cubestore
 .env
-
+.vimspector.json

packages/cubejs-api-gateway/src/gateway.ts

@@ -234,7 +234,7 @@ class ApiGateway {
       const { query, variables } = req.body;
       const compilerApi = await this.getCompilerApi(req.context);
 
-      const metaConfig = await compilerApi.metaConfig({
+      const metaConfig = await compilerApi.metaConfig(req.context, {
         requestId: req.context.requestId,
       });
 
@@ -267,7 +267,7 @@ class ApiGateway {
         const compilerApi = await this.getCompilerApi(req.context);
         let schema = compilerApi.getGraphQLSchema();
         if (!schema) {
-          let metaConfig = await compilerApi.metaConfig({
+          let metaConfig = await compilerApi.metaConfig(req.context, {
             requestId: req.context.requestId,
           });
           metaConfig = this.filterVisibleItemsInMeta(req.context, metaConfig);
@@ -551,7 +551,7 @@ class ApiGateway {
     try {
       await this.assertApiScope('meta', context.securityContext);
       const compilerApi = await this.getCompilerApi(context);
-      const metaConfig = await compilerApi.metaConfig({
+      const metaConfig = await compilerApi.metaConfig(context, {
         requestId: context.requestId,
         includeCompilerId: includeCompilerId || onlyCompilerId
       });
@@ -587,7 +587,7 @@ class ApiGateway {
     try {
       await this.assertApiScope('meta', context.securityContext);
       const compilerApi = await this.getCompilerApi(context);
-      const metaConfigExtended = await compilerApi.metaConfigExtended({
+      const metaConfigExtended = await compilerApi.metaConfigExtended(context, {
         requestId: context.requestId,
       });
       const { metaConfig, cubeDefinitions } = metaConfigExtended;
@@ -1010,7 +1010,7 @@ class ApiGateway {
           } else {
             const metaCacheKey = JSON.stringify(ctx);
             if (!metaCache.has(metaCacheKey)) {
-              metaCache.set(metaCacheKey, await compiler.metaConfigExtended(ctx));
+              metaCache.set(metaCacheKey, await compiler.metaConfigExtended(context, ctx));
             }
 
             // checking and fetching result status
@@ -1180,6 +1180,7 @@ class ApiGateway {
     }, context);
 
     const startTime = new Date().getTime();
+    const compilerApi = await this.getCompilerApi(context);
 
     let normalizedQueries: NormalizedQuery[] = await Promise.all(
       queries.map(
@@ -1195,8 +1196,14 @@ class ApiGateway {
           }
 
           const normalizedQuery = normalizeQuery(currentQuery, persistent);
-          let rewrittenQuery = await this.queryRewrite(
+          // First apply cube/view level security policies
+          const queryWithRlsFilters = await compilerApi.applyRowLevelSecurity(
             normalizedQuery,
+            context
+          );
+          // Then apply user-supplied queryRewrite
+          let rewrittenQuery = await this.queryRewrite(
+            queryWithRlsFilters,
             context,
           );
 
@@ -1693,7 +1700,7 @@ class ApiGateway {
         await this.getNormalizedQueries(query, context);
 
       let metaConfigResult = await (await this
-        .getCompilerApi(context)).metaConfig({
+        .getCompilerApi(context)).metaConfig(request.context, {
         requestId: context.requestId
       });
 
@@ -1803,7 +1810,7 @@ class ApiGateway {
         await this.getNormalizedQueries(query, context, request.streaming, request.memberExpressions);
 
       const compilerApi = await this.getCompilerApi(context);
-      let metaConfigResult = await compilerApi.metaConfig({
+      let metaConfigResult = await compilerApi.metaConfig(request.context, {
         requestId: context.requestId
       });
 

packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts

@@ -48,7 +48,8 @@ const annotation = (
 ) => (member: string | MemberExpression): undefined | [string, ConfigItem] => {
   const [cubeName, fieldName] = (<MemberExpression>member).expression ? [(<MemberExpression>member).cubeName, (<MemberExpression>member).name] : (<string>member).split('.');
   const memberWithoutGranularity = [cubeName, fieldName].join('.');
-  const config: ConfigItem = configMap[cubeName][memberType]
+  const cubeConfig = configMap[cubeName];
+  const config: ConfigItem = cubeConfig && cubeConfig[memberType]
     .find(m => m.name === memberWithoutGranularity);
 
   if (!config) {

packages/cubejs-api-gateway/test/index.test.ts

@@ -471,7 +471,7 @@ describe('API Gateway', () => {
         queryRewrite: async (query, _context) => {
           query.limit = 2;
           return query;
-        }
+        },
       }
     );
 

packages/cubejs-api-gateway/test/mocks.ts

@@ -75,6 +75,10 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({
     return 'postgres';
   },
 
+  async applyRowLevelSecurity(query: any) {
+    return query;
+  },
+
   async metaConfig() {
     return [
       {

packages/cubejs-schema-compiler/src/compiler/CompilerCache.ts

@@ -4,6 +4,8 @@ import { QueryCache } from '../adapter/QueryCache';
 export class CompilerCache extends QueryCache {
   protected readonly queryCache: LRUCache<string, QueryCache>;
 
+  protected readonly rbacCache: LRUCache<string, any>;
+
   public constructor({ maxQueryCacheSize, maxQueryCacheAge }) {
     super();
 
@@ -12,6 +14,15 @@ export class CompilerCache extends QueryCache {
       maxAge: (maxQueryCacheAge * 1000) || 1000 * 60 * 10,
       updateAgeOnGet: true
     });
+
+    this.rbacCache = new LRUCache({
+      max: 10000,
+      maxAge: 1000 * 60 * 5, // 5 minutes
+    });
+  }
+
+  public getRbacCacheInstance(): LRUCache<string, any> {
+    return this.rbacCache;
   }
 
   public getQueryCache(key: unknown): QueryCache {

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -65,6 +65,8 @@ export class CubeEvaluator extends CubeSymbols {
 
   public byFileName: Record<string, any> = {};
 
+  private isRbacEnabledCache: boolean | null = null;
+
   public constructor(
     protected readonly cubeValidator: CubeValidator
   ) {
@@ -112,9 +114,71 @@ export class CubeEvaluator extends CubeSymbols {
 
     this.prepareHierarchies(cube);
 
+    this.prepareAccessPolicy(cube, errorReporter);
+
     return cube;
   }
 
+  private allMembersOrList(cube: any, specifier: string | string[]): string[] {
+    const types = ['measures', 'dimensions', 'segments'];
+    if (specifier === '*') {
+      const allMembers = R.unnest(types.map(type => Object.keys(cube[type] || {})));
+      return allMembers;
+    } else {
+      return specifier as string[] || [];
+    }
+  }
+
+  private prepareAccessPolicy(cube: any, errorReporter: ErrorReporter) {
+    if (!cube.accessPolicy) {
+      return;
+    }
+
+    const memberMapper = (memberType: string) => (member: string) => {
+      if (member.indexOf('.') !== -1) {
+        const cubeName = member.split('.')[0];
+        if (cubeName !== cube.name) {
+          errorReporter.error(
+            `Paths aren't allowed in the accessPolicy policy but '${member}' provided as ${memberType} for ${cube.name}`
+          );
+        }
+        return member;
+      }
+      return this.pathFromArray([cube.name, member]);
+    };
+
+    const filterEvaluator = (filter: any) => {
+      if (filter.member) {
+        filter.memberReference = this.evaluateReferences(cube.name, filter.member);
+        filter.memberReference = memberMapper('a filter member reference')(filter.memberReference);
+      } else {
+        if (filter.and) {
+          filter.and.forEach(filterEvaluator);
+        }
+        if (filter.or) {
+          filter.or.forEach(filterEvaluator);
+        }
+      }
+    };
+
+    for (const policy of cube.accessPolicy) {
+      for (const filter of policy?.rowLevel?.filters || []) {
+        filterEvaluator(filter);
+      }
+
+      if (policy.memberLevel) {
+        policy.memberLevel.includesMembers = this.allMembersOrList(
+          cube,
+          policy.memberLevel.includes || '*'
+        ).map(memberMapper('an includes member'));
+        policy.memberLevel.excludesMembers = this.allMembersOrList(
+          cube,
+          policy.memberLevel.excludes || []
+        ).map(memberMapper('an excludes member'));
+      }
+    }
+  }
+
   private prepareHierarchies(cube: any) {
     if (Array.isArray(cube.hierarchies)) {
       cube.hierarchies = cube.hierarchies.map(hierarchy => ({
@@ -515,6 +579,19 @@ export class CubeEvaluator extends CubeSymbols {
     return path.split('.');
   }
 
+  public isRbacEnabledForCube(cube: any): boolean {
+    return cube.accessPolicy && cube.accessPolicy.length;
+  }
+
+  public isRbacEnabled(): boolean {
+    if (this.isRbacEnabledCache === null) {
+      this.isRbacEnabledCache = this.cubeNames().some(
+        cubeName => this.isRbacEnabledForCube(this.cubeFromPath(cubeName))
+      );
+    }
+    return this.isRbacEnabledCache;
+  }
+
   protected parsePathAnyType(path) {
     // Should throw UserError in case of parse error
     this.byPathAnyType(path);

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js

@@ -10,6 +10,10 @@ import { BaseQuery } from '../adapter';
 const FunctionRegex = /function\s+\w+\(([A-Za-z0-9_,]*)|\(([\s\S]*?)\)\s*=>|\(?(\w+)\)?\s*=>/;
 const CONTEXT_SYMBOLS = {
   SECURITY_CONTEXT: 'securityContext',
+  // SECURITY_CONTEXT has been deprecated, however security_context (lowecase)
+  // is allowed in RBAC policies for query-time attribute matching
+  security_context: 'securityContext',
+  securityContext: 'securityContext',
   FILTER_PARAMS: 'filterParams',
   FILTER_GROUP: 'filterGroup',
   SQL_UTILS: 'sqlUtils'
@@ -139,6 +143,7 @@ export class CubeSymbols {
     this.camelCaseTypes(cube.dimensions);
     this.camelCaseTypes(cube.segments);
     this.camelCaseTypes(cube.preAggregations);
+    this.camelCaseTypes(cube.accessPolicy);
 
     if (cube.preAggregations) {
       this.transformPreAggregations(cube.preAggregations);
@@ -406,6 +411,34 @@ export class CubeSymbols {
     });
   }
 
+  /**
+   * This method is mainly used for evaluating RLS conditions and filters.
+   * It allows referencing security_context (lowecase) in dynamic conditions or filter values.
+   *
+   * It currently does not support async calls because inner resolveSymbol and
+   * resolveSymbolsCall are sync. Async support may be added later with deeper
+   * refactoring.
+   */
+  evaluateContextFunction(cube, contextFn, context = {}) {
+    const cubeEvaluator = this;
+
+    const res = cubeEvaluator.resolveSymbolsCall(contextFn, (name) => {
+      const resolvedSymbol = this.resolveSymbol(cube, name);
+      if (resolvedSymbol) {
+        return resolvedSymbol;
+      }
+      throw new UserError(
+        `Cube references are not allowed when evaluating RLS conditions or filters. Found: ${name} in ${cube.name}`
+      );
+    }, {
+      contextSymbols: {
+        securityContext: context.securityContext,
+      }
+    });
+
+    return res;
+  }
+
   evaluateReferences(cube, referencesFn, options = {}) {
     const cubeEvaluator = this;
 

packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts

@@ -25,7 +25,7 @@ export const nonStringFields = new Set([
   'external',
   'useOriginalSqlPreAggregations',
   'readOnly',
-  'prefix'
+  'prefix',
 ]);
 
 const identifierRegex = /^[_a-zA-Z][_a-zA-Z0-9]*$/;
@@ -615,6 +615,64 @@ const SegmentsSchema = Joi.object().pattern(identifierRegex, Joi.object().keys({
   public: Joi.boolean().strict(),
 }));
 
+const PolicyFilterSchema = Joi.object().keys({
+  member: Joi.func().required(),
+  memberReference: Joi.string(),
+  operator: Joi.any().valid(
+    'equals',
+    'notEquals',
+    'contains',
+    'notContains',
+    'startsWith',
+    'notStartsWith',
+    'endsWith',
+    'notEndsWith',
+    'gt',
+    'gte',
+    'lt',
+    'lte',
+    'inDateRange',
+    'notInDateRange',
+    'beforeDate',
+    'beforeOrOnDate',
+    'afterDate',
+    'afterOrOnDate',
+  ).required(),
+  values: Joi.func().required(),
+});
+
+const PolicyFilterConditionSchema = Joi.object().keys({
+  or: Joi.array().items(PolicyFilterSchema, Joi.link('...').description('Filter Condition schema')),
+  and: Joi.array().items(PolicyFilterSchema, Joi.link('...').description('Filter Condition schema')),
+}).xor('or', 'and');
+
+const MemberLevelPolicySchema = Joi.object().keys({
+  includes: Joi.alternatives([
+    Joi.string().valid('*'),
+    Joi.array().items(Joi.string())
+  ]),
+  excludes: Joi.alternatives([
+    Joi.string().valid('*'),
+    Joi.array().items(Joi.string().required())
+  ]),
+  includesMembers: Joi.array().items(Joi.string().required()),
+  excludesMembers: Joi.array().items(Joi.string().required()),
+});
+
+const RowLevelPolicySchema = Joi.object().keys({
+  filters: Joi.array().items(PolicyFilterSchema, PolicyFilterConditionSchema),
+  allowAll: Joi.boolean().valid(true).strict(),
+}).xor('filters', 'allowAll');
+
+const RolePolicySchema = Joi.object().keys({
+  role: Joi.string().required(),
+  memberLevel: MemberLevelPolicySchema,
+  rowLevel: RowLevelPolicySchema,
+  conditions: Joi.array().items(Joi.object().keys({
+    if: Joi.func().required(),
+  })),
+});
+
 /* *****************************
  * ATTENTION:
  * In case of adding/removing/changing any Joi.func() field that needs to be transpiled,
@@ -692,6 +750,7 @@ const baseSchema = {
     title: Joi.string(),
     levels: Joi.func()
   })),
+  accessPolicy: Joi.array().items(RolePolicySchema.required()),
 };
 
 const cubeSchema = inherit(baseSchema, {
@@ -726,6 +785,7 @@ const viewSchema = inherit(baseSchema, {
       'object.oxor': 'Using split together with prefix is not supported'
     })
   ),
+  accessPolicy: Joi.array().items(RolePolicySchema.required()),
 });
 
 function formatErrorMessageFromDetails(explain, d) {