From 2678a5faf81c3bb0340bf1e59ddd1c112edb0e8b Mon Sep 17 00:00:00 2001 From: Don McCurdy Date: Wed, 21 Aug 2024 06:22:45 -0400 Subject: [PATCH] refactor(json): Remove expression-eval dependency (#9070) * refactor(json): Remove expression-eval dependency * chore(json): Add tests for expression-eval --- modules/json/package.json | 2 +- .../src/helpers/parse-expression-string.ts | 3 +- modules/json/src/utils/expression-eval.ts | 339 ++++++++++++++++++ test/modules/json/index.ts | 1 + .../json/utils/expression-eval.spec.ts | 181 ++++++++++ yarn.lock | 38 +- 6 files changed, 551 insertions(+), 13 deletions(-) create mode 100644 modules/json/src/utils/expression-eval.ts create mode 100644 test/modules/json/utils/expression-eval.spec.ts diff --git a/modules/json/package.json b/modules/json/package.json index a0e6da580f7..09a96001c18 100644 --- a/modules/json/package.json +++ b/modules/json/package.json @@ -38,7 +38,7 @@ "prepublishOnly": "npm run build-bundle && npm run build-bundle -- --env=dev" }, "dependencies": { - "expression-eval": "^5.0.0" + "jsep": "^0.3.0" }, "peerDependencies": { "@deck.gl/core": "^9.0.0" diff --git a/modules/json/src/helpers/parse-expression-string.ts b/modules/json/src/helpers/parse-expression-string.ts index f88ecef4c17..ab1dcf691d8 100644 --- a/modules/json/src/helpers/parse-expression-string.ts +++ b/modules/json/src/helpers/parse-expression-string.ts @@ -1,8 +1,7 @@ import {get} from '../utils/get'; // expression-eval: Small jsep based expression parser that supports array and object indexing -import * as expressionEval from 'expression-eval'; -const {parse, eval: evaluate} = expressionEval; +import {parse, eval as evaluate} from '../utils/expression-eval'; const cachedExpressionMap = { '-': object => object diff --git a/modules/json/src/utils/expression-eval.ts b/modules/json/src/utils/expression-eval.ts new file mode 100644 index 00000000000..e5f7b0857c3 --- /dev/null +++ b/modules/json/src/utils/expression-eval.ts @@ -0,0 +1,339 @@ +import jsep from 'jsep'; + +/** + * Sources: + * - Copyright (c) 2013 Stephen Oney, http://jsep.from.so/, MIT License + * - Copyright (c) 2023 Don McCurdy, https://github.com/donmccurdy/expression-eval, MIT License + */ + +// Default operator precedence from https://github.com/EricSmekens/jsep/blob/master/src/jsep.js#L55 +const DEFAULT_PRECEDENCE = { + '||': 1, + '&&': 2, + '|': 3, + '^': 4, + '&': 5, + '==': 6, + '!=': 6, + '===': 6, + '!==': 6, + '<': 7, + '>': 7, + '<=': 7, + '>=': 7, + '<<': 8, + '>>': 8, + '>>>': 8, + '+': 9, + '-': 9, + '*': 10, + '/': 10, + '%': 10 +}; + +const binops = { + '||': (a: unknown, b: unknown) => { + return a || b; + }, + '&&': (a: unknown, b: unknown) => { + return a && b; + }, + '|': (a: number, b: number) => { + return a | b; + }, + '^': (a: number, b: number) => { + return a ^ b; + }, + '&': (a: number, b: number) => { + return a & b; + }, + '==': (a: unknown, b: unknown) => { + // eslint-disable-next-line eqeqeq + return a == b; + }, + '!=': (a: unknown, b: unknown) => { + // eslint-disable-next-line eqeqeq + return a != b; + }, + '===': (a: unknown, b: unknown) => { + return a === b; + }, + '!==': (a: unknown, b: unknown) => { + return a !== b; + }, + '<': (a: number | string, b: number | string) => { + return a < b; + }, + '>': (a: number | string, b: number | string) => { + return a > b; + }, + '<=': (a: number | string, b: number | string) => { + return a <= b; + }, + '>=': (a: number | string, b: number | string) => { + return a >= b; + }, + '<<': (a: number, b: number) => { + return a << b; + }, + '>>': (a: number, b: number) => { + return a >> b; + }, + '>>>': (a: number, b: number) => { + return a >>> b; + }, + '+': (a: unknown, b: unknown) => { + // @ts-expect-error + return a + b; + }, + '-': (a: number, b: number) => { + return a - b; + }, + '*': (a: number, b: number) => { + return a * b; + }, + '/': (a: number, b: number) => { + return a / b; + }, + '%': (a: number, b: number) => { + return a % b; + } +}; + +const unops = { + '-': (a: number) => { + return -a; + }, + '+': (a: unknown) => { + // @ts-expect-error + // eslint-disable-next-line no-implicit-coercion + return +a; + }, + '~': (a: number) => { + return ~a; + }, + '!': (a: unknown) => { + return !a; + } +}; + +declare type operand = number | string; +declare type unaryCallback = (a: operand) => operand; +declare type binaryCallback = (a: operand, b: operand) => operand; + +type AnyExpression = + | jsep.ArrayExpression + | jsep.BinaryExpression + | jsep.MemberExpression + | jsep.CallExpression + | jsep.ConditionalExpression + | jsep.Identifier + | jsep.Literal + | jsep.LogicalExpression + | jsep.ThisExpression + | jsep.UnaryExpression; + +function evaluateArray(list, context) { + return list.map(function (v) { + return evaluate(v, context); + }); +} + +async function evaluateArrayAsync(list, context) { + const res = await Promise.all(list.map(v => evalAsync(v, context))); + return res; +} + +function evaluateMember(node: jsep.MemberExpression, context: object) { + const object = evaluate(node.object, context); + let key: string; + if (node.computed) { + key = evaluate(node.property, context); + } else { + key = (node.property as jsep.Identifier).name; + } + if (/^__proto__|prototype|constructor$/.test(key)) { + throw Error(`Access to member "${key}" disallowed.`); + } + return [object, object[key]]; +} + +async function evaluateMemberAsync(node: jsep.MemberExpression, context: object) { + const object = await evalAsync(node.object, context); + let key: string; + if (node.computed) { + key = await evalAsync(node.property, context); + } else { + key = (node.property as jsep.Identifier).name; + } + if (/^__proto__|prototype|constructor$/.test(key)) { + throw Error(`Access to member "${key}" disallowed.`); + } + return [object, object[key]]; +} + +// eslint-disable-next-line complexity +function evaluate(_node: jsep.Expression, context: object) { + const node = _node as AnyExpression; + + switch (node.type) { + case 'ArrayExpression': + return evaluateArray(node.elements, context); + + case 'BinaryExpression': + return binops[node.operator](evaluate(node.left, context), evaluate(node.right, context)); + + case 'CallExpression': + let caller: object; + let fn: Function; + let assign: unknown[]; + if (node.callee.type === 'MemberExpression') { + assign = evaluateMember(node.callee as jsep.MemberExpression, context); + caller = assign[0] as object; + fn = assign[1] as Function; + } else { + fn = evaluate(node.callee, context); + } + if (typeof fn !== 'function') { + return undefined; + } + return fn.apply(caller!, evaluateArray(node.arguments, context)); + + case 'ConditionalExpression': + return evaluate(node.test, context) + ? evaluate(node.consequent, context) + : evaluate(node.alternate, context); + + case 'Identifier': + return context[node.name]; + + case 'Literal': + return node.value; + + case 'LogicalExpression': + if (node.operator === '||') { + return evaluate(node.left, context) || evaluate(node.right, context); + } else if (node.operator === '&&') { + return evaluate(node.left, context) && evaluate(node.right, context); + } + return binops[node.operator](evaluate(node.left, context), evaluate(node.right, context)); + + case 'MemberExpression': + return evaluateMember(node, context)[1]; + + case 'ThisExpression': + return context; + + case 'UnaryExpression': + return unops[node.operator](evaluate(node.argument, context)); + + default: + return undefined; + } +} + +// eslint-disable-next-line complexity +async function evalAsync(_node: jsep.Expression, context: object) { + const node = _node as AnyExpression; + + // Brackets used for some case blocks here, to avoid edge cases related to variable hoisting. + // See: https://stackoverflow.com/questions/57759348/const-and-let-variable-shadowing-in-a-switch-statement + switch (node.type) { + case 'ArrayExpression': + return await evaluateArrayAsync(node.elements, context); + + case 'BinaryExpression': { + const [left, right] = await Promise.all([ + evalAsync(node.left, context), + evalAsync(node.right, context) + ]); + return binops[node.operator](left, right); + } + + case 'CallExpression': { + let caller: object; + let fn: Function; + let assign: unknown[]; + if (node.callee.type === 'MemberExpression') { + assign = await evaluateMemberAsync(node.callee as jsep.MemberExpression, context); + caller = assign[0] as object; + fn = assign[1] as Function; + } else { + fn = await evalAsync(node.callee, context); + } + if (typeof fn !== 'function') { + return undefined; + } + return await fn.apply(caller!, await evaluateArrayAsync(node.arguments, context)); + } + + case 'ConditionalExpression': + return (await evalAsync(node.test, context)) + ? await evalAsync(node.consequent, context) + : await evalAsync(node.alternate, context); + + case 'Identifier': + return context[node.name]; + + case 'Literal': + return node.value; + + case 'LogicalExpression': { + if (node.operator === '||') { + return (await evalAsync(node.left, context)) || (await evalAsync(node.right, context)); + } else if (node.operator === '&&') { + return (await evalAsync(node.left, context)) && (await evalAsync(node.right, context)); + } + + const [left, right] = await Promise.all([ + evalAsync(node.left, context), + evalAsync(node.right, context) + ]); + + return binops[node.operator](left, right); + } + + case 'MemberExpression': + return (await evaluateMemberAsync(node, context))[1]; + + case 'ThisExpression': + return context; + + case 'UnaryExpression': + return unops[node.operator](await evalAsync(node.argument, context)); + + default: + return undefined; + } +} + +function compile(expression: string | jsep.Expression): (context: object) => any { + return evaluate.bind(null, jsep(expression)); +} + +function compileAsync(expression: string | jsep.Expression): (context: object) => Promise { + return evalAsync.bind(null, jsep(expression)); +} + +// Added functions to inject Custom Unary Operators (and override existing ones) +function addUnaryOp(operator: string, _function: unaryCallback): void { + jsep.addUnaryOp(operator); + unops[operator] = _function; +} + +// Added functions to inject Custom Binary Operators (and override existing ones) +function addBinaryOp( + operator: string, + precedenceOrFn: number | binaryCallback, + _function: binaryCallback +): void { + if (_function) { + jsep.addBinaryOp(operator, precedenceOrFn as number); + binops[operator] = _function; + } else { + jsep.addBinaryOp(operator, DEFAULT_PRECEDENCE[operator] || 1); + binops[operator] = precedenceOrFn; + } +} + +export {jsep as parse, evaluate as eval, evalAsync, compile, compileAsync, addUnaryOp, addBinaryOp}; diff --git a/test/modules/json/index.ts b/test/modules/json/index.ts index abc460444e9..3da271c0cb5 100644 --- a/test/modules/json/index.ts +++ b/test/modules/json/index.ts @@ -1,3 +1,4 @@ +import './utils/expression-eval.spec'; import './utils/get.spec'; import './utils/shallow-equal-objects.spec'; diff --git a/test/modules/json/utils/expression-eval.spec.ts b/test/modules/json/utils/expression-eval.spec.ts new file mode 100644 index 00000000000..8eb9e8585ee --- /dev/null +++ b/test/modules/json/utils/expression-eval.spec.ts @@ -0,0 +1,181 @@ +import test from 'tape-promise/tape'; +import {compile, compileAsync, addUnaryOp, addBinaryOp} from '@deck.gl/json/utils/expression-eval'; + +const fixtures = [ + // array expression + {expr: '([1,2,3])[0]', expected: 1}, + {expr: '(["one","two","three"])[1]', expected: 'two'}, + {expr: '([true,false,true])[2]', expected: true}, + {expr: '([1,true,"three"]).length', expected: 3}, + {expr: 'isArray([1,2,3])', expected: true}, + {expr: 'list[3]', expected: 4}, + {expr: 'numMap[1 + two]', expected: 'three'}, + + // binary expression + {expr: '1+2', expected: 3}, + {expr: '2-1', expected: 1}, + {expr: '2*2', expected: 4}, + {expr: '6/3', expected: 2}, + {expr: '5|3', expected: 7}, + {expr: '5&3', expected: 1}, + {expr: '5^3', expected: 6}, + {expr: '4<<2', expected: 16}, + {expr: '256>>4', expected: 16}, + {expr: '-14>>>2', expected: 1073741820}, + {expr: '10%6', expected: 4}, + {expr: '"a"+"b"', expected: 'ab'}, + {expr: 'one + three', expected: 4}, + + // call expression + {expr: 'func(5)', expected: 6}, + {expr: 'func(1+2)', expected: 4}, + + // conditional expression + {expr: '(true ? "true" : "false")', expected: 'true'}, + {expr: '( ( bool || false ) ? "true" : "false")', expected: 'true'}, + {expr: '( true ? ( 123*456 ) : "false")', expected: 123 * 456}, + {expr: '( false ? "true" : one + two )', expected: 3}, + + // identifier + {expr: 'string', expected: 'string'}, + {expr: 'number', expected: 123}, + {expr: 'bool', expected: true}, + + // literal + {expr: '"foo"', expected: 'foo'}, // string literal + {expr: "'foo'", expected: 'foo'}, // string literal + {expr: '123', expected: 123}, // numeric literal + {expr: 'true', expected: true}, // boolean literal + + // logical expression + {expr: 'true || false', expected: true}, + {expr: 'true && false', expected: false}, + {expr: '1 == "1"', expected: true}, + {expr: '2 != "2"', expected: false}, + {expr: '1.234 === 1.234', expected: true}, + {expr: '123 !== "123"', expected: true}, + {expr: '1 < 2', expected: true}, + {expr: '1 > 2', expected: false}, + {expr: '2 <= 2', expected: true}, + {expr: '1 >= 2', expected: false}, + + // logical expression lazy evaluation + {expr: 'true || throw()', expected: true}, + {expr: 'false || true', expected: true}, + {expr: 'false && throw()', expected: false}, + {expr: 'true && false', expected: false}, + + // member expression + {expr: 'foo.bar', expected: 'baz'}, + {expr: 'foo["bar"]', expected: 'baz'}, + {expr: 'foo[foo.bar]', expected: 'wow'}, + + // call expression with member + {expr: 'foo.func("bar")', expected: 'baz'}, + + // unary expression + {expr: '-one', expected: -1}, + {expr: '+two', expected: 2}, + {expr: '!false', expected: true}, + {expr: '!!true', expected: true}, + {expr: '~15', expected: -16}, + {expr: '+[]', expected: 0}, + + // 'this' context + {expr: 'this.three', expected: 3}, + + // custom operators + {expr: '@2', expected: 'two'}, + {expr: '3#4', expected: 3.4}, + {expr: '(1 # 2 # 3)', expected: 1.5}, // Fails with undefined precedence, see issue #45 + {expr: '1 + 2 ~ 3', expected: 9} // ~ is * but with low precedence +]; + +const context = { + string: 'string', + number: 123, + bool: true, + one: 1, + two: 2, + three: 3, + foo: { + bar: 'baz', + baz: 'wow', + func: function (x) { + return this[x]; + } + }, + numMap: {10: 'ten', 3: 'three'}, + list: [1, 2, 3, 4, 5], + func: function (x) { + return x + 1; + }, + isArray: Array.isArray, + throw: () => { + throw new Error('Should not be called.'); + } +}; + +addUnaryOp('@', a => { + if (a === 2) { + return 'two'; + } + throw new Error('Unexpected value: ' + a); +}); + +addBinaryOp('#', (a: number, b: number) => a + b / 10); + +addBinaryOp('~', 1, (a: number, b: number) => a * b); + +test('sync', t => { + fixtures.forEach(o => { + const val = compile(o.expr)(context); + t.equal(val, o.expected, `${o.expr} (${val}) === ${o.expected}`); + }); + + t.end(); +}); + +test('async', async t => { + const asyncContext = context; + (asyncContext as Record).asyncFunc = async function ( + a: number | Promise, + b: number + ) { + return (await a) + b; + }; + (asyncContext as Record).promiseFunc = function (a: number, b: number) { + return new Promise(resolve => setTimeout(() => resolve(a + b), 1000)); + }; + const asyncFixtures = fixtures; + asyncFixtures.push( + { + expr: 'asyncFunc(one, two)', + expected: 3 + }, + { + expr: 'promiseFunc(one, two)', + expected: 3 + } + ); + + for (let o of asyncFixtures) { + const val = await compileAsync(o.expr)(asyncContext); + t.equal(val, o.expected, `${o.expr} (${val}) === ${o.expected}`); + } + t.end(); +}); + +test('errors', async t => { + const expectedMsg = /Access to member "\w+" disallowed/; + t.throws(() => compile(`o.__proto__`)({o: {}}), expectedMsg, '.__proto__'); + t.throws(() => compile(`o.prototype`)({o: {}}), expectedMsg, '.prototype'); + t.throws(() => compile(`o.constructor`)({o: {}}), expectedMsg, '.constructor'); + t.throws(() => compile(`o['__proto__']`)({o: {}}), expectedMsg, '["__proto__"]'); + t.throws(() => compile(`o['prototype']`)({o: {}}), expectedMsg, '["prototype"]'); + t.throws(() => compile(`o['constructor']`)({o: {}}), expectedMsg, '["constructor"]'); + t.throws(() => compile(`o[p]`)({o: {}, p: '__proto__'}), expectedMsg, '[~__proto__]'); + t.throws(() => compile(`o[p]`)({o: {}, p: 'prototype'}), expectedMsg, '[~prototype]'); + t.throws(() => compile(`o[p]`)({o: {}, p: 'constructor'}), expectedMsg, '[~constructor]'); + t.end(); +}); diff --git a/yarn.lock b/yarn.lock index 67cb55a9252..ae8420171a8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5610,13 +5610,6 @@ exponential-backoff@^3.1.1: resolved "https://registry.yarnpkg.com/exponential-backoff/-/exponential-backoff-3.1.1.tgz#64ac7526fe341ab18a39016cd22c787d01e00bf6" integrity sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw== -expression-eval@^5.0.0: - version "5.0.1" - resolved "https://registry.yarnpkg.com/expression-eval/-/expression-eval-5.0.1.tgz#845758fa9ba64d9edc7b6804ae404934a6cfee6b" - integrity sha512-7SL4miKp19lI834/F6y156xlNg+i9Q41tteuGNCq9C06S78f1bm3BXuvf0+QpQxv369Pv/P2R7Hb17hzxLpbDA== - dependencies: - jsep "^0.3.0" - extend-shallow@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-2.0.1.tgz#51af7d614ad9a9f610ea1bafbb989d6b1c56890f" @@ -10697,7 +10690,16 @@ streamx@^2.15.0: fast-fifo "^1.1.0" queue-tick "^1.0.1" -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -10850,7 +10852,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -10871,6 +10873,13 @@ strip-ansi@^6.0.0: dependencies: ansi-regex "^5.0.0" +strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -11813,7 +11822,7 @@ wordwrap@^1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus= -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -11831,6 +11840,15 @@ wrap-ansi@^6.0.1: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"