Select: jsonpath support (#150)

Added jsonpath support for ``crud.select``. Part of #12

Author: mRrvz

CHANGELOG.md

@@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 * Option flag `force_map_call` for `select()`/`pairs()`
   to disable the `bucket_id` computation from primary key.
 * `crud.min` and `crud.max` functions to find the minimum and maximum values in the specified index.
+* Added support for jsonpath for select.
 
 ## [0.6.0] - 2021-03-29
 

crud/compare/conditions.lua

@@ -63,7 +63,7 @@ for func_name, operator in pairs(cond_operators_by_func_names) do
         return new_condition({
             operator = operator,
             operand = operand,
-            values = values
+            values = values,
         })
     end
 end

crud/select/filters.lua

@@ -1,12 +1,10 @@
-local json = require('json')
 local errors = require('errors')
 
 local utils = require('crud.common.utils')
 local dev_checks = require('crud.common.dev_checks')
 local collations = require('crud.common.collations')
 local compare_conditions = require('crud.compare.conditions')
 
-local ParseConditionsError = errors.new_class('ParseConditionsError', {capture_stack = false})
 local GenFiltersError = errors.new_class('GenFiltersError', {capture_stack = false})
 
 local filters = {}
@@ -97,31 +95,42 @@ local function parse(space, conditions, opts)
     for i, condition in ipairs(conditions) do
         if i ~= opts.scan_condition_num then
             -- Index check (including one and multicolumn)
-            local fieldnos
-            local fields_types
+            local fields
+            local fields_types = {}
             local values_opts
 
             local index = space_indexes[condition.operand]
 
             if index ~= nil then
-                fieldnos = get_index_fieldnos(index)
+                fields = get_index_fieldnos(index)
                 fields_types = get_index_fields_types(index)
                 values_opts = get_values_opts(index)
-            elseif fieldnos_by_names[condition.operand] ~= nil then
-                local fiendno = fieldnos_by_names[condition.operand]
-                fieldnos = {fiendno}
-                local field_format = space_format[fiendno]
-                fields_types = {field_format.type}
-                local is_nullable = field_format.is_nullable == true
+            else
+                local fieldno = fieldnos_by_names[condition.operand]
+
+                if fieldno ~= nil then
+                    fields = {fieldno}
+                else
+                    -- We assume this is jsonpath, so it is
+                    -- not in fieldnos_by_name map.
+                    fields = {condition.operand}
+                end
+
+                local field_format = space_format[fieldno]
+                local is_nullable
+
+                if field_format ~= nil then
+                    fields_types = {field_format.type}
+                    is_nullable = field_format.is_nullable == true
+                end
+
                 values_opts = {
                     {is_nullable = is_nullable, collation = nil},
                 }
-            else
-                return nil, ParseConditionsError('No field or index is found for condition %s', json.encode(condition))
             end
 
             table.insert(filter_conditions, {
-                fieldnos = fieldnos,
+                fields = fields,
                 operator = condition.operator,
                 values = condition.values,
                 types = fields_types,
@@ -156,12 +165,30 @@ end
 local PARSE_ARGS_TEMPLATE = 'local tuple = ...'
 local LIB_FUNC_HEADER_TEMPLATE = 'function M.%s(%s)'
 
+local function format_path(path)
+    local path_type = type(path)
+    if path_type == 'number' then
+        return tostring(path)
+    elseif path_type == 'string' then
+        return ('%q'):format(path)
+    end
+
+    assert(false, ('Unexpected format: %s'):format(path_type))
+end
+
 local function concat_conditions(conditions, operator)
     return '(' .. table.concat(conditions, (' %s '):format(operator)) .. ')'
 end
 
-local function get_field_variable_name(fieldno)
-    return string.format('field_%s', fieldno)
+local function get_field_variable_name(field)
+    local field_type = type(field)
+    if field_type == 'number' then
+        field = tostring(field)
+    elseif field_type == 'string' then
+        field = string.gsub(field, '([().^$%[%]%+%-%*%?%%\'"])', '_')
+    end
+
+    return string.format('field_%s', field)
 end
 
 local function get_eq_func_name(id)
@@ -173,38 +200,39 @@ local function get_cmp_func_name(id)
 end
 
 local function gen_tuple_fields_def_code(filter_conditions)
-    -- get field numbers
-    local fieldnos_added = {}
-    local fieldnos = {}
+    -- get field names
+    local fields_added = {}
+    local fields = {}
 
     for _, cond in ipairs(filter_conditions) do
         for i = 1, #cond.values do
-            local fieldno = cond.fieldnos[i]
-            if not fieldnos_added[fieldno] then
-                table.insert(fieldnos, fieldno)
-                fieldnos_added[fieldno] = true
+            local field = cond.fields[i]
+
+            if not fields_added[field] then
+                table.insert(fields, field)
+                fields_added[field] = true
             end
         end
     end
 
     -- gen definitions for all used fields
     local fields_def_parts = {}
 
-    for _, fieldno in ipairs(fieldnos) do
+    for _, field in ipairs(fields) do
         table.insert(fields_def_parts, string.format(
             'local %s = tuple[%s]',
-            get_field_variable_name(fieldno), fieldno
+            get_field_variable_name(field), format_path(field)
         ))
     end
 
     return table.concat(fields_def_parts, '\n')
 end
 
-local function format_comp_with_value(fieldno, func_name, value)
+local function format_comp_with_value(field, func_name, value)
     return string.format(
         '%s(%s, %s)',
         func_name,
-        get_field_variable_name(fieldno),
+        get_field_variable_name(field),
         format_value(value)
     )
 end
@@ -238,7 +266,7 @@ local function format_eq(cond)
     local values_opts = cond.values_opts or {}
 
     for j = 1, #cond.values do
-        local fieldno = cond.fieldnos[j]
+        local field = cond.fields[j]
         local value = cond.values[j]
         local value_type = cond.types[j]
         local value_opts = values_opts[j] or {}
@@ -254,7 +282,7 @@ local function format_eq(cond)
             func_name = 'eq_uuid'
         end
 
-        table.insert(cond_strings, format_comp_with_value(fieldno, func_name, value))
+        table.insert(cond_strings, format_comp_with_value(field, func_name, value))
     end
 
     return cond_strings
@@ -265,7 +293,7 @@ local function format_lt(cond)
     local values_opts = cond.values_opts or {}
 
     for j = 1, #cond.values do
-        local fieldno = cond.fieldnos[j]
+        local field = cond.fields[j]
         local value = cond.values[j]
         local value_type = cond.types[j]
         local value_opts = values_opts[j] or {}
@@ -279,9 +307,10 @@ local function format_lt(cond)
         elseif value_type == 'uuid' then
             func_name = 'lt_uuid'
         end
+
         func_name = add_strict_postfix(func_name, value_opts)
 
-        table.insert(cond_strings, format_comp_with_value(fieldno, func_name, value))
+        table.insert(cond_strings, format_comp_with_value(field, func_name, value))
     end
 
     return cond_strings
@@ -366,10 +395,10 @@ local function gen_cmp_array_func_code(operator, func_name, cond, func_args_code
     return table.concat(func_code_lines, '\n')
 end
 
-local function function_args_by_fieldnos(fieldnos)
+local function function_args_by_field(fields)
     local arg_names = {}
-    for _, fieldno in ipairs(fieldnos) do
-        table.insert(arg_names, get_field_variable_name(fieldno))
+    for _, field in ipairs(fields) do
+        table.insert(arg_names, get_field_variable_name(field))
     end
     return table.concat(arg_names, ', ')
 end
@@ -408,8 +437,8 @@ local function gen_filter_code(filter_conditions)
     table.insert(filter_code_parts, '')
 
     for i, cond in ipairs(filter_conditions) do
-        local args_fieldnos = { unpack(cond.fieldnos, 1, #cond.values) }
-        local func_args_code = function_args_by_fieldnos(args_fieldnos)
+        local args_fields = { unpack(cond.fields, 1, #cond.values) }
+        local func_args_code = function_args_by_field(args_fields)
 
         local library_func_name, library_func_code = gen_library_func(i, cond, func_args_code)
         table.insert(library_funcs_code_parts, library_func_code)

crud/select/plan.lua

@@ -6,9 +6,7 @@ local dev_checks = require('crud.common.dev_checks')
 
 local select_plan = {}
 
-local SelectPlanError = errors.new_class('SelectPlanError', {capture_stack = false})
 local IndexTypeError = errors.new_class('IndexTypeError', {capture_stack = false})
-local ValidateConditionsError = errors.new_class('ValidateConditionsError', {capture_stack = false})
 local FilterFieldsError = errors.new_class('FilterFieldsError',  {capture_stack = false})
 
 local function index_is_allowed(index)
@@ -42,34 +40,6 @@ local function get_index_for_condition(space_indexes, space_format, condition)
     end
 end
 
-local function validate_conditions(conditions, space_indexes, space_format)
-    local field_names = {}
-    for _, field_format in ipairs(space_format) do
-        field_names[field_format.name] = true
-    end
-
-    local index_names = {}
-
-    -- If we use # (not table.maxn), we may lose indexes, when user drop some indexes.
-    -- E.g: we have table with indexes id {1, 2, 3, nil, nil, 6}.
-    -- If we use #{1, 2, 3, nil, nil, 6} (== 3) we will lose index with id = 6.
-    -- See details: https://github.com/tarantool/crud/issues/103
-    for i = 0, table.maxn(space_indexes) do
-        local index = space_indexes[i]
-        if index ~= nil then
-            index_names[index.name] = true
-        end
-    end
-
-    for _, condition in ipairs(conditions) do
-        if index_names[condition.operand] == nil and field_names[condition.operand] == nil then
-            return false, ValidateConditionsError:new("No field or index %q found", condition.operand)
-        end
-    end
-
-    return true
-end
-
 local function extract_sharding_key_from_scan_value(scan_value, scan_index, sharding_index)
     if #scan_value < #sharding_index.parts then
         return nil
@@ -157,11 +127,6 @@ function select_plan.new(space, conditions, opts)
     local space_indexes = space.index
     local space_format = space:format()
 
-    local ok, err = validate_conditions(conditions, space_indexes, space_format)
-    if not ok then
-        return nil, SelectPlanError:new('Passed bad conditions: %s', err)
-    end
-
     if conditions == nil then -- also cdata<NULL>
         conditions = {}
     end

test/entrypoint/srv_select.lua

@@ -126,6 +126,31 @@ package.preload['customers-storage'] = function()
                 unique = false,
                 if_not_exists = true,
             })
+
+            local developers_space = box.schema.space.create('developers', {
+                format = {
+                    {name = 'id', type = 'unsigned'},
+                    {name = 'bucket_id', type = 'unsigned'},
+                    {name = 'name', type = 'string'},
+                    {name = 'last_name', type = 'string'},
+                    {name = 'age', type = 'number'},
+                    {name = 'additional', type = 'any'},
+                },
+                if_not_exists = true,
+                engine = engine,
+            })
+
+            -- primary index
+            developers_space:create_index('id_index', {
+                parts = { 'id' },
+                if_not_exists = true,
+            })
+
+            developers_space:create_index('bucket_id', {
+                parts = { 'bucket_id' },
+                unique = false,
+                if_not_exists = true,
+            })
         end,
     }
 end

test/integration/select_test.lua

@@ -31,6 +31,7 @@ pgroup:set_after_all(function(g) helpers.stop_cluster(g.cluster) end)
 
 pgroup:set_before_each(function(g)
     helpers.truncate_space_on_cluster(g.cluster, 'customers')
+    helpers.truncate_space_on_cluster(g.cluster, 'developers')
 end)
 
 
@@ -1199,3 +1200,57 @@ pgroup:add('test_select_force_map_call', function(g)
     table.sort(objects, function(obj1, obj2) return obj1.bucket_id < obj2.bucket_id end)
     t.assert_equals(objects, customers)
 end)
+
+pgroup:add('test_jsonpath', function(g)
+    helpers.insert_objects(g, 'developers', {
+        {
+            id = 1, name = "Alexey", last_name = "Smith",
+            age = 20, additional = { a = { b = 140 } },
+        }, {
+            id = 2, name = "Sergey", last_name = "Choppa",
+            age = 21, additional = { a = { b = 120 } },
+        }, {
+            id = 3, name = "Mikhail", last_name = "Crossman",
+            age = 42, additional = {},
+        }, {
+            id = 4, name = "Pavel", last_name = "White",
+            age = 51, additional = { a = { b = 50 } },
+        }, {
+            id = 5, name = "Tatyana", last_name = "May",
+            age = 17, additional = { a = 55 },
+        },
+    })
+
+    local result, err = g.cluster.main_server.net_box:call('crud.select',
+            {'developers', {{'>=', '[5]', 40}}, {fields = {'name', 'last_name'}}})
+    t.assert_equals(err, nil)
+
+    local objects = crud.unflatten_rows(result.rows, result.metadata)
+    local expected_objects = {
+        {id = 3, name = "Mikhail", last_name = "Crossman"},
+        {id = 4, name = "Pavel", last_name = "White"},
+    }
+    t.assert_equals(objects, expected_objects)
+
+    local result, err = g.cluster.main_server.net_box:call('crud.select',
+            {'developers', {{'<', '["age"]', 21}}, {fields = {'name', 'last_name'}}})
+    t.assert_equals(err, nil)
+
+    local objects = crud.unflatten_rows(result.rows, result.metadata)
+    local expected_objects = {
+        {id = 1, name = "Alexey", last_name = "Smith"},
+        {id = 5, name = "Tatyana", last_name = "May"},
+    }
+    t.assert_equals(objects, expected_objects)
+
+    local result, err = g.cluster.main_server.net_box:call('crud.select',
+        {'developers', {{'>=', '[6].a.b', 55}}, {fields = {'name', 'last_name'}}})
+    t.assert_equals(err, nil)
+
+    local objects = crud.unflatten_rows(result.rows, result.metadata)
+    local expected_objects = {
+        {id = 1, name = "Alexey", last_name = "Smith"},
+        {id = 2, name = "Sergey", last_name = "Choppa"},
+    }
+    t.assert_equals(objects, expected_objects)
+end)

test/integration/simple_operations_test.lua

@@ -1006,3 +1006,4 @@ pgroup:add('test_partial_result_bad_input', function(g)
     t.assert_equals(result, nil)
     t.assert_str_contains(err.err, 'Space format doesn\'t contain field named "lastname"')
 end)
+