feat: support readdir in batch

ignore grpc test case

fix test case

resume ci.yml

try to fix old unit test code

Author: DoubleSpout

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
           - version: 3.4.0
             conf: Procfile-single-enable-mtls
 
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     env:
       OPENRESTY_PREFIX: "/usr/local/openresty"
       AUTH_ENDPOINT_V3: "127.0.0.1:12379"
@@ -41,7 +41,7 @@ jobs:
           go-version: "1.17"
 
       - name: get dependencies
-        run: sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl
+        run: sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3-dev libpcre3
 
       - name: Setup Lua
         uses: leafo/[email protected]

api_v3.md

@@ -1,25 +1,28 @@
 API V3
 ======
 
-* [Methods](#methods)
-    * [new](#new)
-    * [get](#get)
-    * [set](#set)
-    * [setnx](#setnx)
-    * [setx](#setx)
-    * [delete](#delete)
-    * [watch](#watch)
-    * [watchcancel](#watchcancel)
-    * [readdir](#readdir)
-    * [watchdir](#watchdir)
-    * [rmdir](#rmdir)
-    * [txn](#txn)
-    * [version](#version)
-    * [grant](#grant)
-    * [revoke](#revoke)
-    * [keepalive](#keepalive)
-    * [timetolive](#timetolive)
-    * [leases](#leases)
+- [API V3](#api-v3)
+- [Method](#method)
+    - [new](#new)
+    - [get](#get)
+    - [set](#set)
+    - [setnx](#setnx)
+    - [setx](#setx)
+    - [delete](#delete)
+    - [watch](#watch)
+    - [watchcancel](#watchcancel)
+    - [readdir](#readdir)
+    - [watchdir](#watchdir)
+    - [rmdir](#rmdir)
+    - [txn](#txn)
+    - [version](#version)
+    - [grant](#grant)
+    - [revoke](#revoke)
+    - [keepalive](#keepalive)
+    - [timetolive](#timetolive)
+    - [leases](#leases)
+    - [nextkey](#nextkey)
+    - [rangeend](#rangeend)
 
 Method
 ======
@@ -358,3 +361,72 @@ To retrieve lease information.
 To list all existing leases.
 
 [Back to TOP](#api-v3)
+
+### nextkey
+
+`syntax: next_key = etcd.nextkey(key:string)`
+
+- `key`: etcd path to key
+
+return next path to key, only used when `sort_order='ASCEND' and sort_target='KEY'`.
+
+[Back to TOP](#api-v3)
+
+
+### rangeend
+
+`syntax: range_end = etcd.rangeend(key:string)`
+
+- `key`: etcd path to key
+
+return range end path to key.
+
+When using etcd's `cli:readdir`, if the value in this path is too large (by default: exceeding 4MB), you will encounter the following error: 
+
+```
+rpc error: code = ResourceExhausted desc = grpc: received message larger than max (8653851 vs. 4194304).
+```
+
+Therefore, we need to read this directory in batches. 
+
+The code example :
+
+``` lua
+local cli, _ = etcd.new([option:table])
+
+local res, err
+local start_key = '/path/to/dir'
+local last_key = start_key
+local range_end = cli.rangeend(start_key)
+local data = {}
+while(1) do
+    res, err = cli:readdir(last_key, {
+        timeout = 3000,
+        range_end = range_end,
+        revision = -1,
+        limit = 20, -- batch size 
+        sort_order = 'ASCEND',
+        sort_target = 'KEY',
+    })
+    -- error handle
+    if err or not res then
+        break
+    end
+    -- collect res.body
+    table.insert(data, res.body)
+    local last_kv_key = res.body.kvs[#res.body.kvs].key
+    last_key = cli.nextkey(last_key)
+    if not res.body.more then
+        break
+    end
+end
+
+for _, body in ipairs(data) do
+    -- handle data
+end
+```
+
+[Back to TOP](#api-v3)
+  
+
+

lib/resty/etcd/utils.lua

@@ -9,6 +9,8 @@ local select        = select
 local ipairs        = ipairs
 local pairs         = pairs
 local type          = type
+local str_char      = string.char
+
 
 
 if not http.tls_handshake then
@@ -125,4 +127,10 @@ local function is_empty_str(input_str)
 end
 _M.is_empty_str = is_empty_str
 
+--- @param  key string
+--- @return string
+_M.next_key = function(key)
+    return key .. str_char(0)
+end
+
 return _M

lib/resty/etcd/v3.lua

@@ -952,7 +952,8 @@ local function request_chunk(self, method, path, opts, timeout)
     end
 end
 
-
+--- @param  key string
+--- @return string
 local function get_range_end(key)
     if #key == 0 then
         return str_char(0)
@@ -1319,7 +1320,7 @@ function _M.readdir(self, key, opts)
 
     key = utils.get_real_key(self.key_prefix, key)
 
-    attr.range_end = get_range_end(key)
+    attr.range_end = opts and opts.range_end or get_range_end(key)
     attr.revision = opts and opts.revision
     attr.limit        = opts and opts.limit
     attr.sort_order   = opts and opts.sort_order
@@ -1689,6 +1690,9 @@ end
 
 end -- do
 
+_M.rangeend = get_range_end
+_M.nextkey = utils.next_key
+
 
 local implemented_grpc_methods = {
     grant = true,

t/v3/grpc/txn.t

@@ -88,6 +88,10 @@ __DATA__
             check_res(data, err, "ddd")
         }
     }
+--- request
+GET /t
+--- no_error_log
+[error]
 --- response_body
 checked val as expect: abc
 checked val as expect: ddd
@@ -118,6 +122,10 @@ checked val as expect: ddd
             check_res(data, err, "abc")
         }
     }
+--- request
+GET /t
+--- no_error_log
+[error]
 --- response_body
 checked val as expect: abc
 checked val as expect: abc
@@ -145,6 +153,10 @@ checked val as expect: abc
             check_res(data, err, "aaa", 200)
         }
     }
+--- request
+GET /t
+--- no_error_log
+[error]
 --- response_body
 checked val as expect: aaa
 
@@ -201,6 +213,10 @@ checked val as expect: aaa
             check_res(data, err, '{"k":"ddd"}')
         }
     }
+--- request
+GET /t
+--- no_error_log
+[error]
 --- response_body
 checked val as expect: {"k":"abc"}
 checked val as expect: {"k":"ddd"}

t/v3/key.t

@@ -113,6 +113,97 @@ GET /t
 ok
 
 
+=== TEST 2-2: readdir(key) in batch
+--- http_config eval: $::HttpConfig
+--- config
+    location /t {
+        content_by_lua_block {
+            local tab_nkeys     = require "table.nkeys"
+            local etcd, err = require "resty.etcd" .new({protocol = "v3"})
+            ngx.say('new')
+            check_res(etcd, err)
+
+            local res, err = etcd:set("/batch/dir", "abc")
+            ngx.say('set1')
+            check_res(res, err)
+
+            local res, err = etcd:set("/batch/dir/a", "abca")
+            ngx.say('set2')
+            check_res(res, err)
+
+            local res, err = etcd:set("/batch/dir/b", "abcb")
+            ngx.say('set3')
+            check_res(res, err)
+
+
+            local start_key = '/batch/dir'
+            local last_key = start_key
+            local range_end = etcd.rangeend(start_key)
+            local data = {}
+            local kvs = {}
+            local times = 0
+            while(1) do
+                times = times + 1
+                res, err = etcd:readdir(last_key, {
+                    timeout = 5000,
+                    range_end = range_end,
+                    revision = -1,
+                    limit = 1,
+                    sort_order = 'ASCEND',
+                    sort_target = 'KEY',
+                })
+                ngx.say('while')
+                check_res(res, err)
+                
+                table.insert(data, res.body)
+
+                local last_kv_key = res.body.kvs[#res.body.kvs].key
+                last_key = etcd.nextkey(last_kv_key)
+
+                if not res.body.more then
+                    break
+                end
+
+                if times > 4 then
+                    ngx.say("too many times")
+                    ngx.exit(500)
+                    return
+                end
+
+            end
+
+            for _, body in ipairs(data) do
+                for _, kv in ipairs(body.kvs) do
+                    table.insert(kvs, kv)
+                end
+            end
+
+            if tab_nkeys(kvs) == 3 and times == 3 then
+                ngx.say("ok")
+                ngx.exit(200)
+            else
+                ngx.say(string.format("failed len(kvs):%s, times:%s", tab_nkeys(kvs), times))
+                ngx.exit(500)
+            end
+
+        }
+    }
+--- request
+GET /t
+--- no_error_log
+[error]
+--- response_body
+new
+set1
+set2
+set3
+while
+while
+while
+ok
+--- timeout: 5
+
+
 
 === TEST 3: watch(key)
 --- http_config eval: $::HttpConfig

t/v3/tls.t

@@ -103,7 +103,7 @@ GET /t
 --- no_error_log
 [error]
 --- response_body eval
-qr/18: self signed certificate/
+qr/self-signed certificate/