From c073abceb062466efee26df27ebf3d36aee74842 Mon Sep 17 00:00:00 2001 From: wzh Date: Tue, 9 Sep 2025 16:15:17 +0800 Subject: [PATCH] feat: support readdir in batch ignore grpc test case fix test case resume ci.yml try to fix old unit test code --- .github/workflows/ci.yml | 4 +- api_v3.md | 110 ++++++++++++++++++++++++++++++++------- lib/resty/etcd/utils.lua | 8 +++ lib/resty/etcd/v3.lua | 8 ++- t/v3/grpc/txn.t | 16 ++++++ t/v3/key.t | 91 ++++++++++++++++++++++++++++++++ t/v3/tls.t | 2 +- 7 files changed, 215 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec45b012..585246ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - version: 3.4.0 conf: Procfile-single-enable-mtls - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: OPENRESTY_PREFIX: "/usr/local/openresty" AUTH_ENDPOINT_V3: "127.0.0.1:12379" @@ -41,7 +41,7 @@ jobs: go-version: "1.17" - name: get dependencies - run: sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl + run: sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3-dev libpcre3 - name: Setup Lua uses: leafo/gh-actions-lua@v8.0.0 diff --git a/api_v3.md b/api_v3.md index 3dd9e37d..28673007 100644 --- a/api_v3.md +++ b/api_v3.md @@ -1,25 +1,28 @@ API V3 ====== -* [Methods](#methods) - * [new](#new) - * [get](#get) - * [set](#set) - * [setnx](#setnx) - * [setx](#setx) - * [delete](#delete) - * [watch](#watch) - * [watchcancel](#watchcancel) - * [readdir](#readdir) - * [watchdir](#watchdir) - * [rmdir](#rmdir) - * [txn](#txn) - * [version](#version) - * [grant](#grant) - * [revoke](#revoke) - * [keepalive](#keepalive) - * [timetolive](#timetolive) - * [leases](#leases) +- [API V3](#api-v3) +- [Method](#method) + - [new](#new) + - [get](#get) + - [set](#set) + - [setnx](#setnx) + - [setx](#setx) + - [delete](#delete) + - [watch](#watch) + - [watchcancel](#watchcancel) + - [readdir](#readdir) + - [watchdir](#watchdir) + - [rmdir](#rmdir) + - [txn](#txn) + - [version](#version) + - [grant](#grant) + - [revoke](#revoke) + - [keepalive](#keepalive) + - [timetolive](#timetolive) + - [leases](#leases) + - [nextkey](#nextkey) + - [rangeend](#rangeend) Method ====== @@ -358,3 +361,72 @@ To retrieve lease information. To list all existing leases. [Back to TOP](#api-v3) + +### nextkey + +`syntax: next_key = etcd.nextkey(key:string)` + +- `key`: etcd path to key + +return next path to key, only used when `sort_order='ASCEND' and sort_target='KEY'`. + +[Back to TOP](#api-v3) + + +### rangeend + +`syntax: range_end = etcd.rangeend(key:string)` + +- `key`: etcd path to key + +return range end path to key. + +When using etcd's `cli:readdir`, if the value in this path is too large (by default: exceeding 4MB), you will encounter the following error: + +``` +rpc error: code = ResourceExhausted desc = grpc: received message larger than max (8653851 vs. 4194304). +``` + +Therefore, we need to read this directory in batches. + +The code example : + +``` lua +local cli, _ = etcd.new([option:table]) + +local res, err +local start_key = '/path/to/dir' +local last_key = start_key +local range_end = cli.rangeend(start_key) +local data = {} +while(1) do + res, err = cli:readdir(last_key, { + timeout = 3000, + range_end = range_end, + revision = -1, + limit = 20, -- batch size + sort_order = 'ASCEND', + sort_target = 'KEY', + }) + -- error handle + if err or not res then + break + end + -- collect res.body + table.insert(data, res.body) + local last_kv_key = res.body.kvs[#res.body.kvs].key + last_key = cli.nextkey(last_key) + if not res.body.more then + break + end +end + +for _, body in ipairs(data) do + -- handle data +end +``` + +[Back to TOP](#api-v3) + + + diff --git a/lib/resty/etcd/utils.lua b/lib/resty/etcd/utils.lua index d4e7bf05..298b0589 100644 --- a/lib/resty/etcd/utils.lua +++ b/lib/resty/etcd/utils.lua @@ -9,6 +9,8 @@ local select = select local ipairs = ipairs local pairs = pairs local type = type +local str_char = string.char + if not http.tls_handshake then @@ -125,4 +127,10 @@ local function is_empty_str(input_str) end _M.is_empty_str = is_empty_str +--- @param key string +--- @return string +_M.next_key = function(key) + return key .. str_char(0) +end + return _M diff --git a/lib/resty/etcd/v3.lua b/lib/resty/etcd/v3.lua index 8c9b2572..e8a1080b 100644 --- a/lib/resty/etcd/v3.lua +++ b/lib/resty/etcd/v3.lua @@ -952,7 +952,8 @@ local function request_chunk(self, method, path, opts, timeout) end end - +--- @param key string +--- @return string local function get_range_end(key) if #key == 0 then return str_char(0) @@ -1319,7 +1320,7 @@ function _M.readdir(self, key, opts) key = utils.get_real_key(self.key_prefix, key) - attr.range_end = get_range_end(key) + attr.range_end = opts and opts.range_end or get_range_end(key) attr.revision = opts and opts.revision attr.limit = opts and opts.limit attr.sort_order = opts and opts.sort_order @@ -1689,6 +1690,9 @@ end end -- do +_M.rangeend = get_range_end +_M.nextkey = utils.next_key + local implemented_grpc_methods = { grant = true, diff --git a/t/v3/grpc/txn.t b/t/v3/grpc/txn.t index 776d1e27..f498eb95 100644 --- a/t/v3/grpc/txn.t +++ b/t/v3/grpc/txn.t @@ -88,6 +88,10 @@ __DATA__ check_res(data, err, "ddd") } } +--- request +GET /t +--- no_error_log +[error] --- response_body checked val as expect: abc checked val as expect: ddd @@ -118,6 +122,10 @@ checked val as expect: ddd check_res(data, err, "abc") } } +--- request +GET /t +--- no_error_log +[error] --- response_body checked val as expect: abc checked val as expect: abc @@ -145,6 +153,10 @@ checked val as expect: abc check_res(data, err, "aaa", 200) } } +--- request +GET /t +--- no_error_log +[error] --- response_body checked val as expect: aaa @@ -201,6 +213,10 @@ checked val as expect: aaa check_res(data, err, '{"k":"ddd"}') } } +--- request +GET /t +--- no_error_log +[error] --- response_body checked val as expect: {"k":"abc"} checked val as expect: {"k":"ddd"} diff --git a/t/v3/key.t b/t/v3/key.t index 79112178..062f4b71 100644 --- a/t/v3/key.t +++ b/t/v3/key.t @@ -113,6 +113,97 @@ GET /t ok +=== TEST 2-2: readdir(key) in batch +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local tab_nkeys = require "table.nkeys" + local etcd, err = require "resty.etcd" .new({protocol = "v3"}) + ngx.say('new') + check_res(etcd, err) + + local res, err = etcd:set("/batch/dir", "abc") + ngx.say('set1') + check_res(res, err) + + local res, err = etcd:set("/batch/dir/a", "abca") + ngx.say('set2') + check_res(res, err) + + local res, err = etcd:set("/batch/dir/b", "abcb") + ngx.say('set3') + check_res(res, err) + + + local start_key = '/batch/dir' + local last_key = start_key + local range_end = etcd.rangeend(start_key) + local data = {} + local kvs = {} + local times = 0 + while(1) do + times = times + 1 + res, err = etcd:readdir(last_key, { + timeout = 5000, + range_end = range_end, + revision = -1, + limit = 1, + sort_order = 'ASCEND', + sort_target = 'KEY', + }) + ngx.say('while') + check_res(res, err) + + table.insert(data, res.body) + + local last_kv_key = res.body.kvs[#res.body.kvs].key + last_key = etcd.nextkey(last_kv_key) + + if not res.body.more then + break + end + + if times > 4 then + ngx.say("too many times") + ngx.exit(500) + return + end + + end + + for _, body in ipairs(data) do + for _, kv in ipairs(body.kvs) do + table.insert(kvs, kv) + end + end + + if tab_nkeys(kvs) == 3 and times == 3 then + ngx.say("ok") + ngx.exit(200) + else + ngx.say(string.format("failed len(kvs):%s, times:%s", tab_nkeys(kvs), times)) + ngx.exit(500) + end + + } + } +--- request +GET /t +--- no_error_log +[error] +--- response_body +new +set1 +set2 +set3 +while +while +while +ok +--- timeout: 5 + + === TEST 3: watch(key) --- http_config eval: $::HttpConfig diff --git a/t/v3/tls.t b/t/v3/tls.t index c34cfde0..f4e2e118 100644 --- a/t/v3/tls.t +++ b/t/v3/tls.t @@ -103,7 +103,7 @@ GET /t --- no_error_log [error] --- response_body eval -qr/18: self signed certificate/ +qr/self-signed certificate/