diff --git a/src/commands/cmd_bit.cc b/src/commands/cmd_bit.cc index 13be25f3134..26c00285233 100644 --- a/src/commands/cmd_bit.cc +++ b/src/commands/cmd_bit.cc @@ -225,6 +225,14 @@ class CommandBitOp : public Commander { op_flag_ = kBitOpXor; else if (opname == "not") op_flag_ = kBitOpNot; + else if (opname == "diff") + op_flag_ = kBitOpDiff; + else if (opname == "diff1") + op_flag_ = kBitOpDiff1; + else if (opname == "andor") + op_flag_ = kBitOpAndOr; + else if (opname == "one") + op_flag_ = kBitOpOne; else return {Status::RedisInvalidCmd, errInvalidSyntax}; if (op_flag_ == kBitOpNot && args.size() != 4) { diff --git a/src/types/redis_bitmap.cc b/src/types/redis_bitmap.cc index 00c8d1b3fde..f050b3d8403 100644 --- a/src/types/redis_bitmap.cc +++ b/src/types/redis_bitmap.cc @@ -554,7 +554,9 @@ rocksdb::Status Bitmap::BitOp(engine::Context &ctx, BitOpFlags op_flag, const st for (uint64_t i = 0; i < frag_numkeys; i++) { lp[i] = reinterpret_cast(fragments[i].data()); } - memcpy(frag_res.get(), fragments[0].data(), frag_minlen); + if (op_flag != kBitOpDiff && op_flag != kBitOpDiff1 && op_flag != kBitOpAndOr) { + memcpy(frag_res.get(), fragments[0].data(), frag_minlen); + } auto apply_fast_path_op = [&](auto op) { // Note: kBitOpNot cannot use this op, it only applying // to kBitOpAnd, kBitOpOr, kBitOpXor. @@ -589,12 +591,94 @@ rocksdb::Status Bitmap::BitOp(engine::Context &ctx, BitOpFlags op_flag, const st j += sizeof(uint64_t) * 4; frag_minlen -= sizeof(uint64_t) * 4; } + } else if (op_flag == kBitOpDiff || op_flag == kBitOpDiff1 || op_flag == kBitOpAndOr) { + size_t processed = 0; + size_t k = 0; + + while (frag_minlen >= sizeof(uint64_t) * 4) { + for (uint64_t i = 1; i < frag_numkeys; i++) { + lres[0] |= lp[i][k + 0]; + lres[1] |= lp[i][k + 1]; + lres[2] |= lp[i][k + 2]; + lres[3] |= lp[i][k + 3]; + } + k += 4; + lres += 4; + j += sizeof(uint64_t) * 4; + frag_minlen -= sizeof(uint64_t) * 4; + processed += sizeof(uint64_t) * 4; + } + + lres = reinterpret_cast(frag_res.get()); + auto *first_key = reinterpret_cast(fragments[0].data()); + switch (op_flag) { + case kBitOpDiff: + for (uint64_t i = 0; i < processed; i += sizeof(uint64_t) * 4) { + lres[0] = (first_key[0] & ~lres[0]); + lres[1] = (first_key[1] & ~lres[1]); + lres[2] = (first_key[2] & ~lres[2]); + lres[3] = (first_key[3] & ~lres[3]); + lres += 4; + first_key += 4; + } + break; + case kBitOpDiff1: + for (uint64_t i = 0; i < processed; i += sizeof(uint64_t) * 4) { + lres[0] = (~first_key[0] & lres[0]); + lres[1] = (~first_key[1] & lres[1]); + lres[2] = (~first_key[2] & lres[2]); + lres[3] = (~first_key[3] & lres[3]); + lres += 4; + first_key += 4; + } + break; + case kBitOpAndOr: + for (uint64_t i = 0; i < processed; i += sizeof(uint64_t) * 4) { + lres[0] = (first_key[0] & lres[0]); + lres[1] = (first_key[1] & lres[1]); + lres[2] = (first_key[2] & lres[2]); + lres[3] = (first_key[3] & lres[3]); + lres += 4; + first_key += 4; + } + break; + } + } else if (op_flag == kBitOpOne) { + uint64_t lcommon_bits[4]; + size_t k = 0; + + while (frag_minlen >= sizeof(uint64_t) * 4) { + memset(lcommon_bits, 0, sizeof(lcommon_bits)); + + for (size_t i = 1; i < frag_numkeys; i++) { + lcommon_bits[0] |= (lres[0] & lp[i][k + 0]); + lcommon_bits[1] |= (lres[1] & lp[i][k + 1]); + lcommon_bits[2] |= (lres[2] & lp[i][k + 2]); + lcommon_bits[3] |= (lres[3] & lp[i][k + 3]); + + lres[0] ^= lp[i][k + 0]; + lres[1] ^= lp[i][k + 1]; + lres[2] ^= lp[i][k + 2]; + lres[3] ^= lp[i][k + 3]; + } + + lres[0] &= ~lcommon_bits[0]; + lres[1] &= ~lcommon_bits[1]; + lres[2] &= ~lcommon_bits[2]; + lres[3] &= ~lcommon_bits[3]; + + k += 4; + lres += 4; + j += sizeof(uint64_t) * 4; + frag_minlen -= sizeof(uint64_t) * 4; + } } } #endif uint8_t output = 0, byte = 0; for (; j < frag_maxlen; j++) { + uint8_t disjunction = 0, common_bits = 0; output = (fragments[0].size() <= j) ? 0 : fragments[0][j]; if (op_flag == kBitOpNot) output = ~output; for (uint64_t i = 1; i < frag_numkeys; i++) { @@ -609,11 +693,34 @@ rocksdb::Status Bitmap::BitOp(engine::Context &ctx, BitOpFlags op_flag, const st case kBitOpXor: output ^= byte; break; + case kBitOpDiff: + case kBitOpDiff1: + case kBitOpAndOr: + disjunction |= byte; + break; + case kBitOpOne: + common_bits |= (output & byte); + output ^= byte; + output &= ~common_bits; + break; default: break; } } - frag_res[j] = output; + switch (op_flag) { + case kBitOpDiff: + frag_res[j] = (output & ~disjunction); + break; + case kBitOpDiff1: + frag_res[j] = (~output & disjunction); + break; + case kBitOpAndOr: + frag_res[j] = (output & disjunction); + break; + default: + frag_res[j] = output; + break; + } } if (op_flag == kBitOpNot) { diff --git a/src/types/redis_bitmap.h b/src/types/redis_bitmap.h index 32a53cc72ab..6fa0cc16233 100644 --- a/src/types/redis_bitmap.h +++ b/src/types/redis_bitmap.h @@ -34,6 +34,10 @@ enum BitOpFlags { kBitOpOr, kBitOpXor, kBitOpNot, + kBitOpDiff, + kBitOpDiff1, + kBitOpAndOr, + kBitOpOne, }; namespace redis { diff --git a/tests/gocase/go.mod b/tests/gocase/go.mod index d83278ea84d..e82e91d7247 100644 --- a/tests/gocase/go.mod +++ b/tests/gocase/go.mod @@ -3,7 +3,7 @@ module github.com/apache/kvrocks/tests/gocase go 1.23.0 require ( - github.com/redis/go-redis/v9 v9.9.0 + github.com/redis/go-redis/v9 v9.12.0 github.com/shirou/gopsutil/v4 v4.25.4 github.com/stretchr/testify v1.10.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 diff --git a/tests/gocase/go.sum b/tests/gocase/go.sum index eb82efcee2f..620ba443c4f 100644 --- a/tests/gocase/go.sum +++ b/tests/gocase/go.sum @@ -8,8 +8,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= -github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -17,20 +15,14 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/lufia/plan9stats v0.0.0-20250303091104-876f3ea5145d h1:fjMbDVUGsMQiVZnSQsmouYJvMdwsGiDipOZoN66v844= -github.com/lufia/plan9stats v0.0.0-20250303091104-876f3ea5145d/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= -github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= -github.com/redis/go-redis/v9 v9.9.0 h1:URbPQ4xVQSQhZ27WMQVmZSo3uT3pL+4IdHVcYq2nVfM= -github.com/redis/go-redis/v9 v9.9.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= -github.com/shirou/gopsutil/v4 v4.25.2 h1:NMscG3l2CqtWFS86kj3vP7soOczqrQYIEhO/pMvvQkk= -github.com/shirou/gopsutil/v4 v4.25.2/go.mod h1:34gBYJzyqCDT11b6bMHP0XCvWeU3J61XRT7a2EmCRTA= +github.com/redis/go-redis/v9 v9.12.0 h1:XlVPGlflh4nxfhsNXPA8Qp6EmEfTo0rp8oaBzPipXnU= +github.com/redis/go-redis/v9 v9.12.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/shirou/gopsutil/v4 v4.25.4 h1:cdtFO363VEOOFrUCjZRh4XVJkb548lyF0q0uTeMqYPw= github.com/shirou/gopsutil/v4 v4.25.4/go.mod h1:xbuxyoZj+UsgnZrENu3lQivsngRR5BdjbJwf2fv4szA= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -41,19 +33,13 @@ github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfj github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= -golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/tests/gocase/unit/type/bitmap/bitmap_test.go b/tests/gocase/unit/type/bitmap/bitmap_test.go index 6211ae3c836..2137f98e117 100644 --- a/tests/gocase/unit/type/bitmap/bitmap_test.go +++ b/tests/gocase/unit/type/bitmap/bitmap_test.go @@ -271,6 +271,17 @@ func TestBitmap(t *testing.T) { require.EqualValues(t, []string{"\x55\xff\x00\xaa"}, GetBitmap(t, rdb, ctx, "s")) }) + t.Run("BITOP DIFF|DIFF1|ANDOR|ONE don't change the string with single input key", func(t *testing.T) { + Set2SetBit(t, rdb, ctx, "a", []byte("\x01\x02\xff")) + Set2SetBit(t, rdb, ctx, "b", []byte("\x01\x02\xff")) + Set2SetBit(t, rdb, ctx, "c", []byte("\x01\x02\xff")) + require.NoError(t, rdb.BitOpDiff(ctx, "res1", "a", "b", "c").Err()) + require.NoError(t, rdb.BitOpDiff1(ctx, "res2", "a", "b", "c").Err()) + require.NoError(t, rdb.BitOpAndOr(ctx, "res3", "a", "b", "c").Err()) + require.NoError(t, rdb.BitOpOne(ctx, "res4", "a", "b", "c").Err()) + require.EqualValues(t, []string{"\x00\x00\x00", "\x00\x01\x00", "\x01\x02\xff", "\x00\x00\x00"}, GetBitmap(t, rdb, ctx, "res1", "res2", "res3", "res4")) + }) + t.Run("BITOP AND|OR|XOR don't change the string with single input key", func(t *testing.T) { Set2SetBit(t, rdb, ctx, "a", []byte("\x01\x02\xff")) require.NoError(t, rdb.BitOpAnd(ctx, "res1", "a").Err())