feat: optimize key/value manifest (#364)

camellia-redis-proxy/camellia-redis-proxy-core/src/main/java/com/netease/nim/camellia/redis/proxy/upstream/local/storage/key/slot/KeyManifest.java

@@ -160,6 +160,9 @@ public SlotInfo expand(short slot) throws IOException {
             long fileId = slotInfo.fileId();
             long offset = slotInfo.offset();
             int capacity = slotInfo.capacity();
+            if (capacity * 2 <= 0) {
+                throw new IOException("slot capacity exceed");
+            }
             int bitsStep = capacity / LocalStorageConstants._64k;
             BitSet bits = fileBitsMap.get(fileId);
             int bitsStart = (int)(offset / LocalStorageConstants._64k);

camellia-redis-proxy/camellia-redis-proxy-core/src/main/java/com/netease/nim/camellia/redis/proxy/upstream/local/storage/value/block/ValueManifest.java

@@ -14,6 +14,7 @@
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import static com.netease.nim.camellia.redis.proxy.upstream.local.storage.constants.LocalStorageConstants.data_file_size;
 
@@ -35,7 +36,7 @@ public class ValueManifest implements IValueManifest {
     private final ConcurrentHashMap<Long, MappedByteBuffer> bitsMmp = new ConcurrentHashMap<>();
     private final ConcurrentHashMap<Long, MappedByteBuffer> slotsMmp = new ConcurrentHashMap<>();
 
-    private final ReentrantLock lock = new ReentrantLock();
+    private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
 
     public ValueManifest(String dir) {
         this.dir = dir;
@@ -203,7 +204,18 @@ private BlockLocation allocate0(long fileId) {
     }
 
     private long selectFileId(BlockType blockType, int index) throws IOException {
-        lock.lock();
+        lock.readLock().lock();
+        try {
+            List<Long> list = fileIdMap.get(blockType);
+            if (list != null && index < list.size()) {
+                return list.get(index);
+            }
+        } catch (Exception e) {
+            logger.error("select fileId error", e);
+        } finally {
+            lock.readLock().unlock();
+        }
+        lock.writeLock().lock();
         try {
             List<Long> list = fileIdMap.computeIfAbsent(blockType, k -> new ArrayList<>());
             if (index < list.size()) {
@@ -213,7 +225,7 @@ private long selectFileId(BlockType blockType, int index) throws IOException {
             list.add(fileId);
             return fileId;
         } finally {
-            lock.unlock();
+            lock.writeLock().unlock();
         }
     }
 
@@ -285,5 +297,7 @@ private void loadIndexFile(File file) throws IOException {
         FileChannel slotFileChannel = FileChannel.open(Paths.get(slotFile), StandardOpenOption.READ, StandardOpenOption.WRITE);
         MappedByteBuffer map = slotFileChannel.map(FileChannel.MapMode.READ_WRITE, 0, blockType.valueSlotManifestSize(data_file_size));
         slotsMmp.put(fileId, map);
+
+        logger.info("load index file = {}", file.getName());
     }
 }

camellia-redis-proxy/camellia-redis-proxy-core/src/test/java/com/netease/nim/camellia/redis/proxy/test/BitSetTest.java

@@ -0,0 +1,66 @@
+package com.netease.nim.camellia.redis.proxy.test;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.BitSet;
+
+import static com.netease.nim.camellia.redis.proxy.upstream.local.storage.constants.LocalStorageConstants._4k;
+import static com.netease.nim.camellia.redis.proxy.upstream.local.storage.constants.LocalStorageConstants.data_file_size;
+
+/**
+ * Created by caojiajun on 2025/1/14
+ */
+public class BitSetTest {
+
+    @Test
+    public void test() {
+        BitSet bitSet = new BitSet((int) (data_file_size / _4k));
+        for (int i=0; i<100; i++) {
+            bitSet.set(i, true);
+        }
+        for (int i=1000; i<2000; i++) {
+            bitSet.set(i, true);
+        }
+
+        long[] longArray = bitSet.toLongArray();
+
+        BitSet bitSet1 = BitSet.valueOf(longArray);
+        for (int i=0; i<100; i++) {
+            Assert.assertTrue(bitSet1.get(i));
+        }
+        for (int i=200; i<900; i++) {
+            Assert.assertFalse(bitSet1.get(i));
+        }
+        for (int i=1000; i<2000; i++) {
+            Assert.assertTrue(bitSet1.get(i));
+        }
+        for (int i=3000; i<5000; i++) {
+            Assert.assertFalse(bitSet1.get(i));
+        }
+
+        for (int i=6000; i<7000; i++) {
+            bitSet1.set(i, true);
+        }
+
+        long[] longArray1 = bitSet1.toLongArray();
+        BitSet bitSet2 = BitSet.valueOf(longArray1);
+
+        for (int i=0; i<100; i++) {
+            Assert.assertTrue(bitSet2.get(i));
+        }
+        for (int i=200; i<900; i++) {
+            Assert.assertFalse(bitSet2.get(i));
+        }
+        for (int i=1000; i<2000; i++) {
+            Assert.assertTrue(bitSet2.get(i));
+        }
+        for (int i=3000; i<5000; i++) {
+            Assert.assertFalse(bitSet2.get(i));
+        }
+        for (int i=6000; i<7000; i++) {
+            Assert.assertTrue(bitSet2.get(i));
+        }
+
+    }
+}