Varints

Author: dima-dencep

gradle.properties

@@ -1,3 +1,3 @@
 group=team.unnamed
-version=1.1.0
+version=1.1.1
 description=A lightweight, fast and efficient Molang lexer, parser, interpreter and compiler for Java 8+

src/main/java/team/unnamed/mocha/util/ExprBytesUtils.java

@@ -24,6 +24,7 @@
 package team.unnamed.mocha.util;
 
 import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufUtil;
 import org.jetbrains.annotations.NotNull;
 import team.unnamed.mocha.parser.ast.*;
 
@@ -68,7 +69,7 @@ public static void writeExpression(Expression expression, ByteBuf buf) {
     }
 
     public static <T> List<T> readList(ByteBuf buf, Function<ByteBuf, T> reader) {
-        int count = buf.readInt();
+        int count = VarIntUtils.readVarInt(buf);
         List<T> list = new ArrayList<>(count);
         for (int i = 0; i < count; i++) {
             list.add(reader.apply(buf));
@@ -77,7 +78,7 @@ public static <T> List<T> readList(ByteBuf buf, Function<ByteBuf, T> reader) {
     }
 
     public static <T> void writeList(ByteBuf buf, List<T> list, BiConsumer<T, ByteBuf> writer) {
-        buf.writeInt(list.size());
+        VarIntUtils.writeVarInt(buf, list.size());
         for (T entry : list) {
             writer.accept(entry, buf);
         }
@@ -92,20 +93,16 @@ public static <T> T getEnum(T[] values, ByteBuf buf) {
     }
 
     public static String readString(ByteBuf buf) {
-        int len = buf.readInt();
-        if (len <= 0) return null;
-        byte[] b = new byte[len];
-        buf.readBytes(b); //that is safe to use.
-        return new String(b, StandardCharsets.UTF_8);
+        int length = VarIntUtils.readVarInt(buf);
+        if (length <= 0) return null;
+        String str = buf.toString(buf.readerIndex(), length, StandardCharsets.UTF_8);
+        buf.skipBytes(length);
+        return str;
     }
 
     public static void writeString(ByteBuf buf, String str) {
-        if (str == null || str.isBlank()) { // Minor optimization to avoid writing empty lines
-            buf.writeInt(0);
-            return;
-        }
-        byte[] b = str.getBytes(StandardCharsets.UTF_8);
-        buf.writeInt(b.length);
-        buf.writeBytes(b);
+        int size = ByteBufUtil.utf8Bytes(str);
+        VarIntUtils.writeVarInt(buf, size);
+        buf.writeCharSequence(str, StandardCharsets.UTF_8);
     }
 }

src/main/java/team/unnamed/mocha/util/VarIntUtils.java

@@ -0,0 +1,105 @@
+/*
+ * This file is part of mocha, licensed under the MIT license
+ *
+ * Copyright (c) 2021-2025 Unnamed Team
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package team.unnamed.mocha.util;
+
+import io.netty.buffer.ByteBuf;
+
+public class VarIntUtils {
+    private static final int MAXIMUM_VARINT_SIZE = 5;
+
+    /**
+     * Reads a Minecraft-style VarInt from the specified {@code buf}.
+     *
+     * @param buf the buffer to read from
+     * @return the decoded VarInt
+     */
+    public static int readVarInt(ByteBuf buf) {
+        int readable = buf.readableBytes();
+        if (readable == 0) {
+            // special case for empty buffer
+            throw new IllegalStateException("Bad VarInt decoded");
+        }
+
+        // we can read at least one byte, and this should be a common case
+        int k = buf.readByte();
+        if ((k & 0x80) != 128) {
+            return k;
+        }
+
+        // in case decoding one byte was not enough, use a loop to decode up to the next 4 bytes
+        int maxRead = Math.min(MAXIMUM_VARINT_SIZE, readable);
+        int i = k & 0x7F;
+        for (int j = 1; j < maxRead; j++) {
+            k = buf.readByte();
+            i |= (k & 0x7F) << j * 7;
+            if ((k & 0x80) != 128) {
+                return i;
+            }
+        }
+        throw new IllegalStateException("Bad VarInt decoded");
+    }
+
+    /**
+     * Writes a Minecraft-style VarInt to the specified {@code buf}.
+     *
+     * @param buf   the buffer to read from
+     * @param value the integer to write
+     */
+    public static void writeVarInt(ByteBuf buf, int value) {
+        // Peel the one and two byte count cases explicitly as they are the most common VarInt sizes
+        // that the proxy will write, to improve inlining.
+        if ((value & (0xFFFFFFFF << 7)) == 0) {
+            buf.writeByte(value);
+        } else if ((value & (0xFFFFFFFF << 14)) == 0) {
+            int w = (value & 0x7F | 0x80) << 8 | (value >>> 7);
+            buf.writeShort(w);
+        } else {
+            writeVarIntFull(buf, value);
+        }
+    }
+
+    private static void writeVarIntFull(ByteBuf buf, int value) {
+        // See https://steinborn.me/posts/performance/how-fast-can-you-write-a-varint/
+
+        // This essentially is an unrolled version of the "traditional" VarInt encoding.
+        if ((value & (0xFFFFFFFF << 7)) == 0) {
+            buf.writeByte(value);
+        } else if ((value & (0xFFFFFFFF << 14)) == 0) {
+            int w = (value & 0x7F | 0x80) << 8 | (value >>> 7);
+            buf.writeShort(w);
+        } else if ((value & (0xFFFFFFFF << 21)) == 0) {
+            int w = (value & 0x7F | 0x80) << 16 | ((value >>> 7) & 0x7F | 0x80) << 8 | (value >>> 14);
+            buf.writeMedium(w);
+        } else if ((value & (0xFFFFFFFF << 28)) == 0) {
+            int w = (value & 0x7F | 0x80) << 24 | (((value >>> 7) & 0x7F | 0x80) << 16)
+                    | ((value >>> 14) & 0x7F | 0x80) << 8 | (value >>> 21);
+            buf.writeInt(w);
+        } else {
+            int w = (value & 0x7F | 0x80) << 24 | ((value >>> 7) & 0x7F | 0x80) << 16
+                    | ((value >>> 14) & 0x7F | 0x80) << 8 | ((value >>> 21) & 0x7F | 0x80);
+            buf.writeInt(w);
+            buf.writeByte(value >>> 28);
+        }
+    }
+}