UTF8 encoding support. (Unicode 16.0.0)

Author: yhirose

Makefile

@@ -1,7 +1,7 @@
-linenoise_example: linenoise.h linenoise.c
+linenoise_example: linenoise.h linenoise.c encodings/utf8.h encodings/utf8.c
 
-linenoise_example: linenoise.c example.c
-	$(CC) -Wall -W -Os -g -o linenoise_example linenoise.c example.c
+linenoise_example: linenoise.c example.c encodings/utf8.c
+	$(CC) -Wall -W -Os -g -o linenoise_example linenoise.c example.c encodings/utf8.c
 
 clean:
 	rm -f linenoise_example

README.markdown

@@ -1,4 +1,4 @@
-# Linenoise
+# Linenoise UTF-8
 
 A minimal, zero-config, BSD licensed, readline replacement used in Redis,
 MongoDB, Android and many other projects.

encodings/tools/EastAsianWidth.txt

@@ -0,0 +1,5 @@
+ARGF.each do |line|
+    if line =~ /^(.*);.*;Mn;/
+        puts "0x#{$1},"
+    end
+end

encodings/tools/UnicodeData.txt

@@ -0,0 +1,15 @@
+ranges = []
+ARGF.each do |line|
+    if line =~ /^(.*?)(?:\.\.(.*?))?\s+;\s+[FW]\s+# .*$/
+        first = $1.to_i(16)
+        last = $2 ? $2.to_i(16) : first
+        if ranges.last && ranges.last[:l] + 1 == first
+            ranges.last[:l] = last
+        else
+            ranges.push({ f: first, l: last })
+        end
+    end
+end
+ranges.each do |range|
+    puts "{ #{'0x%X' % range[:f]}, #{'0x%X' % range[:l]} },"
+end

encodings/tools/generate_combining_char_table.rb

@@ -0,0 +1,55 @@
+/* encodings/utf8.h -- VERSION 1.0
+ *
+ * Guerrilla line editing library against the idea that a line editing lib
+ * needs to be 20,000 lines of C code.
+ *
+ * See linenoise.c for more information.
+ *
+ * ------------------------------------------------------------------------
+ *
+ * Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *  *  Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *
+ *  *  Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __LINENOISE_ENCODINGS_UTF8_H
+#define __LINENOISE_ENCODINGS_UTF8_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+size_t linenoiseUtf8PrevCharLen(const char* buf, size_t buf_len, size_t pos, size_t *col_len);
+size_t linenoiseUtf8NextCharLen(const char* buf, size_t buf_len, size_t pos, size_t *col_len);
+size_t linenoiseUtf8ReadCode(int fd, char* buf, size_t buf_len, int* cp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __LINENOISE_ENCODINGS_UTF8_H */
+

encodings/tools/generate_wide_char_table.rb

@@ -4,10 +4,22 @@
 #include <sys/select.h>
 #include "linenoise.h"
 
+#define UTF8
+
+#ifdef UTF8
+#include "encodings/utf8.h"
+#endif
+
 void completion(const char *buf, linenoiseCompletions *lc) {
     if (buf[0] == 'h') {
+#ifdef UTF8
+        linenoiseAddCompletion(lc,"hello こんにちは");
+        linenoiseAddCompletion(lc,"hello こんにちは there");
+        linenoiseAddCompletion(lc,"hello こんにちは 👨‍💻");
+#else
         linenoiseAddCompletion(lc,"hello");
         linenoiseAddCompletion(lc,"hello there");
+#endif
     }
 }
 
@@ -17,6 +29,11 @@ char *hints(const char *buf, int *color, int *bold) {
         *bold = 0;
         return " World";
     }
+    if (!strcasecmp(buf,"こんにちは")) {
+        *color = 35;
+        *bold = 0;
+        return " 世界";
+    }
     return NULL;
 }
 
@@ -43,6 +60,13 @@ int main(int argc, char **argv) {
         }
     }
 
+#ifdef UTF8
+    linenoiseSetEncodingFunctions(
+        linenoiseUtf8PrevCharLen,
+        linenoiseUtf8NextCharLen,
+        linenoiseUtf8ReadCode);
+#endif
+
     /* Set the completion callback. This will be called every time the
      * user uses the <tab> key. */
     linenoiseSetCompletionCallback(completion);
@@ -61,7 +85,11 @@ int main(int argc, char **argv) {
 
     while(1) {
         if (!async) {
-            line = linenoise("hello> ");
+#ifdef UTF8
+            line = linenoise("\033[32mこんにちは\x1b[0m> ");
+#else
+            line = linenoise("\033[32mhello\x1b[0m> ");
+#endif
             if (line == NULL) break;
         } else {
             /* Asynchronous mode using the multiplexing API: wait for