UTF8 encoding support.

Author: yhirose

Makefile

@@ -1,7 +1,7 @@
-linenoise_example: linenoise.h linenoise.c
+linenoise_example: linenoise.h linenoise.c encodings/utf8.h encodings/utf8.c
 
-linenoise_example: linenoise.c example.c
-	$(CC) -Wall -W -Os -g -o linenoise_example linenoise.c example.c
+linenoise_example: linenoise.c example.c encodings/utf8.c
+	$(CC) -Wall -W -Os -g -o linenoise_example linenoise.c example.c encodings/utf8.c
 
 clean:
 	rm -f linenoise_example

README.markdown

@@ -1,4 +1,4 @@
-# Linenoise
+# Linenoise UTF-8
 
 A minimal, zero-config, BSD licensed, readline replacement used in Redis,
 MongoDB, and Android.

encodings/tools/EastAsianWidth.txt

@@ -0,0 +1,5 @@
+ARGF.each do |line|
+    if line =~ /^(.*);.*;Mn;/
+        puts "0x#{$1},"
+    end
+end

encodings/tools/UnicodeData.txt

@@ -0,0 +1,15 @@
+ranges = []
+ARGF.each do |line|
+    if line =~ /^(.*?)(?:\.\.(.*?))?;[FW]\s+# .*$/
+        first = $1.to_i(16)
+        last = $2 ? $2.to_i(16) : first
+        if ranges.last && ranges.last[:l] + 1 == first
+            ranges.last[:l] = last
+        else
+            ranges.push({ f: first, l: last })
+        end
+    end
+end
+ranges.each do |range|
+    puts "{ #{'0x%X' % range[:f]}, #{'0x%X' % range[:l]} },"
+end

encodings/tools/generate_combining_char_table.rb

@@ -0,0 +1,55 @@
+/* encodings/utf8.h -- VERSION 1.0
+ *
+ * Guerrilla line editing library against the idea that a line editing lib
+ * needs to be 20,000 lines of C code.
+ *
+ * See linenoise.c for more information.
+ *
+ * ------------------------------------------------------------------------
+ *
+ * Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *  *  Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *
+ *  *  Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __LINENOISE_ENCODINGS_UTF8_H
+#define __LINENOISE_ENCODINGS_UTF8_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+size_t linenoiseUtf8PrevCharLen(const char* buf, size_t buf_len, size_t pos, size_t *col_len);
+size_t linenoiseUtf8NextCharLen(const char* buf, size_t buf_len, size_t pos, size_t *col_len);
+size_t linenoiseUtf8ReadCode(int fd, char* buf, size_t buf_len, int* cp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __LINENOISE_ENCODINGS_UTF8_H */
+

encodings/tools/generate_wide_char_table.rb

@@ -3,11 +3,22 @@
 #include <string.h>
 #include "linenoise.h"
 
+#define UTF8
+
+#ifdef UTF8
+#include "encodings/utf8.h"
+#endif
 
 void completion(const char *buf, linenoiseCompletions *lc) {
     if (buf[0] == 'h') {
+#ifdef UTF8
+        linenoiseAddCompletion(lc,"hello こんにちは");
+        linenoiseAddCompletion(lc,"hello こんにちは there");
+        linenoiseAddCompletion(lc,"hello こんにちは 👨‍💻");
+#else
         linenoiseAddCompletion(lc,"hello");
         linenoiseAddCompletion(lc,"hello there");
+#endif
     }
 }
 
@@ -17,6 +28,11 @@ char *hints(const char *buf, int *color, int *bold) {
         *bold = 0;
         return " World";
     }
+    if (!strcasecmp(buf,"こんにちは")) {
+        *color = 35;
+        *bold = 0;
+        return " 世界";
+    }
     return NULL;
 }
 
@@ -40,6 +56,13 @@ int main(int argc, char **argv) {
         }
     }
 
+#ifdef UTF8
+    linenoiseSetEncodingFunctions(
+        linenoiseUtf8PrevCharLen,
+        linenoiseUtf8NextCharLen,
+        linenoiseUtf8ReadCode);
+#endif
+
     /* Set the completion callback. This will be called every time the
      * user uses the <tab> key. */
     linenoiseSetCompletionCallback(completion);
@@ -55,8 +78,11 @@ int main(int argc, char **argv) {
      *
      * The typed string is returned as a malloc() allocated string by
      * linenoise, so the user needs to free() it. */
-    
-    while((line = linenoise("hello> ")) != NULL) {
+#ifdef UTF8
+    while((line = linenoise("\033[32mこんにちは\x1b[0m> ")) != NULL) {
+#else
+    while((line = linenoise("\033[32mhello\x1b[0m> ")) != NULL) {
+#endif
         /* Do something with the string. */
         if (line[0] != '\0' && line[0] != '/') {
             printf("echo: '%s'\n", line);