Add UTF8 support

This is the second half of the PR that I'm merging in from upstream.
It adds UTF8 13.0 support by including an extra library to get the
char width of those characters (the visual width that is). During
testing things seem to work quite well. I'm impressed.

Author: https://github.com/yhirose
PR: antirez/linenoise#187

Author: apainintheneck

Makefile

@@ -7,11 +7,14 @@ default: extension lint test
 example: extension
 	crystal run example/example.cr
 
-extension: src/lib/linenoise.o
+extension: src/lib/linenoise.o src/lib/utf8.o
 
 src/lib/linenoise.o: ext/linenoise.c ext/linenoise.h
 	$(CC) -o $@ $< $(CFLAGS)
 
+src/lib/utf8.o: ext/utf8.c ext/utf8.h
+	$(CC) -o $@ $< $(CFLAGS)
+
 check: extension
 	crystal build --no-codegen -o example_program example/example.cr
 
@@ -35,3 +38,4 @@ test: specs expect
 
 clean:
 	rm -f src/lib/linenoise.o
+	rm -f src/lib/utf8.o

ext/linenoise.c

@@ -119,6 +119,7 @@
 #include <sys/types.h>
 #include <sys/ioctl.h>
 #include "linenoise.h"
+#include "utf8.h"
 
 #define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
 #define LINENOISE_MAX_LINE 4096
@@ -140,7 +141,7 @@ static int history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
 static int history_len = 0;
 static char **history = NULL;
 
-enum KEY_ACTION{
+enum KEY_ACTION {
 	KEY_NULL = 0,	    /* NULL */
 	CTRL_A = 1,         /* Ctrl+a */
 	CTRL_B = 2,         /* Ctrl-b */
@@ -189,41 +190,52 @@ FILE *lndebug_fp = NULL;
 #endif
 
 /* ========================== Encoding functions ============================= */
+
+typedef size_t (linenoisePrevCharLen)(const char *buf, size_t buf_len, size_t pos, size_t *col_len);
+typedef size_t (linenoiseNextCharLen)(const char *buf, size_t buf_len, size_t pos, size_t *col_len);
+typedef size_t (linenoiseReadCode)(int fd, char *buf, size_t buf_len, int* c);
+
+/* Set default encoding functions */
+static linenoisePrevCharLen *prevCharLen = linenoiseUtf8PrevCharLen;
+static linenoiseNextCharLen *nextCharLen = linenoiseUtf8NextCharLen;
+static linenoiseReadCode *readCode = linenoiseUtf8ReadCode;
+
 /* Get byte length and column length of the previous character */
-static size_t defaultPrevCharLen(const char *buf, size_t buf_len, size_t pos, size_t *col_len) {
+static size_t asciiPrevCharLen(const char *buf, size_t buf_len, size_t pos, size_t *col_len) {
     UNUSED(buf); UNUSED(buf_len); UNUSED(pos);
     if (col_len != NULL) *col_len = 1;
     return 1;
 }
 
 /* Get byte length and column length of the next character */
-static size_t defaultNextCharLen(const char *buf, size_t buf_len, size_t pos, size_t *col_len) {
+static size_t asciiNextCharLen(const char *buf, size_t buf_len, size_t pos, size_t *col_len) {
     UNUSED(buf); UNUSED(buf_len); UNUSED(pos);
     if (col_len != NULL) *col_len = 1;
     return 1;
 }
 
 /* Read bytes of the next character */
-static size_t defaultReadCode(int fd, char *buf, size_t buf_len, int* c) {
+static size_t asciiReadCode(int fd, char *buf, size_t buf_len, int* c) {
     if (buf_len < 1) return -1;
     int nread = read(fd,&buf[0],1);
     if (nread == 1) *c = buf[0];
     return nread;
 }
 
-/* Set default encoding functions */
-static linenoisePrevCharLen *prevCharLen = defaultPrevCharLen;
-static linenoiseNextCharLen *nextCharLen = defaultNextCharLen;
-static linenoiseReadCode *readCode = defaultReadCode;
-
-/* Set used defined encoding functions */
-void linenoiseSetEncodingFunctions(
-    linenoisePrevCharLen *prevCharLenFunc,
-    linenoiseNextCharLen *nextCharLenFunc,
-    linenoiseReadCode *readCodeFunc) {
-    prevCharLen = prevCharLenFunc;
-    nextCharLen = nextCharLenFunc;
-    readCode = readCodeFunc;
+/* Set string encoding functions */
+void linenoiseSetEncoding(enum STRING_ENCODING encoding) {
+    switch(encoding) {
+        case ASCII:
+            prevCharLen = asciiPrevCharLen;
+            nextCharLen = asciiNextCharLen;
+            readCode = asciiReadCode;
+            break;
+        case UTF8:
+            prevCharLen = linenoiseUtf8PrevCharLen;
+            nextCharLen = linenoiseUtf8NextCharLen;
+            readCode = linenoiseUtf8ReadCode;
+            break;
+    }
 }
 
 /* Get column length from begining of buffer to current byte position */

ext/linenoise.h

@@ -106,14 +106,13 @@ void linenoisePrintKeyCodes(void);
 void linenoiseMaskModeEnable(void);
 void linenoiseMaskModeDisable(void);
 
-typedef size_t (linenoisePrevCharLen)(const char *buf, size_t buf_len, size_t pos, size_t *col_len);
-typedef size_t (linenoiseNextCharLen)(const char *buf, size_t buf_len, size_t pos, size_t *col_len);
-typedef size_t (linenoiseReadCode)(int fd, char *buf, size_t buf_len, int* c);
+/* String encoding. */
+enum STRING_ENCODING {
+    ASCII,
+    UTF8 // default
+};
 
- void linenoiseSetEncodingFunctions(
-     linenoisePrevCharLen *prevCharLenFunc,
-     linenoiseNextCharLen *nextCharLenFunc,
-     linenoiseReadCode *readCodeFunc);
+void linenoiseSetEncoding(enum STRING_ENCODING);
 
 #ifdef __cplusplus
 }