Merge pull request #15 from secretGeek/unicode-fix

Unicode fix

.ok

@@ -10,6 +10,7 @@ _OK_PROMPT="% "; _OK_VERBOSE=0; _OK_PROMPT_DEFAULT=0; _OK_COMMENT_ALIGN=0 # Show
 # Tests arguments passing (you can pass arguments after <number>, both at the bash-prompt and the ok-prompt)
 echo "Passed arguments: 1:[$1], 2:[$2], 3:[$3], 4+:[${*:4}] (nr args: $#)" # Comment-color starts too early; clearly a bug (so better
 echo "All passed arguments (no comment on this line): [$*]"
+cd test
 
 ok help --verbose # Show help page of 🆗, including environment variables
 set | grep "^_OK_" # Show all set environment variables, used with ok-bash

demo/fmt/.ok

@@ -1,5 +1,7 @@
+termtosvg termtosvg_demo_fmt.svg -g 15x75 -t window_frame_js
 export PS1='> ' # Hide computer details in demo
 export _OK_VERBOSE=0 # No echoing the command
+printf '\e[8;15;75t\e[2J\e[;H' # Resize current window + clear screen
 # Demo of comment indentation
 ok list --comment_align 0 # No indentation
 ok l --comment_align 1 # Group adjecent indentations

ok-show.py

@@ -143,15 +143,21 @@ def main():
         print('comment_align:', args.comment_align)
         print('terminal_width:', args.terminal_width)
 
-    #setup UTF-8
-    if sys.stdin.encoding != 'UTF-8':
-        sys.stdin = codecs.getreader('utf8')(sys.stdin, 'strict')
-    if sys.stdout.encoding != 'UTF-8':
-        sys.stdout = codecs.getwriter('utf-8')(sys.stdout, 'strict')
-    if sys.stderr.encoding != 'UTF-8':
-        sys.stderr = codecs.getwriter('utf-8')(sys.stderr, 'strict')
-    # prepare
-    lines = sys.stdin.readlines()
+    # prepare (read stdin parse, transform, and calculate stuff)
+    # Unicode: best to ignore other encodings? SO doesn't seem to give good advice
+    # See https://stackoverflow.com/q/2737966/56
+    try:
+        lines = sys.stdin.readlines()
+    except UnicodeDecodeError as err:
+        print('ERROR: UTF-8 (unicode) should be used as sole encoding for .ok-files', file=sys.stderr)
+        if args.verbose > 1:
+            print('UnicodeDecodeError exception properties (error on: %s):' % err.object[err.start:err.end], file=sys.stderr)
+            print('* encoding: %s' % err.encoding, file=sys.stderr)
+            print('* reason__: %s' % err.reason,   file=sys.stderr)
+            print('* object__: %s' % err.object,   file=sys.stderr)
+            print('* start___: %s' % err.start,    file=sys.stderr)
+            print('* end_____: %s' % err.end,      file=sys.stderr)
+        exit(1)
     p_lines = parse_lines(lines)
     cmd_lines = [pl.line_nr for pl in p_lines if pl.line_nr]
     nr_positions_line_nr = len(str(max(cmd_lines))) if len(cmd_lines)>0 else 0

test/.ok

@@ -0,0 +1,13 @@
+# Control what python to use
+_OK__PATH_TO_PYTHON=$(command -v python) # set python2
+_OK__PATH_TO_PYTHON=$(command -v python3) # set python3
+echo "Path to python: '$_OK__PATH_TO_PYTHON'" # check path
+$_OK__PATH_TO_PYTHON --version # check python version
+# Control encoding or not
+echo "Python IO Encoding: ${PYTHONIOENCODING:--=not-set=-}"
+export PYTHONIOENCODING="UTF-8" # only python3 will choke on non-UTF-8, when this is set
+unset PYTHONIOENCODING # nonetheless, python will treat everything as ASCII
+# Go to the test cases
+cd iso_latin_1
+cd mac_os_roman
+cd utf_8

test/iso_latin_1/.ok

@@ -0,0 +1,8 @@
+# This file is encoded with ISO-8859-1 (or Western ISO Latin 1)
+export PYTHONIOENCODING="latin_1"
+echo "A0-AF: ��������������� (including NO-BREAK SPACE and SOFT HYPHEN)"
+echo "B0-BF: ����������������"
+echo "C0-CF: ����������������"
+echo "D0-DF: ���������������� (look at that MULTIPLICATION SIGN)"
+echo "E0-EF: ����������������"
+echo "F0-FF: ���������������� (the DIVISION SIGN is placed in the middle)"

test/mac_os_roman/.ok

@@ -0,0 +1,3 @@
+# only one comment
+export PYTHONIOENCODING="mac_roman"
+echo 'H�, �r �s g��n b�l �p de TV!'

test/utf_8/.ok

@@ -0,0 +1,4 @@
+# A full Unicode UTF-8 encoded .ok-file
+export PYTHONIOENCODING="UTF-8" 
+echo "Hi ㋛, how is it going?"
+echo "Ⓘ  ⓐ ⓜ   ⓦ ⓔ ⓛ ⓛ ,   🅣 🅗 🅐 🅽 🅚   🆈 🅾 🆄 ❕"