Fix potential segfault

If more than ~20 objects were being returned at once, Lua would
segfault because the default stack size is 20 and nobody was
resizing the stack.

Now we can return up to ~8,000 objects at once before
erroring out the function properly instead of segfaulting.

Also added test for segfault. All tests currently pass.

Author: mattsta

lua_cmsgpack.c

@@ -561,6 +561,16 @@ void mp_decode_to_lua_hash(lua_State *L, mp_cur *c, size_t len) {
  * a Lua type, that is left as the only result on the stack. */
 void mp_decode_to_lua_type(lua_State *L, mp_cur *c) {
     mp_cur_need(c,1);
+
+    /* If we return more than 18 elements, we must resize the stack to
+     * fit all our return values.  But, there is no way to
+     * determine how many objects a msgpack will unpack to up front, so
+     * we request a +1 larger stack on each iteration (noop if stack is
+     * big enough, and when stack does require resize it doubles in size) */
+    luaL_checkstack(L, 1,
+        "too many return values at once; "
+        "use unpack_one or unpack_limit instead.");
+
     switch(c->p[0]) {
     case 0xcc:  /* uint 8 */
         mp_cur_need(c,2);

test.lua

@@ -369,6 +369,10 @@ pack = cmsgpack.pack(a)
 test_pack("regression for issue #4 output matching",a,"82a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a178c0", "82a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a178c0a17905a17905a17905a17905a17905a17905a17905a17905")
 test_circular("regression for issue #4 circular",a)
 
+-- test unpacking malformed input without crashing.  This actually returns one integer value (the ASCII code)
+-- for each character in the string.  We don't care about the return value, just that we don't segfault.
+cmsgpack.unpack("82a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17")
+
 -- Tests from github.com/moteus
 test_circular("map with number keys", {[1] = {1,2,3}})
 test_circular("map with float keys", {[1.5] = {1,2,3}})