fixed embedded JSON in JSON direct mode

supporting JSON objects in arrays

Author: eisber

cs/cs_json/Serializer/JsonRawStringListConverter.cs

@@ -41,8 +41,10 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
             var valueStringEnumerable = value as List<string>;
             if (valueStringEnumerable != null)
             {
+                writer.WriteStartArray();
                 foreach (var str in valueStringEnumerable)
                     writer.WriteRawValue(str);
+                writer.WriteEndArray();
                 return;
             }
 

cs/cs_json/Serializer/VowpalWabbitJsonBuilder.cs

@@ -165,6 +165,7 @@ internal void Parse(List<VowpalWabbitJsonParseContext> path, VowpalWabbitMarshal
         /// </summary>
         public void Parse(JsonReader reader, VowpalWabbitMarshalContext context, Namespace ns, List<VowpalWabbitJsonExtension> extensions = null)
         {
+            this.namespaceStrings.Clear();
             this.reader = reader;
             this.extensions = extensions;
 
@@ -345,9 +346,10 @@ private void ParseSpecialProperty(VowpalWabbitJsonParseContext context, string p
                     this.foundMulti = true;
 
                 // forward to handler
-                foreach (var extension in this.extensions)
-                    if (extension(this.extensionState, propertyName))
-                        return;
+                if (this.extensions != null)
+                    foreach (var extension in this.extensions)
+                        if (extension(this.extensionState, propertyName))
+                            return;
 
                 // if not handled, skip it
                 reader.Skip();
@@ -563,7 +565,7 @@ private void ParseFeature(List<VowpalWabbitJsonParseContext> path, VowpalWabbitM
                     // probably best to ignore?
                     break;
                 case JsonToken.StartArray:
-                    this.WrapInNamespace(path, featureName, lastContext => this.ParseFeatureArray(lastContext.Context, lastContext.Namespace));
+                    this.WrapInNamespace(path, featureName, lastContext => this.ParseFeatureArray(path));
                     break;
                 default:
                     throw new VowpalWabbitJsonException(this.reader, "Unexpected token " + reader.TokenType + " while deserializing primitive feature");
@@ -573,8 +575,11 @@ private void ParseFeature(List<VowpalWabbitJsonParseContext> path, VowpalWabbitM
         /// <summary>
         /// Expects: "1,2.2,3]" (excluding the leading [)
         /// </summary>
-        private void ParseFeatureArray(VowpalWabbitMarshalContext context, Namespace ns)
+        private void ParseFeatureArray(List<VowpalWabbitJsonParseContext> path)
         {
+            var context = path.Last().Context;
+            var ns = path.Last().Namespace;
+
             ulong index = 0;
 
             while (reader.Read())
@@ -587,6 +592,9 @@ private void ParseFeatureArray(VowpalWabbitMarshalContext context, Namespace ns)
                     case JsonToken.Float:
                         MarshalFloatFeature(context, ns, index, (float)(double)reader.Value);
                         break;
+                    case JsonToken.StartObject:
+                        ParseProperties(path);
+                        break;
                     case JsonToken.EndArray:
                         return;
                     default:

cs/unittest/TestJsonDirect.cs

@@ -274,27 +274,48 @@ public void TestJsonConvertibleMulti()
                     }
                 });
 
-                vw.Validate(new[] {
-                    "shared | Bar:5",
-                    " | Foo:1 |A test:1.2 |B bar:2 |D d:1.2 |E e"
-                },
-                    new JsonRawAdfString
-                    {
-                        Bar = 5,
-                        _multi = new[]
-                        {
-                                new JsonRawString
-                                {
-                                    Foo = 1,
-                                    Value = JsonConvert.SerializeObject(new { A = new { test = 1.2 }, B = new { bar= 2 } }),
-                                    Values = new []
+                var adf = new JsonRawString
+                {
+                    Foo = 1,
+                    Value = JsonConvert.SerializeObject(new { A = new { test = 1.2 }, B = new { bar = 2 } }),
+                    Values = new[]
                                     {
                                         JsonConvert.SerializeObject(new { D = new { d = 1.2 } }),
                                         JsonConvert.SerializeObject(new { E = new { e = true } }),
                                     }.ToList()
-                                }
-                        }
-                    });
+                };
+
+                var ctx = new JsonRawAdfString
+                {
+                    Bar = 5,
+                    _multi = new[] { adf }
+                };
+
+                vw.Validate(new[] {
+                    "shared | Bar:5",
+                    " | Foo:1 |A test:1.2 |B bar:2 |D d:1.2 |E e"
+                }, ctx);
+            }
+        }
+
+        [TestMethod]
+        [TestCategory("JSON")]
+        public void TestJsonConvertible()
+        {
+            using (var vw = new VowpalWabbitExampleValidator<JsonRawString>(new VowpalWabbitSettings("") { TypeInspector = JsonTypeInspector.Default }))
+            {
+                var adf = new JsonRawString
+                {
+                    Foo = 1,
+                    Value = JsonConvert.SerializeObject(new { A = new { test = 1.2 }, B = new { bar = 2 } }),
+                    Values = new[]
+                    {
+                        JsonConvert.SerializeObject(new { D = new { d = 1.2 } }),
+                        JsonConvert.SerializeObject(new { E = new { e = true } }),
+                    }.ToList()
+                };
+
+                vw.Validate(" | Foo:1 |A test:1.2 |B bar:2 |D d:1.2 |E e", adf);
             }
         }
 
@@ -312,7 +333,7 @@ public void TestJsonDictArray()
                             { "B", new float[] { 2, 3, 4.1f} }
                         }
                 };
-                
+
                 vw.Validate(" |A :1 :2 :3.1 |B :2 :3 :4.1", ex);
             }
         }

cs/version.props

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 <PropertyGroup>
-<VowpalWabbitAssemblyVersion>8.3.0.6</VowpalWabbitAssemblyVersion>
+<VowpalWabbitAssemblyVersion>8.3.0.7</VowpalWabbitAssemblyVersion>
 </PropertyGroup>
 </Project>

vowpalwabbit/parse_example_json.h

@@ -495,6 +495,14 @@ class ArrayState : public BaseState<audit>
 		return Float(ctx, (float)f);
 	}
 
+	BaseState<audit>* StartObject(Context<audit>& ctx)
+	{
+		// parse properties
+		ctx.PushNamespace(ctx.CurrentNamespace().name.c_str(), this);
+
+		return &ctx.default_state;
+	}
+
 	BaseState<audit>* EndArray(Context<audit>& ctx, rapidjson::SizeType elementCount)
 	{
 		return ctx.PopNamespace();