Make hashing stable w.r.t. indentation settings and property ordering. (#6476)

* Make hashing stable w.r.t. indentation settings and property ordering.

* Add indentation invariance test

Author: eiriktsarpalis

src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.cs

@@ -6,8 +6,10 @@
 using System.Diagnostics;
 #endif
 using System.IO;
+using System.Linq;
 using System.Security.Cryptography;
 using System.Text.Json;
+using System.Text.Json.Nodes;
 using System.Text.Json.Serialization.Metadata;
 #if NET
 using System.Threading;
@@ -112,7 +114,9 @@ public static string HashDataToString(ReadOnlySpan<object?> values, JsonSerializ
         {
             foreach (object? value in values)
             {
-                JsonSerializer.Serialize(stream, value, jti);
+                JsonNode? jsonNode = JsonSerializer.SerializeToNode(value, jti);
+                NormalizeJsonNode(jsonNode);
+                JsonSerializer.Serialize(stream, jsonNode, JsonContextNoIndentation.Default.JsonNode!);
             }
 
             stream.GetHashAndReset(hashData);
@@ -130,7 +134,9 @@ public static string HashDataToString(ReadOnlySpan<object?> values, JsonSerializ
         MemoryStream stream = new();
         foreach (object? value in values)
         {
-            JsonSerializer.Serialize(stream, value, jti);
+            JsonNode? jsonNode = JsonSerializer.SerializeToNode(value, jti);
+            NormalizeJsonNode(jsonNode);
+            JsonSerializer.Serialize(stream, jsonNode, JsonContextNoIndentation.Default.JsonNode!);
         }
 
         using var hashAlgorithm = SHA384.Create();
@@ -156,6 +162,31 @@ static string ConvertToHexString(ReadOnlySpan<byte> hashData)
             return new string(chars);
         }
 #endif
+        static void NormalizeJsonNode(JsonNode? node)
+        {
+            switch (node)
+            {
+                case JsonArray array:
+                    foreach (JsonNode? item in array)
+                    {
+                        NormalizeJsonNode(item);
+                    }
+
+                    break;
+
+                case JsonObject obj:
+                    var entries = obj.OrderBy(e => e.Key, StringComparer.Ordinal).ToArray();
+                    obj.Clear();
+
+                    foreach (var entry in entries)
+                    {
+                        obj.Add(entry.Key, entry.Value);
+                        NormalizeJsonNode(entry.Value);
+                    }
+
+                    break;
+            }
+        }
     }
 
     private static void AddAIContentTypeCore(JsonSerializerOptions options, Type contentType, string typeDiscriminatorId)

src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs

@@ -123,7 +123,7 @@ protected override async Task WriteCacheStreamingAsync(string key, IReadOnlyList
     protected override string GetCacheKey(IEnumerable<ChatMessage> messages, ChatOptions? options, params ReadOnlySpan<object?> additionalValues)
     {
         // Bump the cache version to invalidate existing caches if the serialization format changes in a breaking way.
-        const int CacheVersion = 1;
+        const int CacheVersion = 2;
 
         return AIJsonUtilities.HashDataToString([CacheVersion, messages, options, .. additionalValues], _jsonSerializerOptions);
     }

test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs

@@ -557,13 +557,30 @@ public static void HashData_Idempotent()
             string key2 = AIJsonUtilities.HashDataToString(["a", 'b', 42], options);
             string key3 = AIJsonUtilities.HashDataToString([TimeSpan.FromSeconds(1), null, 1.23], options);
             string key4 = AIJsonUtilities.HashDataToString([TimeSpan.FromSeconds(1), null, 1.23], options);
+            string key5 = AIJsonUtilities.HashDataToString([new Dictionary<string, object> { ["key1"] = 1, ["key2"] = 2 }], options);
+            string key6 = AIJsonUtilities.HashDataToString([new Dictionary<string, object> { ["key2"] = 2, ["key1"] = 1 }], options);
 
             Assert.Equal(key1, key2);
             Assert.Equal(key3, key4);
+            Assert.Equal(key5, key6);
             Assert.NotEqual(key1, key3);
+            Assert.NotEqual(key1, key5);
         }
     }
 
+    [Fact]
+    public static void HashData_IndentationInvariant()
+    {
+        JsonSerializerOptions indentOptions = new(AIJsonUtilities.DefaultOptions) { WriteIndented = true };
+        JsonSerializerOptions noIndentOptions = new(AIJsonUtilities.DefaultOptions) { WriteIndented = false };
+
+        Dictionary<string, object> dict = new() { ["key1"] = 1, ["key2"] = 2 };
+        string key1 = AIJsonUtilities.HashDataToString([dict], indentOptions);
+        string key2 = AIJsonUtilities.HashDataToString([dict], noIndentOptions);
+
+        Assert.Equal(key1, key2);
+    }
+
     [Fact]
     public static void CreateFunctionJsonSchema_InvokesIncludeParameterCallbackForEveryParameter()
     {