Skip to content

Commit 4fd0ff1

Browse files
MpdreamzMpdreamz
committed
Merge pull request #410 from Tasteful/AddSettingsSerialization
IndexSettings are not reading analyzers etc.
2 parents 49512ce + 6b4d304 commit 4fd0ff1

13 files changed

+600
-58
lines changed

‎src/Nest.Tests.Integration/Indices/IndicesTests.cs

Lines changed: 113 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Collections.Generic;
33
using System.Linq;
44
using FluentAssertions;
5+
using Nest.Domain.Settings;
56
using Nest.Tests.MockData;
67
using Nest.Tests.MockData.Domain;
78
using NUnit.Framework;
@@ -45,7 +46,34 @@ public void GetIndexSettingsComplex()
4546
var settings = new IndexSettings();
4647
settings.NumberOfReplicas = 4;
4748
settings.NumberOfShards = 8;
49+
4850
settings.Analysis.Analyzers.Add("snowball", new SnowballAnalyzer { Language = "English" });
51+
settings.Analysis.Analyzers.Add("standard", new StandardAnalyzer { StopWords = new[]{"word1", "word2"}});
52+
settings.Analysis.Analyzers.Add("swedishlanguage", new LanguageAnalyzer(Language.Swedish) { StopWords = new[] { "word1", "word2" }, StemExclusionList = new[] { "stem1", "stem2" } });
53+
54+
settings.Analysis.CharFilters.Add("char1", new HtmlStripCharFilter());
55+
settings.Analysis.CharFilters.Add("char2", new MappingCharFilter{ Mappings = new []{"ph=>f", "qu=>q"}});
56+
57+
settings.Analysis.TokenFilters.Add("tokenfilter1", new EdgeNGramTokenFilter());
58+
settings.Analysis.TokenFilters.Add("tokenfilter2", new SnowballTokenFilter());
59+
60+
settings.Analysis.Tokenizers.Add("token1", new KeywordTokenizer());
61+
settings.Analysis.Tokenizers.Add("token2", new PathHierarchyTokenizer());
62+
63+
settings.Similarity = new SimilaritySettings();
64+
var dfr = new CustomSimilaritySettings("test1", "DFR");
65+
dfr.SimilarityParameters.Add("basic_model", "g");
66+
dfr.SimilarityParameters.Add("after_effect", "l");
67+
dfr.SimilarityParameters.Add("normalization", "h2");
68+
dfr.SimilarityParameters.Add("normalization.h2.c", 3);
69+
settings.Similarity.CustomSimilarities.Add(dfr);
70+
71+
var ib = new CustomSimilaritySettings("test2", "IB");
72+
ib.SimilarityParameters.Add("distribution", "spl");
73+
ib.SimilarityParameters.Add("lambda", "ttf");
74+
ib.SimilarityParameters.Add("normalization", "h1");
75+
settings.Similarity.CustomSimilarities.Add(ib);
76+
4977
var typeMapping = this._client.GetMapping(ElasticsearchConfiguration.DefaultIndex, "elasticsearchprojects");
5078
typeMapping.TypeNameMarker = index;
5179
settings.Mappings.Add(typeMapping);
@@ -61,7 +89,91 @@ public void GetIndexSettingsComplex()
6189
Assert.AreEqual(r.Settings.NumberOfShards, 8);
6290
Assert.Greater(r.Settings.Count(), 0);
6391
Assert.True(r.Settings.ContainsKey("merge.policy.merge_factor"));
64-
92+
Assert.AreEqual("10", r.Settings["merge.policy.merge_factor"]);
93+
94+
Assert.AreEqual(3, r.Settings.Analysis.Analyzers.Count);
95+
{ // assert analyzers
96+
Assert.True(r.Settings.Analysis.Analyzers.ContainsKey("snowball"));
97+
var snoballAnalyser = r.Settings.Analysis.Analyzers["snowball"] as SnowballAnalyzer;
98+
Assert.NotNull(snoballAnalyser);
99+
Assert.AreEqual("English", snoballAnalyser.Language);
100+
101+
Assert.True(r.Settings.Analysis.Analyzers.ContainsKey("standard"));
102+
var standardAnalyser = r.Settings.Analysis.Analyzers["standard"] as StandardAnalyzer;
103+
Assert.NotNull(standardAnalyser);
104+
Assert.NotNull(standardAnalyser.StopWords);
105+
Assert.AreEqual(2, standardAnalyser.StopWords.Count());
106+
Assert.True(standardAnalyser.StopWords.Contains("word1"));
107+
Assert.True(standardAnalyser.StopWords.Contains("word2"));
108+
109+
Assert.True(r.Settings.Analysis.Analyzers.ContainsKey("swedishlanguage"));
110+
var languageAnalyser = r.Settings.Analysis.Analyzers["swedishlanguage"] as LanguageAnalyzer;
111+
Assert.NotNull(languageAnalyser);
112+
Assert.AreEqual(Language.Swedish.ToString().ToLower(), languageAnalyser.Type);
113+
Assert.NotNull(languageAnalyser.StopWords);
114+
Assert.AreEqual(2, languageAnalyser.StopWords.Count());
115+
Assert.True(languageAnalyser.StopWords.Contains("word1"));
116+
Assert.True(languageAnalyser.StopWords.Contains("word2"));
117+
Assert.AreEqual(2, languageAnalyser.StemExclusionList.Count());
118+
Assert.True(languageAnalyser.StemExclusionList.Contains("stem1"));
119+
Assert.True(languageAnalyser.StemExclusionList.Contains("stem2"));
120+
}
121+
122+
Assert.AreEqual(2, r.Settings.Analysis.CharFilters.Count);
123+
{ // assert char filters
124+
Assert.True(r.Settings.Analysis.CharFilters.ContainsKey("char1"));
125+
var filter1 = r.Settings.Analysis.CharFilters["char1"] as HtmlStripCharFilter;
126+
Assert.NotNull(filter1);
127+
Assert.True(r.Settings.Analysis.CharFilters.ContainsKey("char2"));
128+
var filter2 = r.Settings.Analysis.CharFilters["char2"] as MappingCharFilter;
129+
Assert.NotNull(filter2);
130+
Assert.AreEqual(2, filter2.Mappings.Count());
131+
Assert.True(filter2.Mappings.Contains("ph=>f"));
132+
Assert.True(filter2.Mappings.Contains("qu=>q"));
133+
}
134+
135+
Assert.AreEqual(2, r.Settings.Analysis.TokenFilters.Count);
136+
{ // assert token filters
137+
Assert.True(r.Settings.Analysis.TokenFilters.ContainsKey("tokenfilter1"));
138+
var filter1 = r.Settings.Analysis.TokenFilters["tokenfilter1"] as EdgeNGramTokenFilter;
139+
Assert.NotNull(filter1);
140+
Assert.True(r.Settings.Analysis.TokenFilters.ContainsKey("tokenfilter2"));
141+
var filter2 = r.Settings.Analysis.TokenFilters["tokenfilter2"] as SnowballTokenFilter;
142+
Assert.NotNull(filter2);
143+
}
144+
145+
Assert.AreEqual(2, r.Settings.Analysis.Tokenizers.Count);
146+
{ // assert tokenizers
147+
Assert.True(r.Settings.Analysis.Tokenizers.ContainsKey("token1"));
148+
var tokenizer1 = r.Settings.Analysis.Tokenizers["token1"] as KeywordTokenizer;
149+
Assert.NotNull(tokenizer1);
150+
Assert.True(r.Settings.Analysis.Tokenizers.ContainsKey("token2"));
151+
var tokenizer2 = r.Settings.Analysis.Tokenizers["token2"] as PathHierarchyTokenizer;
152+
Assert.NotNull(tokenizer2);
153+
}
154+
155+
156+
Assert.NotNull(r.Settings.Similarity);
157+
Assert.NotNull(r.Settings.Similarity.CustomSimilarities);
158+
Assert.AreEqual(2, r.Settings.Similarity.CustomSimilarities.Count);
159+
{ // assert similarity
160+
var similarity1 = r.Settings.Similarity.CustomSimilarities.FirstOrDefault(x => x.Name.Equals("test1", StringComparison.InvariantCultureIgnoreCase));
161+
Assert.NotNull(similarity1);
162+
Assert.AreEqual("DFR", similarity1.Type);
163+
Assert.AreEqual(4, similarity1.SimilarityParameters.Count);
164+
Assert.True(similarity1.SimilarityParameters.Any(x => x.Key.Equals("basic_model") && x.Value.ToString().Equals("g")));
165+
Assert.True(similarity1.SimilarityParameters.Any(x => x.Key.Equals("after_effect") && x.Value.ToString().Equals("l")));
166+
Assert.True(similarity1.SimilarityParameters.Any(x => x.Key.Equals("normalization") && x.Value.ToString().Equals("h2")));
167+
Assert.True(similarity1.SimilarityParameters.Any(x => x.Key.Equals("normalization.h2.c") && x.Value.ToString().Equals("3")));
168+
169+
var similarity2 = r.Settings.Similarity.CustomSimilarities.FirstOrDefault(x => x.Name.Equals("test2", StringComparison.InvariantCultureIgnoreCase));
170+
Assert.NotNull(similarity2);
171+
Assert.AreEqual("IB", similarity2.Type);
172+
Assert.AreEqual(3, similarity2.SimilarityParameters.Count);
173+
Assert.True(similarity2.SimilarityParameters.Any(x => x.Key.Equals("distribution") && x.Value.ToString().Equals("spl")));
174+
Assert.True(similarity2.SimilarityParameters.Any(x => x.Key.Equals("lambda") && x.Value.ToString().Equals("ttf")));
175+
Assert.True(similarity2.SimilarityParameters.Any(x => x.Key.Equals("normalization") && x.Value.ToString().Equals("h1")));
176+
}
65177
this._client.DeleteIndex(index);
66178
}
67179
[Test]

‎src/Nest/Domain/Analysis/Analyzers/AnalysisSettings.cs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44

55
namespace Nest
66
{
7-
public class AnalysisSettings
7+
[JsonConverter(typeof(AnalysisSettingsConverter))]
8+
public class AnalysisSettings
89
{
910
public AnalysisSettings()
1011
{
@@ -14,16 +15,16 @@ public AnalysisSettings()
1415
this.CharFilters = new Dictionary<string, CharFilterBase>();
1516
}
1617

17-
[JsonConverter(typeof(DictionaryKeysAreNotPropertyNamesJsonConverter))]
18+
[JsonConverter(typeof(AnalyzerCollectionConverter))]
1819
public IDictionary<string, AnalyzerBase> Analyzers { get; set; }
1920

20-
[JsonConverter(typeof(DictionaryKeysAreNotPropertyNamesJsonConverter))]
21+
[JsonConverter(typeof(TokenFilterCollectionConverter))]
2122
public IDictionary<string, TokenFilterBase> TokenFilters { get; set; }
2223

23-
[JsonConverter(typeof(DictionaryKeysAreNotPropertyNamesJsonConverter))]
24+
[JsonConverter(typeof(TokenizerCollectionConverter))]
2425
public IDictionary<string, TokenizerBase> Tokenizers { get; set; }
2526

26-
[JsonConverter(typeof(DictionaryKeysAreNotPropertyNamesJsonConverter))]
27+
[JsonConverter(typeof(CharFilterCollectionConverter))]
2728
public IDictionary<string, CharFilterBase> CharFilters { get; set; }
2829
}
2930
}

‎src/Nest/Domain/Settings/CustomSimilaritySettings.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ public CustomSimilaritySettings(string name, string type)
1515
{
1616
Name = name;
1717
Type = type;
18+
19+
SimilarityParameters = new Dictionary<string, object>();
1820
}
1921

2022
}

‎src/Nest/ElasticClient-MappingIndex.cs

Lines changed: 64 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
using System.Linq;
1+
using System;
2+
using System.ComponentModel;
3+
using System.Linq;
24
using Newtonsoft.Json;
35
using Newtonsoft.Json.Linq;
46
using System.Text.RegularExpressions;
@@ -33,21 +35,37 @@ public IIndexSettingsResponse GetIndexSettings(string index)
3335
{
3436
var o = JObject.Parse(status.Result);
3537
var settingsObject = o.First.First.First.First;
36-
var settings = new IndexSettings(); //this.Deserialize<IndexSettings>(settingsObject.ToString());
3738

39+
var settingsContainer = new JObject();
40+
// In indexsettings response all analyzers etc are delivered as settings so need to split up the settings key and make proper json
3841
foreach (JProperty s in settingsObject.Children<JProperty>())
3942
{
40-
settings.Add(StripIndex.Replace(s.Name, ""), s.Value.ToString());
43+
var name = StripIndex.Replace(s.Name, "");
44+
if (name.StartsWith("analysis."))
45+
{
46+
var keys = name.Split('.');
47+
RewriteIndexSettingsResponseToIndexSettingsJSon(settingsContainer, keys, s.Value);
48+
}
49+
else if (name.StartsWith("similarity."))
50+
{
51+
var keys = name.Split('.');
52+
var similaryKeys = new[] { keys[0], keys[1], string.Join(".", keys.Skip(2).ToArray()) };
53+
RewriteIndexSettingsResponseToIndexSettingsJSon(settingsContainer, similaryKeys, s.Value);
54+
}
55+
else
56+
{
57+
RewriteIndexSettingsResponseToIndexSettingsJSon(settingsContainer, new[] { name }, s.Value);
58+
}
4159
}
4260

43-
44-
response.Settings = settings;
61+
response.Settings = this.Deserialize<IndexSettings>(settingsContainer);
4562
response.IsValid = true;
4663
}
4764
catch { }
4865
response.ConnectionStatus = status;
4966
return response;
5067
}
68+
5169
/// <summary>
5270
/// Update the index settings for the default index
5371
/// </summary>
@@ -64,29 +82,9 @@ public ISettingsOperationResponse UpdateSettings(string index, IndexSettings set
6482

6583
string path = this.PathResolver.CreateIndexPath(index, "_settings");
6684
settings.Settings = settings.Settings
67-
.Where(kv => IndexSettings.UpdateWhiteList.Any(p =>
68-
{
69-
return kv.Key.StartsWith(p);
70-
}
71-
)).ToDictionary(kv => kv.Key, kv => kv.Value);
72-
73-
var sb = new StringBuilder();
74-
var sw = new StringWriter(sb);
75-
using (JsonWriter jsonWriter = new JsonTextWriter(sw))
76-
{
77-
jsonWriter.WriteStartObject();
78-
jsonWriter.WritePropertyName("index");
79-
jsonWriter.WriteStartObject();
80-
foreach (var kv in settings.Settings)
81-
{
82-
jsonWriter.WritePropertyName(kv.Key);
83-
jsonWriter.WriteValue(kv.Value);
84-
}
85-
86-
jsonWriter.WriteEndObject();
87-
}
88-
string data = sb.ToString();
85+
.Where(kv => IndexSettings.UpdateWhiteList.Any(p => kv.Key.StartsWith(p))).ToDictionary(kv => kv.Key, kv => kv.Value);
8986

87+
string data = this.Serializer.Serialize(settings, Formatting.None);
9088
var status = this.Connection.PutSync(path, data);
9189

9290
var r = new SettingsOperationResponse();
@@ -122,5 +120,44 @@ public IIndicesResponse DeleteIndex(string index)
122120
return r;
123121
}
124122

123+
/// <summary>
124+
/// Rewrites the index settings response to index settings json.
125+
/// </summary>
126+
/// <param name="container">The container.</param>
127+
/// <param name="key">The key.</param>
128+
/// <param name="value">The value.</param>
129+
private void RewriteIndexSettingsResponseToIndexSettingsJSon(JContainer container, string[] key, JToken value)
130+
{
131+
var thisKey = key.First();
132+
int indexer;
133+
134+
if (key.Length > 2 || (key.Length == 2 && !int.TryParse(key.Last(), out indexer)))
135+
{
136+
var property = (JContainer)((JObject)container).GetValue(thisKey);
137+
if (property == null)
138+
{
139+
property = new JObject();
140+
((JObject)container).Add(thisKey, property);
141+
}
142+
RewriteIndexSettingsResponseToIndexSettingsJSon(property, key.Skip(1).ToArray(), value);
143+
}
144+
else if (key.Length == 2 && int.TryParse(key.Last(), out indexer))
145+
{
146+
var property = ((JObject)container).Property(thisKey);
147+
if (property == null)
148+
{
149+
property = new JProperty(thisKey, new JArray());
150+
container.Add(property);
151+
}
152+
var jArray = (JArray)property.Value;
153+
jArray.Add(value);
154+
}
155+
else
156+
{
157+
var property = new JProperty(thisKey, value);
158+
container.Add(property);
159+
}
160+
}
161+
125162
}
126163
}

‎src/Nest/ExposedInternals/ElasticSerializer.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using Nest.Resolvers.Converters;
99
using Newtonsoft.Json;
1010
using Newtonsoft.Json.Converters;
11+
using Newtonsoft.Json.Linq;
1112

1213
namespace Nest
1314
{
@@ -79,6 +80,11 @@ internal T DeserializeInternal<T>(
7980
var jsonSettings = extraConverters.HasAny() || piggyBackJsonConverter != null
8081
? this.CreateSettings(extraConverters, piggyBackJsonConverter)
8182
: this._serializationSettings;
83+
84+
var jTokenValue = value as JToken;
85+
if (jTokenValue != null)
86+
return JsonSerializer.Create(jsonSettings).Deserialize<T>(jTokenValue.CreateReader());
87+
8288
var status = value as ConnectionStatus;
8389
if (status == null || !typeof(BaseResponse).IsAssignableFrom(typeof(T)))
8490
return JsonConvert.DeserializeObject<T>(value.ToString(), jsonSettings);

‎src/Nest/Nest.csproj

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,9 @@
132132
<Compile Include="Extensions\TypeExtensions.cs" />
133133
<Compile Include="Extensions\UriExtensions.cs" />
134134
<Compile Include="IRawElasticClient.Generated.cs" />
135+
<Compile Include="Resolvers\Converters\AnalysisSettingsConverter.cs" />
136+
<Compile Include="Resolvers\Converters\AnalyzerCollectionConverter.cs" />
137+
<Compile Include="Resolvers\Converters\CharFilterCollectionConverter.cs" />
135138
<Compile Include="Resolvers\Converters\ConcreteTypeConverter.cs" />
136139
<Compile Include="Domain\Mapping\Attributes\IElasticPropertyAttribute.cs" />
137140
<Compile Include="Domain\Mapping\Attributes\IElasticPropertyVisitor.cs" />
@@ -520,6 +523,8 @@
520523
<Compile Include="Resolvers\Converters\DynamicMappingOptionConverter.cs" />
521524
<Compile Include="Resolvers\Converters\DictionaryKeysAreNotPropertyNamesJsonConverter.cs" />
522525
<Compile Include="Resolvers\Converters\IndexNameMarkerConverter.cs" />
526+
<Compile Include="Resolvers\Converters\TokenFilterCollectionConverter.cs" />
527+
<Compile Include="Resolvers\Converters\TokenizerCollectionConverter.cs" />
523528
<Compile Include="Resolvers\Converters\TypeNameMarkerConverter.cs" />
524529
<Compile Include="Resolvers\Converters\WarmerResponseConverter.cs" />
525530
<Compile Include="Resolvers\Converters\WarmerMappingConverter.cs" />

0 commit comments

Comments
 (0)