diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java index 9dd320a1a0a85..f1b4391546598 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java @@ -369,7 +369,7 @@ public void benchmark() { blockFactory, ByteSizeValue.ofMb(1).getBytes(), fields(name), - new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException("can't load _source here"); }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))), 0 diff --git a/docs/changelog/136438.yaml b/docs/changelog/136438.yaml new file mode 100644 index 0000000000000..148f9108d07c0 --- /dev/null +++ b/docs/changelog/136438.yaml @@ -0,0 +1,5 @@ +pr: 136438 +summary: '`BlockSourceReader` should always apply source filtering' +area: "ES|QL" +type: enhancement +issues: [] diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 70c28ee18184f..628e84173496f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -605,7 +605,7 @@ protected String delegatingTo() { } // fallback to _source (synthetic or not) - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()), blContext.indexSettings()); // MatchOnlyText never has norms, so we have to use the field names field BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup); @@ -636,7 +636,7 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name()), fieldDataContext.indexSettings()), fieldDataContext.lookupSupplier().get(), TextDocValuesField::new ); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index 8e4eda87abfb8..7a1a03cc537d7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -148,11 +148,12 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher(context); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(SearchExecutionContext context) { + Set sourcePaths = context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(); + return new SourceValueFetcher(sourcePaths, nullValue, context.getIndexSettings().getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { return objectToFloat(value); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index d0bbf3059395d..89af161ce5aab 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; @@ -396,8 +397,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - - ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name())); + var valueFetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) @@ -489,7 +489,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedDoubleIndexFieldData.Builder( name(), valuesSourceType, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, ScaledFloatDocValuesField::new ); @@ -503,11 +503,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected Double parseSourceValue(Object value) { double doubleValue; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java index 74f1f9a16a6d6..88f14aa5ecc90 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java @@ -59,6 +59,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class MatchOnlyTextFieldTypeTests extends FieldTypeTestCase { @@ -315,6 +316,7 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet // when MappedFieldType.BlockLoaderContext blContext = mock(MappedFieldType.BlockLoaderContext.class); doReturn(FieldNamesFieldMapper.FieldNamesFieldType.get(false)).when(blContext).fieldNames(); + when(blContext.indexSettings()).thenReturn(indexSettings); BlockLoader blockLoader = ft.blockLoader(blContext); // then @@ -362,6 +364,7 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet // when MappedFieldType.BlockLoaderContext blContext = mock(MappedFieldType.BlockLoaderContext.class); + when(blContext.indexSettings()).thenReturn(indexSettings); doReturn(FieldNamesFieldMapper.FieldNamesFieldType.get(false)).when(blContext).fieldNames(); BlockLoader blockLoader = ft.blockLoader(blContext); diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 039f8c049e0c5..d04f2067d8aed 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -99,6 +99,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 50f3925c425e7..d4cac1b4f0649 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -1828,6 +1828,14 @@ public SourceFieldMapper.Mode getIndexMappingSourceMode() { return indexMappingSourceMode; } + public IgnoredSourceFieldMapper.IgnoredSourceFormat getIgnoredSourceFormat() { + if (getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC) { + return IgnoredSourceFieldMapper.ignoredSourceFormat(getIndexVersionCreated()); + } else { + return IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + } + /** * @return Whether recovery source should be enabled if needed. * Note that this is a node setting, and this setting is not sourced from index settings. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index f348debeca1c6..07338cad43a96 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.geo.GeometryFormatterFactory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -172,9 +173,9 @@ protected Object parseSourceValue(Object value) { }; } - public ValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format) { + public ValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format, IndexSettings indexSettings) { Function, List> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON); - return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue)) { + return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue), indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { final List values = new ArrayList<>(); @@ -185,7 +186,7 @@ protected Object parseSourceValue(Object value) { } protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { - ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); + var fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB, blContext.indexSettings()); // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index fa835e4356667..c73b4fefa8a4a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; @@ -30,6 +31,7 @@ public abstract class ArraySourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; + private final IgnoredSourceFormat ignoredSourceFormat; public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -43,15 +45,17 @@ public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { this.sourcePaths = context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(); this.nullValue = nullValue; + this.ignoredSourceFormat = context.getIndexSettings().getIgnoredSourceFormat(); } /** * @param sourcePaths The paths to pull source values from * @param nullValue An optional substitute value if the _source value is `null` */ - public ArraySourceValueFetcher(Set sourcePaths, Object nullValue) { + public ArraySourceValueFetcher(Set sourcePaths, Object nullValue, IgnoredSourceFormat ignoredSourceFormat) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -76,7 +80,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 3b467e383f20d..858b054012fe2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -107,7 +107,7 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) @Override public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return fetcher.storedFieldsSpec(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 56ff91eef69f0..d118683ba5942 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -281,11 +282,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected Boolean parseSourceValue(Object value) { if (value instanceof Boolean) { @@ -364,7 +368,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); + var fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); BlockSourceReader.LeafIteratorLookup lookup = indexType.hasTerms() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); @@ -431,7 +435,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedBooleanIndexFieldData.Builder( name(), CoreValuesSourceType.BOOLEAN, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, BooleanDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 9991abba7ba6a..e69793fb799cf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -662,8 +662,8 @@ public String parseSourceValue(Object value) { } // returns a Long to support source fallback which emulates numeric doc values for dates - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override public Long parseSourceValue(Object value) { String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString(); @@ -995,11 +995,13 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - BlockSourceReader.LeafIteratorLookup lookup = isStored() || indexType.hasPoints() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return new BlockSourceReader.LongsBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()), + lookup + ); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { @@ -1067,7 +1069,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedNumericIndexFieldData.Builder( name(), resolution.numericType().getValuesSourceType(), - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, resolution.getDefaultToScriptFieldFactory() ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index b66730d6704e0..f97a4faafd5c8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -65,7 +65,7 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep @Override public StoredFieldsSpec rowStrideStoredFieldSpec() { - return new StoredFieldsSpec(false, false, Set.of(), new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat)); + return new StoredFieldsSpec(false, false, Set.of(), new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat), Set.of()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 29edf41e5760b..0e278e95ce1f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -494,7 +494,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherMultiGeoPointIndexFieldData.Builder( name(), valuesSourceType, - valueFetcher(sourcePaths, null, null), + valueFetcher(sourcePaths, null, null, fieldDataContext.indexSettings()), searchLookup, GeoPointDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 6db87007747ad..644ab73a0217b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -53,7 +53,6 @@ import java.util.Collections; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.BiFunction; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; @@ -481,12 +480,11 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (isSyntheticSource && blContext.parentField(name()) == null) { return blockLoaderFromFallbackSyntheticSource(blContext); } - // see #indexValue BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && hasPoints ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext), lookup); } private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { @@ -503,8 +501,8 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(BlockLoaderContext blContext) { + return new SourceValueFetcher(blContext.sourcePaths(name()), nullValue, blContext.indexSettings().getIgnoredSourceFormat()) { @Override public InetAddress parseSourceValue(Object value) { return parse(value); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 89a158cf3bdb4..b337cdd41dcb3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -820,7 +821,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext)); } @@ -908,7 +909,7 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), fieldDataContext.lookupSupplier().get(), KeywordDocValuesField::new ); @@ -930,11 +931,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 6ca488e137d60..27f4ace5cc739 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -2055,7 +2056,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { // We only write the field names field if there aren't doc values or norms ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()), lookup); } @Override @@ -2077,7 +2078,12 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation == FielddataOperation.SCRIPT) { SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - return type.getValueFetcherFieldDataBuilder(name(), valuesSourceType, searchLookup, sourceValueFetcher(sourcePaths)); + return type.getValueFetcherFieldDataBuilder( + name(), + valuesSourceType, + searchLookup, + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()) + ); } throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); @@ -2099,11 +2105,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 7d1e722200ba3..a35215310a85e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -10,6 +10,8 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; @@ -31,6 +33,7 @@ public abstract class SourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; + private final IgnoredSourceFormat ignoredSourceFormat; public SourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -41,16 +44,22 @@ public SourceValueFetcher(String fieldName, SearchExecutionContext context) { * @param nullValue An optional substitute value if the _source value is 'null'. */ public SourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { - this(context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(), nullValue); + this( + context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(), + nullValue, + context.getIndexSettings().getIgnoredSourceFormat() + ); } /** - * @param sourcePaths The paths to pull source values from - * @param nullValue An optional substitute value if the _source value is `null` + * @param sourcePaths The paths to pull source values from + * @param nullValue An optional substitute value if the _source value is `null` + * @param ignoredSourceFormat */ - public SourceValueFetcher(Set sourcePaths, Object nullValue) { + public SourceValueFetcher(Set sourcePaths, Object nullValue, IgnoredSourceFormat ignoredSourceFormat) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -98,7 +107,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); } /** @@ -142,8 +151,8 @@ protected Object parseSourceValue(Object value) { * Creates a {@link SourceValueFetcher} that converts source values to Strings * @param sourcePaths the paths to fetch values from in the source */ - public static SourceValueFetcher toString(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + public static SourceValueFetcher toString(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { return value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 5acb088040485..6c5231fdfb9a1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1116,9 +1116,8 @@ protected String delegatingTo() { if (isSyntheticSourceEnabled() && syntheticSourceDelegate.isEmpty() && parentField == null) { return fallbackSyntheticSourceBlockLoader(blContext); } - // otherwise, load values from _source (synthetic or not) - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()), blContext.indexSettings()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext)); } @@ -1247,7 +1246,7 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name()), fieldDataContext.indexSettings()), fieldDataContext.lookupSupplier().get(), TextDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 69a520d2bf518..de8c9020c2f3e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -753,15 +754,18 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } public IgnoreAbove ignoreAbove() { return ignoreAbove; } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) { @Override @SuppressWarnings("unchecked") protected Object parseSourceValue(Object value) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 33d99c5628732..84d32874a17b3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2691,13 +2691,16 @@ public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) { return new BlockDocValuesReader.DenseVectorFromBinaryBlockLoader(name(), dims, indexVersionCreated, element.elementType()); } - BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DenseVectorBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, dims); + return new BlockSourceReader.DenseVectorBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()), + lookup, + dims + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index f494fdd0acfab..6eb4c10c68b11 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.DocumentParser; @@ -279,11 +280,12 @@ private static Fields generateTermVectors( } } if (source != null) { + IndexSettings indexSettings = indexShard.indexSettings(); MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); Source s = Source.fromMap(source, XContentType.JSON); for (String field : fields) { if (values.containsKey(field) == false) { - SourceValueFetcher valueFetcher = SourceValueFetcher.toString(mappingLookup.sourcePaths(field)); + SourceValueFetcher valueFetcher = SourceValueFetcher.toString(mappingLookup.sourcePaths(field), indexSettings); List ignoredValues = new ArrayList<>(); List v = valueFetcher.fetchValues(s, -1, ignoredValues); if (v.isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java index ce52fabb20049..ca085264d00d2 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -10,6 +10,7 @@ package org.elasticsearch.search.fetch; import org.elasticsearch.index.mapper.IgnoredFieldsSpec; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import java.util.Collection; import java.util.HashSet; @@ -25,10 +26,11 @@ public record StoredFieldsSpec( boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields, - IgnoredFieldsSpec ignoredFieldsSpec + IgnoredFieldsSpec ignoredFieldsSpec, + Set sourcePaths ) { public StoredFieldsSpec(boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields) { - this(requiresSource, requiresMetadata, requiredStoredFields, IgnoredFieldsSpec.NONE); + this(requiresSource, requiresMetadata, requiredStoredFields, IgnoredFieldsSpec.NONE, Set.of()); } public boolean noRequirements() { @@ -52,6 +54,18 @@ public boolean onlyRequiresIgnoredFields() { */ public static final StoredFieldsSpec NEEDS_SOURCE = new StoredFieldsSpec(true, false, Set.of()); + /** + * @return a stored field spec that requires source only for the specified source paths. + * This is more efficient than using {@link #NEEDS_SOURCE}. + */ + public static StoredFieldsSpec withSourcePaths(IgnoredSourceFormat ignoredSourceFormat, Set sourcePaths) { + // The fields in source paths might also be in ignored source, so include source paths there as well. + IgnoredFieldsSpec ignoredFieldsSpec = ignoredSourceFormat == IgnoredSourceFormat.NO_IGNORED_SOURCE + ? IgnoredFieldsSpec.NONE + : new IgnoredFieldsSpec(sourcePaths, ignoredSourceFormat); + return new StoredFieldsSpec(true, false, Set.of(), ignoredFieldsSpec, sourcePaths); + } + /** * Combine these stored field requirements with those from another StoredFieldsSpec */ @@ -70,11 +84,23 @@ public StoredFieldsSpec merge(StoredFieldsSpec other) { mergedFields = new HashSet<>(this.requiredStoredFields); mergedFields.addAll(other.requiredStoredFields); } + Set mergedSourcePaths; + if (this.sourcePaths.isEmpty() == false && other.sourcePaths.isEmpty() == false) { + mergedSourcePaths = new HashSet<>(this.sourcePaths); + mergedSourcePaths.addAll(other.sourcePaths); + } else if (this.sourcePaths.isEmpty() == false) { + mergedSourcePaths = this.sourcePaths; + } else if (other.sourcePaths.isEmpty() == false) { + mergedSourcePaths = other.sourcePaths; + } else { + mergedSourcePaths = Set.of(); + } return new StoredFieldsSpec( this.requiresSource || other.requiresSource, this.requiresMetadata || other.requiresMetadata, mergedFields, - ignoredFieldsSpec.merge(other.ignoredFieldsSpec) + ignoredFieldsSpec.merge(other.ignoredFieldsSpec), + mergedSourcePaths ); } diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java index e460b5a460194..913e51aa807ea 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java @@ -39,7 +39,8 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + Set.of() ) ) ); @@ -50,7 +51,8 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + Set.of() ) ) ); @@ -61,7 +63,8 @@ public void testSupports() { true, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + Set.of() ) ) ); @@ -129,7 +132,8 @@ private void testLoader( false, false, Set.of(), - new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + Set.of() ); assertTrue(IgnoredSourceFieldLoader.supports(spec)); iw.addDocument(doc); diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java index 0255506816799..7cff817f26217 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java @@ -48,7 +48,7 @@ private StoredFieldsSpec fieldsSpec( Set ignoredFields, IgnoredSourceFieldMapper.IgnoredSourceFormat format ) { - return new StoredFieldsSpec(false, false, storedFields, new IgnoredFieldsSpec(ignoredFields, format)); + return new StoredFieldsSpec(false, false, storedFields, new IgnoredFieldsSpec(ignoredFields, format), Set.of()); } public void testEmpty() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 1fa9c85a5c738..0c0b5b6dbf4de 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -11,9 +11,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -29,35 +33,66 @@ public class BlockSourceReaderTests extends MapperServiceTestCase { public void testSingle() throws IOException { withIndex( source -> source.field("field", "foo"), - ctx -> loadBlock(ctx, block -> assertThat(block.get(0), equalTo(new BytesRef("foo")))) + (mapperService, ctx) -> loadBlock(mapperService, ctx, block -> assertThat(block.get(0), equalTo(new BytesRef("foo")))) ); } public void testMissing() throws IOException { - withIndex(source -> {}, ctx -> loadBlock(ctx, block -> assertThat(block.get(0), nullValue()))); + withIndex(source -> {}, (mapperService, ctx) -> loadBlock(mapperService, ctx, block -> assertThat(block.get(0), nullValue()))); } public void testArray() throws IOException { withIndex( source -> source.startArray("field").value("foo").value("bar").endArray(), - ctx -> loadBlock(ctx, block -> assertThat(block.get(0), equalTo(List.of(new BytesRef("foo"), new BytesRef("bar"))))) + (mapperService, ctx) -> loadBlock( + mapperService, + ctx, + block -> assertThat(block.get(0), equalTo(List.of(new BytesRef("foo"), new BytesRef("bar")))) + ) ); } public void testEmptyArray() throws IOException { - withIndex(source -> source.startArray("field").endArray(), ctx -> loadBlock(ctx, block -> assertThat(block.get(0), nullValue()))); + withIndex( + source -> source.startArray("field").endArray(), + (mapperService, ctx) -> loadBlock(mapperService, ctx, block -> assertThat(block.get(0), nullValue())) + ); } - private void loadBlock(LeafReaderContext ctx, Consumer test) throws IOException { - ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field")); - BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field"); + public void testMoreFields() throws IOException { + withIndex( + source -> source.field("field", "foo").field("other_field", "bar").field("other_field_2", 1L), + (mapperService, ctx) -> loadBlock(mapperService, ctx, block -> assertThat(block.get(0), equalTo(new BytesRef("foo")))) + ); + } + + private void loadBlock(MapperService mapperService, LeafReaderContext ctx, Consumer test) throws IOException { + boolean syntheticSource = mapperService.mappingLookup().isSourceSynthetic(); + var valueFetcher = SourceValueFetcher.toString(Set.of("field"), mapperService.getIndexSettings()); + boolean hasNorms = mapperService.mappingLookup().getFieldType("field").getTextSearchInfo().hasNorms(); + BlockSourceReader.LeafIteratorLookup lookup = hasNorms + ? BlockSourceReader.lookupFromNorms("field") + : BlockSourceReader.lookupMatchingAll(); BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup); assertThat(loader.columnAtATimeReader(ctx), nullValue()); BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx); - assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE)); - BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( + assertThat( + loader.rowStrideStoredFieldSpec(), + equalTo( + StoredFieldsSpec.withSourcePaths( + syntheticSource ? IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE : IgnoredSourceFormat.NO_IGNORED_SOURCE, + Set.of("field") + ) + ) + ); + var sourceLoader = mapperService.mappingLookup() + .newSourceLoader(new SourceFilter(new String[] { "field" }, null), SourceFieldMetrics.NOOP); + var sourceLoaderLeaf = sourceLoader.leaf(ctx.reader(), null); + + assertThat(loader.rowStrideStoredFieldSpec().requiresSource(), equalTo(true)); + var storedFields = new BlockLoaderStoredFieldsFromLeafLoader( StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null), - loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null + sourceLoaderLeaf ); BlockLoader.Builder builder = loader.builder(TestBlock.factory(), 1); storedFields.advanceTo(0); @@ -67,15 +102,37 @@ private void loadBlock(LeafReaderContext ctx, Consumer test) throws I test.accept(block); } - private void withIndex(CheckedConsumer buildSource, CheckedConsumer test) - throws IOException { - MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "text"))); + private void withIndex( + CheckedConsumer buildSource, + CheckedBiConsumer test + ) throws IOException { + boolean syntheticSource = randomBoolean(); + String fieldType = randomFrom("text", "keyword"); + MapperService mapperService = syntheticSource + ? createMapperServiceWithSyntheticSource(mapping(fieldType)) + : createMapperService(mapping(fieldType)); withLuceneIndex(mapperService, writer -> { ParsedDocument parsed = mapperService.documentParser().parseDocument(source(buildSource), mapperService.mappingLookup()); writer.addDocuments(parsed.docs()); }, reader -> { assertThat(reader.leaves(), hasSize(1)); - test.accept(reader.leaves().get(0)); + test.accept(mapperService, reader.leaves().get(0)); + }); + } + + MapperService createMapperServiceWithSyntheticSource(XContentBuilder mappings) throws IOException { + var settings = Settings.builder() + .put("index.mapping.source.mode", "synthetic") + .put("index.mapping.synthetic_source_keep", "arrays") + .build(); + return createMapperService(getVersion(), settings, () -> true, mappings); + } + + static XContentBuilder mapping(String type) throws IOException { + return mapping(b -> { + b.startObject("field").field("type", type).endObject(); + b.startObject("other_field").field("type", "keyword").endObject(); + b.startObject("other_field_2").field("type", "long").endObject(); }); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 332c727ffadde..cee149071134f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1446,8 +1446,9 @@ public void testEmpty() throws Exception { throw new IllegalArgumentException("Can't load source in scripts in synthetic mode"); } : SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()); SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider); + var indexSettings = mapperService.getIndexSettings(); IndexFieldData sfd = ft.fielddataBuilder( - new FieldDataContext("", null, () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) + new FieldDataContext("", indexSettings, () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) ).build(null, null); LeafFieldData lfd = sfd.load(getOnlyLeafReader(searcher.getIndexReader()).getContext()); TextDocValuesField scriptDV = (TextDocValuesField) lfd.getScriptFieldFactory("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index fcab101da9baa..4787b444d732f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -55,6 +55,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TextFieldTypeTests extends FieldTypeTestCase { @@ -374,7 +375,9 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet ); // when - BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + var context = mock(MappedFieldType.BlockLoaderContext.class); + when(context.indexSettings()).thenReturn(indexSettings); + BlockLoader blockLoader = ft.blockLoader(context); // then // verify that we don't delegate anything @@ -423,7 +426,9 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet ); // when - BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + var context = mock(MappedFieldType.BlockLoaderContext.class); + when(context.indexSettings()).thenReturn(indexSettings); + BlockLoader blockLoader = ft.blockLoader(context); // then // verify that we don't delegate anything diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index c8d7ad8127b55..dadeae3cc654c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -18,12 +18,14 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -186,6 +188,7 @@ public void testFetchSourceValue() throws IOException { Map sourceValue = Map.of("key", "value"); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath("field.key")).thenReturn(Set.of("field.key")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 5932180ac3c03..8d2787ab76dc9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -2403,6 +2403,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; SourceToParse source = source(b -> b.field(ft.name(), value)); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java index 7cbef28bd9532..4ca7fad26114b 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.fetch; +import org.elasticsearch.index.mapper.IgnoredFieldsSpec; import org.elasticsearch.script.Script; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourcePhase; @@ -20,6 +21,9 @@ import java.util.Set; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; @@ -84,6 +88,39 @@ public void testNoCloneOnMerge() { assertThat(spec.requiredStoredFields(), sameInstance(withCat.requiredStoredFields())); } + public void testMergeSourcePaths() { + StoredFieldsSpec spec = StoredFieldsSpec.NO_REQUIREMENTS; + spec = spec.merge(new StoredFieldsSpec(true, false, Set.of(), IgnoredFieldsSpec.NONE, Set.of("cat"))); + assertThat(spec.ignoredFieldsSpec(), equalTo(IgnoredFieldsSpec.NONE)); + assertThat(spec.requiresSource(), equalTo(true)); + assertThat(spec.requiresMetadata(), equalTo(false)); + assertThat(spec.requiredStoredFields(), empty()); + assertThat(spec.sourcePaths(), containsInAnyOrder("cat")); + + spec = spec.merge(new StoredFieldsSpec(true, false, Set.of(), IgnoredFieldsSpec.NONE, Set.of("dog"))); + assertThat(spec.ignoredFieldsSpec(), equalTo(IgnoredFieldsSpec.NONE)); + assertThat(spec.requiresSource(), equalTo(true)); + assertThat(spec.requiresMetadata(), equalTo(false)); + assertThat(spec.requiredStoredFields(), empty()); + assertThat(spec.sourcePaths(), containsInAnyOrder("cat", "dog")); + + spec = spec.merge(new StoredFieldsSpec(true, false, Set.of(), IgnoredFieldsSpec.NONE, Set.of("hamster"))); + assertThat(spec.ignoredFieldsSpec(), equalTo(IgnoredFieldsSpec.NONE)); + assertThat(spec.requiresSource(), equalTo(true)); + assertThat(spec.requiresMetadata(), equalTo(false)); + assertThat(spec.requiredStoredFields(), empty()); + assertThat(spec.sourcePaths(), containsInAnyOrder("cat", "dog", "hamster")); + var pref = spec.sourcePaths(); + + spec = spec.merge(new StoredFieldsSpec(true, false, Set.of("other_field"), IgnoredFieldsSpec.NONE, Set.of())); + assertThat(spec.ignoredFieldsSpec(), equalTo(IgnoredFieldsSpec.NONE)); + assertThat(spec.requiresSource(), equalTo(true)); + assertThat(spec.requiresMetadata(), equalTo(false)); + assertThat(spec.requiredStoredFields(), containsInAnyOrder("other_field")); + assertThat(spec.sourcePaths(), containsInAnyOrder("cat", "dog", "hamster")); + assertThat(spec.sourcePaths(), sameInstance(pref)); + } + private static SearchContext searchContext(SearchSourceBuilder sourceBuilder) { SearchContext sc = mock(SearchContext.class); when(sc.fetchSourceContext()).thenReturn(sourceBuilder.fetchSource()); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 5f26b046e0c8e..5311c1664c9c3 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import static java.util.Collections.emptyMap; @@ -1585,9 +1586,13 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException { } public void testStoredFieldsSpec() throws IOException { + var mapperService = createMapperService(); List fields = List.of(new FieldAndFormat("field", null)); - FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(createMapperService()), fields); - assertEquals(StoredFieldsSpec.NEEDS_SOURCE, fieldFetcher.storedFieldsSpec()); + FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(mapperService), fields); + assertEquals( + StoredFieldsSpec.withSourcePaths(mapperService.getIndexSettings().getIgnoredSourceFormat(), Set.of("field")), + fieldFetcher.storedFieldsSpec() + ); } private List fieldAndFormatList(String name, String format, boolean includeUnmapped) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index fae33f51740f1..757ff99e0d508 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.Query; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.FieldLookup; @@ -24,6 +25,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -63,6 +65,7 @@ public static List fetchSourceValue(MappedFieldType fieldType, Object sourceV public static List fetchSourceValue(MappedFieldType fieldType, Object sourceValue, String format) throws IOException { String field = fieldType.name(); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); @@ -74,6 +77,7 @@ public static List fetchSourceValue(MappedFieldType fieldType, Object sourceV public static List fetchSourceValues(MappedFieldType fieldType, Object... values) throws IOException { String field = fieldType.name(); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 98d6576f59a44..a2967973fb102 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -859,6 +859,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java index d20a002407be6..965d1a5037205 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.Set; /** * Context of each shard we're operating against. @@ -48,7 +49,7 @@ public interface ShardContext extends RefCounted { /** * Build something to load source {@code _source}. */ - SourceLoader newSourceLoader(); + SourceLoader newSourceLoader(Set sourcePaths); /** * Returns something to load values from this field into a {@link Block}. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java index 84cafb09a8d4f..9e667978559c4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java @@ -191,7 +191,7 @@ private void fieldsMoved(LeafReaderContext ctx, int shard) throws IOException { } SourceLoader sourceLoader = null; if (storedFieldsSpec.requiresSource()) { - sourceLoader = operator.shardContexts.get(shard).newSourceLoader().get(); + sourceLoader = operator.shardContexts.get(shard).newSourceLoader().apply(storedFieldsSpec.sourcePaths()); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } storedFields = new BlockLoaderStoredFieldsFromLeafLoader( diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java index 59e54103eb310..6203842dc9e0d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java @@ -139,7 +139,7 @@ private void loadFromRowStrideReaders( SourceLoader sourceLoader = null; ValuesSourceReaderOperator.ShardContext shardContext = operator.shardContexts.get(shard); if (storedFieldsSpec.requiresSource()) { - sourceLoader = shardContext.newSourceLoader().get(); + sourceLoader = shardContext.newSourceLoader().apply(storedFieldsSpec.sourcePaths()); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } if (storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java index cea156c072dd5..d5f13bee0bdb1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java @@ -29,9 +29,10 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.function.Function; import java.util.function.IntFunction; -import java.util.function.Supplier; /** * Operator that extracts doc_values from a Lucene index out of pages that have been produced by {@link LuceneSourceOperator} @@ -92,7 +93,11 @@ public String describe() { */ public record FieldInfo(String name, ElementType type, boolean nullsFiltered, IntFunction blockLoader) {} - public record ShardContext(IndexReader reader, Supplier newSourceLoader, double storedFieldsSequentialProportion) {} + public record ShardContext( + IndexReader reader, + Function, SourceLoader> newSourceLoader, + double storedFieldsSequentialProportion + ) {} final BlockFactory blockFactory; /** diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java index 490fc76747623..84db6369ce011 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java @@ -191,7 +191,7 @@ public void testPushRoundToToQuery() throws IOException { f -> new BlockDocValuesReader.LongsBlockLoader("v") ) ), - new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException(); }, 0.8)), 0 diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java index 9262fe19ade68..5c2cd244f36c5 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java @@ -210,7 +210,7 @@ private List runQuery(Set values, Query query, boolean shuffleDocs unused -> new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(FIELD) ) ), - new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException(); }, 0.2)), 0 diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java index 7a87772362223..3212430c93d11 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java @@ -54,6 +54,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -457,7 +458,7 @@ public IndexSearcher searcher() { } @Override - public SourceLoader newSourceLoader() { + public SourceLoader newSourceLoader(Set sourcePaths) { return SourceLoader.FROM_STORED_SOURCE; } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index 070ed6d8f0f15..c4a3079d866ba 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -210,7 +210,7 @@ private MapperService mapperService(String indexKey) { private List initShardContexts() { return INDICES.keySet() .stream() - .map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), () -> SourceLoader.FROM_STORED_SOURCE, 0.2)) + .map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2)) .toList(); } @@ -1337,7 +1337,11 @@ public void testWithNulls() throws IOException { LuceneOperator.NO_LIMIT, false // no scoring ); - var vsShardContext = new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2); + var vsShardContext = new ValuesSourceReaderOperator.ShardContext( + reader(indexKey), + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, + 0.2 + ); try ( Driver driver = TestDriverFactory.create( driverContext, @@ -1468,7 +1472,7 @@ public void testDescriptionOfMany() throws IOException { ByteSizeValue.ofGb(1), cases.stream().map(c -> c.info).toList(), new IndexedByShardIdFromSingleton<>( - new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2) + new ValuesSourceReaderOperator.ShardContext(reader(indexKey), (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2) ), 0 ); @@ -1498,7 +1502,7 @@ public void testManyShards() throws IOException { for (int s = 0; s < shardCount; s++) { contexts.add(new LuceneSourceOperatorTests.MockShardContext(readers[s], s)); readerShardContexts.add( - new ValuesSourceReaderOperator.ShardContext(readers[s], () -> SourceLoader.FROM_STORED_SOURCE, 0.2) + new ValuesSourceReaderOperator.ShardContext(readers[s], (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2) ); } var luceneFactory = new LuceneSourceOperator.Factory( diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java index 91f648ff850cd..97379a9ac8480 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java @@ -166,7 +166,7 @@ static Operator.OperatorFactory factory(IndexReader reader, String name, Element new IndexedByShardIdFromSingleton<>( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -502,7 +502,7 @@ public void testManySingleDocPages() { new IndexedByShardIdFromSingleton<>( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -622,7 +622,7 @@ private void loadSimpleAndAssert( new IndexedByShardIdFromSingleton<>( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -641,7 +641,7 @@ private void loadSimpleAndAssert( new IndexedByShardIdFromSingleton<>( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -740,7 +740,7 @@ private void testLoadAllStatus(boolean allInOnePage) { new IndexedByShardIdFromSingleton<>( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -974,7 +974,7 @@ private void testLoadLong(boolean shuffle, boolean manySegments) throws IOExcept new IndexedByShardIdFromSingleton<>( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1643,7 +1643,7 @@ public void testNullsShared() { new IndexedByShardIdFromSingleton<>( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1695,7 +1695,7 @@ private void testSequentialStoredFields(boolean sequential, int docCount) throws new IndexedByShardIdFromSingleton<>( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1730,7 +1730,7 @@ public void testDescriptionOfMany() throws IOException { new IndexedByShardIdFromSingleton<>( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1763,7 +1763,7 @@ public void testManyShards() throws IOException { readerShardContexts.add( new ValuesSourceReaderOperator.ShardContext( readers[s], - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java index a3c52374ef859..9af360dd1695c 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java @@ -362,7 +362,7 @@ private void runLookup(List keyTypes, PopulateIndices populateIndices, PlannerSettings.VALUES_LOADING_JUMBO_SIZE.getDefault(Settings.EMPTY), fieldInfos, new IndexedByShardIdFromSingleton<>( - new ValuesSourceReaderOperator.ShardContext(searchContext.getSearchExecutionContext().getIndexReader(), () -> { + new ValuesSourceReaderOperator.ShardContext(searchContext.getSearchExecutionContext().getIndexReader(), (paths) -> { throw new IllegalStateException("can't load source here"); }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY)) ), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index ac49ec93f9cc3..76cfa1914d396 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -60,6 +60,7 @@ import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.xpack.esql.core.expression.Attribute; @@ -453,8 +454,9 @@ public String shardIdentifier() { } @Override - public SourceLoader newSourceLoader() { - return ctx.newSourceLoader(null, false); + public SourceLoader newSourceLoader(Set sourcePaths) { + var filter = sourcePaths != null ? new SourceFilter(sourcePaths.toArray(new String[0]), null) : null; + return ctx.newSourceLoader(filter, false); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java index f5865a8291f97..cf05f240b43c5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Function; /** A rather roundabout way of ignoring shard ref counters in tests. */ @@ -60,7 +61,7 @@ public String shardIdentifier() { } @Override - public SourceLoader newSourceLoader() { + public SourceLoader newSourceLoader(Set sourcePaths) { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 1a8b162eb1b46..6e16bc4a43512 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -1053,7 +1053,7 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes @Override public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { String name = useLegacyFormat ? name().concat(".text") : name(); - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name)); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name), blContext.indexSettings()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 74459072b8979..b11eb62b9db7d 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -398,7 +399,8 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher valueFetcher = new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted) { + var ignoredSourceFormat = blContext.indexSettings().getIgnoredSourceFormat(); + var valueFetcher = new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted, ignoredSourceFormat) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { @@ -511,7 +513,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedUnsignedLongIndexFieldData.Builder( name(), valuesSourceType, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, UnsignedLongDocValuesField::new ); @@ -525,11 +527,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValueFormatted) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValueFormatted, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index ad29a191aace3..5834aca5fa0a5 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -374,6 +374,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; SourceToParse source = source(b -> b.field(ft.name(), value)); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> {