Skip to content

Commit 4f77a07

Browse files
normen662normen662
authored
implicit group bys (#2943)
* make GroupByTest more interesting * predicate mappings use translated query predicates * pullUp and propagation of pull up structure during compensate call * save point * refactored pruning iterator * save point before refactoring max match maps to query value post order traversal * save point * save-point * progress on the pullup of predicates * pull up seems to be code-complete * save point 2 * added translations to result values * fixing things * changed matched quantifier map back to matched quantifier set * pull up shenanigans * pull up seems to work for residuals * nullability problems * make selects not match if not all for each quantifiers are matched * proper rebase * existential planning works now * major fixes so far done * fixing stuff * ranked indexes work * fix order from plan for aggregate index scans * save point before m3 calculate refactoring * some refactoring to enumerate all expansions * dp algorithm for m3 matching * simple testcases work * deeplyy nested test case works * refactored Values explain and M3 maps calculate * all known bugs fixed * fixing more bugs * memoization for predicate pull ups * adding test cases * all tests pass * version value fix * adding documentation to MaxMatchMap * adjusts due to the rebase * adjust for null on empty * enable implicit groupings * add unit test and address more comments * performance improvements to short circuit the computation of max match maps * addressing yhatem's comments
1 parent 79a5e63 commit 4f77a07

File tree

182 files changed

+5305
-1722
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

182 files changed

+5305
-1722
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
3131
import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
3232
import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
33+
import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedRecordValue;
3334
import com.apple.foundationdb.record.query.plan.cascades.values.Value;
3435
import com.apple.foundationdb.record.query.plan.cascades.values.VersionValue;
3536
import com.apple.foundationdb.record.util.HashUtils;
@@ -116,7 +117,7 @@ public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpr
116117
@Nonnull
117118
@Override
118119
public Value toValue(@Nonnull final CorrelationIdentifier baseAlias, @Nonnull final Type baseType) {
119-
return new VersionValue(baseAlias);
120+
return new VersionValue(QuantifiedRecordValue.of(baseAlias, baseType));
120121
}
121122

122123
@Override

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScanComparisons.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ public Set<CorrelationIdentifier> getCorrelatedTo() {
147147
@Nonnull
148148
@Override
149149
public IndexScanParameters rebase(@Nonnull final AliasMap translationMap) {
150-
return translateCorrelations(TranslationMap.rebaseWithAliasMap(translationMap));
150+
return translateCorrelations(TranslationMap.rebaseWithAliasMap(translationMap), false);
151151
}
152152

153153
@Override
@@ -178,8 +178,10 @@ public int semanticHashCode() {
178178
@Nonnull
179179
@Override
180180
@SuppressWarnings("PMD.CompareObjectsWithEquals")
181-
public IndexScanParameters translateCorrelations(@Nonnull final TranslationMap translationMap) {
182-
final var translatedScanComparisons = scanComparisons.translateCorrelations(translationMap);
181+
public IndexScanParameters translateCorrelations(@Nonnull final TranslationMap translationMap,
182+
final boolean shouldSimplifyValues) {
183+
final var translatedScanComparisons =
184+
scanComparisons.translateCorrelations(translationMap, shouldSimplifyValues);
183185
if (translatedScanComparisons != scanComparisons) {
184186
return withScanComparisons(translatedScanComparisons);
185187
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScanParameters.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public interface IndexScanParameters extends PlanHashable, Correlated<IndexScanP
8686
void getPlannerGraphDetails(@Nonnull ImmutableList.Builder<String> detailsBuilder, @Nonnull ImmutableMap.Builder<String, Attribute> attributeMapBuilder);
8787

8888
@Nonnull
89-
IndexScanParameters translateCorrelations(@Nonnull TranslationMap translationMap);
89+
IndexScanParameters translateCorrelations(@Nonnull TranslationMap translationMap, boolean shouldSimplifyValues);
9090

9191
@Nonnull
9292
PIndexScanParameters toIndexScanParametersProto(@Nonnull PlanSerializationContext serializationContext);

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/MultidimensionalIndexScanComparisons.java

+9-5
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ public Set<CorrelationIdentifier> getCorrelatedTo() {
193193
@Nonnull
194194
@Override
195195
public IndexScanParameters rebase(@Nonnull final AliasMap translationMap) {
196-
return translateCorrelations(TranslationMap.rebaseWithAliasMap(translationMap));
196+
return translateCorrelations(TranslationMap.rebaseWithAliasMap(translationMap), false);
197197
}
198198

199199
@Override
@@ -236,20 +236,24 @@ public int semanticHashCode() {
236236
@Nonnull
237237
@Override
238238
@SuppressWarnings("PMD.CompareObjectsWithEquals")
239-
public IndexScanParameters translateCorrelations(@Nonnull final TranslationMap translationMap) {
240-
final ScanComparisons translatedPrefixScanComparisons = prefixScanComparisons.translateCorrelations(translationMap);
239+
public IndexScanParameters translateCorrelations(@Nonnull final TranslationMap translationMap,
240+
final boolean shouldSimplifyValues) {
241+
final ScanComparisons translatedPrefixScanComparisons =
242+
prefixScanComparisons.translateCorrelations(translationMap, shouldSimplifyValues);
241243

242244
final ImmutableList.Builder<ScanComparisons> translatedDimensionScanComparisonBuilder = ImmutableList.builder();
243245
boolean isSameDimensionsScanComparisons = true;
244246
for (final ScanComparisons dimensionScanComparisons : dimensionsScanComparisons) {
245-
final ScanComparisons translatedDimensionScanComparison = dimensionScanComparisons.translateCorrelations(translationMap);
247+
final ScanComparisons translatedDimensionScanComparison =
248+
dimensionScanComparisons.translateCorrelations(translationMap, shouldSimplifyValues);
246249
if (translatedDimensionScanComparison != dimensionScanComparisons) {
247250
isSameDimensionsScanComparisons = false;
248251
}
249252
translatedDimensionScanComparisonBuilder.add(translatedDimensionScanComparison);
250253
}
251254

252-
final ScanComparisons translatedSuffixKeyScanComparisons = suffixScanComparisons.translateCorrelations(translationMap);
255+
final ScanComparisons translatedSuffixKeyScanComparisons =
256+
suffixScanComparisons.translateCorrelations(translationMap, shouldSimplifyValues);
253257

254258
if (translatedPrefixScanComparisons != prefixScanComparisons || !isSameDimensionsScanComparisons ||
255259
translatedSuffixKeyScanComparisons != suffixScanComparisons) {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java

+110-15
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,11 @@
101101
import java.util.List;
102102
import java.util.Map;
103103
import java.util.Objects;
104+
import java.util.Optional;
104105
import java.util.Queue;
105106
import java.util.Set;
106107
import java.util.UUID;
108+
import java.util.function.Function;
107109
import java.util.function.Supplier;
108110
import java.util.stream.Collectors;
109111

@@ -800,6 +802,9 @@ default Comparison withValue(@Nonnull Value value) {
800802
throw new RecordCoreException("withValue is not implemented");
801803
}
802804

805+
@Nonnull
806+
Optional<Comparison> replaceValuesMaybe(@Nonnull Function<Value, Optional<Value>> replacementFunction);
807+
803808
/**
804809
* Get the comparison value without any bindings.
805810
* @return the value to be compared
@@ -848,11 +853,11 @@ default Set<CorrelationIdentifier> getCorrelatedTo() {
848853
@Nonnull
849854
@Override
850855
default Comparison rebase(@Nonnull AliasMap translationMap) {
851-
return translateCorrelations(TranslationMap.rebaseWithAliasMap(translationMap));
856+
return translateCorrelations(TranslationMap.rebaseWithAliasMap(translationMap), false);
852857
}
853858

854859
@Nonnull
855-
Comparison translateCorrelations(@Nonnull TranslationMap translationMap);
860+
Comparison translateCorrelations(@Nonnull TranslationMap translationMap, boolean shouldSimplifyValues);
856861

857862
@Override
858863
default boolean semanticEquals(@Nullable Object other, @Nonnull AliasMap aliasMap) {
@@ -984,9 +989,19 @@ public Value getValue() {
984989
@Nonnull
985990
@Override
986991
public Comparison withValue(@Nonnull final Value value) {
992+
if (value instanceof LiteralValue<?>) {
993+
return new SimpleComparison(getType(),
994+
Objects.requireNonNull(((LiteralValue<?>)value).getLiteralValue()));
995+
}
987996
return new ValueComparison(getType(), value);
988997
}
989998

999+
@Nonnull
1000+
@Override
1001+
public Optional<Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
1002+
return Optional.of(this);
1003+
}
1004+
9901005
@Nullable
9911006
@Override
9921007
public Boolean eval(@Nullable FDBRecordStoreBase<?> store, @Nonnull EvaluationContext context, @Nullable Object value) {
@@ -1055,7 +1070,8 @@ public int queryHash(@Nonnull final QueryHashKind hashKind) {
10551070

10561071
@Nonnull
10571072
@Override
1058-
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
1073+
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap,
1074+
final boolean shouldSimplifyValues) {
10591075
return this;
10601076
}
10611077
}
@@ -1220,10 +1236,19 @@ public Object getComparand(@Nullable FDBRecordStoreBase<?> store, @Nullable Eval
12201236
}
12211237
}
12221238

1239+
@Nonnull
1240+
@Override
1241+
public Optional<Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
1242+
if (isCorrelation()) {
1243+
return Optional.empty();
1244+
}
1245+
return Optional.of(this);
1246+
}
1247+
12231248
@Nonnull
12241249
@Override
12251250
@SuppressWarnings("PMD.CompareObjectsWithEquals")
1226-
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
1251+
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap, final boolean shouldSimplifyValues) {
12271252
if (isCorrelation()) {
12281253
final var alias = CorrelationIdentifier.of(Bindings.Internal.CORRELATION.identifier(parameter));
12291254
final var quantifiedObjectValue = QuantifiedObjectValue.of(alias,
@@ -1234,7 +1259,7 @@ public Comparison translateCorrelations(@Nonnull final TranslationMap translatio
12341259
//
12351260
final var translatedQuantifiedObjectValue =
12361261
(QuantifiedObjectValue)quantifiedObjectValue
1237-
.translateCorrelations(translationMap);
1262+
.translateCorrelations(translationMap, shouldSimplifyValues);
12381263
if (quantifiedObjectValue == translatedQuantifiedObjectValue) {
12391264
return this;
12401265
}
@@ -1550,6 +1575,19 @@ public Value getComparandValue() {
15501575
return comparandValue;
15511576
}
15521577

1578+
@Nonnull
1579+
@Override
1580+
@SuppressWarnings("PMD.CompareObjectsWithEquals")
1581+
public Optional<Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
1582+
return replacementFunction.apply(getValue())
1583+
.map(replacedComparandValue -> {
1584+
if (replacedComparandValue == getValue()) {
1585+
return this;
1586+
}
1587+
return withValue(replacedComparandValue);
1588+
});
1589+
}
1590+
15531591
@Nullable
15541592
@Override
15551593
public Object getComparand(@Nullable FDBRecordStoreBase<?> store, @Nullable EvaluationContext context) {
@@ -1561,14 +1599,15 @@ public Object getComparand(@Nullable FDBRecordStoreBase<?> store, @Nullable Eval
15611599

15621600
@Nonnull
15631601
@Override
1564-
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
1602+
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap, final boolean shouldSimplifyValues) {
15651603
if (comparandValue.getCorrelatedTo()
15661604
.stream()
15671605
.noneMatch(translationMap::containsSourceAlias)) {
15681606
return this;
15691607
}
15701608

1571-
return new ValueComparison(type, comparandValue.translateCorrelations(translationMap), parameterRelationshipGraph);
1609+
return new ValueComparison(type, comparandValue.translateCorrelations(translationMap, shouldSimplifyValues),
1610+
parameterRelationshipGraph);
15721611
}
15731612

15741613
@Nonnull
@@ -1814,7 +1853,14 @@ public List getComparandWithRealEquals() {
18141853

18151854
@Nonnull
18161855
@Override
1817-
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
1856+
public Optional<Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
1857+
return Optional.of(this);
1858+
}
1859+
1860+
@Nonnull
1861+
@Override
1862+
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap,
1863+
final boolean shouldSimplifyValues) {
18181864
return this;
18191865
}
18201866

@@ -2003,7 +2049,14 @@ public Object getComparand(@Nullable FDBRecordStoreBase<?> store, @Nullable Eval
20032049

20042050
@Nonnull
20052051
@Override
2006-
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
2052+
public Optional<Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
2053+
return Optional.of(this);
2054+
}
2055+
2056+
@Nonnull
2057+
@Override
2058+
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap,
2059+
final boolean shouldSimplifyValues) {
20072060
return this;
20082061
}
20092062

@@ -2131,7 +2184,14 @@ public Object getComparand(@Nullable FDBRecordStoreBase<?> store, @Nullable Eval
21312184

21322185
@Nonnull
21332186
@Override
2134-
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
2187+
public Optional<Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
2188+
return Optional.of(this);
2189+
}
2190+
2191+
@Nonnull
2192+
@Override
2193+
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap,
2194+
final boolean shouldSimplifyValues) {
21352195
return this;
21362196
}
21372197

@@ -2250,7 +2310,14 @@ public TextComparison(@Nonnull Type type, @Nonnull List<String> tokens, @Nullabl
22502310

22512311
@Nonnull
22522312
@Override
2253-
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
2313+
public Optional<Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
2314+
return Optional.of(this);
2315+
}
2316+
2317+
@Nonnull
2318+
@Override
2319+
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap,
2320+
final boolean shouldSimplifyValues) {
22542321
return this;
22552322
}
22562323

@@ -2705,8 +2772,22 @@ public Comparison withValue(@Nonnull final Value value) {
27052772
@Nonnull
27062773
@Override
27072774
@SuppressWarnings("PMD.CompareObjectsWithEquals")
2708-
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
2709-
final var translatedInner = inner.translateCorrelations(translationMap);
2775+
public Optional<Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
2776+
return inner.replaceValuesMaybe(replacementFunction)
2777+
.map(replacedInner -> {
2778+
if (replacedInner == inner) {
2779+
return this;
2780+
}
2781+
return new MultiColumnComparison(replacedInner);
2782+
});
2783+
}
2784+
2785+
@Nonnull
2786+
@Override
2787+
@SuppressWarnings("PMD.CompareObjectsWithEquals")
2788+
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap,
2789+
final boolean shouldSimplifyValues) {
2790+
final var translatedInner = inner.translateCorrelations(translationMap, shouldSimplifyValues);
27102791
if (inner == translatedInner) {
27112792
return this;
27122793
} else {
@@ -2985,8 +3066,22 @@ public ExplainTokensWithPrecedence explain() {
29853066
@Nonnull
29863067
@Override
29873068
@SuppressWarnings({"PMD.CompareObjectsWithEquals"}) // used here for referential equality
2988-
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
2989-
Comparison translated = originalComparison.translateCorrelations(translationMap);
3069+
public Optional<Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
3070+
return originalComparison.replaceValuesMaybe(replacementFunction)
3071+
.map(translatedOriginalComparison -> {
3072+
if (translatedOriginalComparison == originalComparison) {
3073+
return this;
3074+
}
3075+
return new InvertedFunctionComparison(function, translatedOriginalComparison, type);
3076+
});
3077+
}
3078+
3079+
@Nonnull
3080+
@Override
3081+
@SuppressWarnings({"PMD.CompareObjectsWithEquals"}) // used here for referential equality
3082+
public Comparison translateCorrelations(@Nonnull final TranslationMap translationMap,
3083+
final boolean shouldSimplifyValues) {
3084+
Comparison translated = originalComparison.translateCorrelations(translationMap, shouldSimplifyValues);
29903085
if (translated == originalComparison) {
29913086
return this;
29923087
} else {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/QueryRecordFunctionWithComparison.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import com.apple.foundationdb.record.query.plan.cascades.predicates.ValuePredicate;
3939
import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
4040
import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue;
41+
import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedRecordValue;
4142
import com.apple.foundationdb.record.query.plan.cascades.values.RankValue;
4243
import com.apple.foundationdb.record.query.plan.cascades.values.VersionValue;
4344
import com.apple.foundationdb.record.util.HashUtils;
@@ -186,7 +187,7 @@ public GraphExpansion expand(@Nonnull final Quantifier.ForEach baseQuantifier,
186187

187188
return GraphExpansion.ofExists(rankComparisonQuantifier);
188189
} else if (function instanceof StoreRecordFunction<?> && FunctionNames.VERSION.equals(function.getName())) {
189-
final VersionValue versionValue = new VersionValue(baseQuantifier.getAlias());
190+
final VersionValue versionValue = new VersionValue(QuantifiedRecordValue.of(baseQuantifier));
190191
return GraphExpansion.builder()
191192
.addPredicate(new ValuePredicate(versionValue, comparison))
192193
.build();

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/RecordTypeKeyComparison.java

+11-1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
4040
import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue;
4141
import com.apple.foundationdb.record.query.plan.cascades.values.RecordTypeValue;
42+
import com.apple.foundationdb.record.query.plan.cascades.values.Value;
4243
import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap;
4344
import com.apple.foundationdb.record.util.HashUtils;
4445
import com.google.auto.service.AutoService;
@@ -50,7 +51,9 @@
5051
import java.util.Collections;
5152
import java.util.List;
5253
import java.util.Objects;
54+
import java.util.Optional;
5355
import java.util.Set;
56+
import java.util.function.Function;
5457
import java.util.function.Supplier;
5558

5659
/**
@@ -186,7 +189,14 @@ public void validate(@Nonnull Descriptors.FieldDescriptor descriptor, boolean fa
186189

187190
@Nonnull
188191
@Override
189-
public Comparisons.Comparison translateCorrelations(@Nonnull final TranslationMap translationMap) {
192+
public Optional<Comparisons.Comparison> replaceValuesMaybe(@Nonnull final Function<Value, Optional<Value>> replacementFunction) {
193+
return Optional.of(this);
194+
}
195+
196+
@Nonnull
197+
@Override
198+
public Comparisons.Comparison translateCorrelations(@Nonnull final TranslationMap translationMap,
199+
final boolean shouldSimplifyValues) {
190200
return this;
191201
}
192202

0 commit comments

Comments
 (0)