diff --git a/metamorphic/cockroachkvs.go b/metamorphic/cockroachkvs.go index 77b096a163..0cbfbd0820 100644 --- a/metamorphic/cockroachkvs.go +++ b/metamorphic/cockroachkvs.go @@ -64,6 +64,16 @@ var CockroachKeyFormat = KeyFormat{ } return cockroachkvs.NewMVCCTimeIntervalFilter(minWallTime, maxWallTime) }, + ParseMaximumSuffixProperty: func(s string) pebble.MaximumSuffixProperty { + return cockroachkvs.MaxMVCCTimestampProperty{} + }, + FormatMaximumSuffixProperty: func(prop pebble.MaximumSuffixProperty) string { + if prop == nil { + return "" + } + return "maxsuffixprop" + }, + MaximumSuffixProperty: cockroachkvs.MaxMVCCTimestampProperty{}, } type cockroachKeyGenerator struct { diff --git a/metamorphic/config.go b/metamorphic/config.go index cfb37f7bec..831aef5c05 100644 --- a/metamorphic/config.go +++ b/metamorphic/config.go @@ -374,6 +374,9 @@ type KeyFormat struct { NewGenerator func(*keyManager, *rand.Rand, OpConfig) KeyGenerator NewSuffixFilterMask func() pebble.BlockPropertyFilterMask NewSuffixBlockPropertyFilter func(min []byte, max []byte) sstable.BlockPropertyFilter + ParseMaximumSuffixProperty func(string) pebble.MaximumSuffixProperty + FormatMaximumSuffixProperty func(pebble.MaximumSuffixProperty) string + MaximumSuffixProperty pebble.MaximumSuffixProperty } func (kf KeyFormat) apply(ro *runAndCompareOptions) { ro.keyFormat = kf } diff --git a/metamorphic/diagram.go b/metamorphic/diagram.go index 8d5e482132..50749afdee 100644 --- a/metamorphic/diagram.go +++ b/metamorphic/diagram.go @@ -15,6 +15,7 @@ func TryToGenerateDiagram(keyFormat KeyFormat, opsData []byte) (string, error) { ops, err := parse(opsData, parserOpts{ parseFormattedUserKey: keyFormat.ParseFormattedKey, parseFormattedUserKeySuffix: keyFormat.ParseFormattedKeySuffix, + parseMaximumSuffixProperty: keyFormat.ParseMaximumSuffixProperty, }) if err != nil { return "", err diff --git a/metamorphic/generator.go b/metamorphic/generator.go index 4f50e37f87..97f00ec250 100644 --- a/metamorphic/generator.go +++ b/metamorphic/generator.go @@ -48,12 +48,16 @@ type iterOpts struct { // see IterOptions.UseL6Filters. useL6Filters bool + // MaximumSuffixProperty is the maximum suffix property used during the lazy + // position of SeekPrefixGE optimization. + maximumSuffixProperty pebble.MaximumSuffixProperty + // NB: If adding or removing fields, ensure IsZero is in sync. } func (o iterOpts) IsZero() bool { return o.lower == nil && o.upper == nil && o.keyTypes == 0 && - o.maskSuffix == nil && o.filterMin == nil && o.filterMax == nil && !o.useL6Filters + o.maskSuffix == nil && o.filterMin == nil && o.filterMax == nil && !o.useL6Filters && o.maximumSuffixProperty == nil } // GenerateOps generates n random operations, drawing randomness from the @@ -566,6 +570,11 @@ func (g *generator) newIter() { opts.useL6Filters = true } + // With 20% probability, enable the lazy positioning SeekPrefixGE optimization. + if g.rng.Float64() <= 0.2 { + opts.maximumSuffixProperty = g.keyManager.kf.MaximumSuffixProperty + } + g.itersLastOpts[iterID] = opts g.iterVisibleKeys[iterID] = g.keyManager.getSetOfVisibleKeys(readerID) g.iterReaderID[iterID] = readerID @@ -1562,6 +1571,10 @@ func (g *generator) maybeMutateOptions(readerID objID, opts *iterOpts) { if g.rng.Float64() <= 0.1 { opts.useL6Filters = !opts.useL6Filters } + // With 20% probability, clear existing maximum suffix property. + if opts.maximumSuffixProperty != nil && g.rng.IntN(5) == 0 { + opts.maximumSuffixProperty = nil + } } } diff --git a/metamorphic/meta.go b/metamorphic/meta.go index 414dab1554..ce93807bb3 100644 --- a/metamorphic/meta.go +++ b/metamorphic/meta.go @@ -509,6 +509,7 @@ func RunOnce(t TestingT, runDir string, seed uint64, historyPath string, rOpts . ops, err := parse(opsData, parserOpts{ parseFormattedUserKey: testOpts.KeyFormat.ParseFormattedKey, parseFormattedUserKeySuffix: testOpts.KeyFormat.ParseFormattedKeySuffix, + parseMaximumSuffixProperty: testOpts.KeyFormat.ParseMaximumSuffixProperty, }) require.NoError(t, err) diff --git a/metamorphic/ops.go b/metamorphic/ops.go index a094092b07..2727a99862 100644 --- a/metamorphic/ops.go +++ b/metamorphic/ops.go @@ -1206,10 +1206,10 @@ func (o *newIterOp) run(t *Test, h historyRecorder) { } func (o *newIterOp) formattedString(kf KeyFormat) string { - return fmt.Sprintf("%s = %s.NewIter(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */)", + return fmt.Sprintf("%s = %s.NewIter(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */, %q /* maximum suffix property */)", o.iterID, o.readerID, kf.FormatKey(o.lower), kf.FormatKey(o.upper), o.keyTypes, kf.FormatKeySuffix(o.filterMax), kf.FormatKeySuffix(o.filterMin), - o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix)) + o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix), kf.FormatMaximumSuffixProperty(o.maximumSuffixProperty)) } func (o *newIterOp) receiver() objID { return o.readerID } @@ -1273,10 +1273,10 @@ func (o *newIterUsingCloneOp) run(t *Test, h historyRecorder) { } func (o *newIterUsingCloneOp) formattedString(kf KeyFormat) string { - return fmt.Sprintf("%s = %s.Clone(%t, %q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */)", + return fmt.Sprintf("%s = %s.Clone(%t, %q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */, %q /* maximum suffix property */)", o.iterID, o.existingIterID, o.refreshBatch, kf.FormatKey(o.lower), kf.FormatKey(o.upper), o.keyTypes, kf.FormatKeySuffix(o.filterMax), - kf.FormatKeySuffix(o.filterMin), o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix)) + kf.FormatKeySuffix(o.filterMin), o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix), kf.FormatMaximumSuffixProperty(o.maximumSuffixProperty)) } func (o *newIterUsingCloneOp) receiver() objID { return o.existingIterID } @@ -1370,10 +1370,10 @@ func (o *iterSetOptionsOp) run(t *Test, h historyRecorder) { } func (o *iterSetOptionsOp) formattedString(kf KeyFormat) string { - return fmt.Sprintf("%s.SetOptions(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */)", + return fmt.Sprintf("%s.SetOptions(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */, %q /* maximum suffix property */)", o.iterID, kf.FormatKey(o.lower), kf.FormatKey(o.upper), o.keyTypes, kf.FormatKeySuffix(o.filterMax), kf.FormatKeySuffix(o.filterMin), - o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix)) + o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix), kf.FormatMaximumSuffixProperty(o.maximumSuffixProperty)) } func iterOptions(kf KeyFormat, o iterOpts) *pebble.IterOptions { @@ -1394,8 +1394,9 @@ func iterOptions(kf KeyFormat, o iterOpts) *pebble.IterOptions { RangeKeyMasking: pebble.RangeKeyMasking{ Suffix: o.maskSuffix, }, - UseL6Filters: o.useL6Filters, - DebugRangeKeyStack: debugIterators, + UseL6Filters: o.useL6Filters, + DebugRangeKeyStack: debugIterators, + MaximumSuffixProperty: o.maximumSuffixProperty, } if opts.RangeKeyMasking.Suffix != nil { opts.RangeKeyMasking.Filter = kf.NewSuffixFilterMask diff --git a/metamorphic/parser.go b/metamorphic/parser.go index 549aeb8b11..a114717feb 100644 --- a/metamorphic/parser.go +++ b/metamorphic/parser.go @@ -93,9 +93,9 @@ func opArgs(op op) (receiverID *objID, targetID *objID, args []interface{}) { case *newIndexedBatchOp: return &t.dbID, &t.batchID, nil case *newIterOp: - return &t.readerID, &t.iterID, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix} + return &t.readerID, &t.iterID, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix, &t.maximumSuffixProperty} case *newIterUsingCloneOp: - return &t.existingIterID, &t.iterID, []interface{}{&t.refreshBatch, &t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix} + return &t.existingIterID, &t.iterID, []interface{}{&t.refreshBatch, &t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix, &t.maximumSuffixProperty} case *newSnapshotOp: return &t.dbID, &t.snapID, []interface{}{&t.bounds} case *newExternalObjOp: @@ -119,7 +119,7 @@ func opArgs(op op) (receiverID *objID, targetID *objID, args []interface{}) { case *iterSetBoundsOp: return &t.iterID, nil, []interface{}{&t.lower, &t.upper} case *iterSetOptionsOp: - return &t.iterID, nil, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix} + return &t.iterID, nil, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix, &t.maximumSuffixProperty} case *singleDeleteOp: return &t.writerID, nil, []interface{}{&t.key, &t.maybeReplaceDelete} case *rangeKeyDeleteOp: @@ -197,6 +197,7 @@ type parserOpts struct { allowUndefinedObjs bool parseFormattedUserKey func(string) UserKey parseFormattedUserKeySuffix func(string) UserKeySuffix + parseMaximumSuffixProperty func(string) pebble.MaximumSuffixProperty } func parse(src []byte, opts parserOpts) (_ []op, err error) { @@ -319,6 +320,22 @@ func (p *parser) parseUserKeySuffix(pos token.Pos, lit string) UserKeySuffix { return p.opts.parseFormattedUserKeySuffix(s) } +func (p *parser) parseMaximumSuffixProperty( + pos token.Pos, lit string, +) pebble.MaximumSuffixProperty { + s, err := strconv.Unquote(lit) + if err != nil { + panic(p.errorf(pos, "%s", err)) + } + if len(s) == 0 { + return nil + } + if p.opts.parseMaximumSuffixProperty == nil { + return nil + } + return p.opts.parseMaximumSuffixProperty(s) +} + func unquoteBytes(lit string) []byte { s, err := strconv.Unquote(lit) if err != nil { @@ -408,6 +425,10 @@ func (p *parser) parseArgs(op op, methodName string, args []interface{}) { } *t = pebble.FormatMajorVersion(val) + case *pebble.MaximumSuffixProperty: + elem.expectToken(p, token.STRING) + *t = p.parseMaximumSuffixProperty(elem.pos, elem.lit) + default: panic(p.errorf(pos, "%s: unsupported arg[%d] type: %T", methodName, i, args[i])) } diff --git a/metamorphic/parser_test.go b/metamorphic/parser_test.go index 5969fd6328..d3a300b87b 100644 --- a/metamorphic/parser_test.go +++ b/metamorphic/parser_test.go @@ -21,6 +21,7 @@ func TestParser(t *testing.T) { ops, err := parse([]byte(d.Input), parserOpts{ parseFormattedUserKey: kf.ParseFormattedKey, parseFormattedUserKeySuffix: kf.ParseFormattedKeySuffix, + parseMaximumSuffixProperty: kf.ParseMaximumSuffixProperty, }) if err != nil { return err.Error() @@ -46,7 +47,9 @@ func TestParserRandom(t *testing.T) { ops := g.generate(10000) src := formatOps(km.kf, ops) - parsedOps, err := parse([]byte(src), parserOpts{}) + parsedOps, err := parse([]byte(src), parserOpts{ + parseMaximumSuffixProperty: km.kf.ParseMaximumSuffixProperty, + }) if err != nil { t.Fatalf("%s\n%s", err, src) } @@ -67,6 +70,7 @@ func TestParserNilBounds(t *testing.T) { parsedOps, err := parse([]byte(formatted), parserOpts{ parseFormattedUserKey: kf.ParseFormattedKey, parseFormattedUserKeySuffix: kf.ParseFormattedKeySuffix, + parseMaximumSuffixProperty: kf.ParseMaximumSuffixProperty, }) require.NoError(t, err) require.Equal(t, 1, len(parsedOps)) diff --git a/metamorphic/simplify.go b/metamorphic/simplify.go index bc1b5fce5f..0b4460b040 100644 --- a/metamorphic/simplify.go +++ b/metamorphic/simplify.go @@ -21,6 +21,7 @@ func TryToSimplifyKeys(keyFormat KeyFormat, opsData []byte, retainSuffixes bool) ops, err := parse(opsData, parserOpts{ parseFormattedUserKey: keyFormat.ParseFormattedKey, parseFormattedUserKeySuffix: keyFormat.ParseFormattedKeySuffix, + parseMaximumSuffixProperty: keyFormat.ParseMaximumSuffixProperty, }) if err != nil { panic(err) diff --git a/metamorphic/testdata/diagram b/metamorphic/testdata/diagram index 0714841655..35c01c453c 100644 --- a/metamorphic/testdata/diagram +++ b/metamorphic/testdata/diagram @@ -5,7 +5,7 @@ snap9 = db2.NewSnapshot("a", "z") db2.RangeKeyDelete("d", "f") db2.Compact("g", "i", true /* parallelize */) db2.Replicate(db1, "b", "f") -iter25 = db1.NewIter("", "", 2 /* key types */, "", "", false /* use L6 filters */, "@7" /* masking suffix */) +iter25 = db1.NewIter("", "", 2 /* key types */, "", "", false /* use L6 filters */, "@7" /* masking suffix */, "" /* maximum suffix property */) iter25.SeekGE("e", "") ---- Init(2 /* dbs */, 49 /* batches */, 63 /* iters */, 45 /* snapshots */, 0 /* externalObjs */) @@ -14,7 +14,7 @@ iter25.SeekGE("e", "") |-----------| db2.RangeKeyDelete("d", "f") |-----------| db2.Compact("g", "i", true /* parallelize */) |-----------------------| db2.Replicate(db1, "b", "f") - iter25 = db1.NewIter("", "", 2 /* key types */, "", "", false /* use L6 filters */, "@7" /* masking suffix */) + iter25 = db1.NewIter("", "", 2 /* key types */, "", "", false /* use L6 filters */, "@7" /* masking suffix */, "" /* maximum suffix property */) | iter25.SeekGE("e", "") |-----|-----|-----|-----|-----|-----|-----|-----|-----| a b c d e f g h i z diff --git a/metamorphic/testkeys.go b/metamorphic/testkeys.go index 41506fdace..5667cfe375 100644 --- a/metamorphic/testkeys.go +++ b/metamorphic/testkeys.go @@ -61,6 +61,16 @@ var TestkeysKeyFormat = KeyFormat{ } return sstable.NewTestKeysBlockPropertyFilter(uint64(high), uint64(low)) }, + ParseMaximumSuffixProperty: func(s string) pebble.MaximumSuffixProperty { + return sstable.MaxTestKeysSuffixProperty{} + }, + FormatMaximumSuffixProperty: func(prop pebble.MaximumSuffixProperty) string { + if prop == nil { + return "" + } + return "maxsuffixprop" + }, + MaximumSuffixProperty: sstable.MaxTestKeysSuffixProperty{}, } type testkeyKeyGenerator struct {