diff --git a/config/config.go b/config/config.go index 3ec0029dd..469fbbe74 100644 --- a/config/config.go +++ b/config/config.go @@ -129,24 +129,25 @@ var knownDataTableContext = map[string]bool{ // DataTable configs type DataTable struct { - Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"` - Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"` - MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"` - MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"` - MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"` - MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"` - TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"` - TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"` - TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"` - TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"` - RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"` - RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"` - RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"` - RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"` - RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"` - Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"` - ContextMap map[string]bool `toml:"-" json:"-"` - Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"` + Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"` + Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"` + MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"` + MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"` + MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"` + MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"` + TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"` + TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"` + TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"` + TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"` + RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"` + RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"` + RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"` + RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"` + RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"` + RollupRuleTypeAutodetect bool `toml:"rollup-rule-type-autodetect" json:"rollup-rule-type-autodetect"` + Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"` + ContextMap map[string]bool `toml:"-" json:"-"` + Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"` } // Debug config @@ -453,7 +454,7 @@ func (c *Config) ProcessDataTables() (err error) { } else if c.DataTable[i].RollupConf == "none" { c.DataTable[i].Rollup, err = rollup.NewDefault(rdp, rdf) } else { - c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf) + c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf, c.DataTable[i].RollupRuleTypeAutodetect) } if err != nil { diff --git a/helper/rollup/compact.go b/helper/rollup/compact.go index 9478b46f5..b0b9ed60e 100644 --- a/helper/rollup/compact.go +++ b/helper/rollup/compact.go @@ -12,11 +12,12 @@ compact form of rollup rules for tests regexp;function;age:precision,age:precision,... */ -func parseCompact(body string) (*Rules, error) { +func parseCompact(body string, auto bool) (*Rules, error) { lines := strings.Split(body, "\n") patterns := make([]Pattern, 0) for _, line := range lines { + var ruleType RuleType if strings.TrimSpace(line) == "" { continue } @@ -29,6 +30,30 @@ func parseCompact(body string) (*Rules, error) { return nil, fmt.Errorf("can't parse line: %#v", line) } regexp := strings.TrimSpace(line[:p1]) + if len(regexp) > 8 && regexp[0] == '<' && regexp[1] == '!' && regexp[7] == '>' { + typeStr := regexp[1:7] + switch typeStr { + case "!ALL_T": + ruleType = RuleAll + case "!PLAIN": + ruleType = RulePlain + case "!TAG_R": + ruleType = RuleTaggedRegex + //case "!TAG_T": + // ruleType = RuleTagged + default: + return nil, fmt.Errorf("not realised rule type for line: %#v", line) + } + regexp = regexp[8:] + } else { + if ruleType == RuleAuto { + if auto && len(regexp) > 0 { + ruleType = AutoDetectRuleType(regexp) + } else { + ruleType = RuleAll + } + } + } function := strings.TrimSpace(line[p1+1 : p2]) retention := make([]Retention, 0) @@ -55,7 +80,7 @@ func parseCompact(body string) (*Rules, error) { } } - patterns = append(patterns, Pattern{Regexp: regexp, Function: function, Retention: retention}) + patterns = append(patterns, Pattern{RuleType: ruleType, Regexp: regexp, Function: function, Retention: retention}) } return (&Rules{Pattern: patterns}).compile() diff --git a/helper/rollup/compact_test.go b/helper/rollup/compact_test.go index 83c06efb3..629c3412b 100644 --- a/helper/rollup/compact_test.go +++ b/helper/rollup/compact_test.go @@ -9,15 +9,186 @@ import ( func TestParseCompact(t *testing.T) { config := ` click_cost;any;0:3600,86400:60 - ;max;0:60,3600:300,86400:3600` + \.max$;max;0:3600,86400:60 + \.max\?;max;0:3600 + \.min$;min;0:3600,86400:60 + \.min\?;min;0:3600 + env=cloud;avg;0:3600 + ;avg;0:60,3600:300,86400:3600` expected, _ := (&Rules{ Pattern: []Pattern{ - Pattern{Regexp: "click_cost", Function: "any", Retention: []Retention{ + Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{ Retention{Age: 0, Precision: 3600}, Retention{Age: 86400, Precision: 60}, }}, - Pattern{Regexp: "", Function: "max", Retention: []Retention{ + Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + expectedPlain, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + expectedTagged, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + assert := assert.New(t) + r, err := parseCompact(config, false) + assert.NoError(err) + assert.Equal(expected, r) + + assert.Equal(len(expected.patternPlain), 6) + assert.Equal(expectedPlain.Pattern, r.patternPlain) + + assert.Equal(len(expected.patternTagged), 6) + assert.Equal(expectedTagged.Pattern, r.patternTagged) +} + +func TestParseCompactAutoDetect(t *testing.T) { + config := ` + click_cost;any;0:3600,86400:60 + \.max$;max;0:3600,86400:60 + \.max\?;max;0:3600 + \.min$;min;0:3600,86400:60 + \.min\?;min;0:3600 + env=cloud;avg;0:3600 + ;avg;0:60,3600:300,86400:3600` + + expected, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + expectedPlain, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + expectedTagged, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ Retention{Age: 0, Precision: 60}, Retention{Age: 3600, Precision: 300}, Retention{Age: 86400, Precision: 3600}, @@ -26,7 +197,11 @@ func TestParseCompact(t *testing.T) { }).compile() assert := assert.New(t) - r, err := parseCompact(config) + r, err := parseCompact(config, true) assert.NoError(err) assert.Equal(expected, r) + + assert.Equal(expectedPlain.Pattern, r.patternPlain) + + assert.Equal(expectedTagged.Pattern, r.patternTagged) } diff --git a/helper/rollup/remote.go b/helper/rollup/remote.go index 86ef80da0..1349a0abb 100644 --- a/helper/rollup/remote.go +++ b/helper/rollup/remote.go @@ -14,11 +14,12 @@ import ( ) type rollupRulesResponseRecord struct { - Regexp string `json:"regexp"` - Function string `json:"function"` - Age string `json:"age"` - Precision string `json:"precision"` - IsDefault int `json:"is_default"` + RuleType RuleType `json:"type"` + Regexp string `json:"regexp"` + Function string `json:"function"` + Age string `json:"age"` + Precision string `json:"precision"` + IsDefault int `json:"is_default"` } type rollupRulesResponse struct { Data []rollupRulesResponseRecord `json:"data"` @@ -74,7 +75,11 @@ func parseJson(body []byte) (*Rules, error) { } } else { if last() == nil || last().Regexp != d.Regexp || last().Function != d.Function { + if d.RuleType == RuleAuto { + d.RuleType = RuleAll // for remote rules - no auto-detect rule type + } r.Pattern = append(r.Pattern, Pattern{ + RuleType: d.RuleType, Retention: make([]Retention, 0), Regexp: d.Regexp, Function: d.Function, @@ -92,6 +97,7 @@ func parseJson(body []byte) (*Rules, error) { if defaultFunction != "" || len(defaultRetention) != 0 { r.Pattern = append(r.Pattern, Pattern{ + RuleType: RuleAll, Regexp: "", Function: defaultFunction, Retention: defaultRetention, diff --git a/helper/rollup/remote_test.go b/helper/rollup/remote_test.go index 490ffc488..8be1846f3 100644 --- a/helper/rollup/remote_test.go +++ b/helper/rollup/remote_test.go @@ -87,9 +87,25 @@ func TestParseJson(t *testing.T) { "precision": "0", "is_default": 0 }, + { + "type": "plain", + "regexp": "\\.min$", + "function": "min", + "age": "0", + "precision": "3600", + "is_default": 0 + }, + { + "type": "tagged_regex", + "regexp": "\\.min\\?", + "function": "min", + "age": "0", + "precision": "3600", + "is_default": 0 + }, { "regexp": "", - "function": "max", + "function": "avg", "age": "0", "precision": "60", "is_default": 1 @@ -112,11 +128,13 @@ func TestParseJson(t *testing.T) { total$;sum; min$;min; max$;max; - ;max;0:60 + \.min$;min;0:3600 + \.min\?;min;0:3600 + ;avg;0:60 ` assert := assert.New(t) - expected, err := parseCompact(compact) + expected, err := parseCompact(compact, false) assert.NoError(err) r, err := parseJson([]byte(response)) diff --git a/helper/rollup/rollup.go b/helper/rollup/rollup.go index 9d11ee721..84b211922 100644 --- a/helper/rollup/rollup.go +++ b/helper/rollup/rollup.go @@ -34,13 +34,13 @@ func NewAuto(addr string, table string, interval time.Duration, defaultPrecision return r, nil } -func NewXMLFile(filename string, defaultPrecision uint32, defaultFunction string) (*Rollup, error) { +func NewXMLFile(filename string, defaultPrecision uint32, defaultFunction string, auto bool) (*Rollup, error) { rollupConfBody, err := ioutil.ReadFile(filename) if err != nil { return nil, err } - rules, err := parseXML(rollupConfBody) + rules, err := parseXML(rollupConfBody, auto) if err != nil { return nil, err } diff --git a/helper/rollup/rules.go b/helper/rollup/rules.go index 3bbc71266..0adce8f6c 100644 --- a/helper/rollup/rules.go +++ b/helper/rollup/rules.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "sort" + "strings" "time" "github.com/lomik/graphite-clickhouse/pkg/dry" @@ -16,7 +17,44 @@ type Retention struct { Precision uint32 `json:"precision"` } +type RuleType uint16 + +const ( + RuleAuto RuleType = iota + RuleAll // rule for plain and tagged + RulePlain // regex for non-tagged + RuleTaggedRegex // regex for tagged + //RuleTagged // complex rule for tagged, separated by tags, like name?tag1=value1&tag2=~value2_regex +) + +func (n *RuleType) UnmarshalText(text []byte) error { + s := strings.ToLower(string(text)) + switch s { + case "plain": + *n = RulePlain + case "tagged_regex": + *n = RuleTaggedRegex + // case "tagged": + // *s = RuleTagged + case "": + case "all": + *n = RuleAll + default: + return fmt.Errorf("not realised rule type: %s", s) + } + return nil +} + +func AutoDetectRuleType(ruleRegexp string) RuleType { + if strings.IndexAny(ruleRegexp, "?&=") == -1 { + return RulePlain + } else { + return RuleTaggedRegex + } +} + type Pattern struct { + RuleType RuleType `json:"type"` Regexp string `json:"regexp"` Function string `json:"function"` Retention []Retention `json:"retention"` @@ -25,8 +63,10 @@ type Pattern struct { } type Rules struct { - Pattern []Pattern `json:"pattern"` - Updated int64 `json:"updated"` + Pattern []Pattern `json:"pattern"` + patternPlain []Pattern + patternTagged []Pattern + Updated int64 `json:"updated"` } // NewMockRulles creates mock rollup for tests @@ -87,6 +127,15 @@ func (r *Rules) compile() (*Rules, error) { if err := r.Pattern[i].compile(); err != nil { return r, err } + + if r.Pattern[i].RuleType == RulePlain { + r.patternPlain = append(r.patternPlain, r.Pattern[i]) + } else if r.Pattern[i].RuleType == RuleTaggedRegex { + r.patternTagged = append(r.patternTagged, r.Pattern[i]) + } else { + r.patternPlain = append(r.patternPlain, r.Pattern[i]) + r.patternTagged = append(r.patternTagged, r.Pattern[i]) + } } return r, nil @@ -110,6 +159,7 @@ func (r *Rules) withDefault(defaultPrecision uint32, defaultFunction *Aggr) *Rul } patterns = append(patterns, Pattern{ + RuleType: RuleAll, Regexp: ".*", Function: defaultFunction.Name(), Retention: retention, @@ -131,7 +181,15 @@ func (r *Rules) withSuperDefault() *Rules { func (r *Rules) Lookup(metric string, age uint32) (precision uint32, ag *Aggr) { precisionFound := false - for _, p := range r.Pattern { + tagged := strings.IndexAny(metric, "?&=") + var patterns []Pattern + if tagged == -1 { + patterns = r.patternPlain + } else { + patterns = r.patternTagged + } + + for _, p := range patterns { // pattern hasn't interested data if (ag != nil || p.aggr == nil) && (precisionFound || len(p.Retention) == 0) { continue diff --git a/helper/rollup/rules_test.go b/helper/rollup/rules_test.go index ffc5b68a4..18dd5858b 100644 --- a/helper/rollup/rules_test.go +++ b/helper/rollup/rules_test.go @@ -35,8 +35,13 @@ func TestLookup(t *testing.T) { ^hourly;;3600:60,86400:3600 ^live;;0:1 total$;sum; - min$;min; - max$;max; + sum$;sum; + sum\?;sum; + sum\?(.*&)*sampling=hourly(&|$);sum;3600:60,86400:600 + min$;min; + min\?;min; + max$;max; + max\?;max; ;avg; ;;60:10 ;;0:42` @@ -57,9 +62,11 @@ func TestLookup(t *testing.T) { {"hourly.rps_min", "86399", "min", "60"}, {"hourly.rps_min", "86400", "min", "3600"}, {"hourly.rps_min", "86401", "min", "3600"}, + {"min?env=stage", "86401", "min", "10"}, // tagged + {"sum?env=stage&sampling=hourly", "86401", "sum", "600"}, // tagged } - r, err := parseCompact(config) + r, err := parseCompact(config, false) assert.NoError(t, err) for _, c := range table { @@ -74,3 +81,248 @@ func TestLookup(t *testing.T) { }) } } + +var benchConfig = ` +^hourly;;3600:60,86400:3600 +^live;;0:1 +\.fake1\..*\.Fake1\.;;3600:60,86400:3600 +fake1\?(.*&)*tag=Fake1(&|$);;3600:60,86400:3600 +\.fake3\..*\.Fake3\.;;3600:60,86400:3600 +fake2\?(.*&)*tag=Fake2(&|$);;3600:60,86400:3600 +\.fake3\..*\.Fake3\.;;3600:60,86400:3600 +fake3\?(.*&)*tag=Fake3(&|$);;3600:60,86400:3600 +\.fake4\..*\.Fake4\.;;3600:60,86400:3600 +fake4\?(.*&)*tag=Fake4(&|$);;3600:60,86400:3600 +total$;sum; +sum$;sum; +sum\?(.*&)*sampling=hourly(&|$);sum;3600:60,86400:3600 +sum\?;sum; +min$;min; +min\?;min; +max$;max; +max\?;max; +;avg; +;;60:10 +;;0:42` + +// BenchmarkSumSeparated-6 Rules with type (all/plain/tagged_regex) +var benchConfigSeparated = ` +^hourly;;3600:60,86400:3600 +^live;;0:1 +\.fake1\..*\.Fake1\.;;3600:60,86400:3600 +fake1\?(.*&)*tag=Fake1(&|$);;3600:60,86400:3600 +\.fake3\..*\.Fake3\.;;3600:60,86400:3600 +fake2\?(.*&)*tag=Fake2(&|$);;3600:60,86400:3600 +\.fake3\..*\.Fake3\.;;3600:60,86400:3600 +fake3\?(.*&)*tag=Fake3(&|$);;3600:60,86400:3600 +\.fake4\..*\.Fake4\.;;3600:60,86400:3600 +fake4\?(.*&)*tag=Fake4(&|$);;3600:60,86400:3600 +total$;sum; +sum$;sum; +sum\?(.*&)*sampling=hourly(&|$);sum;3600:60,86400:3600 +sum\?;sum; +min$;min; +min\?;min; +max$;max; +max\?;max; +;avg; +;;60:10 +;;0:42` + +func BenchmarkSum(b *testing.B) { + r, err := parseCompact(benchConfig, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("test.sum", 1) + _ = precision + _ = ag + } +} + +func BenchmarkSumAuto(b *testing.B) { + r, err := parseCompact(benchConfig, true) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("test.sum", 1) + _ = precision + _ = ag + } +} + +func BenchmarkSumSeparated(b *testing.B) { + r, err := parseCompact(benchConfigSeparated, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("test.sum", 1) + _ = precision + _ = ag + } +} + +func BenchmarkSumTagged(b *testing.B) { + r, err := parseCompact(benchConfig, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("sum?env=test&tag=Fake5", 1) + _ = precision + _ = ag + } +} + +func BenchmarkSumTaggedAuto(b *testing.B) { + r, err := parseCompact(benchConfig, true) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("sum?env=test&tag=Fake5", 1) + _ = precision + _ = ag + } +} + +func BenchmarkSumTaggedSeparated(b *testing.B) { + r, err := parseCompact(benchConfigSeparated, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("sum?env=test&tag=Fake5", 1) + _ = precision + _ = ag + } +} + +func BenchmarkMax(b *testing.B) { + r, err := parseCompact(benchConfig, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("test.max", 1) + _ = precision + _ = ag + } +} + +func BenchmarkMaxAuto(b *testing.B) { + r, err := parseCompact(benchConfig, true) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("test.max", 1) + _ = precision + _ = ag + } +} + +func BenchmarkMaxSeparated(b *testing.B) { + r, err := parseCompact(benchConfigSeparated, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("test.max", 1) + _ = precision + _ = ag + } +} + +func BenchmarkMaxTagged(b *testing.B) { + r, err := parseCompact(benchConfig, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("max?env=test&tag=Fake5", 1) + _ = precision + _ = ag + } +} + +func BenchmarkMaxTaggedAuto(b *testing.B) { + r, err := parseCompact(benchConfig, true) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("max?env=test&tag=Fake5", 1) + _ = precision + _ = ag + } +} + +func BenchmarkMaxTaggedSeparated(b *testing.B) { + r, err := parseCompact(benchConfigSeparated, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("max?env=test&tag=Fake5", 1) + _ = precision + _ = ag + } +} + +func BenchmarkDefault(b *testing.B) { + r, err := parseCompact(benchConfigSeparated, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("test.p95", 1) + _ = precision + _ = ag + } +} + +func BenchmarkDefaultAuto(b *testing.B) { + r, err := parseCompact(benchConfig, true) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("test.p95", 1) + _ = precision + _ = ag + } +} + +func BenchmarkDefaultSeparated(b *testing.B) { + r, err := parseCompact(benchConfigSeparated, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("test.p95", 1) + _ = precision + _ = ag + } +} + +func BenchmarkDefaultTagged(b *testing.B) { + r, err := parseCompact(benchConfig, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("p95?env=test&tag=Fake5", 1) + _ = precision + _ = ag + } +} + +func BenchmarkDefaultTaggedAuto(b *testing.B) { + r, err := parseCompact(benchConfig, true) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("p95?env=test&tag=Fake5", 1) + _ = precision + _ = ag + } +} + +func BenchmarkDefaultTaggedSeparated(b *testing.B) { + r, err := parseCompact(benchConfigSeparated, false) + assert.NoError(b, err) + + for i := 0; i < b.N; i++ { + precision, ag := r.Lookup("p95?env=test&tag=Fake5", 1) + _ = precision + _ = ag + } +} diff --git a/helper/rollup/xml.go b/helper/rollup/xml.go index 6db62860a..2402e430b 100644 --- a/helper/rollup/xml.go +++ b/helper/rollup/xml.go @@ -46,6 +46,7 @@ type RetentionXML struct { } type PatternXML struct { + RuleType RuleType `xml:"type"` Regexp string `xml:"regexp"` Function string `xml:"function"` Retention []*RetentionXML `xml:"retention"` @@ -62,6 +63,7 @@ func (r *RetentionXML) retention() Retention { func (p *PatternXML) pattern() Pattern { result := Pattern{ + RuleType: p.RuleType, Regexp: p.Regexp, Function: p.Function, Retention: make([]Retention, 0, len(p.Retention)), @@ -74,7 +76,7 @@ func (p *PatternXML) pattern() Pattern { return result } -func parseXML(body []byte) (*Rules, error) { +func parseXML(body []byte, auto bool) (*Rules, error) { r := &RulesXML{} err := xml.Unmarshal(body, r) if err != nil { @@ -94,9 +96,19 @@ func parseXML(body []byte) (*Rules, error) { patterns := make([]Pattern, 0, uint64(len(r.Pattern))+4) for _, p := range r.Pattern { patterns = append(patterns, p.pattern()) + for i := range patterns { + if patterns[i].RuleType == RuleAuto { + if auto { + patterns[i].RuleType = AutoDetectRuleType(patterns[i].Regexp) + } else { + patterns[i].RuleType = RuleAll + } + } + } } if r.Default != nil { + r.Default.RuleType = RuleAll patterns = append(patterns, r.Default.pattern()) } diff --git a/helper/rollup/xml_test.go b/helper/rollup/xml_test.go index 33c7de7d4..3ac3a6ee4 100644 --- a/helper/rollup/xml_test.go +++ b/helper/rollup/xml_test.go @@ -10,7 +10,7 @@ import ( func TestParseXML(t *testing.T) { config := ` - + click_cost any @@ -22,7 +22,30 @@ func TestParseXML(t *testing.T) { 60 - + + plain + \.max$ + max + + 0 + 3600 + + + 86400 + 60 + + + + tagged_regex + \.max\? + max + + 0 + 3600 + + + + plain without_function 0 @@ -33,12 +56,223 @@ func TestParseXML(t *testing.T) { 60 - + + plain without_retention min + avg + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + +` + + compact := ` + click_cost;any;0:3600,86400:60 + \.max$;max;0:3600,86400:60 + \.max\?;max;0:3600 + without_function;;0:3600,86400:60 + without_retention;min; + ;avg;0:60,3600:300,86400:3600 + ` + + expected, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RulePlain, Regexp: "without_function", Function: "", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: "without_retention", Function: "min"}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + expectedPlain, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: "without_function", Function: "", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: "without_retention", Function: "min"}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + expectedTagged, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + t.Run("default", func(t *testing.T) { + assert := assert.New(t) + r, err := parseXML([]byte(config), false) + assert.NoError(err) + assert.Equal(expected, r) + + // check sorting + assert.Equal(uint32(0), r.Pattern[0].Retention[0].Age) + assert.Equal(uint32(3600), r.Pattern[0].Retention[0].Precision) + + assert.Equal(len(expected.patternPlain), 5) + assert.Equal(expectedPlain.Pattern, r.patternPlain) + + assert.Equal(len(expected.patternTagged), 3) + assert.Equal(expectedTagged.Pattern, r.patternTagged) + }) + + t.Run("inside yandex tag", func(t *testing.T) { + assert := assert.New(t) + r, err := parseXML([]byte(fmt.Sprintf("%s", config)), false) + assert.NoError(err) + assert.Equal(expected, r) + }) + + t.Run("compare with compact", func(t *testing.T) { + assert := assert.New(t) + expectedCompact, err := parseCompact(compact, false) + assert.NoError(err) + + r, err := parseXML([]byte(fmt.Sprintf("%s", config)), false) + assert.NoError(err) + assert.Equal(expectedCompact, r) + }) +} + +func TestParseXMLAutoDetect(t *testing.T) { + config := ` + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + plain + \.max$ max + + 0 + 3600 + + + 86400 + 60 + + + + tagged_regex + \.max\? + max + + 0 + 3600 + + + + \.min$ + min + + 0 + 3600 + + + 86400 + 60 + + + + \.min\? + min + + 0 + 3600 + + + + env=cloud + avg + + 0 + 3600 + + + + plain + without_function + + 0 + 3600 + + + 86400 + 60 + + + + plain + without_retention + min + + + avg 0 60 @@ -57,23 +291,91 @@ func TestParseXML(t *testing.T) { compact := ` click_cost;any;0:3600,86400:60 - without_function;;0:3600,86400:60 - without_retention;min; - ;max;0:60,3600:300,86400:3600 + \.max$;max;0:3600,86400:60 + \.max\?;max;0:3600 + \.min$;min;0:3600,86400:60 + \.min\?;min;0:3600 + env=cloud;avg;0:3600 + without_function;;0:3600,86400:60 + without_retention;min; + ;avg;0:60,3600:300,86400:3600 ` expected, _ := (&Rules{ Pattern: []Pattern{ - Pattern{Regexp: "click_cost", Function: "any", Retention: []Retention{ + Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{ Retention{Age: 0, Precision: 3600}, }}, - Pattern{Regexp: "without_function", Function: "", Retention: []Retention{ + Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{ Retention{Age: 0, Precision: 3600}, Retention{Age: 86400, Precision: 60}, }}, - Pattern{Regexp: "without_retention", Function: "min", Retention: nil}, - Pattern{Regexp: "", Function: "max", Retention: []Retention{ + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RulePlain, Regexp: "without_function", Function: "", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: "without_retention", Function: "min"}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + expectedPlain, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: "without_function", Function: "", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + Retention{Age: 86400, Precision: 60}, + }}, + Pattern{RuleType: RulePlain, Regexp: "without_retention", Function: "min"}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 60}, + Retention{Age: 3600, Precision: 300}, + Retention{Age: 86400, Precision: 3600}, + }}, + }, + }).compile() + + expectedTagged, _ := (&Rules{ + Pattern: []Pattern{ + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{ + Retention{Age: 0, Precision: 3600}, + }}, + Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{ Retention{Age: 0, Precision: 60}, Retention{Age: 3600, Precision: 300}, Retention{Age: 86400, Precision: 3600}, @@ -83,28 +385,32 @@ func TestParseXML(t *testing.T) { t.Run("default", func(t *testing.T) { assert := assert.New(t) - r, err := parseXML([]byte(config)) + r, err := parseXML([]byte(config), true) assert.NoError(err) assert.Equal(expected, r) // check sorting assert.Equal(uint32(0), r.Pattern[0].Retention[0].Age) assert.Equal(uint32(3600), r.Pattern[0].Retention[0].Precision) + + assert.Equal(expectedPlain.Pattern, r.patternPlain) + + assert.Equal(expectedTagged.Pattern, r.patternTagged) }) t.Run("inside yandex tag", func(t *testing.T) { assert := assert.New(t) - r, err := parseXML([]byte(fmt.Sprintf("%s", config))) + r, err := parseXML([]byte(fmt.Sprintf("%s", config)), true) assert.NoError(err) assert.Equal(expected, r) }) t.Run("compare with compact", func(t *testing.T) { assert := assert.New(t) - expectedCompact, err := parseCompact(compact) + expectedCompact, err := parseCompact(compact, true) assert.NoError(err) - r, err := parseXML([]byte(fmt.Sprintf("%s", config))) + r, err := parseXML([]byte(fmt.Sprintf("%s", config)), true) assert.NoError(err) assert.Equal(expectedCompact, r) })