Skip to content

rollup: split rules for plain/tagged metrics (for perfomance) #162

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

rollup: split rules for plain/tagged metrics (for perfomance) #162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 20 additions & 19 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,24 +129,25 @@ var knownDataTableContext = map[string]bool{

// DataTable configs
type DataTable struct {
Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"`
Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"`
MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"`
MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"`
MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"`
MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"`
TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"`
TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"`
TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"`
TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"`
RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"`
ContextMap map[string]bool `toml:"-" json:"-"`
Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"`
Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"`
Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"`
MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"`
MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"`
MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"`
MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"`
TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"`
TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"`
TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"`
TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"`
RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
RollupRuleTypeAutodetect bool `toml:"rollup-rule-type-autodetect" json:"rollup-rule-type-autodetect"`
Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"`
ContextMap map[string]bool `toml:"-" json:"-"`
Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"`
}

// Debug config
Expand Down Expand Up @@ -453,7 +454,7 @@ func (c *Config) ProcessDataTables() (err error) {
} else if c.DataTable[i].RollupConf == "none" {
c.DataTable[i].Rollup, err = rollup.NewDefault(rdp, rdf)
} else {
c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf)
c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf, c.DataTable[i].RollupRuleTypeAutodetect)
}

if err != nil {
Expand Down
29 changes: 27 additions & 2 deletions helper/rollup/compact.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@ compact form of rollup rules for tests
regexp;function;age:precision,age:precision,...
*/

func parseCompact(body string) (*Rules, error) {
func parseCompact(body string, auto bool) (*Rules, error) {
lines := strings.Split(body, "\n")
patterns := make([]Pattern, 0)

for _, line := range lines {
var ruleType RuleType
if strings.TrimSpace(line) == "" {
continue
}
Expand All @@ -29,6 +30,30 @@ func parseCompact(body string) (*Rules, error) {
return nil, fmt.Errorf("can't parse line: %#v", line)
}
regexp := strings.TrimSpace(line[:p1])
if len(regexp) > 8 && regexp[0] == '<' && regexp[1] == '!' && regexp[7] == '>' {
typeStr := regexp[1:7]
switch typeStr {
case "!ALL_T":
ruleType = RuleAll
case "!PLAIN":
ruleType = RulePlain
case "!TAG_R":
ruleType = RuleTaggedRegex
//case "!TAG_T":
// ruleType = RuleTagged
default:
return nil, fmt.Errorf("not realised rule type for line: %#v", line)
}
regexp = regexp[8:]
} else {
if ruleType == RuleAuto {
if auto && len(regexp) > 0 {
ruleType = AutoDetectRuleType(regexp)
} else {
ruleType = RuleAll
}
}
}
function := strings.TrimSpace(line[p1+1 : p2])
retention := make([]Retention, 0)

Expand All @@ -55,7 +80,7 @@ func parseCompact(body string) (*Rules, error) {
}
}

patterns = append(patterns, Pattern{Regexp: regexp, Function: function, Retention: retention})
patterns = append(patterns, Pattern{RuleType: ruleType, Regexp: regexp, Function: function, Retention: retention})
}

return (&Rules{Pattern: patterns}).compile()
Expand Down
183 changes: 179 additions & 4 deletions helper/rollup/compact_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,186 @@ import (
func TestParseCompact(t *testing.T) {
config := `
click_cost;any;0:3600,86400:60
;max;0:60,3600:300,86400:3600`
<!PLAIN>\.max$;max;0:3600,86400:60
<!TAG_R>\.max\?;max;0:3600
\.min$;min;0:3600,86400:60
\.min\?;min;0:3600
env=cloud;avg;0:3600
;avg;0:60,3600:300,86400:3600`

expected, _ := (&Rules{
Pattern: []Pattern{
Pattern{Regexp: "click_cost", Function: "any", Retention: []Retention{
Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{Regexp: "", Function: "max", Retention: []Retention{
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

expectedPlain, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

expectedTagged, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

assert := assert.New(t)
r, err := parseCompact(config, false)
assert.NoError(err)
assert.Equal(expected, r)

assert.Equal(len(expected.patternPlain), 6)
assert.Equal(expectedPlain.Pattern, r.patternPlain)

assert.Equal(len(expected.patternTagged), 6)
assert.Equal(expectedTagged.Pattern, r.patternTagged)
}

func TestParseCompactAutoDetect(t *testing.T) {
config := `
click_cost;any;0:3600,86400:60
<!PLAIN>\.max$;max;0:3600,86400:60
<!TAG_R>\.max\?;max;0:3600
\.min$;min;0:3600,86400:60
\.min\?;min;0:3600
env=cloud;avg;0:3600
;avg;0:60,3600:300,86400:3600`

expected, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

expectedPlain, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

expectedTagged, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
Expand All @@ -26,7 +197,11 @@ func TestParseCompact(t *testing.T) {
}).compile()

assert := assert.New(t)
r, err := parseCompact(config)
r, err := parseCompact(config, true)
assert.NoError(err)
assert.Equal(expected, r)

assert.Equal(expectedPlain.Pattern, r.patternPlain)

assert.Equal(expectedTagged.Pattern, r.patternTagged)
}
16 changes: 11 additions & 5 deletions helper/rollup/remote.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@ import (
)

type rollupRulesResponseRecord struct {
Regexp string `json:"regexp"`
Function string `json:"function"`
Age string `json:"age"`
Precision string `json:"precision"`
IsDefault int `json:"is_default"`
RuleType RuleType `json:"type"`
Regexp string `json:"regexp"`
Function string `json:"function"`
Age string `json:"age"`
Precision string `json:"precision"`
IsDefault int `json:"is_default"`
}
type rollupRulesResponse struct {
Data []rollupRulesResponseRecord `json:"data"`
Expand Down Expand Up @@ -74,7 +75,11 @@ func parseJson(body []byte) (*Rules, error) {
}
} else {
if last() == nil || last().Regexp != d.Regexp || last().Function != d.Function {
if d.RuleType == RuleAuto {
d.RuleType = RuleAll // for remote rules - no auto-detect rule type
}
r.Pattern = append(r.Pattern, Pattern{
RuleType: d.RuleType,
Retention: make([]Retention, 0),
Regexp: d.Regexp,
Function: d.Function,
Expand All @@ -92,6 +97,7 @@ func parseJson(body []byte) (*Rules, error) {

if defaultFunction != "" || len(defaultRetention) != 0 {
r.Pattern = append(r.Pattern, Pattern{
RuleType: RuleAll,
Regexp: "",
Function: defaultFunction,
Retention: defaultRetention,
Expand Down
24 changes: 21 additions & 3 deletions helper/rollup/remote_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,25 @@ func TestParseJson(t *testing.T) {
"precision": "0",
"is_default": 0
},
{
"type": "plain",
"regexp": "\\.min$",
"function": "min",
"age": "0",
"precision": "3600",
"is_default": 0
},
{
"type": "tagged_regex",
"regexp": "\\.min\\?",
"function": "min",
"age": "0",
"precision": "3600",
"is_default": 0
},
{
"regexp": "",
"function": "max",
"function": "avg",
"age": "0",
"precision": "60",
"is_default": 1
Expand All @@ -112,11 +128,13 @@ func TestParseJson(t *testing.T) {
total$;sum;
min$;min;
max$;max;
;max;0:60
<!PLAIN>\.min$;min;0:3600
<!TAG_R>\.min\?;min;0:3600
;avg;0:60
`

assert := assert.New(t)
expected, err := parseCompact(compact)
expected, err := parseCompact(compact, false)
assert.NoError(err)

r, err := parseJson([]byte(response))
Expand Down
Loading