From 3dbb99466eaf739f38a0984ffa08b2df61a7d772 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 21 Dec 2020 12:48:15 +0000 Subject: [PATCH 01/72] [wip] redistimeseries v1.4 support --- .circleci/config.yml | 43 ++ .dockerignore | 24 ++ .gitignore | 9 +- Makefile | 65 +++ README.md | 8 + .../databases/redistimeseries/common.go | 53 +++ .../databases/redistimeseries/devops.go | 281 ++++++++++++ .../uses/devops/common.go | 11 + cmd/tsbs_load_redistimeseries/creator.go | 30 ++ cmd/tsbs_load_redistimeseries/main.go | 238 +++++++++++ cmd/tsbs_load_redistimeseries/main_test.go | 11 + cmd/tsbs_load_redistimeseries/scan.go | 90 ++++ .../query_executor.go | 2 +- cmd/tsbs_run_queries_redistimeseries/main.go | 325 ++++++++++++++ docker/Dockerfile | 17 + docker/docker_entrypoint.sh | 10 + docs/redistimeseries.md | 400 ++++++++++++++++++ go.mod | 5 +- go.sum | 11 + internal/inputs/generator_data.go | 1 - internal/inputs/generator_queries.go | 1 + pkg/data/usecases/common/simulator.go | 26 ++ .../usecases/devops/cpu_only_generate_data.go | 33 +- pkg/data/usecases/iot/simulator.go | 4 + pkg/query/redistimeseries.go | 96 +++++ pkg/query/redistimeseries_functors.go | 297 +++++++++++++ pkg/query/redistimeseries_functors_test.go | 210 +++++++++ pkg/targets/constants/constants.go | 2 + .../redistimeseries/implemented_target.go | 37 ++ .../redistimeseries/redistimeseries.go | 130 ++++++ .../redistimeseries/redistimeseries_test.go | 38 ++ pkg/targets/redistimeseries/serializer.go | 133 ++++++ .../redistimeseries/serializer_test.go | 41 ++ .../full_cycle_minitest_timescaledb.sh | 39 +- ...ug_responses_full_cycle_minitest_influx.sh | 73 ++++ ...ses_full_cycle_minitest_redistimeseries.sh | 61 +++ ...sponses_full_cycle_minitest_timescaledb.sh | 61 +++ scripts/generate_data.sh | 32 +- scripts/generate_queries.sh | 59 +-- scripts/load/load_common.sh | 8 +- scripts/load_redistimeseries.sh | 58 +++ scripts/query_common.sh | 22 + scripts/run_queries_redistimeseries.sh | 83 ++++ 43 files changed, 3115 insertions(+), 63 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .dockerignore create mode 100644 Makefile create mode 100644 cmd/tsbs_generate_queries/databases/redistimeseries/common.go create mode 100644 cmd/tsbs_generate_queries/databases/redistimeseries/devops.go create mode 100644 cmd/tsbs_load_redistimeseries/creator.go create mode 100644 cmd/tsbs_load_redistimeseries/main.go create mode 100644 cmd/tsbs_load_redistimeseries/main_test.go create mode 100644 cmd/tsbs_load_redistimeseries/scan.go create mode 100644 cmd/tsbs_run_queries_redistimeseries/main.go create mode 100644 docker/Dockerfile create mode 100644 docker/docker_entrypoint.sh create mode 100644 docs/redistimeseries.md create mode 100644 pkg/query/redistimeseries.go create mode 100644 pkg/query/redistimeseries_functors.go create mode 100644 pkg/query/redistimeseries_functors_test.go create mode 100644 pkg/targets/redistimeseries/implemented_target.go create mode 100644 pkg/targets/redistimeseries/redistimeseries.go create mode 100644 pkg/targets/redistimeseries/redistimeseries_test.go create mode 100644 pkg/targets/redistimeseries/serializer.go create mode 100644 pkg/targets/redistimeseries/serializer_test.go create mode 100755 scripts/functional/debug_responses_full_cycle_minitest_influx.sh create mode 100755 scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh create mode 100755 scripts/functional/debug_responses_full_cycle_minitest_timescaledb.sh create mode 100755 scripts/load_redistimeseries.sh create mode 100644 scripts/query_common.sh create mode 100755 scripts/run_queries_redistimeseries.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..e0af09b3a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,43 @@ +# Golang CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-go/ for more details +version: 2 +jobs: + build: # test with redisearch:latest + docker: + - image: circleci/golang:1.13 + + working_directory: /go/src/github.com/RedisTimeSeries/tsbs + steps: + - checkout + - run: go get -t -v ./... + - run: GO111MODULE=on go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... + - run: bash <(curl -s https://codecov.io/bash) -t ${CODECOV_TOKEN} + + build-multiarch-docker: + machine: + enabled: true + steps: + - checkout + - run: | + echo "$DOCKER_REDISBENCH_PWD" | base64 --decode | docker login --username $DOCKER_REDISBENCH_USER --password-stdin + - run: + name: Build + command: | + make docker-release + no_output_timeout: 20m + +workflows: + version: 2 + build_and_package: + jobs: + - build: + filters: + tags: + only: /.*/ + - build-multiarch-docker: + filters: + tags: + only: /.*/ + branches: + only: master diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..290031ac2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,24 @@ +# ignore .git and .cache folders +.git +.editorconfig +.gitignore +.dockerignore +.travis.yml +coverage.txt +AUTHORS.md +CONTRIBUTING.md +LICENSE +NOTICE +README.md + +*.out +*.log +.DS_Store +.idea +.vscode +**/bin +**/docs +**/scripts + +# High Dynamic Range (HDR) Histogram files +*.hdr diff --git a/.gitignore b/.gitignore index 04dd8bed8..8af262ac9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,12 @@ .idea .vscode +bin + # High Dynamic Range (HDR) Histogram files -*.hdr \ No newline at end of file +*.hdr + +/docs/responses +coverage.txt + +bin/ \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..3969e095b --- /dev/null +++ b/Makefile @@ -0,0 +1,65 @@ +# Go parameters +GOCMD=go +GOBUILD=$(GOCMD) build +GOINSTALL=$(GOCMD) install +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get +GOMOD=$(GOCMD) mod +GOFMT=$(GOCMD) fmt + +# DOCKER +DOCKER_APP_NAME=tsbs +DOCKER_ORG=redisbench +DOCKER_REPO:=${DOCKER_ORG}/${DOCKER_APP_NAME} +DOCKER_IMG:="$(DOCKER_REPO):$(DOCKER_TAG)" +DOCKER_LATEST:="${DOCKER_REPO}:latest" + +.PHONY: all generators loaders runners +all: generators loaders runners + +generators: tsbs_generate_data tsbs_generate_queries + +loaders: tsbs_load_redistimeseries tsbs_load_cassandra tsbs_load_clickhouse tsbs_load_influx tsbs_load_mongo tsbs_load_siridb tsbs_load_timescaledb + +runners: tsbs_run_queries_redistimeseries tsbs_run_queries_cassandra tsbs_run_queries_clickhouse tsbs_run_queries_influx tsbs_run_queries_mongo tsbs_run_queries_siridb tsbs_run_queries_timescaledb + +test: + GO111MODULE=on $(GOTEST) -v -race -coverprofile=coverage.txt -covermode=atomic ./... + +tsbs_%: $(wildcard ./cmd/$@/*.go) + $(GOGET) ./cmd/$@ + $(GOBUILD) -o bin/$@ ./cmd/$@ + $(GOINSTALL) ./cmd/$@ + +# DOCKER TASKS +# Build the container +docker-build: + docker build -t $(DOCKER_APP_NAME):latest -f docker/Dockerfile . + +# Build the container without caching +docker-build-nc: + docker build --no-cache -t $(DOCKER_APP_NAME):latest -f docker/Dockerfile . + +# Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR +docker-release: docker-build-nc docker-publish + +# Docker publish +docker-publish: docker-publish-latest + +## login to DockerHub with credentials found in env +docker-repo-login: + docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} + +## Publish the `latest` tagged container to ECR +docker-publish-latest: docker-tag-latest + @echo 'publish latest to $(DOCKER_REPO)' + docker push $(DOCKER_LATEST) + +# Docker tagging +docker-tag: docker-tag-latest + +## Generate container `{version}` tag +docker-tag-latest: + @echo 'create tag latest' + docker tag $(DOCKER_APP_NAME) $(DOCKER_LATEST) diff --git a/README.md b/README.md index cdcf948b7..be8839626 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,14 @@ scripts). The easiest way to get and install the Go programs is to use $ go get github.com/timescale/tsbs $ cd $GOPATH/src/github.com/timescale/tsbs/cmd $ go get ./... +$ git remote add redistimeseries https://github.com/RedisTimeSeries/tsbs.git +$ git fetch +$ git checkout redistimeseries +$ cd $GOPATH/src/github.com/timescale/tsbs +$ make +$ FORMATS="redistimeseries" SCALE=100 SEED=123 TS_END="2016-01-31T00:00:00Z" ./scripts/generate_data.sh +# Benchmark part. in PREFIX add a text that will be prefixed on the output results file +$ NUM_WORKERS=1 BATCH_SIZE=10000 PIPELINE=200 CONNECTIONS=10 PREFIX=1.2.0 ./scripts/load_redistimeseries.sh # Install desired binaries. At a minimum this includes tsbs_generate_data, # tsbs_generate_queries, one tsbs_load_* binary, and one tsbs_run_queries_* diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/common.go b/cmd/tsbs_generate_queries/databases/redistimeseries/common.go new file mode 100644 index 000000000..0f172d087 --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/common.go @@ -0,0 +1,53 @@ +package redistimeseries + +import ( + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/utils" + "github.com/timescale/tsbs/query" + "time" +) + +// BaseGenerator contains settings specific for RedisTimeSeries database. +type BaseGenerator struct { +} + +// GenerateEmptyQuery returns an empty query.Cassandra. +func (g *BaseGenerator) GenerateEmptyQuery() query.Query { + return query.NewRedisTimeSeries() +} + +// fill Query fills the query struct with data +func (d *BaseGenerator) fillInQueryStrings(qi query.Query, humanLabel, humanDesc string) { + q := qi.(*query.RedisTimeSeries) + q.HumanLabel = []byte(humanLabel) + q.HumanDescription = []byte(humanDesc) +} + +// AddQuery adds a command to be executed in the full flow of this Query +func (d *BaseGenerator) AddQuery(qi query.Query, tq [][]byte, commandname []byte) { + q := qi.(*query.RedisTimeSeries) + q.AddQuery(tq, commandname) +} + +// SetApplyFunctor sets SetApplyFunctor used for this Query +func (d *BaseGenerator) SetApplyFunctor(qi query.Query, value bool, functor string) { + q := qi.(*query.RedisTimeSeries) + q.SetApplyFunctor(value) + q.SetFunctor(functor) +} + +// NewDevops creates a new devops use case query generator. +func (g *BaseGenerator) NewDevops(start, end time.Time, scale int) (utils.QueryGenerator, error) { + core, err := devops.NewCore(start, end, scale) + + if err != nil { + return nil, err + } + + var devops utils.QueryGenerator = &Devops{ + BaseGenerator: g, + Core: core, + } + + return devops, nil +} diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go new file mode 100644 index 000000000..092522fa7 --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go @@ -0,0 +1,281 @@ +package redistimeseries + +import ( + "fmt" + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" + "github.com/timescale/tsbs/query" +) + +// TODO: Remove the need for this by continuing to bubble up errors +func panicIfErr(err error) { + if err != nil { + panic(err.Error()) + } +} + +const ( + oneMinuteMillis = 60 * 1000 + oneHourMillis = oneMinuteMillis * 60 +) + +// Devops produces RedisTimeSeries-specific queries for all the devops query types. +type Devops struct { + *BaseGenerator + *devops.Core +} + +// GenerateEmptyQuery returns an empty query.RedisTimeSeries +func (d *Devops) GenerateEmptyQuery() query.Query { + return query.NewRedisTimeSeries() +} + +// GroupByTime fetches the MAX for numMetrics metrics under 'cpu', per minute for nhosts hosts, +// every 5 mins for 1 hour +func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) { + interval := d.Interval.MustRandWindow(timeRange) + redisQuery := [][]byte{ + //[]byte("TS.MRANGE"), Just to help understanding + []byte(fmt.Sprintf("%d", interval.StartUnixMillis())), + []byte(fmt.Sprintf("%d", interval.EndUnixMillis())), + []byte("WITHLABELS"), + []byte("AGGREGATION"), + []byte("MAX"), + []byte(fmt.Sprintf("%d", oneMinuteMillis)), + []byte("FILTER"), + []byte("measurement=cpu"), + } + + metrics, err := devops.GetCPUMetricsSlice(numMetrics) + panicIfErr(err) + + // we only need to filter if we we dont want all of them + if numMetrics != devops.GetCPUMetricsLen() { + redisArg := "fieldname=" + if numMetrics > 1 { + redisArg += "(" + } + for idx, value := range metrics { + redisArg += value + if idx != (numMetrics - 1) { + redisArg += "," + } + } + if numMetrics > 1 { + redisArg += ")" + } + redisQuery = append(redisQuery, []byte(redisArg)) + } + + hostnames, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + + // add specific fieldname if needed. + redisArg := "hostname=" + if nHosts > 1 { + redisArg += "(" + } + for idx, value := range hostnames { + redisArg += value + if idx != (nHosts - 1) { + redisArg += "," + } + } + if nHosts > 1 { + redisArg += ")" + } + redisQuery = append(redisQuery, []byte(redisArg)) + + humanLabel := devops.GetSingleGroupByLabel("RedisTimeSeries", numMetrics, nHosts, string(timeRange)) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQueryStrings(qi, humanLabel, humanDesc) + d.AddQuery(qi, redisQuery, []byte("TS.MRANGE")) + if numMetrics > 1 && nHosts == 1 { + functorName := query.GetFunctionName(query.SingleGroupByTime) + d.SetApplyFunctor(qi, true, functorName) + } + if nHosts > 1 && numMetrics == 1 { + functorName := query.GetFunctionName(query.GroupByTimeAndMax) + d.SetApplyFunctor(qi, true, functorName) + } + if nHosts > 1 && numMetrics > 1 { + functorName := query.GetFunctionName(query.GroupByTimeAndTagMax) + d.SetApplyFunctor(qi, true, functorName) + } +} + +// GroupByTimeAndPrimaryTag selects the AVG of numMetrics metrics under 'cpu' per device per hour for a day +func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { + interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration) + redisQuery := [][]byte{ + //[]byte("TS.MRANGE"), Just to help understanding + []byte(fmt.Sprintf("%d", interval.StartUnixMillis())), + []byte(fmt.Sprintf("%d", interval.EndUnixMillis())), + []byte("WITHLABELS"), + []byte("AGGREGATION"), + []byte("AVG"), + []byte(fmt.Sprintf("%d", oneHourMillis)), + []byte("FILTER"), + []byte("measurement=cpu"), + } + + metrics, err := devops.GetCPUMetricsSlice(numMetrics) + panicIfErr(err) + + // add specific fieldname if needed. + if numMetrics != devops.GetCPUMetricsLen() { + redisArg := "fieldname=" + if numMetrics > 1 { + redisArg += "(" + } + for idx, value := range metrics { + redisArg += value + if idx != (numMetrics - 1) { + redisArg += "," + } + } + if numMetrics > 1 { + redisArg += ")" + } + redisQuery = append(redisQuery, []byte(redisArg)) + } + + humanLabel := devops.GetDoubleGroupByLabel("RedisTimeSeries", numMetrics) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQueryStrings(qi, humanLabel, humanDesc) + d.AddQuery(qi, redisQuery, []byte("TS.MRANGE")) + functorName := query.GetFunctionName(query.GroupByTimeAndTagHostname) + d.SetApplyFunctor(qi, true, functorName) +} + +// MaxAllCPU fetches the aggregate across all CPU metrics per hour over 1 hour for a single host. +// Currently only one host is supported +func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) { + interval := d.Interval.MustRandWindow(devops.MaxAllDuration) + hostnames, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + redisQuery := [][]byte{ + //[]byte("TS.MRANGE"), Just to help understanding + []byte(fmt.Sprintf("%d", interval.StartUnixMillis())), + []byte(fmt.Sprintf("%d", interval.EndUnixMillis())), + []byte("WITHLABELS"), + []byte("AGGREGATION"), + []byte("MAX"), + []byte(fmt.Sprintf("%d", oneHourMillis)), + []byte("FILTER"), + []byte("measurement=cpu"), + } + + redisArg := "hostname=" + if nHosts > 1 { + redisArg += "(" + } + for idx, value := range hostnames { + redisArg += value + if idx != (nHosts - 1) { + redisArg += "," + } + } + if nHosts > 1 { + redisArg += ")" + } + redisQuery = append(redisQuery, []byte(redisArg)) + + humanLabel := devops.GetMaxAllLabel("RedisTimeSeries", nHosts) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQueryStrings(qi, humanLabel, humanDesc) + d.AddQuery(qi, redisQuery, []byte("TS.MRANGE")) + if nHosts == 1 { + functorName := query.GetFunctionName(query.SingleGroupByTime) + d.SetApplyFunctor(qi, true, functorName) + } else { + functorName := query.GetFunctionName(query.GroupByTimeAndTagMax) + d.SetApplyFunctor(qi, true, functorName) + } +} + +// LastPointPerHost finds the last row for every host in the dataset +func (d *Devops) LastPointPerHost(qi query.Query) { + redisQuery := [][]byte{ + //[]byte("TS.MREVRANGE"), Just to help understanding + []byte("-"), + []byte("+"), + []byte("COUNT"), + []byte("1"), + []byte("WITHLABELS"), + []byte("FILTER"), + []byte("measurement=cpu"), + []byte("hostname!="), + } + + humanLabel := "RedisTimeSeries last row per host" + humanDesc := fmt.Sprintf("%s", humanLabel) + d.fillInQueryStrings(qi, humanLabel, humanDesc) + d.AddQuery(qi, redisQuery, []byte("TS.MREVRANGE")) + functorName := query.GetFunctionName(query.GroupByTimeAndTagHostname) + d.SetApplyFunctor(qi, true, functorName) + +} + +func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { + hostnames, err := d.GetRandomHosts(nHosts) + interval := d.Interval.MustRandWindow(devops.HighCPUDuration) + redisQuery := [][]byte{ + //[]byte("TS.MRANGE"), Just to help understanding + []byte(fmt.Sprintf("%d", interval.StartUnixMillis())), + []byte(fmt.Sprintf("%d", interval.EndUnixMillis())), + []byte("FILTER"), + []byte("measurement=cpu"), + } + + if nHosts > 0 { + redisArg := "hostname=" + if nHosts > 1 { + redisArg += "(" + } + for idx, value := range hostnames { + redisArg += value + if idx != (nHosts - 1) { + redisArg += "," + } + } + if nHosts > 1 { + redisArg += ")" + } + redisQuery = append(redisQuery, []byte(redisArg)) + } + humanLabel, err := devops.GetHighCPULabel("RedisTimeSeries", nHosts) + panicIfErr(err) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQueryStrings(qi, humanLabel, humanDesc) + d.AddQuery(qi, redisQuery, []byte("TS.MRANGE")) + functorName := query.GetFunctionName(query.HighCpu) + d.SetApplyFunctor(qi, true, functorName) +} + +// GroupByOrderByLimit populates a query.Query that has a time WHERE clause, that groups by a truncated date, orders by that date, and takes a limit: +func (d *Devops) GroupByOrderByLimit(qi query.Query) { + + interval := d.Interval.MustRandWindow(time.Hour) + redisQuery := [][]byte{ + //[]byte("TS.MREVRANGE"), Just to help understanding + []byte("-"), + []byte(fmt.Sprintf("%d", interval.EndUnixMillis())), + []byte("COUNT"), + []byte("5"), + []byte("AGGREGATION"), + []byte("MAX"), + []byte(fmt.Sprintf("%d", oneMinuteMillis)), + []byte("FILTER"), + []byte("measurement=cpu"), + []byte("fieldname=usage_user"), + } + + humanLabel := devops.GetGroupByOrderByLimitLabel("RedisTimeSeries") + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.EndString()) + + d.fillInQueryStrings(qi, humanLabel, humanDesc) + d.AddQuery(qi, redisQuery, []byte("TS.MREVRANGE")) + +} diff --git a/cmd/tsbs_generate_queries/uses/devops/common.go b/cmd/tsbs_generate_queries/uses/devops/common.go index 0e95b7298..e3d2ca1fd 100644 --- a/cmd/tsbs_generate_queries/uses/devops/common.go +++ b/cmd/tsbs_generate_queries/uses/devops/common.go @@ -120,6 +120,12 @@ type HighCPUFiller interface { HighCPUForHosts(query.Query, int) } +// GetSingleGroupByLabel returns the Query human-readable label for DoubleGroupBy queries +func GetSingleGroupByLabel(dbName string, numMetrics, nHosts int, timerange string) string { + return fmt.Sprintf("%s MAX of %d metrics,random %4d hosts, random %s by 1m", dbName, numMetrics, nHosts, timerange) + +} + // GetDoubleGroupByLabel returns the Query human-readable label for DoubleGroupBy queries func GetDoubleGroupByLabel(dbName string, numMetrics int) string { return fmt.Sprintf("%s mean of %d metrics, all hosts, random %s by 1h", dbName, numMetrics, DoubleGroupByDuration) @@ -143,6 +149,11 @@ func GetMaxAllLabel(dbName string, nHosts int) string { return fmt.Sprintf("%s max of all CPU metrics, random %4d hosts, random %s by 1h", dbName, nHosts, MaxAllDuration) } +// GetMaxAllLabel returns the Query human-readable label for MaxAllCPU queries +func GetGroupByOrderByLimitLabel(dbName string) string { + return fmt.Sprintf("%s max cpu over last 5 min-intervals (random end)", dbName) +} + // getRandomHosts returns a subset of numHosts hostnames of a permutation of hostnames, // numbered from 0 to totalHosts. // Ex.: host_12, host_7, host_25 for numHosts=3 and totalHosts=30 (3 out of 30) diff --git a/cmd/tsbs_load_redistimeseries/creator.go b/cmd/tsbs_load_redistimeseries/creator.go new file mode 100644 index 000000000..6d7bf6e32 --- /dev/null +++ b/cmd/tsbs_load_redistimeseries/creator.go @@ -0,0 +1,30 @@ +package main + +import ( + redistimeseries "github.com/RedisTimeSeries/redistimeseries-go" +) + +type dbCreator struct { + client *redistimeseries.Client +} + +func (d *dbCreator) Init() { + d.client = redistimeseries.NewClient(host, "test_client", nil) +} + +func (d *dbCreator) DBExists(dbName string) bool { + return true +} + +func (d *dbCreator) RemoveOldDB(dbName string) error { + return nil +} + +func (d *dbCreator) CreateDB(dbName string) error { + return nil +} + +func (d *dbCreator) Close() { + conn := d.client.Pool.Get() + conn.Close() +} diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go new file mode 100644 index 000000000..2fe564517 --- /dev/null +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -0,0 +1,238 @@ +package main + +import ( + "bufio" + "crypto/md5" + "encoding/binary" + "fmt" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "github.com/timescale/tsbs/internal/utils" + "io" + "log" + "strconv" + "strings" + "sync" + + "github.com/gomodule/redigo/redis" + "github.com/timescale/tsbs/load" +) + +// Program option vars: +var ( + host string + connections uint64 + pipeline uint64 + checkChunks uint64 + singleQueue bool + dataModel string + compressionEnabled bool +) + +// Global vars +var ( + loader *load.BenchmarkRunner + //bufPool sync.Pool +) + +// allows for testing +var fatal = log.Fatal +var md5h = md5.New() + +// Parse args: +func init() { + var config load.BenchmarkRunnerConfig + config.AddToFlagSet(pflag.CommandLine) + + pflag.StringVar(&host, "host", "localhost:6379", "The host:port for Redis connection") + pflag.Uint64Var(&connections, "connections", 10, "The number of connections per worker") + pflag.Uint64Var(&pipeline, "pipeline", 50, "The pipeline's size") + pflag.BoolVar(&singleQueue, "single-queue", true, "Whether to use a single queue") + pflag.BoolVar(&compressionEnabled, "compression-enabled", true, "Whether to use compressed time series") + pflag.Uint64Var(&checkChunks, "check-chunks", 0, "Whether to perform post ingestion chunck count") + pflag.StringVar(&dataModel, "data-model", "redistimeseries", "Data model (redistimeseries, rediszsetdevice, rediszsetmetric, redisstream)") + pflag.Parse() + + err := utils.SetupConfigFile() + + if err != nil { + panic(fmt.Errorf("fatal error config file: %s", err)) + } + + if err := viper.Unmarshal(&config); err != nil { + panic(fmt.Errorf("unable to decode config: %s", err)) + } + loader = load.GetBenchmarkRunner(config) + +} + +type benchmark struct { + dbc *dbCreator +} + +type RedisIndexer struct { + partitions uint +} + +func (i *RedisIndexer) GetIndex(p *load.Point) int { + row := p.Data.(string) + key := strings.Split(row, " ")[1] + start := strings.Index(key, "{") + end := strings.Index(key, "}") + _, _ = io.WriteString(md5h, key[start+1:end]) + hash := binary.LittleEndian.Uint32(md5h.Sum(nil)) + md5h.Reset() + return int(uint(hash) % i.partitions) +} + +func (b *benchmark) GetPointDecoder(br *bufio.Reader) load.PointDecoder { + return &decoder{scanner: bufio.NewScanner(br)} +} + +func (b *benchmark) GetBatchFactory() load.BatchFactory { + return &factory{} +} + +func (b *benchmark) GetPointIndexer(maxPartitions uint) load.PointIndexer { + return &RedisIndexer{partitions: maxPartitions} +} + +func (b *benchmark) GetProcessor() load.Processor { + return &processor{b.dbc, nil, nil, nil} +} + +func (b *benchmark) GetDBCreator() load.DBCreator { + return b.dbc +} + +type processor struct { + dbc *dbCreator + rows []chan string + metrics chan uint64 + wg *sync.WaitGroup +} + +func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint64, conn redis.Conn, id uint64) { + curPipe := uint64(0) + //fmt.Println(fmt.Sprintf("wg started for id %d\n",id)) + + for row := range rows { + cmdname, s := buildCommand(row, compressionEnabled == false) + var err error + + if curPipe == pipeline { + cnt, err := sendRedisFlush(curPipe, conn) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- cnt + curPipe = 0 + } + err = sendRedisCommand(conn, cmdname, s) + if err != nil { + log.Fatalf("sendRedisCommand failed with %v", err) + } + curPipe++ + + } + if curPipe > 0 { + cnt, err := sendRedisFlush(curPipe, conn) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- cnt + } + wg.Done() + //fmt.Println(fmt.Sprintf("wg done for id %d\n",id)) +} + +func (p *processor) Init(_ int, _ bool) {} + +// ProcessBatch reads eventsBatches which contain rows of data for TS.ADD redis command string +func (p *processor) ProcessBatch(b load.Batch, doLoad bool) (uint64, uint64) { + events := b.(*eventsBatch) + rowCnt := uint64(len(events.rows)) + metricCnt := uint64(0) + // indexer := &RedisIndexer{partitions: uint(connections)} + if doLoad { + buflen := rowCnt + 1 + p.rows = make([]chan string, connections) + p.metrics = make(chan uint64, buflen) + p.wg = &sync.WaitGroup{} + for i := uint64(0); i < connections; i++ { + conn := p.dbc.client.Pool.Get() + defer conn.Close() + p.rows[i] = make(chan string, buflen) + p.wg.Add(1) + go connectionProcessor(p.wg, p.rows[i], p.metrics, conn, i) + } + for _, row := range events.rows { + key := strings.Split(row, " ")[1] + start := strings.Index(key, "{") + end := strings.Index(key, "}") + tag, _ := strconv.ParseUint(key[start+1:end], 10, 64) + i := tag % connections + p.rows[i] <- row + } + + for i := uint64(0); i < connections; i++ { + close(p.rows[i]) + } + p.wg.Wait() + close(p.metrics) + //fmt.Println("out\n") + + for val := range p.metrics { + metricCnt += val + } + } + events.rows = events.rows[:0] + ePool.Put(events) + return metricCnt, rowCnt +} + +func (p *processor) Close(_ bool) { +} + +func runCheckData() { + log.Println("Running post ingestion data check") + conn, err := redis.DialURL(fmt.Sprintf("redis://%s", host)) + if err != nil { + log.Fatalf("Error while dialing %v", err) + } + _, err = conn.Do("PING") + if err != nil { + log.Fatalf("Error while PING %v", err) + } + + cursor := 0 + total := 0 + for { + rep, _ := redis.Values(conn.Do("SCAN", cursor)) + cursor, _ = redis.Int(rep[0], nil) + keys, _ := redis.Strings(rep[1], nil) + for _, key := range keys { + total++ + info, _ := redis.Values(conn.Do("TS.INFO", key)) + chunks, _ := redis.Int(info[5], nil) + if chunks != int(checkChunks) { + log.Printf("Verification error: key %v has %v chunks", key, chunks) + } + } + if cursor == 0 { + break + } + } + log.Printf("Verified %v keys", total) +} + +func main() { + workQueues := uint(load.WorkerPerQueue) + if singleQueue { + workQueues = load.SingleQueue + } + loader.RunBenchmark(&benchmark{dbc: &dbCreator{}}, workQueues) + if checkChunks > 0 { + runCheckData() + } +} diff --git a/cmd/tsbs_load_redistimeseries/main_test.go b/cmd/tsbs_load_redistimeseries/main_test.go new file mode 100644 index 000000000..3dbd6f12e --- /dev/null +++ b/cmd/tsbs_load_redistimeseries/main_test.go @@ -0,0 +1,11 @@ +package main + +import ( + "testing" + + "github.com/timescale/tsbs/load" +) + +func TestRedisTimeSeriesLoader(t *testing.T) { + loader.RunBenchmark(&benchmark{dbc: &dbCreator{}}, load.WorkerPerQueue) +} diff --git a/cmd/tsbs_load_redistimeseries/scan.go b/cmd/tsbs_load_redistimeseries/scan.go new file mode 100644 index 000000000..818d325f0 --- /dev/null +++ b/cmd/tsbs_load_redistimeseries/scan.go @@ -0,0 +1,90 @@ +package main + +import ( + "bufio" + //"fmt" + "log" + "strings" + "sync" + + "github.com/gomodule/redigo/redis" + "github.com/timescale/tsbs/load" +) + +type decoder struct { + scanner *bufio.Scanner +} + +// Reads and returns a text line that encodes a data point for a specif field name. +// Since scanning happens in a single thread, we hold off on transforming it +// to an INSERT statement until it's being processed concurrently by a worker. +func (d *decoder) Decode(_ *bufio.Reader) *load.Point { + ok := d.scanner.Scan() + if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF + return nil + } else if !ok { + log.Fatalf("scan error: %v", d.scanner.Err()) + } + return load.NewPoint(d.scanner.Text()) +} + +func sendRedisCommand(conn redis.Conn, cmdName string, s redis.Args) (err error) { + err = conn.Send(cmdName, s...) + if err != nil { + log.Fatalf("sendRedisCommand %s failed: %s\n", cmdName, err) + } + return +} + +func buildCommand(line string, forceUncompressed bool) (cmdname string, s redis.Args) { + t := strings.Split(line, " ") + cmdname = t[0] + if cmdname == "TS.CREATE" && forceUncompressed { + t = append(t, "UNCOMPRESSED") + s = s.Add(t[1]) + s = s.Add("UNCOMPRESSED") + s = s.AddFlat(t[2:]) + } else { + s = s.AddFlat(t[1:]) + } + return +} + +func sendRedisFlush(count uint64, conn redis.Conn) (metrics uint64, err error) { + metrics = uint64(0) + err = conn.Flush() + if err != nil { + return + } + for i := uint64(0); i < count; i++ { + _, err := conn.Receive() + //fmt.Println(r) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } else { + metrics += 10 // ts.madd + } + } + return metrics, err +} + +type eventsBatch struct { + rows []string +} + +func (eb *eventsBatch) Len() int { + return len(eb.rows) +} + +func (eb *eventsBatch) Append(item *load.Point) { + that := item.Data.(string) + eb.rows = append(eb.rows, that) +} + +var ePool = &sync.Pool{New: func() interface{} { return &eventsBatch{rows: []string{}} }} + +type factory struct{} + +func (f *factory) New() load.Batch { + return ePool.Get().(*eventsBatch) +} diff --git a/cmd/tsbs_run_queries_cassandra/query_executor.go b/cmd/tsbs_run_queries_cassandra/query_executor.go index 392fd67d2..28dcda5d3 100644 --- a/cmd/tsbs_run_queries_cassandra/query_executor.go +++ b/cmd/tsbs_run_queries_cassandra/query_executor.go @@ -90,7 +90,7 @@ func (qe *HLQueryExecutor) Do(q *HLQuery, opts HLQueryExecutorDoOptions) (qpLagM // optionally, print reponses for query validation: if opts.PrettyPrintResponses { for _, r := range results { - fmt.Fprintf(os.Stderr, "ID %d: [%s, %s] -> %v\n", q.GetID(), r.TimeInterval.Start(), r.TimeInterval.End(), r.Values) + fmt.Fprintf(os.Stdout, "ID %d: [%s, %s] -> %v\n", q.GetID(), r.TimeInterval.Start(), r.TimeInterval.End(), r.Values) } } return diff --git a/cmd/tsbs_run_queries_redistimeseries/main.go b/cmd/tsbs_run_queries_redistimeseries/main.go new file mode 100644 index 000000000..254f00e6a --- /dev/null +++ b/cmd/tsbs_run_queries_redistimeseries/main.go @@ -0,0 +1,325 @@ +// tsbs_run_queries_redistimeseries speed tests RedisTimeSeries using requests from stdin or file +// + +// This program has no knowledge of the internals of the endpoint. +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "fmt" + redistimeseries "github.com/RedisTimeSeries/redistimeseries-go" + _ "github.com/lib/pq" + "github.com/pkg/errors" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "github.com/timescale/tsbs/internal/utils" + "github.com/timescale/tsbs/query" + "log" + "sort" + "strings" + "time" +) + +// Program option vars: +var ( + host string + showExplain bool + // scale uint64 +) + +// Global vars: +var ( + runner *query.BenchmarkRunner + cmdMrange = []byte("TS.MRANGE") + cmdMRevRange = []byte("TS.MREVRANGE") + cmdQueryIndex = []byte("TS.QUERYINDEX") + reflect_SingleGroupByTime = query.GetFunctionName(query.SingleGroupByTime) + reflect_GroupByTimeAndMax = query.GetFunctionName(query.GroupByTimeAndMax) + reflect_GroupByTimeAndTagMax = query.GetFunctionName(query.GroupByTimeAndTagMax) + reflect_GroupByTimeAndTagHostname = query.GetFunctionName(query.GroupByTimeAndTagHostname) + reflect_HighCpu = query.GetFunctionName(query.HighCpu) +) + +var ( + redisConnector *redistimeseries.Client +) + +// Parse args: +func init() { + var config query.BenchmarkRunnerConfig + config.AddToFlagSet(pflag.CommandLine) + + pflag.StringVar(&host, "host", "localhost:6379", "Redis host address and port") + + pflag.Parse() + + err := utils.SetupConfigFile() + + if err != nil { + panic(fmt.Errorf("fatal error config file: %s", err)) + } + + if err := viper.Unmarshal(&config); err != nil { + panic(fmt.Errorf("unable to decode config: %s", err)) + } + runner = query.NewBenchmarkRunner(config) + + redisConnector = redistimeseries.NewClient( + host, runner.DatabaseName(), nil) +} + +func main() { + runner.Run(&query.RedisTimeSeriesPool, newProcessor) +} + +type queryExecutorOptions struct { + showExplain bool + debug bool + printResponse bool +} + +type processor struct { + opts *queryExecutorOptions +} + +func newProcessor() query.Processor { return &processor{} } + +func (p *processor) Init(numWorker int) { + p.opts = &queryExecutorOptions{ + showExplain: showExplain, + debug: runner.DebugLevel() > 0, + printResponse: runner.DoPrintResponses(), + } +} + +func mapRows(r *sql.Rows) []map[string]interface{} { + rows := []map[string]interface{}{} + cols, _ := r.Columns() + for r.Next() { + row := make(map[string]interface{}) + values := make([]interface{}, len(cols)) + for i := range values { + values[i] = new(interface{}) + } + + err := r.Scan(values...) + if err != nil { + panic(errors.Wrap(err, "error while reading values")) + } + + for i, column := range cols { + row[column] = *values[i].(*interface{}) + } + rows = append(rows, row) + } + return rows +} + +// prettyPrintResponseRange prints a Query and its response in JSON format with two +// keys: 'query' which has a value of the RedisTimeseries query used to generate the second key +// 'results' which is an array of each element in the return set. +func prettyPrintResponseRange(responses []interface{}, q *query.RedisTimeSeries) { + full := make(map[string]interface{}) + for idx, qry := range q.RedisQueries { + resp := make(map[string]interface{}) + fullcmd := append([][]byte{q.CommandNames[idx]}, qry...) + resp["query"] = strings.Join(ByteArrayToStringArray(fullcmd), " ") + + res := responses[idx] + switch v := res.(type) { + case []redistimeseries.Range: + resp["client_side_work"] = q.ApplyFunctor + rows := []map[string]interface{}{} + for _, r := range res.([]redistimeseries.Range) { + row := make(map[string]interface{}) + values := make(map[string]interface{}) + values["datapoints"] = r.DataPoints + values["labels"] = r.Labels + row[r.Name] = values + rows = append(rows, row) + } + resp["results"] = rows + case redistimeseries.Range: + resp["client_side_work"] = q.ApplyFunctor + resp["results"] = res.(redistimeseries.Range) + case []query.MultiRange: + resp["client_side_work"] = q.ApplyFunctor + rows := []map[string]interface{}{} + for _, converted := range res.([]query.MultiRange) { + query_result := map[string]interface{}{} + //converted := r.(query.MultiRange) + query_result["names"] = converted.Names + query_result["labels"] = converted.Labels + datapoints := make([]query.MultiDataPoint, 0, len(converted.DataPoints)) + var keys []int + for k := range converted.DataPoints { + keys = append(keys, int(k)) + } + sort.Ints(keys) + for _, k := range keys { + dp := converted.DataPoints[int64(k)] + time_str := time.Unix(dp.Timestamp/1000, 0).Format(time.RFC3339) + dp.HumanReadbleTime = &time_str + datapoints = append(datapoints, dp) + } + query_result["datapoints"] = datapoints + rows = append(rows, query_result) + } + resp["results"] = rows + case query.MultiRange: + resp["client_side_work"] = q.ApplyFunctor + query_result := map[string]interface{}{} + converted := res.(query.MultiRange) + query_result["names"] = converted.Names + query_result["labels"] = converted.Labels + datapoints := make([]query.MultiDataPoint, 0, len(converted.DataPoints)) + var keys []int + for k := range converted.DataPoints { + keys = append(keys, int(k)) + } + sort.Ints(keys) + for _, k := range keys { + dp := converted.DataPoints[int64(k)] + time_str := time.Unix(dp.Timestamp/1000, 0).Format(time.RFC3339) + dp.HumanReadbleTime = &time_str + datapoints = append(datapoints, dp) + } + query_result["datapoints"] = datapoints + resp["results"] = query_result + default: + fmt.Printf("I don't know about type %T!\n", v) + } + + full[fmt.Sprintf("query %d", idx+1)] = resp + } + + line, err := json.MarshalIndent(full, "", " ") + if err != nil { + panic(err) + } + + fmt.Println(string(line) + "\n") +} + +func (p *processor) ProcessQuery(q query.Query, isWarm bool) (queryStats []*query.Stat, err error) { + + // No need to run again for EXPLAIN + if isWarm && p.opts.showExplain { + return nil, nil + } + tq := q.(*query.RedisTimeSeries) + var parsedResponses = make([]interface{}, 0, 0) + + var cmds = make([][]interface{}, 0, 0) + for _, qry := range tq.RedisQueries { + cmds = append(cmds, ByteArrayToInterfaceArray(qry)) + } + conn := redisConnector.Pool.Get() + + start := time.Now() + for idx, commandArgs := range cmds { + var result interface{} + if p.opts.debug { + fmt.Println(fmt.Sprintf("Issuing command (%s %s)", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "))) + } + res, err := conn.Do(string(tq.CommandNames[idx]), commandArgs...) + if err != nil { + log.Fatalf("Command (%s %s) failed with error: %v\n", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "), err) + } + if err != nil { + return nil, err + } + if bytes.Compare(tq.CommandNames[idx], cmdMrange) == 0 || bytes.Compare(tq.CommandNames[idx], cmdMRevRange) == 0 { + + if err != nil { + return nil, err + } + if tq.ApplyFunctor { + if p.opts.debug { + fmt.Println(fmt.Sprintf("Applying functor %s on %s", tq.Functor, tq.HumanLabel)) + } + switch tq.Functor { + case reflect_SingleGroupByTime: + if p.opts.debug { + fmt.Println(fmt.Sprintf("Applying functor reflect_SingleGroupByTime %s", reflect_SingleGroupByTime)) + } + result, err = query.SingleGroupByTime(res) + if err != nil { + return nil, err + } + case reflect_GroupByTimeAndMax: + if p.opts.debug { + fmt.Println(fmt.Sprintf("Applying functor reflect_GroupByTimeAndMax %s", reflect_GroupByTimeAndMax)) + } + result, err = query.GroupByTimeAndMax(res) + if err != nil { + return nil, err + } + case reflect_GroupByTimeAndTagMax: + if p.opts.debug { + fmt.Println(fmt.Sprintf("Applying functor reflect_GroupByTimeAndTagMax %s", reflect_GroupByTimeAndTagMax)) + } + result, err = query.GroupByTimeAndTagMax(res) + if err != nil { + return nil, err + } + case reflect_GroupByTimeAndTagHostname: + if p.opts.debug { + fmt.Println(fmt.Sprintf("Applying functor reflect_GroupByTimeAndTagHostname %s", reflect_GroupByTimeAndTagHostname)) + } + result, err = query.GroupByTimeAndTagHostname(res) + if err != nil { + return nil, err + } + case reflect_HighCpu: + if p.opts.debug { + fmt.Println(fmt.Sprintf("Applying functor reflect_HighCpu %s", reflect_HighCpu)) + } + result, err = query.HighCpu(res) + if err != nil { + return nil, err + } + default: + errors.Errorf("The selected functor %s is not known!\n", tq.Functor) + } + } else { + result, err = redistimeseries.ParseRanges(res) + if err != nil { + return nil, err + } + } + + } else if bytes.Compare(tq.CommandNames[idx], cmdQueryIndex) == 0 { + var parsedRes = make([]redistimeseries.Range, 0, 0) + parsedResponses = append(parsedResponses, parsedRes) + } + parsedResponses = append(parsedResponses, result) + } + took := float64(time.Since(start).Nanoseconds()) / 1e6 + if p.opts.printResponse { + prettyPrintResponseRange(parsedResponses, tq) + } + stat := query.GetStat() + stat.Init(q.HumanLabelName(), took) + queryStats = []*query.Stat{stat} + + return queryStats, err +} + +func ByteArrayToInterfaceArray(qry [][]byte) []interface{} { + commandArgs := make([]interface{}, len(qry)) + for i := 0; i < len(qry); i++ { + commandArgs[i] = qry[i] + } + return commandArgs +} + +func ByteArrayToStringArray(qry [][]byte) []string { + commandArgs := make([]string, len(qry)) + for i := 0; i < len(qry); i++ { + commandArgs[i] = string(qry[i]) + } + return commandArgs +} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..647d8fe9f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:1.13.5-alpine AS builder + +# Copy the code from the host and compile it +WORKDIR $GOPATH/src/github.com/timescale/tsbs +COPY . ./ +RUN apk add --no-cache git make bash +RUN make all + +FROM golang:1.13.5-alpine +# install bash shell +RUN apk add --update bash && rm -rf /var/cache/apk/* + +ENV PATH ./:$PATH +COPY --from=builder $GOPATH/src/github.com/timescale/tsbs/bin/tsbs_* ./ +COPY ./docker/docker_entrypoint.sh ./ +RUN chmod 751 docker_entrypoint.sh +ENTRYPOINT ["./docker_entrypoint.sh"] \ No newline at end of file diff --git a/docker/docker_entrypoint.sh b/docker/docker_entrypoint.sh new file mode 100644 index 000000000..27bddcdf1 --- /dev/null +++ b/docker/docker_entrypoint.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Ensure runner is available +EXE_FILE_NAME=$(which $1) +if [[ -z "$EXE_FILE_NAME" ]]; then + echo "$1 not available. It is not specified explicitly and not found in \$PATH" + exit 1 +else + "$@" + exit 0 +fi diff --git a/docs/redistimeseries.md b/docs/redistimeseries.md new file mode 100644 index 000000000..76655d500 --- /dev/null +++ b/docs/redistimeseries.md @@ -0,0 +1,400 @@ +# TSBS Supplemental Guide: RedisTimeSeries + +RedisTimeSeries is a Redis Module adding a Time Series data structure to Redis. This supplemental guide explains how +the data generated for TSBS is stored, additional flags available when +using the data importer (`tsbs_load_redistimeseries`). **This +should be read *after* the main README.** + + +--- + + +## Installation + +TSBS is a collection of Go programs (with some auxiliary bash and Python +scripts). The easiest way to get and install the Go programs is to use +`go get` and then `go install`: +```bash +# Fetch TSBS and its dependencies +$ go get github.com/timescale/tsbs +$ cd $GOPATH/src/github.com/timescale/tsbs/cmd +$ go get ./... + +# Install redistimeseries binaries. +cd $GOPATH/src/github.com/timescale/tsbs +make +``` + +## Full cycle TSBS RedisTimeSeries scripts + +Instead of calling tsbs redistimeseries binaries directly, we also supply scripts/*.sh for convenience with many of the flags set to a reasonable default for RedisTimeSeries database. + +So for a Full cycle TSBS RedisTimeSeries benchmark, ensure that RedisTimeSeries is running and then use: + +# functional full cycle + +``` +scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh +``` + + +# benchmark commands +``` +# generate the dataset +FORMATS="redistimeseries" SKIP_IF_EXISTS=FALSE SCALE=100 SEED=123 \ + scripts/generate_data.sh + +# generate the queries +FORMATS="redistimeseries" SKIP_IF_EXISTS=FALSE SCALE=4000 SEED=123 \ + scripts/generate_queries.sh + +# load the data into RedisTimeSeries +scripts/load_redistimeseries.sh + +# benchmark RedisTimeSeries query performance +scripts/run_queries_redistimeseries.sh + +``` + +## Data format + +Data generated by `tsbs_generate_data` for RedisTimeSeries is serialized in +`key timestamp value [key timestamp value ...]` format. Each metric reading is composed of `key timestamp value`. In addition to metric readings, 'tags' (including the location of the host, its operating system, etc) are added to each distinct time series at the moment of the first insertion, using [TS.ADD](https://oss.redislabs.com/redistimeseries/commands/#tsadd). + +An example for the `cpu-only` use case, on the first metric reading for the `cpu_usage_user{3297394792}`, `cpu_usage_system{3297394792}`, and `cpu_usage_idle{3297394792}` metrics, they would be insert using [TS.ADD](https://oss.redislabs.com/redistimeseries/commands/#tsadd), in the following manner: +```text +TS.ADD cpu_usage_user{3297394792} 1451606400000 58 LABELS hostname host_0 region eu-central-1 datacenter eu-central-1a rack 6 os Ubuntu15.10 arch x86 team SF service 19 service_version 1 service_environment test measurement cpu fieldname usage_user +TS.ADD cpu_usage_system{3297394792} 1451606400000 2 LABELS hostname host_0 region eu-central-1 datacenter eu-central-1a rack 6 os Ubuntu15.10 arch x86 team SF service 19 service_version 1 service_environment test measurement cpu fieldname usage_system +TS.ADD cpu_usage_idle{3297394792} 1451606400000 24 LABELS hostname host_0 region eu-central-1 datacenter eu-central-1a rack 6 os Ubuntu15.10 arch x86 team SF service 19 service_version 1 service_environment test measurement cpu fieldname usage_idle +``` + +The second and following metric readings for the `cpu_usage_user{3297394792}`, `cpu_usage_system{3297394792}`, and `cpu_usage_idle{3297394792}` metrics would be inserted using [TS.MADD](https://oss.redislabs.com/redistimeseries/commands/#tsmadd) command: + +```text +TS.MADD cpu_usage_user{3297394792} 1451606410000 57 cpu_usage_system{3297394792} 1451606410000 3 cpu_usage_idle{3297394792} +TS.MADD cpu_usage_user{3297394792} 1451606420000 58 cpu_usage_system{3297394792} 1451606420000 2 cpu_usage_idle{3297394792} +``` + +## Query types mapping to RedisTimeSeries query language + +### Client side work functors: + - MergeSeriesOnTimestamp - self explanatory + - FilterRangesByThresholdAbove - filter a MultiRange ( multiple series merged by timestamp ) on a threshold for one of the metrics in the multiseries + - ReduceSeriesOnTimestampBy - reduce a MultiRange over a given function ( in our specific case, Max and Avg ) + +### Devops / cpu-only + +#### Simple Rollups + +##### q1) single-groupby-1-1-1 +Simple aggregrate (MAX) on one metric for 1 host, every 1 minute for 1 hour + +###### Query language +``` +TS.MRANGE 1451679382646 1451682982646 AGGREGATION MAX 60000 FILTER measurement=cpu fieldname=usage_user hostname=host_9 +``` + +###### Sample Responses: +- [InfluxDB](./responses/influx_single-groupby-1-1-1.json) +- [TimescaleDB](./responses/timescaledb_single-groupby-1-1-1.json) +- [RedistimeSeries](./responses/redistimeseries_single-groupby-1-1-1.json) + +##### q2) single-groupby-1-1-12 + +Simple aggregrate (MAX) on one metric for 1 host, every 1 minute for 12 hours + +###### Query language +``` +TS.MRANGE 1451628982646 1451672182646 AGGREGATION MAX 60000 FILTER measurement=cpu fieldname=usage_user hostname=host_9 +``` + +###### Sample Responses: +- [InfluxDB](./responses/influx_single-groupby-1-1-12.json) +- [TimescaleDB](./responses/timescaledb_single-groupby-1-1-12.json) +- [RedistimeSeries](./responses/redistimeseries_single-groupby-1-1-12.json) + +##### q3) single-groupby-1-8-1 ( *client side work for RedisTimeSeries ) +Simple aggregrate (MAX) on one metric for 8 hosts, every 1 minute for 1 hour + +###### Query language +``` +TS.MRANGE 1451679382646 1451682982646 AGGREGATION MAX 60000 FILTER measurement=cpu fieldname=usage_user hostname=(host_9,host_3,host_5,host_1,host_7,host_2,host_8,host_4) +``` +###### Client side work description: +Max Reduction over time on multiple time-series with the same metric +###### Code: [github.com/timescale/tsbs/query.GroupByTimeAndMax](./../query/redistimeseries_functors.go#L42) + +###### Sample Responses: +- [InfluxDB](./responses/influx_single-groupby-1-8-1.json) +- [TimescaleDB](./responses/timescaledb_single-groupby-1-8-1.json) +- [RedistimeSeries](./responses/redistimeseries_single-groupby-1-8-1.json) + + + +##### q4) single-groupby-5-1-1 ( *client side work for RedisTimeSeries ) + Simple aggregrate (MAX) on 5 metrics for 1 host, every 5 mins for 1 hour + +###### Query language +``` +TS.MRANGE 1451679382646 1451682982646 AGGREGATION MAX 60000 FILTER measurement=cpu fieldname=(usage_user,usage_system,usage_idle,usage_nice,usage_iowait) hostname=host_9 +``` +###### Client side work description: +Aggregation over time on multiple time-series datapoints with the same timestamp + +###### Code: [github.com/timescale/tsbs/query.SingleGroupByTime](./../query/redistimeseries_functors.go#L33) + +###### Sample Responses: +- [InfluxDB](./responses/influx_single-groupby-5-1-1.json) +- [TimescaleDB](./responses/timescaledb_single-groupby-5-1-1.json) +- [RedistimeSeries](./responses/redistimeseries_single-groupby-5-1-1.json) + + + +##### q5) single-groupby-5-1-12 ( *client side work for RedisTimeSeries ) +Simple aggregrate (MAX) on 5 metrics for 1 host, every 5 mins for 12 hours + + +###### Query language +``` +TS.MRANGE 1451628982646 1451672182646 AGGREGATION MAX 60000 FILTER measurement=cpu fieldname=(usage_user,usage_system,usage_idle,usage_nice,usage_iowait) hostname=host_9 +``` +###### Client side work description: +Aggregation over time on multiple time-series datapoints with the same timestamp + +###### Code: [github.com/timescale/tsbs/query.SingleGroupByTime](./../query/redistimeseries_functors.go#L33) + +###### Sample Responses: +- [InfluxDB](./responses/influx_single-groupby-5-1-12.json) +- [TimescaleDB](./responses/timescaledb_single-groupby-5-1-12.json) +- [RedistimeSeries](./responses/redistimeseries_single-groupby-5-1-12.json) + + + +##### q6) single-groupby-5-8-1 ( *client side work for RedisTimeSeries ) +Simple aggregrate (MAX) on 5 metrics for 8 hosts, every 5 mins for 1 hour + + + +###### Query language +``` +TS.MRANGE 1451679382646 1451682982646 AGGREGATION MAX 60000 FILTER measurement=cpu fieldname=(usage_user,usage_system,usage_idle,usage_nice,usage_iowait) hostname=(host_9,host_3,host_5,host_1,host_7,host_2,host_8,host_4) +``` +###### Client side work description: +Max Reduction over time on multiple time-series with the same metric, and aggregation over time on multiple time-series datapoints with the same timestamp but different metrics + +###### Code: [github.com/timescale/tsbs/query.GroupByTimeAndTagMax](./../query/redistimeseries_functors.go#L51) + +###### Sample Responses: +- [InfluxDB](./responses/influx_single-groupby-5-8-1.json) +- [TimescaleDB](./responses/timescaledb_single-groupby-5-8-1.json) +- [RedistimeSeries](./responses/redistimeseries_single-groupby-5-8-1.json) + + + +#### Simple Aggregations + +##### q7) cpu-max-all-1 ( *client side work for RedisTimeSeries ) +Aggregate across all CPU metrics per hour over 1 hour for a single host + + + +###### Query language +``` +TS.MRANGE 1451614582646 1451643382646 AGGREGATION MAX 3600000 FILTER measurement=cpu hostname=host_9 +``` +###### Client side work description: +Aggregation over time on multiple time-series datapoints with the same timestamp + +###### Code: [github.com/timescale/tsbs/query.SingleGroupByTime](./../query/redistimeseries_functors.go#L33) + +###### Sample Responses: +- [InfluxDB](./responses/influx_cpu-max-all-1.json) *this query is being generated with the wrong parameters for InfluxDB ( will create issue upwards )* +- [TimescaleDB](./responses/timescaledb_cpu-max-all-1.json) +- [RedistimeSeries](./responses/redistimeseries_cpu-max-all-1.json) + + + +##### q8) cpu-max-all-8 ( *client side work for RedisTimeSeries ) +Aggregate across all CPU metrics per hour over 1 hour for eight hosts + + +###### Query language +``` +TS.MRANGE 1451614582646 1451643382646 AGGREGATION MAX 3600000 FILTER measurement=cpu hostname=(host_9,host_3,host_5,host_1,host_7,host_2,host_8,host_4) +``` +###### Client side work description: +Max Reduction over time on multiple time-series with the same metric, and aggregation over time on multiple time-series datapoints with the same timestamp but different metrics + +###### Code: [github.com/timescale/tsbs/query.GroupByTimeAndTagMax](./../query/redistimeseries_functors.go#L51) + +###### Sample Responses: +- [InfluxDB](./responses/influx_cpu-max-all-8.json) +- [TimescaleDB](./responses/timescaledb_cpu-max-all-8.json) +- [RedistimeSeries](./responses/redistimeseries_cpu-max-all-8.json) + + + +#### Double Rollups +##### q9) double-groupby-1 ( *client side work for RedisTimeSeries ) +Aggregate on across both time and host, giving the average of 1 CPU metric per host per hour for 24 hours + + +###### Query language +``` +TS.MRANGE 1451628982646 1451672182646 AGGREGATION AVG 3600000 FILTER measurement=cpu fieldname=usage_user +``` +###### Client side work description: +Group on multiple time-series with the same tag ( hostname ), and aggregation over time on multiple time-series datapoints with the same timestamp but same metric + +###### Code: [github.com/timescale/tsbs/query.GroupByTimeAndTagHostname](./../query/redistimeseries_functors.go#L76) + +###### Sample Responses: +- [InfluxDB](./responses/influx_double-groupby-1.json) +- [TimescaleDB](./responses/timescaledb_double-groupby-1.json) +- [RedistimeSeries](./responses/redistimeseries_double-groupby-1.json) + + + +##### q10) double-groupby-5 ( *client side work for RedisTimeSeries ) + Aggregate on across both time and host, giving the average of 5 CPU metrics per host per hour for 24 hours + +###### Query language +``` +TS.MRANGE 1451628982646 1451672182646 AGGREGATION AVG 3600000 FILTER measurement=cpu fieldname=(usage_user,usage_system,usage_idle,usage_nice,usage_iowait) +``` +###### Client side work description: +Group on multiple time-series with the same tag ( hostname ), and aggregation over time on multiple time-series datapoints with the same timestamp but different metrics + +###### Code: [github.com/timescale/tsbs/query.GroupByTimeAndTagHostname](./../query/redistimeseries_functors.go#L76) + +###### Sample Responses: +- [InfluxDB](./responses/influx_double-groupby-5.json) +- [TimescaleDB](./responses/timescaledb_double-groupby-5.json) +- [RedistimeSeries](./responses/redistimeseries_double-groupby-5.json) + + + +##### q11) double-groupby-all ( *client side work for RedisTimeSeries ) + Aggregate on across both time and host, giving the average of all (10) CPU metrics per host per hour for 24 hours + +###### Query language +``` +TS.MRANGE 1451628982646 1451672182646 AGGREGATION AVG 3600000 FILTER measurement=cpu +``` +###### Client side work description: +Group on multiple time-series with the same tag ( hostname ), and aggregation over time on multiple time-series datapoints with the same timestamp but different metrics + +###### Code: [github.com/timescale/tsbs/query.GroupByTimeAndTagHostname](./../query/redistimeseries_functors.go#L76) + +###### Sample Responses: +- [InfluxDB](./responses/influx_double-groupby-all.json) +- [TimescaleDB](./responses/timescaledb_double-groupby-all.json) +- [RedistimeSeries](./responses/redistimeseries_double-groupby-all.json) + + +#### Thresholds + +##### q12) high-cpu-all ( *client side work for RedisTimeSeries ) +All the readings where one metric is above a threshold across all hosts + +###### Query language +``` +TS.MRANGE 1451628982646 1451672182646 FILTER measurement=cpu +``` +###### Client side work description: +Group on multiple time-series with the same tag ( hostname ), and aggregation over time on multiple time-series datapoints with the same timestamp but different metrics, if a specific time-series with one of the metrics (usage_user) is above a threshold + +###### Code: [github.com/timescale/tsbs/query.HighCpu](./../query/redistimeseries_functors.go#L98) + +###### Sample Responses: +- [InfluxDB](./responses/influx_high-cpu-all.json) +- [TimescaleDB](./responses/timescaledb_high-cpu-all.json) +- [RedistimeSeries](./responses/redistimeseries_high-cpu-all.json) + + +##### q13) high-cpu-1 ( *client side work for RedisTimeSeries ) + All the readings where one metric is above a threshold for a particular host + +###### Query language +``` +TS.MRANGE 1451649250138 1451692450138 FILTER measurement=cpu hostname=host_5 +``` +###### Client side work description: +Group on multiple time-series with the same tag ( hostname ), and aggregation over time on multiple time-series datapoints with the same timestamp but different metrics, if a specific time-series with one of the metrics (usage_user) is above a threshold + +###### Code: [github.com/timescale/tsbs/query.HighCpu](./../query/redistimeseries_functors.go#L98) + +###### Sample Responses: +- [InfluxDB](./responses/influx_high-cpu-1.json) +- [TimescaleDB](./responses/timescaledb_high-cpu-1.json) +- [RedistimeSeries](./responses/redistimeseries_high-cpu-1.json) + + +#### Complex queries + +##### q14) lastpoint ( *client side work for RedisTimeSeries )[REQUIRES MREVRANGE] +The last reading for each host + +###### Query language +``` +TS.MREVRANGE + - COUNT 1 FILTER measurement=cpu hostname!= +``` +###### Client side work description: + +Group on multiple time-series with the same tag ( hostname ), and aggregation over time on multiple time-series datapoints with the same timestamp but different metrics + +###### Code: [github.com/timescale/tsbs/query.GroupByTimeAndTagHostname](./../query/redistimeseries_functors.go#L76) + +###### Sample Responses: +- [InfluxDB](./responses/influx_lastpoint.json) +- [TimescaleDB](./responses/timescaledb_lastpoint.json) +- [RedistimeSeries](./responses/redistimeseries_lastpoint.json) + + +##### q15) groupby-orderby-limit ( *client side work for RedisTimeSeries )[REQUIRES MREVRANGE] [STILL WIP] + +The last 5 aggregate readings (across time) before a randomly chosen endpoint + +###### Query language +``` +TS.MREVRANGE 1451682982646 - COUNT 5 AGGREGATION MAX 60000 FILTER measurement=cpu fieldname=usage_user +``` +###### Client side work description: +WIP + +###### Code: WIP + +###### Sample Responses: +- [InfluxDB](./responses/influx_groupby-orderby-limit.json) +- [TimescaleDB](./responses/timescaledb_groupby-orderby-limit.json) +- [RedistimeSeries](./responses/redistimeseries_groupby-orderby-limit.json) + + + +--- + +## `tsbs_load_redistimeseries` Additional Flags + +### Database related + +#### `-host` (type: `string`, default: `localhost:6379`) + +The host:port for Redis connection. + +#### `-connections` (type: `int`, default: `10`) + +The number of connections per worker. + +#### `-pipeline` (type: `int`, default: `50`) + +The pipeline's size. Read the full documentation [here](https://redis.io/topics/pipelining). + +### Miscellaneous + +#### `-single-queue` (type: `boolean`, default: `true`) + +Whether to use a single queue. + +#### `-check-chunks` (type: `int`, default: `0`) + +Whether to perform post ingestion chunck count. diff --git a/go.mod b/go.mod index bd99fe622..f49b9b9a8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/timescale/tsbs go 1.14 require ( + github.com/RedisTimeSeries/redistimeseries-go v1.4.3 github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 @@ -14,6 +15,7 @@ require ( github.com/gocql/gocql v0.0.0-20190810123941-df4b9cc33030 github.com/golang/protobuf v1.4.2 github.com/golang/snappy v0.0.1 + github.com/gomodule/redigo v1.8.2 github.com/google/flatbuffers v1.11.0 github.com/google/go-cmp v0.5.2 github.com/jackc/pgx/v4 v4.8.0 @@ -26,11 +28,12 @@ require ( github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/spf13/cobra v0.0.7 github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.4.0 github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 github.com/transceptor-technology/go-qpack v0.0.0-20190116123619-49a14b216a45 github.com/valyala/fasthttp v1.4.0 - golang.org/x/net v0.0.0-20200904194848-62affa334b73 go.uber.org/atomic v1.6.0 + golang.org/x/net v0.0.0-20200904194848-62affa334b73 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 8ce0d78e8..e4c29703d 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,10 @@ github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/RedisTimeSeries/redistimeseries-go v0.3.1-0.20191211172356-add367a3fed3 h1:jP6nxChl290N7amKzCx15fJhzcs7jMayt33oNggjH/Q= +github.com/RedisTimeSeries/redistimeseries-go v0.3.1-0.20191211172356-add367a3fed3/go.mod h1:mkrqzk8fT/m6GtZvyyK5+rSxvXP59/M+p2CPtr+Fl6g= +github.com/RedisTimeSeries/redistimeseries-go v1.4.3 h1:jTs0sUsBJvCiPy788gIOEzyh9nYezsY70Dm4Q/q1yyA= +github.com/RedisTimeSeries/redistimeseries-go v1.4.3/go.mod h1:qbwAboI+9BGBAWLLLJO2XwoLwkUpcPJ6GRRBEWJ10as= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921 h1:GIWNb0z3t/YKr7xcGNhFgxasaTpnsX91Z0Zt4CeLk+c= @@ -384,6 +388,11 @@ github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/gomodule/redigo/redis v0.0.0-do-not-use h1:J7XIp6Kau0WoyT4JtXHT3Ei0gA1KkSc6bc87j9v9WIo= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A= @@ -394,6 +403,7 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -634,6 +644,7 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= diff --git a/internal/inputs/generator_data.go b/internal/inputs/generator_data.go index ee639659c..bca8cb755 100644 --- a/internal/inputs/generator_data.go +++ b/internal/inputs/generator_data.go @@ -121,7 +121,6 @@ func (g *DataGenerator) runSimulator(sim common.Simulator, serializer serialize. } } point.Reset() - currGroupID = (currGroupID + 1) % dgc.InterleavedNumGroups } return nil diff --git a/internal/inputs/generator_queries.go b/internal/inputs/generator_queries.go index 44a9ec58c..6e6a3b425 100644 --- a/internal/inputs/generator_queries.go +++ b/internal/inputs/generator_queries.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/gob" "fmt" + "io" "math/rand" "os" diff --git a/pkg/data/usecases/common/simulator.go b/pkg/data/usecases/common/simulator.go index d984866e6..48fbe69ad 100644 --- a/pkg/data/usecases/common/simulator.go +++ b/pkg/data/usecases/common/simulator.go @@ -1,6 +1,7 @@ package common import ( + "fmt" "github.com/timescale/tsbs/pkg/data" "reflect" "time" @@ -99,6 +100,31 @@ type BaseSimulator struct { simulatedMeasurementIndex int } +func (s *BaseSimulator) MaxPoints() uint64 { + return s.maxPoints +} + +func (s *BaseSimulator) Epochs() uint64 { + return s.epochs +} + +func (s *BaseSimulator) TimestampEnd() time.Time { + return s.timestampEnd +} + +func (s *BaseSimulator) TimestampStart() time.Time { + return s.timestampStart +} + +func (s *BaseSimulator) MadePoints() uint64 { + return s.madePoints +} + +// GetSummary returns a summary string after data has been generated. +func (s *BaseSimulator) GetSummary() string { + return fmt.Sprintf("Generated a total of %d points.", s.madePoints) +} + // Finished tells whether we have simulated all the necessary points. func (s *BaseSimulator) Finished() bool { return s.madePoints >= s.maxPoints diff --git a/pkg/data/usecases/devops/cpu_only_generate_data.go b/pkg/data/usecases/devops/cpu_only_generate_data.go index b03e84283..8e7caf0ac 100644 --- a/pkg/data/usecases/devops/cpu_only_generate_data.go +++ b/pkg/data/usecases/devops/cpu_only_generate_data.go @@ -9,6 +9,7 @@ import ( // A CPUOnlySimulator generates data similar to telemetry from Telegraf for only CPU metrics. // It fulfills the Simulator interface. type CPUOnlySimulator struct { + base CPUOnlySimulatorConfig *commonDevopsSimulator } @@ -41,6 +42,10 @@ func (d *CPUOnlySimulator) Next(p *data.Point) bool { return d.populatePoint(p, 0) } +func (s *CPUOnlySimulator) MaxPoints() uint64 { + return s.maxPoints +} + // CPUOnlySimulatorConfig is used to create a CPUOnlySimulator. type CPUOnlySimulatorConfig commonDevopsSimulatorConfig @@ -57,21 +62,23 @@ func (c *CPUOnlySimulatorConfig) NewSimulator(interval time.Duration, limit uint // Set specified points number limit maxPoints = limit } - sim := &CPUOnlySimulator{&commonDevopsSimulator{ - madePoints: 0, - maxPoints: maxPoints, + sim := &CPUOnlySimulator{ + *c, + &commonDevopsSimulator{ + madePoints: 0, + maxPoints: maxPoints, - hostIndex: 0, - hosts: hostInfos, + hostIndex: 0, + hosts: hostInfos, - epoch: 0, - epochs: epochs, - epochHosts: c.InitHostCount, - initHosts: c.InitHostCount, - timestampStart: c.Start, - timestampEnd: c.End, - interval: interval, - }} + epoch: 0, + epochs: epochs, + epochHosts: c.InitHostCount, + initHosts: c.InitHostCount, + timestampStart: c.Start, + timestampEnd: c.End, + interval: interval, + }} return sim } diff --git a/pkg/data/usecases/iot/simulator.go b/pkg/data/usecases/iot/simulator.go index c76188ba1..203b7c85b 100644 --- a/pkg/data/usecases/iot/simulator.go +++ b/pkg/data/usecases/iot/simulator.go @@ -36,6 +36,10 @@ func (sc *SimulatorConfig) NewSimulator(interval time.Duration, limit uint64) co } } +func (s *Simulator) MaxPoints() uint64 { + return s.MaxPoints() +} + // Simulator is responsible for simulating entries for the IoT use case. // It will run on batches of entries and apply the generated batch configuration // which it gets from the config generator. That way it can introduce things like diff --git a/pkg/query/redistimeseries.go b/pkg/query/redistimeseries.go new file mode 100644 index 000000000..7b948743d --- /dev/null +++ b/pkg/query/redistimeseries.go @@ -0,0 +1,96 @@ +package query + +import ( + "fmt" + "sync" +) + +// RedisTimeSeries encodes a RedisTimeSeries request. This will be serialized for use +// by the tsbs_run_queries_redistimeseries program. +type RedisTimeSeries struct { + HumanLabel []byte + HumanDescription []byte + + RedisQueries [][][]byte + CommandNames [][]byte + id uint64 + ApplyFunctor bool + Functor string +} + +// RedisTimeSeriesPool is a sync.Pool of RedisTimeSeries Query types +var RedisTimeSeriesPool = sync.Pool{ + New: func() interface{} { + queries := make([][][]byte, 0, 0) + commands := make([][]byte, 0, 0) + return &RedisTimeSeries{ + HumanLabel: make([]byte, 0, 1024), + HumanDescription: make([]byte, 0, 1024), + RedisQueries: queries, + CommandNames: commands, + ApplyFunctor: false, + } + }, +} + +// NewRedisTimeSeries returns a new RedisTimeSeries Query instance +func NewRedisTimeSeries() *RedisTimeSeries { + return RedisTimeSeriesPool.Get().(*RedisTimeSeries) +} + +// GetID returns the ID of this Query +func (q *RedisTimeSeries) GetID() uint64 { + return q.id +} + +// SetID sets the ID for this Query +func (q *RedisTimeSeries) SetID(n uint64) { + q.id = n +} + +// SetApplyFunctor sets the flag for group by timestamp on a MultiRange Serie +func (q *RedisTimeSeries) SetApplyFunctor(value bool) { + q.ApplyFunctor = value +} + +func (q *RedisTimeSeries) SetFunctor(f string) { + q.Functor = f +} + +// GetCommandName returns the command used for this Query +func (q *RedisTimeSeries) AddQuery(query [][]byte, commandname []byte) { + q.RedisQueries = append(q.RedisQueries, query) + q.CommandNames = append(q.CommandNames, commandname) +} + +// GetCommandName returns the command used for this Query +func (q *RedisTimeSeries) GetCommandName(pos int) []byte { + return q.CommandNames[pos] +} + +// String produces a debug-ready description of a Query. +func (q *RedisTimeSeries) String() string { + return fmt.Sprintf("HumanLabel: %s, HumanDescription: %s, Query: %s", q.HumanLabel, q.HumanDescription, q.RedisQueries) +} + +// HumanLabelName returns the human readable name of this Query +func (q *RedisTimeSeries) HumanLabelName() []byte { + return q.HumanLabel +} + +// HumanDescriptionName returns the human readable description of this Query +func (q *RedisTimeSeries) HumanDescriptionName() []byte { + return q.HumanDescription +} + +// Release resets and returns this Query to its pool +func (q *RedisTimeSeries) Release() { + q.HumanLabel = q.HumanLabel[:0] + q.HumanDescription = q.HumanDescription[:0] + q.id = 0 + + q.RedisQueries = q.RedisQueries[:0] + q.CommandNames = q.CommandNames[:0] + + RedisTimeSeriesPool.Put(q) +} diff --git a/pkg/query/redistimeseries_functors.go b/pkg/query/redistimeseries_functors.go new file mode 100644 index 000000000..41c1bea74 --- /dev/null +++ b/pkg/query/redistimeseries_functors.go @@ -0,0 +1,297 @@ +package query + +import ( + "fmt" + redistimeseries "github.com/RedisTimeSeries/redistimeseries-go" + "reflect" + "runtime" + "sort" + "strings" +) + +type ResponseFunctor func(interface{}) (interface{}, error) +type void struct{} + +var member void + +type MultiDataPoint struct { + Timestamp int64 + HumanReadbleTime *string + Values []*float64 +} + +type MultiRange struct { + Names []string + Labels []map[string]string + DataPoints map[int64]MultiDataPoint +} + +func GetFunctionName(i interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() +} + +func SingleGroupByTime(res interface{}) (result interface{}, err error) { + parsedRes, err := redistimeseries.ParseRanges(res) + if err != nil { + return + } + result = MergeSeriesOnTimestamp(parsedRes) + return +} + +func GroupByTimeAndMax(res interface{}) (result interface{}, err error) { + parsedRes, err := redistimeseries.ParseRanges(res) + if err != nil { + return + } + result, err = ReduceSeriesOnTimestampBy(parsedRes, MaxReducerSeriesDatapoints) + return +} + +func GroupByTimeAndTagMax(res interface{}) (result interface{}, err error) { + parsedRes, err := redistimeseries.ParseRanges(res) + if err != nil { + return + } + labels, err := GetUniqueLabelValue(parsedRes, "fieldname") + if err != nil { + return + } + var outseries = make([]redistimeseries.Range, 0, 0) + for _, label := range labels { + filteredSeries, err := FilterRangesByLabelValue(parsedRes, "fieldname", label, true) + if err != nil { + return result, err + } + reducedSerie, err := ReduceSeriesOnTimestampBy(filteredSeries, MaxReducerSeriesDatapoints) + if err != nil { + return result, err + } + outseries = append(outseries, reducedSerie) + } + result = MergeSeriesOnTimestamp(outseries) + return +} + +func GroupByTimeAndTagHostname(res interface{}) (result interface{}, err error) { + parsedRes, err := redistimeseries.ParseRanges(res) + if err != nil { + return + } + labels, err := GetUniqueLabelValue(parsedRes, "hostname") + if err != nil { + return + } + var outseries = make([]MultiRange, 0, 0) + for _, label := range labels { + filteredSeries, err := FilterRangesByLabelValue(parsedRes, "hostname", label, true) + if err != nil { + return result, err + } + merged := MergeSeriesOnTimestamp(filteredSeries) + outseries = append(outseries, merged) + } + result = outseries + return +} + +func HighCpu(res interface{}) (result interface{}, err error) { + parsedRes, err := redistimeseries.ParseRanges(res) + if err != nil { + return + } + labels, err := GetUniqueLabelValue(parsedRes, "hostname") + if err != nil { + return + } + var outseries = make([]MultiRange, 0, 0) + for _, label := range labels { + filteredSeries, err := FilterRangesByLabelValue(parsedRes, "hostname", label, true) + if err != nil { + return result, err + } + merged := MergeSeriesOnTimestamp(filteredSeries) + above, err := FilterRangesByThresholdAbove(merged, "fieldname", "usage_user", 90) + if err != nil { + return result, err + } + if len(above.DataPoints) > 0 { + outseries = append(outseries, above) + } + } + result = outseries + return +} + +func FilterRangesByLabelValue(series []redistimeseries.Range, labelname, labelvalue string, keepMatches bool) (result []redistimeseries.Range, err error) { + result = make([]redistimeseries.Range, 0, 1) + for _, serie := range series { + flagged := false + value, labelExists := serie.Labels[labelname] + if labelExists == true && value == labelvalue { + flagged = true + } + if flagged == true && keepMatches == true { + result = append(result, serie) + } + if flagged == false && keepMatches == false { + result = append(result, serie) + + } + } + return +} + +func FilterRangesByThresholdAbove(serie MultiRange, labelname, labelvalue string, above float64) (result MultiRange, err error) { + datapoints := make(map[int64]MultiDataPoint) + thresholdIdx := -1 + for idx, labels := range serie.Labels { + v, found := labels[labelname] + if found { + if labelvalue == v { + thresholdIdx = idx + } + } + } + + for ts, datapoint := range serie.DataPoints { + vp := datapoint.Values[thresholdIdx] + if vp != nil && *vp > above { + datapoints[ts] = datapoint + } + } + result = MultiRange{serie.Names, serie.Labels, datapoints} + return +} + +func GetUniqueLabelValue(series []redistimeseries.Range, label string) (result []string, err error) { + set := make(map[string]void) // New empty set + result = make([]string, 0, 0) + for _, serie := range series { + value, found := serie.Labels[label] + if found == true { + set[value] = member + } + } + for k := range set { + result = append(result, k) + } + return +} + +func MergeSeriesOnTimestamp(series []redistimeseries.Range) MultiRange { + names := make([]string, len(series), len(series)) + labels := make([]map[string]string, len(series), len(series)) + datapoints := make(map[int64]MultiDataPoint) + for idx, serie := range series { + names[idx] = serie.Name + labels[idx] = serie.Labels + for _, datapoint := range serie.DataPoints { + _, found := datapoints[datapoint.Timestamp] + if found == true { + var v = datapoint.Value + datapoints[datapoint.Timestamp].Values[idx] = &v + } else { + multipointValues := make([]*float64, len(series), len(series)) + for ii := range multipointValues { + multipointValues[ii] = nil + } + var v = datapoint.Value + multipointValues[idx] = &v + datapoints[datapoint.Timestamp] = MultiDataPoint{datapoint.Timestamp, nil, multipointValues} + } + } + } + return MultiRange{names, labels, datapoints} +} + +func AvgReducerSeriesDatapoints(series []redistimeseries.Range) (c redistimeseries.Range, err error) { + allNames := make([]string, 0, len(series)) + for _, serie := range series { + allNames = append(allNames, serie.Name) + } + var vPoints = make(map[int64]float64) + var fPoints = make(map[int64]float64) + var cPoints = make(map[int64]int64) + pos := 0 + for pos < len(series) { + serie := series[pos] + for _, v := range serie.DataPoints { + _, found := cPoints[v.Timestamp] + if found == true { + cPoints[v.Timestamp] = cPoints[v.Timestamp] + 1 + vPoints[v.Timestamp] = vPoints[v.Timestamp] + v.Value + fPoints[v.Timestamp] = vPoints[v.Timestamp] / float64(cPoints[v.Timestamp]) + } else { + cPoints[v.Timestamp] = 1 + vPoints[v.Timestamp] = v.Value + fPoints[v.Timestamp] = v.Value + } + } + pos = pos + 1 + } + var keys []int + for k := range cPoints { + keys = append(keys, int(k)) + } + sort.Ints(keys) + datapoints := make([]redistimeseries.DataPoint, 0, len(keys)) + for _, k := range keys { + dp := fPoints[int64(k)] + datapoints = append(datapoints, redistimeseries.DataPoint{int64(k), dp}) + } + name := fmt.Sprintf("avg reduction over %s", strings.Join(allNames, " ")) + c = redistimeseries.Range{name, nil, datapoints} + return +} + +func MaxReducerSeriesDatapoints(series []redistimeseries.Range) (c redistimeseries.Range, err error) { + allNames := make([]string, 0, len(series)) + for _, serie := range series { + allNames = append(allNames, serie.Name) + } + var cPoints = make(map[int64]float64) + pos := 0 + for pos < len(series) { + serie := series[pos] + for _, v := range serie.DataPoints { + _, found := cPoints[v.Timestamp] + if found == true { + if cPoints[v.Timestamp] < v.Value { + cPoints[v.Timestamp] = v.Value + } + } else { + cPoints[v.Timestamp] = v.Value + } + } + pos = pos + 1 + } + var keys []int + for k := range cPoints { + keys = append(keys, int(k)) + } + sort.Ints(keys) + datapoints := make([]redistimeseries.DataPoint, 0, len(keys)) + for _, k := range keys { + dp := cPoints[int64(k)] + datapoints = append(datapoints, redistimeseries.DataPoint{int64(k), dp}) + } + name := fmt.Sprintf("max reduction over %s", strings.Join(allNames, " ")) + c = redistimeseries.Range{name, nil, datapoints} + return +} + +func ReduceSeriesOnTimestampBy(series []redistimeseries.Range, reducer func(series []redistimeseries.Range) (redistimeseries.Range, error)) (outserie redistimeseries.Range, err error) { + allNames := make([]string, 0, len(series)) + for _, serie := range series { + allNames = append(allNames, serie.Name) + } + if len(series) == 0 { + return + } + if len(series) == 1 { + outserie = series[0] + return + } + return reducer(series) +} diff --git a/pkg/query/redistimeseries_functors_test.go b/pkg/query/redistimeseries_functors_test.go new file mode 100644 index 000000000..1c52f197e --- /dev/null +++ b/pkg/query/redistimeseries_functors_test.go @@ -0,0 +1,210 @@ +package query + +import ( + redistimeseries "github.com/RedisTimeSeries/redistimeseries-go" + "reflect" + "sort" + "testing" +) + +func TestMergeSeriesOnTimestamp(t *testing.T) { + type args struct { + series []redistimeseries.Range + } + ts := []int64{1, 2, 3, 4, 5} + vals := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + tests := []struct { + name string + args args + want MultiRange + }{ + //Name string + //Labels map[string]string + //DataPoints []DataPoint + {"test 1 series empty labels and datapoints", + args{ + []redistimeseries.Range{ + {"serie1", map[string]string{}, + []redistimeseries.DataPoint{}, + }}}, + MultiRange{[]string{"serie1"}, []map[string]string{{}}, map[int64]MultiDataPoint{}}, + }, + {"test 2 series empty labels and datapoints", + args{ + []redistimeseries.Range{ + {"serie1", map[string]string{}, []redistimeseries.DataPoint{}}, + {"serie2", map[string]string{}, []redistimeseries.DataPoint{}}, + }}, + MultiRange{[]string{"serie1", "serie2"}, []map[string]string{{}, {}}, map[int64]MultiDataPoint{}}, + }, + {"test 2 series with labels and empty datapoints", + args{ + []redistimeseries.Range{ + {"serie1", map[string]string{"host": "1"}, []redistimeseries.DataPoint{}}, + {"serie2", map[string]string{"host": "2"}, []redistimeseries.DataPoint{}}, + }}, + MultiRange{[]string{"serie1", "serie2"}, []map[string]string{{"host": "1"}, {"host": "2"}}, map[int64]MultiDataPoint{}}, + }, + {"test 2 series with labels and datapoints", + args{ + []redistimeseries.Range{ + {"serie1", map[string]string{"host": "1"}, []redistimeseries.DataPoint{{ts[0], vals[0]}}}, + {"serie2", map[string]string{"host": "2"}, []redistimeseries.DataPoint{{ts[0], vals[0]}}}}, + }, + MultiRange{ + []string{"serie1", "serie2"}, + []map[string]string{{"host": "1"}, {"host": "2"}}, + map[int64]MultiDataPoint{ts[0]: { + Timestamp: ts[0], + Values: []*float64{&vals[0], &vals[0]}, + }}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := MergeSeriesOnTimestamp(tt.args.series) + if !reflect.DeepEqual(got.Names, tt.want.Names) { + t.Errorf("MergeSeriesOnTimestamp() Error on Names got %v, want %v", got.Names, tt.want.Names) + } + if !reflect.DeepEqual(got.Labels, tt.want.Labels) { + t.Errorf("MergeSeriesOnTimestamp() Error on Labels got %v, want %v", got.Labels, tt.want.Labels) + } + if !reflect.DeepEqual(got.DataPoints, tt.want.DataPoints) { + t.Errorf("MergeSeriesOnTimestamp() Error on DataPoints got %v, want %v", got.DataPoints, tt.want.DataPoints) + } + }) + } +} + +func TestReduceSeriesOnTimestampBy(t *testing.T) { + type args struct { + series []redistimeseries.Range + reducer func(series []redistimeseries.Range) (redistimeseries.Range, error) + } + ts := []int64{1, 2, 3, 4, 5} + vals := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + tests := []struct { + name string + args args + wantOutserie redistimeseries.Range + wantErr bool + }{ + {"test 1 series with labels and datapoints", + args{ + []redistimeseries.Range{ + {"serie1", map[string]string{"host": "1"}, []redistimeseries.DataPoint{{ts[0], vals[0]}, {ts[1], vals[0]}}}, + }, + MaxReducerSeriesDatapoints, + }, + redistimeseries.Range{"serie1", map[string]string{"host": "1"}, []redistimeseries.DataPoint{{ts[0], vals[0]}, {ts[1], vals[0]}}}, + false, + }, + {"test 2 series with labels and datapoints", + args{ + []redistimeseries.Range{ + {"serie1", map[string]string{"host": "1"}, []redistimeseries.DataPoint{{ts[0], vals[1]}, {ts[1], vals[0]}}}, + {"serie2", map[string]string{"host": "2"}, []redistimeseries.DataPoint{{ts[0], vals[0]}}}, + }, + MaxReducerSeriesDatapoints, + }, + redistimeseries.Range{"max reduction over serie1 serie2", nil, []redistimeseries.DataPoint{{ts[0], vals[1]}, {ts[1], vals[0]}}}, + false, + }, + {"test 3 series with labels and datapoints", + args{ + []redistimeseries.Range{ + {"serie1", map[string]string{"host": "1"}, []redistimeseries.DataPoint{{ts[0], vals[1]}, {ts[1], vals[0]}}}, + {"serie2", map[string]string{"host": "2"}, []redistimeseries.DataPoint{{ts[0], vals[0]}}}, + {"serie3", map[string]string{"host": "3"}, []redistimeseries.DataPoint{{ts[0], vals[2]}}}, + }, + MaxReducerSeriesDatapoints, + }, + redistimeseries.Range{"max reduction over serie1 serie2 serie3", nil, []redistimeseries.DataPoint{{ts[0], vals[2]}, {ts[1], vals[0]}}}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotOutserie, err := ReduceSeriesOnTimestampBy(tt.args.series, tt.args.reducer) + if (err != nil) != tt.wantErr { + t.Errorf("ReduceSeriesOnTimestampBy() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotOutserie.Name, tt.wantOutserie.Name) { + t.Errorf("MergeSeriesOnTimestamp() Error on Names got %v, want %v", gotOutserie.Name, tt.wantOutserie.Name) + } + if !reflect.DeepEqual(gotOutserie.Labels, tt.wantOutserie.Labels) { + t.Errorf("MergeSeriesOnTimestamp() Error on Labels got %v, want %v", gotOutserie.Labels, tt.wantOutserie.Labels) + } + if !reflect.DeepEqual(gotOutserie.DataPoints, tt.wantOutserie.DataPoints) { + t.Errorf("MergeSeriesOnTimestamp() Error on DataPoints got %v, want %v", gotOutserie.DataPoints, tt.wantOutserie.DataPoints) + } + }) + } +} + +func TestGetUniqueLabelValue(t *testing.T) { + type args struct { + series []redistimeseries.Range + label string + } + ts := []int64{1, 2, 3, 4, 5} + vals := []float64{1.0, 2.0, 3.0, 4.0, 5.0} + tests := []struct { + name string + args args + wantResult []string + wantErr bool + }{ + {"test empty label series with distinct labels and datapoints", + args{ + []redistimeseries.Range{ + {"serie1", nil, []redistimeseries.DataPoint{{ts[0], vals[1]}, {ts[1], vals[0]}}}, + {"serie2", nil, []redistimeseries.DataPoint{{ts[0], vals[0]}}}, + {"serie3", nil, []redistimeseries.DataPoint{{ts[0], vals[2]}}}, + }, + "host", + }, + []string{}, + false, + }, + {"test 3 series with equal labels and datapoints", + args{ + []redistimeseries.Range{ + {"serie1", map[string]string{"host": "1"}, []redistimeseries.DataPoint{{ts[0], vals[1]}, {ts[1], vals[0]}}}, + {"serie2", map[string]string{"host": "1"}, []redistimeseries.DataPoint{{ts[0], vals[0]}}}, + {"serie3", map[string]string{"host": "1"}, []redistimeseries.DataPoint{{ts[0], vals[2]}}}, + }, + "host", + }, + []string{"1"}, + false, + }, + {"test 3 series with distinct labels and datapoints", + args{ + []redistimeseries.Range{ + {"serie1", map[string]string{"host": "1"}, []redistimeseries.DataPoint{{ts[0], vals[1]}, {ts[1], vals[0]}}}, + {"serie2", map[string]string{"host": "2"}, []redistimeseries.DataPoint{{ts[0], vals[0]}}}, + {"serie3", map[string]string{"host": "3"}, []redistimeseries.DataPoint{{ts[0], vals[2]}}}, + }, + "host", + }, + []string{"1", "2", "3"}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotResult, err := GetUniqueLabelValue(tt.args.series, tt.args.label) + sort.Strings(gotResult) + sort.Strings(tt.wantResult) + if (err != nil) != tt.wantErr { + t.Errorf("GetUniqueLabelValue() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotResult, tt.wantResult) { + t.Errorf("GetUniqueLabelValue() gotResult = %v, want %v", gotResult, tt.wantResult) + } + }) + } +} diff --git a/pkg/targets/constants/constants.go b/pkg/targets/constants/constants.go index a7e3035f3..e1bb44c0e 100644 --- a/pkg/targets/constants/constants.go +++ b/pkg/targets/constants/constants.go @@ -13,6 +13,7 @@ const ( FormatPrometheus = "prometheus" FormatVictoriaMetrics = "victoriametrics" FormatTimestream = "timestream" + FormatRedisTimeSeries = "redistimeseries" ) func SupportedFormats() []string { @@ -28,5 +29,6 @@ func SupportedFormats() []string { FormatPrometheus, FormatVictoriaMetrics, FormatTimestream, + FormatRedisTimeSeries, } } diff --git a/pkg/targets/redistimeseries/implemented_target.go b/pkg/targets/redistimeseries/implemented_target.go new file mode 100644 index 000000000..2ec34ebfd --- /dev/null +++ b/pkg/targets/redistimeseries/implemented_target.go @@ -0,0 +1,37 @@ +package redistimeseries + +import ( + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/pkg/data/serialize" + "github.com/timescale/tsbs/pkg/data/source" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/constants" +) + +func NewTarget() targets.ImplementedTarget { + return &redistimeseriesTarget{} +} + +type redistimeseriesTarget struct { +} + +func (t *redistimeseriesTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.FlagSet) { + flagSet.String(flagPrefix+"host", "localhost:6379", "The host:port for Redis connection") + pflag.Uint64(flagPrefix+"pipeline", 50, "The pipeline's size") + pflag.Bool(flagPrefix+ "compression-enabled", true, "Whether to use compressed time series") + pflag.Bool(flagPrefix+ "cluster", false, "Whether to use OSS cluster API") + +} + +func (t *redistimeseriesTarget) TargetName() string { + return constants.FormatRedisTimeSeries +} + +func (t *redistimeseriesTarget) Serializer() serialize.PointSerializer { + return &Serializer{} +} + +func (t *redistimeseriesTarget) Benchmark(string, *source.DataSourceConfig, *viper.Viper) (targets.Benchmark, error) { + panic("not implemented") +} diff --git a/pkg/targets/redistimeseries/redistimeseries.go b/pkg/targets/redistimeseries/redistimeseries.go new file mode 100644 index 000000000..ef6c4e936 --- /dev/null +++ b/pkg/targets/redistimeseries/redistimeseries.go @@ -0,0 +1,130 @@ +package serialize + +import ( + "crypto/md5" + "encoding/binary" + "fmt" + "io" +) + +// RedisTimeSeriesSerializer writes a Point in a serialized form for RedisTimeSeries +type RedisTimeSeriesSerializer struct{} + +var keysSoFar map[string]bool +var hashSoFar map[string][]byte + +// Serialize writes Point data to the given writer, in a format that will be easy to create a redis-timeseries command +// from. +// +// This function writes output that looks like: +//cpu_usage_user{md5(hostname=host_0|region=eu-central-1...)} 1451606400 58 LABELS hostname host_0 region eu-central-1 ... measurement cpu fieldname usage_user +// +// Which the loader will decode into a set of TS.ADD commands for each fieldKey. Once labels have been created for a each fieldKey, +// subsequent rows are ommitted with them and are ingested with TS.MADD for a row's metrics. +func (s *RedisTimeSeriesSerializer) Serialize(p *Point, w io.Writer) (err error) { + if keysSoFar == nil { + keysSoFar = make(map[string]bool) + } + + if hashSoFar == nil { + hashSoFar = make(map[string][]byte) + } + + var hashBytes []byte + //var hashExists bool + hostname := p.tagValues[0] + + for fieldID := 0; fieldID < len(p.fieldKeys); fieldID++ { + fieldName := p.fieldKeys[fieldID] + keyName := fmt.Sprintf("%s%s", hostname, fieldName) + //fmt.Errorf("%s\n",fieldName) + //if hashBytes, hashExists = hashSoFar[keyName]; hashExists == false { + //do something here + labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) + hashBytes = fastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) + //hashSoFar[keyName] = hashBytes + //} + + // if this key was already inserted and created, we don't to specify the labels again + if keysSoFar[keyName] == false { + w.Write([]byte("TS.CREATE ")) + writeKeyName(w, p, fieldName, hashBytes) + w.Write([]byte("LABELS")) + for i, v := range p.tagValues { + w.Write([]byte(" ")) + w.Write(p.tagKeys[i]) + w.Write([]byte(" ")) + w.Write(fastFormatAppend(v, []byte{})) + } + w.Write([]byte(" measurement ")) + // add measurement name as additional label to be used in queries + w.Write(p.measurementName) + + // additional label of fieldname + w.Write([]byte(" fieldname ")) + w.Write(fieldName) + w.Write([]byte("\n")) + keysSoFar[keyName] = true + } + } + w.Write([]byte("TS.MADD ")) + + for fieldID := 0; fieldID < len(p.fieldKeys); fieldID++ { + fieldName := p.fieldKeys[fieldID] + + //keyName := fmt.Sprintf("%s%s", hostname, fieldName) + //fmt.Fprint(os.Stderr, fmt.Sprintf("%s\n", keyName)) + + labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) + hashBytes = fastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) + + fieldValue := p.fieldValues[fieldID] + writeKeyName(w, p, fieldName, hashBytes) + writeTS_and_Value(w, p, fieldValue) + if fieldID < len(p.fieldKeys)-1 { + w.Write([]byte(" ")) + } + } + w.Write([]byte("\n")) + + return err +} + +func appendTS_and_Value(lbuf []byte, p *Point, fieldValue interface{}) []byte { + // write timestamp in ms + lbuf = fastFormatAppend(p.timestamp.UTC().Unix()*1000, lbuf) + lbuf = append(lbuf, ' ') + // write value + lbuf = fastFormatAppend(fieldValue, lbuf) + return lbuf +} + +func writeTS_and_Value(w io.Writer, p *Point, fieldValue interface{}) (err error) { + // write timestamp in ms + w.Write(fastFormatAppend(p.timestamp.UTC().Unix()*1000, []byte{})) + w.Write([]byte(" ")) + // write value + _, err = w.Write(fastFormatAppend(fieldValue, []byte{})) + return +} + +func appendKeyName(lbuf []byte, p *Point, fieldName []byte, hashBytes []byte) []byte { + lbuf = append(lbuf, p.measurementName...) + lbuf = append(lbuf, '_') + lbuf = append(lbuf, fieldName...) + + lbuf = append(lbuf, '{') + lbuf = append(lbuf, hashBytes...) + lbuf = append(lbuf, '}', ' ') + return lbuf +} + +func writeKeyName(w io.Writer, p *Point, fieldName []byte, hashBytes []byte) (err error) { + w.Write(p.measurementName) + w.Write([]byte("_")) + w.Write(fieldName) + w.Write([]byte("{")) + w.Write(hashBytes) + _, err = w.Write([]byte("} ")) + return +} diff --git a/pkg/targets/redistimeseries/redistimeseries_test.go b/pkg/targets/redistimeseries/redistimeseries_test.go new file mode 100644 index 000000000..9aa2924f8 --- /dev/null +++ b/pkg/targets/redistimeseries/redistimeseries_test.go @@ -0,0 +1,38 @@ +package serialize + +import ( + "testing" +) + +func TestRedisTimeSeriesSerializer(t *testing.T) { + cases := []serializeCase{ + { + desc: "a regular Point", + inputPoint: testPointDefault, + output: "TS.CREATE cpu_usage_guest_nice{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest_nice\nTS.MADD cpu_usage_guest_nice{1998426147} 1451606400000 38.24311829\n", + }, + { + desc: "a regular Point using int as value", + inputPoint: testPointInt, + output: "TS.CREATE cpu_usage_guest{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest\nTS.MADD cpu_usage_guest{1998426147} 1451606400000 38\n", + }, + //{ + // desc: "a Point with no tags", + // inputPoint: testPointNoTags, + // output: "TS.ADD cpu_usage_guest_nice{3558706393} 1451606400000 38.24311829 LABELS measurement cpu fieldname usage_guest_nice\n", + //}, + } + + testSerializer(t, cases, &RedisTimeSeriesSerializer{}) +} + +func TestRedisTimeSeriesSerializerErr(t *testing.T) { + p := testPointMultiField + s := &RedisTimeSeriesSerializer{} + err := s.Serialize(p, &errWriter{}) + if err == nil { + t.Errorf("no error returned when expected") + } else if err.Error() != errWriterAlwaysErr { + t.Errorf("unexpected writer error: %v", err) + } +} diff --git a/pkg/targets/redistimeseries/serializer.go b/pkg/targets/redistimeseries/serializer.go new file mode 100644 index 000000000..3d1857821 --- /dev/null +++ b/pkg/targets/redistimeseries/serializer.go @@ -0,0 +1,133 @@ +package redistimeseries + +import ( + "crypto/md5" + "encoding/binary" + "fmt" + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/serialize" + "io" +) + +var keysSoFar map[string]bool +var hashSoFar map[string][]byte + +// Serializer writes a Point in a serialized form for RedisTimeSeries +type Serializer struct{} + +// Serialize writes Point data to the given writer, in a format that will be easy to create a redis-timeseries command +// from. +// +// This function writes output that looks like: +//cpu_usage_user{md5(hostname=host_0|region=eu-central-1...)} 1451606400 58 LABELS hostname host_0 region eu-central-1 ... measurement cpu fieldname usage_user +// +// Which the loader will decode into a set of TS.ADD commands for each fieldKey. Once labels have been created for a each fieldKey, +// subsequent rows are ommitted with them and are ingested with TS.MADD for a row's metrics. +func (s *Serializer) Serialize(p *data.Point, w io.Writer) (err error) { + if keysSoFar == nil { + keysSoFar = make(map[string]bool) + } + + if hashSoFar == nil { + hashSoFar = make(map[string][]byte) + } + + var hashBytes []byte + //var hashExists bool + p.TagValues() + hostname := p.TagValues()[0] + + for fieldID := 0; fieldID < len(p.FieldKeys()); fieldID++ { + fieldName := p.FieldKeys()[fieldID] + keyName := fmt.Sprintf("%s%s", hostname, fieldName) + //fmt.Errorf("%s\n",fieldName) + //if hashBytes, hashExists = hashSoFar[keyName]; hashExists == false { + //do something here + labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) + hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) + //hashSoFar[keyName] = hashBytes + //} + + // if this key was already inserted and created, we don't to specify the labels again + if keysSoFar[keyName] == false { + w.Write([]byte("TS.CREATE ")) + writeKeyName(w, p, fieldName, hashBytes) + w.Write([]byte("LABELS")) + for i, v := range p.TagValues() { + w.Write([]byte(" ")) + w.Write(p.TagKeys()[i]) + w.Write([]byte(" ")) + w.Write(serialize.FastFormatAppend(v, []byte{})) + } + w.Write([]byte(" measurement ")) + // add measurement name as additional label to be used in queries + w.Write(p.MeasurementName()) + + // additional label of fieldname + w.Write([]byte(" fieldname ")) + w.Write(fieldName) + w.Write([]byte("\n")) + keysSoFar[keyName] = true + } + } + w.Write([]byte("TS.MADD ")) + + for fieldID := 0; fieldID < len(p.FieldKeys()); fieldID++ { + fieldName := p.FieldKeys()[fieldID] + + //keyName := fmt.Sprintf("%s%s", hostname, fieldName) + //fmt.Fprint(os.Stderr, fmt.Sprintf("%s\n", keyName)) + + labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) + hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) + + fieldValue := p.FieldValues()[fieldID] + writeKeyName(w, p, fieldName, hashBytes) + writeTS_and_Value(w, p, fieldValue) + if fieldID < len(p.FieldKeys())-1 { + w.Write([]byte(" ")) + } + } + w.Write([]byte("\n")) + + return err +} + +func appendTS_and_Value(lbuf []byte, p *data.Point, fieldValue interface{}) []byte { + // write timestamp in ms + lbuf = serialize.FastFormatAppend(p.Timestamp().UTC().Unix()*1000, lbuf) + lbuf = append(lbuf, ' ') + // write value + lbuf = serialize.FastFormatAppend(fieldValue, lbuf) + return lbuf +} + +func writeTS_and_Value(w io.Writer, p *data.Point, fieldValue interface{}) (err error) { + // write timestamp in ms + w.Write(serialize.FastFormatAppend(p.Timestamp().UTC().Unix()*1000, []byte{})) + w.Write([]byte(" ")) + // write value + _, err = w.Write(serialize.FastFormatAppend(fieldValue, []byte{})) + return +} + +func appendKeyName(lbuf []byte, p *data.Point, fieldName []byte, hashBytes []byte) []byte { + lbuf = append(lbuf, p.MeasurementName()...) + lbuf = append(lbuf, '_') + lbuf = append(lbuf, fieldName...) + + lbuf = append(lbuf, '{') + lbuf = append(lbuf, hashBytes...) + lbuf = append(lbuf, '}', ' ') + return lbuf +} + +func writeKeyName(w io.Writer, p *data.Point, fieldName []byte, hashBytes []byte) (err error) { + w.Write(p.MeasurementName()) + w.Write([]byte("_")) + w.Write(fieldName) + w.Write([]byte("{")) + w.Write(hashBytes) + _, err = w.Write([]byte("} ")) + return +} diff --git a/pkg/targets/redistimeseries/serializer_test.go b/pkg/targets/redistimeseries/serializer_test.go new file mode 100644 index 000000000..fd2a402c6 --- /dev/null +++ b/pkg/targets/redistimeseries/serializer_test.go @@ -0,0 +1,41 @@ +package redistimeseries + +import ( + "github.com/timescale/tsbs/pkg/data/serialize" + "testing" +) + +func TestInfluxSerializerSerialize(t *testing.T) { + cases := []serialize.SerializeCase{ + { + Desc: "a regular Point", + InputPoint: serialize.TestPointDefault(), + Output: "TS.CREATE cpu_usage_guest_nice{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest_nice\\nTS.MADD cpu_usage_guest_nice{1998426147} 1451606400000 38.24311829\\n", + }, + //{ + // Desc: "a regular Point using int as value", + // InputPoint: serialize.TestPointInt(), + // Output: "cpu,hostname=host_0,region=eu-west-1,datacenter=eu-west-1b usage_guest=38i 1451606400000000000\n", + //}, + //{ + // Desc: "a regular Point with multiple fields", + // InputPoint: serialize.TestPointMultiField(), + // Output: "cpu,hostname=host_0,region=eu-west-1,datacenter=eu-west-1b big_usage_guest=5000000000i,usage_guest=38i,usage_guest_nice=38.24311829 1451606400000000000\n", + //}, + //{ + // Desc: "a Point with no tags", + // InputPoint: serialize.TestPointNoTags(), + // Output: "cpu usage_guest_nice=38.24311829 1451606400000000000\n", + //}, { + // Desc: "a Point with a nil tag", + // InputPoint: serialize.TestPointWithNilTag(), + // Output: "cpu usage_guest_nice=38.24311829 1451606400000000000\n", + //}, { + // Desc: "a Point with a nil field", + // InputPoint: serialize.TestPointWithNilField(), + // Output: "cpu usage_guest_nice=38.24311829 1451606400000000000\n", + //}, + } + + serialize.SerializerTest(t, cases, &Serializer{}) +} diff --git a/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh b/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh index c892180d4..2ef33fc1b 100755 --- a/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh +++ b/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh @@ -6,7 +6,28 @@ MAX_RPS=${MAX_RPS:-"0"} MAX_QUERIES=${MAX_QUERIES:-"1000"} -PASSWORD=${PASSWORD:-""} +PASSWORD=${PASSWORD:-"password"} + +# All available query types (sorted alphabetically) +QUERY_TYPES_ALL="\ + cpu-max-all-1 \ + cpu-max-all-8 \ + double-groupby-1 \ + double-groupby-5 \ + double-groupby-all \ + groupby-orderby-limit \ + high-cpu-1 \ + high-cpu-all \ + lastpoint \ + single-groupby-1-1-1 \ + single-groupby-1-1-12 \ + single-groupby-1-8-1 \ + single-groupby-5-1-1 \ + single-groupby-5-1-12 \ + single-groupby-5-8-1" + +# What query types to generate +QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} mkdir -p /tmp/bulk_data @@ -14,14 +35,14 @@ mkdir -p /tmp/bulk_data $GOPATH/bin/tsbs_generate_data --format timescaledb --use-case cpu-only --scale 10 --seed 123 --file /tmp/bulk_data/timescaledb_data # generate queries -$GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format timescaledb --use-case cpu-only --scale 10 --seed 123 --query-type lastpoint --file /tmp/bulk_data/timescaledb_query_lastpoint -$GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format timescaledb --use-case cpu-only --scale 10 --seed 123 --query-type cpu-max-all-1 --file /tmp/bulk_data/timescaledb_query_cpu-max-all-1 -$GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format timescaledb --use-case cpu-only --scale 10 --seed 123 --query-type high-cpu-1 --file /tmp/bulk_data/timescaledb_query_high-cpu-1 - +for QUERY_TYPE in ${QUERY_TYPES}; do + $GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format timescaledb --use-case cpu-only --scale 10 --seed 123 --query-type ${QUERY_TYPE} --file /tmp/bulk_data/timescaledb_query_${QUERY_TYPE} +done # insert benchmark $GOPATH/bin/tsbs_load_timescaledb --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --host=127.0.0.1 --user=postgres --workers=1 --file=/tmp/bulk_data/timescaledb_data -# queries benchmark -$GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_lastpoint.hdr" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_lastpoint -$GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_cpu-max-all-1.hdr" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_cpu-max-all-1 -$GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_high-cpu-1.hdr" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_high-cpu-1 +# Loop over all requested queries types and generate data +for QUERY_TYPE in ${QUERY_TYPES}; do + # queries benchmark + $GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_${QUERY_TYPE}.hdr" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_${QUERY_TYPE} +done diff --git a/scripts/functional/debug_responses_full_cycle_minitest_influx.sh b/scripts/functional/debug_responses_full_cycle_minitest_influx.sh new file mode 100755 index 000000000..0966ba53c --- /dev/null +++ b/scripts/functional/debug_responses_full_cycle_minitest_influx.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# showcases the ftsb 3 phases for timescaledb +# - 1) data and query generation +# - 2) data loading/insertion +# - 3) query execution + +SCALE=${SCALE:-"10"} +SEED=${SEED:-"123"} +FORMAT="influx" + +mkdir -p /tmp/bulk_data +rm /tmp/bulk_data/${FORMAT}_* +rm docs/responses/${FORMAT}_* + +# Load parameters - common +DATABASE_PORT=${DATABASE_PORT:-8086} +DATABASE_HOST=${DATABASE_HOST:-localhost} + +# All available query types (sorted alphabetically) +QUERY_TYPES_ALL="\ + cpu-max-all-1 \ + cpu-max-all-8 \ + double-groupby-1 \ + double-groupby-5 \ + double-groupby-all \ + groupby-orderby-limit \ + high-cpu-1 \ + high-cpu-all \ + lastpoint \ + single-groupby-1-1-1 \ + single-groupby-1-1-12 \ + single-groupby-1-8-1 \ + single-groupby-5-1-1 \ + single-groupby-5-1-12 \ + single-groupby-5-8-1" + +# What query types to generate +QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} + +# generate data +$GOPATH/bin/tsbs_generate_data --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} --file /tmp/bulk_data/${FORMAT}_data + +for queryName in $QUERY_TYPES; do + echo "generating query: $queryName" + $GOPATH/bin/tsbs_generate_queries --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} \ + --queries=1 \ + --query-type $queryName \ + --file /tmp/bulk_data/${FORMAT}_query_$queryName +done + +until curl http://${DATABASE_HOST}:${DATABASE_PORT}/ping 2>/dev/null; do + echo "Waiting for InfluxDB" + sleep 1 +done + +# Remove previous database +curl -X POST http://${DATABASE_HOST}:${DATABASE_PORT}/query?q=drop%20database%20benchmark + +# insert benchmark +$GOPATH/bin/tsbs_load_${FORMAT} \ + --db-name=benchmark \ + --backoff=1s \ + --workers=1 \ + --urls=http://${DATABASE_HOST}:${DATABASE_PORT} \ + --file=/tmp/bulk_data/${FORMAT}_data + +# queries benchmark +for queryName in $QUERY_TYPES; do + echo "running query: $queryName" + $GOPATH/bin/tsbs_run_queries_${FORMAT} --print-responses \ + --workers=1 \ + --file /tmp/bulk_data/${FORMAT}_query_$queryName >docs/responses/${FORMAT}_$queryName.json +done diff --git a/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh b/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh new file mode 100755 index 000000000..597a04bc3 --- /dev/null +++ b/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# showcases the ftsb 3 phases for timescaledb +# - 1) data and query generation +# - 2) data loading/insertion +# - 3) query execution + +SCALE=${SCALE:-"10"} +SEED=${SEED:-"123"} +PASSWORD=${PASSWORD:-"password"} +FORMAT="redistimeseries" + +# All available query types (sorted alphabetically) +QUERY_TYPES_ALL="\ + cpu-max-all-1 \ + cpu-max-all-8 \ + double-groupby-1 \ + double-groupby-5 \ + double-groupby-all \ + groupby-orderby-limit \ + high-cpu-1 \ + high-cpu-all \ + lastpoint \ + single-groupby-1-1-1 \ + single-groupby-1-1-12 \ + single-groupby-1-8-1 \ + single-groupby-5-1-1 \ + single-groupby-5-1-12 \ + single-groupby-5-8-1" + +# What query types to generate +QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} + +mkdir -p /tmp/bulk_data +rm /tmp/bulk_data/${FORMAT}_* +rm docs/responses/${FORMAT}_* + +redis-cli flushall + +# generate data +$GOPATH/bin/tsbs_generate_data --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} --file /tmp/bulk_data/${FORMAT}_data + +for queryName in $QUERY_TYPES; do + $GOPATH/bin/tsbs_generate_queries --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} \ + --queries=1 \ + --query-type $queryName \ + --file /tmp/bulk_data/${FORMAT}_query_$queryName +done + +# insert benchmark +$GOPATH/bin/tsbs_load_${FORMAT} \ + --workers=1 \ + --file=/tmp/bulk_data/${FORMAT}_data + +# queries benchmark +for queryName in $QUERY_TYPES; do + echo "running query: $queryName" + $GOPATH/bin/tsbs_run_queries_${FORMAT} --print-responses \ + --workers=1 \ + --debug=3 \ + --file /tmp/bulk_data/${FORMAT}_query_$queryName >docs/responses/${FORMAT}_$queryName.json +done diff --git a/scripts/functional/debug_responses_full_cycle_minitest_timescaledb.sh b/scripts/functional/debug_responses_full_cycle_minitest_timescaledb.sh new file mode 100755 index 000000000..6c7c3ebe3 --- /dev/null +++ b/scripts/functional/debug_responses_full_cycle_minitest_timescaledb.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# showcases the ftsb 3 phases for timescaledb +# - 1) data and query generation +# - 2) data loading/insertion +# - 3) query execution + +SCALE=${SCALE:-"10"} +SEED=${SEED:-"123"} +PASSWORD=${PASSWORD:-"password"} +FORMAT="timescaledb" + +# All available query types (sorted alphabetically) +QUERY_TYPES_ALL="\ + cpu-max-all-1 \ + cpu-max-all-8 \ + double-groupby-1 \ + double-groupby-5 \ + double-groupby-all \ + groupby-orderby-limit \ + high-cpu-1 \ + high-cpu-all \ + lastpoint \ + single-groupby-1-1-1 \ + single-groupby-1-1-12 \ + single-groupby-1-8-1 \ + single-groupby-5-1-1 \ + single-groupby-5-1-12 \ + single-groupby-5-8-1" + +# What query types to generate +QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} + +mkdir -p /tmp/bulk_data +rm /tmp/bulk_data/${FORMAT}_* +rm docs/responses/${FORMAT}_* + +# generate data +$GOPATH/bin/tsbs_generate_data --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} --file /tmp/bulk_data/${FORMAT}_data + +for queryName in $QUERY_TYPES; do + echo "generating query: $queryName" + $GOPATH/bin/tsbs_generate_queries --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} \ + --queries=1 \ + --query-type $queryName \ + --file /tmp/bulk_data/${FORMAT}_query_$queryName +done + +# insert benchmark +$GOPATH/bin/tsbs_load_${FORMAT} --pass=${PASSWORD} \ + --postgres="sslmode=disable port=5433" --db-name=benchmark \ + --host=127.0.0.1 --user=postgres --workers=1 \ + --file=/tmp/bulk_data/${FORMAT}_data + +# queries benchmark +for queryName in $QUERY_TYPES; do + echo "running query: $queryName" + $GOPATH/bin/tsbs_run_queries_${FORMAT} --print-responses \ + --pass=${PASSWORD} --postgres="sslmode=disable port=5433" \ + --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 \ + --file /tmp/bulk_data/${FORMAT}_query_$queryName >docs/responses/${FORMAT}_$queryName.json +done diff --git a/scripts/generate_data.sh b/scripts/generate_data.sh index e2888992b..a16608a9a 100755 --- a/scripts/generate_data.sh +++ b/scripts/generate_data.sh @@ -7,6 +7,8 @@ if [[ -z "${EXE_FILE_NAME}" ]]; then exit 1 fi +set -x + # Data folder BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_data"} @@ -19,6 +21,11 @@ SCALE=${SCALE:-"4000"} # Rand seed SEED=${SEED:-"123"} +# Control level of debug output +DEBUG=${DEBUG:-"0"} + +INTERLEAVED_GENERATION_GROUPS=${INTERLEAVED_GENERATION_GROUPS:-"1"} + # Start and stop time for generated timeseries TS_START=${TS_START:-"2016-01-01T00:00:00Z"} TS_END=${TS_END:-"2016-01-04T00:00:00Z"} @@ -32,6 +39,9 @@ LOG_INTERVAL=${LOG_INTERVAL:-"10s"} # Max number of points to generate data. 0 means "use TS_START TS_END with LOG_INTERVAL" MAX_DATA_POINTS=${MAX_DATA_POINTS:-"0"} +# Whether to skip data generation if it already exists +SKIP_IF_EXISTS=${SKIP_IF_EXISTS:-"TRUE"} + # Ensure DATA DIR available mkdir -p ${BULK_DATA_DIR} chmod a+rwx ${BULK_DATA_DIR} @@ -42,7 +52,7 @@ set -eo pipefail # Loop over all requested target formats and generate data for FORMAT in ${FORMATS}; do DATA_FILE_NAME="data_${FORMAT}_${USE_CASE}_${SCALE}_${TS_START}_${TS_END}_${LOG_INTERVAL}_${SEED}.dat.gz" - if [ -f "${DATA_FILE_NAME}" ]; then + if [ -f "${DATA_FILE_NAME}" ] && [ "${SKIP_IF_EXISTS}" == "TRUE" ]; then echo "WARNING: file ${DATA_FILE_NAME} already exists, skip generating new data" else cleanup() { @@ -53,15 +63,17 @@ for FORMAT in ${FORMATS}; do echo "Generating ${DATA_FILE_NAME}:" ${EXE_FILE_NAME} \ - --format ${FORMAT} \ - --use-case ${USE_CASE} \ - --scale ${SCALE} \ - --timestamp-start ${TS_START} \ - --timestamp-end ${TS_END} \ - --seed ${SEED} \ - --log-interval ${LOG_INTERVAL} \ - --max-data-points ${MAX_DATA_POINTS} \ - | gzip > ${DATA_FILE_NAME} + --format=${FORMAT} \ + --use-case=${USE_CASE} \ + --scale=${SCALE} \ + --timestamp-start=${TS_START} \ + --timestamp-end=${TS_END} \ + --debug=${DEBUG} \ + --seed=${SEED} \ + --log-interval=${LOG_INTERVAL} \ + --interleaved-generation-groups=${INTERLEAVED_GENERATION_GROUPS} \ + --max-data-points=${MAX_DATA_POINTS} > ${DATA_FILE_NAME} +# | gzip > ${DATA_FILE_NAME} trap - EXIT # Make short symlink for convenience diff --git a/scripts/generate_queries.sh b/scripts/generate_queries.sh index 511dbf018..da8183e24 100755 --- a/scripts/generate_queries.sh +++ b/scripts/generate_queries.sh @@ -17,45 +17,56 @@ USE_TAGS=${USE_TAGS:-true} USE_TIME_BUCKET=${USE_TIME_BUCKET:-true} # Space-separated list of target DB formats to generate -FORMATS=${FORMATS:-"timescaledb"} +FORMATS=${FORMATS:-"redistimeseries"} + +# All available for generation query types (sorted alphabetically) +#QUERY_TYPES_ALL="\ +#cpu-max-all-1 \ +#cpu-max-all-8 \ +#double-groupby-1 \ +#double-groupby-5 \ +#double-groupby-all \ +#groupby-orderby-limit \ +#high-cpu-1 \ +#high-cpu-all \ +#lastpoint \ +#single-groupby-1-1-1 \ +#single-groupby-1-1-12 \ +#single-groupby-1-8-1 \ +#single-groupby-5-1-1 \ +#single-groupby-5-1-12 \ +#single-groupby-5-8-1" # All available for generation query types (sorted alphabetically) QUERY_TYPES_ALL="\ cpu-max-all-1 \ -cpu-max-all-8 \ -double-groupby-1 \ -double-groupby-5 \ -double-groupby-all \ -groupby-orderby-limit \ -high-cpu-1 \ -high-cpu-all \ -lastpoint \ single-groupby-1-1-1 \ single-groupby-1-1-12 \ -single-groupby-1-8-1 \ single-groupby-5-1-1 \ -single-groupby-5-1-12 \ -single-groupby-5-8-1" +single-groupby-5-1-12" # What query types to generate QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} # Number of hosts to generate data about -SCALE=${SCALE:-"4000"} +SCALE=${SCALE:-"100"} # Number of queries to generate -QUERIES=${QUERIES:-"1000"} +QUERIES=${QUERIES:-"1000000"} # Rand seed SEED=${SEED:-"123"} # Start and stop time for generated timeseries TS_START=${TS_START:-"2016-01-01T00:00:00Z"} -TS_END=${TS_END:-"2016-01-04T00:00:01Z"} +TS_END=${TS_END:-"2016-01-31T00:00:01Z"} # What set of data to generate: devops (multiple data), cpu-only (cpu-usage data) USE_CASE=${USE_CASE:-"cpu-only"} +# Whether to skip data generation if it already exists +SKIP_IF_EXISTS=${SKIP_IF_EXISTS:-"TRUE"} + # Ensure DATA DIR available mkdir -p ${BULK_DATA_DIR} chmod a+rwx ${BULK_DATA_DIR} @@ -67,7 +78,7 @@ set -eo pipefail for QUERY_TYPE in ${QUERY_TYPES}; do for FORMAT in ${FORMATS}; do DATA_FILE_NAME="queries_${FORMAT}_${QUERY_TYPE}_${EXE_FILE_VERSION}_${QUERIES}_${SCALE}_${SEED}_${TS_START}_${TS_END}_${USE_CASE}.dat.gz" - if [ -f "${DATA_FILE_NAME}" ]; then + if [ -f "${DATA_FILE_NAME}" ] && [ "${SKIP_IF_EXISTS}" == "TRUE" ]; then echo "WARNING: file ${DATA_FILE_NAME} already exists, skip generating new data" else cleanup() { @@ -78,14 +89,14 @@ for QUERY_TYPE in ${QUERY_TYPES}; do echo "Generating ${DATA_FILE_NAME}:" ${EXE_FILE_NAME} \ - --format ${FORMAT} \ - --queries ${QUERIES} \ - --query-type ${QUERY_TYPE} \ - --scale ${SCALE} \ - --seed ${SEED} \ - --timestamp-start ${TS_START} \ - --timestamp-end ${TS_END} \ - --use-case ${USE_CASE} \ + --format=${FORMAT} \ + --queries=${QUERIES} \ + --query-type=${QUERY_TYPE} \ + --scale=${SCALE} \ + --seed=${SEED} \ + --timestamp-start=${TS_START} \ + --timestamp-end=${TS_END} \ + --use-case=${USE_CASE} \ --timescale-use-json=${USE_JSON} \ --timescale-use-tags=${USE_TAGS} \ --timescale-use-time-bucket=${USE_TIME_BUCKET} \ diff --git a/scripts/load/load_common.sh b/scripts/load/load_common.sh index a4ec1c09e..d4652f946 100644 --- a/scripts/load/load_common.sh +++ b/scripts/load/load_common.sh @@ -11,10 +11,12 @@ DATA_FILE=${DATA_FILE:-${BULK_DATA_DIR}/${DATA_FILE_NAME}} # Load parameters BATCH_SIZE=${BATCH_SIZE:-10000} -# How many concurrent worker would load data - match num of cores, or default to 4 -NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 4)} +# Debug +DEBUG=${DEBUG:-0} +# How many concurrent worker would load data - match num of cores, or default to 8 +NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 8)} BACKOFF_SECS=${BACKOFF_SECS:-1s} -REPORTING_PERIOD=${REPORTING_PERIOD:-10s} +REPORTING_PERIOD=${REPORTING_PERIOD:-1s} # Ensure data file is in place if [ ! -f ${DATA_FILE} ]; then diff --git a/scripts/load_redistimeseries.sh b/scripts/load_redistimeseries.sh new file mode 100755 index 000000000..e92329990 --- /dev/null +++ b/scripts/load_redistimeseries.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Ensure loader is available +EXE_FILE_NAME=${EXE_FILE_NAME:-$(which tsbs_load_redistimeseries)} +if [[ -z "$EXE_FILE_NAME" ]]; then + echo "tsbs_load_redistimeseries not available. It is not specified explicitly and not found in \$PATH" + exit 1 +fi +FORMAT="redistimeseries" + +DATA_FILE_NAME=${DATA_FILE_NAME:-${FORMAT}-data.gz} +DATABASE_PORT=${DATABASE_PORT:-6379} +CONNECTIONS=${CONNECTIONS:-10} +REPETITIONS=${REPETITIONS:-3} +PIPELINE=${PIPELINE:-100} +EXTENSION="${DATA_FILE_NAME##*.}" +DIR=$(dirname "${DATA_FILE_NAME}") +NO_EXT_DATA_FILE_NAME="${DATA_FILE_NAME%.*}" +PREFIX=${PREFIX:-""} + +EXE_DIR=${EXE_DIR:-$(dirname $0)} +COMPRESSION_ENABLED=${COMPRESSION_ENABLED:-true} +SLEEP_BETWEEN_RUNS=${SLEEP_BETWEEN_RUNS:-"60"} + +# Load parameters - common +source ${EXE_DIR}/load_common.sh + +for run in $(seq ${REPETITIONS}); do + echo "Running RUN $run" + OUT_FULL_FILE_NAME="${DIR}/${PREFIX}_load_result_${NO_EXT_DATA_FILE_NAME}_run_${run}.out" + echo "Using only 1 worker" + echo "Saving results to ${OUT_FULL_FILE_NAME}" + + # Remove previous database + redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} flushall + + # Retrieve command stats output + redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} config resetstat + + # Load new data + cat ${DATA_FILE} | $EXE_FILE_NAME \ + --workers=1 \ + --batch-size=${BATCH_SIZE} \ + --reporting-period=${REPORTING_PERIOD} \ + --host=${DATABASE_HOST}:${DATABASE_PORT} \ + --compression-enabled=${COMPRESSION_ENABLED} \ + --connections=${CONNECTIONS} --pipeline=${PIPELINE} | + tee ${OUT_FULL_FILE_NAME} + + # Retrieve command stats output + redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info commandstats >> ${OUT_FULL_FILE_NAME} + redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info >> ${OUT_FULL_FILE_NAME} + redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info commandstats + + echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" + sleep ${SLEEP_BETWEEN_RUNS} + +done \ No newline at end of file diff --git a/scripts/query_common.sh b/scripts/query_common.sh new file mode 100644 index 000000000..1a812ee87 --- /dev/null +++ b/scripts/query_common.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Database credentials +DATABASE_HOST=${DATABASE_HOST:-"localhost"} +DATABASE_NAME=${DATABASE_NAME:-"benchmark"} + +# Data folder +BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"} + +# Data folder +RESULTS_DIR=${RESULTS_DIR:-"/tmp/results_queries"} + +# Load parameters +BATCH_SIZE=${BATCH_SIZE:-10000} +# Debug +DEBUG=${DEBUG:-0} +# How many concurrent worker would load data - match num of cores, or default to 8 +NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 8)} +BACKOFF_SECS=${BACKOFF_SECS:-1s} +REPORTING_PERIOD=${REPORTING_PERIOD:-10s} +# +#set -x diff --git a/scripts/run_queries_redistimeseries.sh b/scripts/run_queries_redistimeseries.sh new file mode 100755 index 000000000..92269f0ca --- /dev/null +++ b/scripts/run_queries_redistimeseries.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +set -e + +# Ensure runner is available +EXE_FILE_NAME=${EXE_FILE_NAME:-$(which tsbs_run_queries_redistimeseries)} +if [[ -z "$EXE_FILE_NAME" ]]; then + echo "tsbs_run_queries_redistimeseries not available. It is not specified explicitly and not found in \$PATH" + exit 1 +fi + +EXE_DIR=${EXE_DIR:-$(dirname $0)} +source ${EXE_DIR}/query_common.sh + +DATABASE_PORT=${DATABASE_PORT:-6379} + +# Print timing stats to stderr after this many queries (0 to disable) +QUERIES_PRINT_INTERVAL=${QUERIES_PRINT_INTERVAL:-"0"} + +# How many queries would be run +MAX_QUERIES=${MAX_QUERIES:-"0"} +REPETITIONS=${REPETITIONS:-3} +PREFIX=${PREFIX:-""} + +# How many queries would be run +SLEEP_BETWEEN_RUNS=${SLEEP_BETWEEN_RUNS:-"60"} + +# Ensure DATA DIR available +mkdir -p ${RESULTS_DIR} +chmod a+rwx ${RESULTS_DIR} + +for run in $(seq ${REPETITIONS}); do + for FULL_DATA_FILE_NAME in ${BULK_DATA_DIR}/queries_redistimeseries*; do + # $FULL_DATA_FILE_NAME: /full/path/to/file_with.ext + # $DATA_FILE_NAME: file_with.ext + # $DIR: /full/path/to + # $EXTENSION: ext + # NO_EXT_DATA_FILE_NAME: file_with + + DATA_FILE_NAME=$(basename -- "${FULL_DATA_FILE_NAME}") + DIR=$(dirname "${FULL_DATA_FILE_NAME}") + EXTENSION="${DATA_FILE_NAME##*.}" + NO_EXT_DATA_FILE_NAME="${DATA_FILE_NAME%.*}" + + OUT_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}_result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + HDR_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}_HDR_TXT_result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + + if [ "${EXTENSION}" == "gz" ]; then + GUNZIP="gunzip" + else + GUNZIP="cat" + fi + + echo "Reseting Redis command stats" + redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} config resetstat + + echo "Running ${DATA_FILE_NAME}" + echo "Saving output to ${OUT_FULL_FILE_NAME}" + echo "Saving HDR Latencies to ${HDR_FULL_FILE_NAME}" + + cat $FULL_DATA_FILE_NAME | + $GUNZIP | + $EXE_FILE_NAME \ + --max-queries=${MAX_QUERIES} \ + --workers=${NUM_WORKERS} \ + --print-interval=${QUERIES_PRINT_INTERVAL} \ + --debug=${DEBUG} \ + --hdr-latencies=${HDR_FULL_FILE_NAME} \ + --host=${DATABASE_HOST}:${DATABASE_PORT} | + tee $OUT_FULL_FILE_NAME + + # Retrieve command stats output + redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info commandstats >> $OUT_FULL_FILE_NAME + redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info commandstats + + echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" + sleep ${SLEEP_BETWEEN_RUNS} + done + echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" + sleep ${SLEEP_BETWEEN_RUNS} + +done \ No newline at end of file From 3006d29f40280dde10becbd8b7e590aa9ab98a17 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 3 Jan 2021 22:39:12 +0000 Subject: [PATCH 02/72] [add] Added LoaderTestResult to persist ingestion benchmark results in a common format across targets --- load/loader.go | 68 +++++++++++++++---- load/loader_test_result.go | 20 ++++++ .../full_cycle_minitest_clickhouse.sh | 2 +- .../full_cycle_minitest_cratedb.sh | 2 +- .../full_cycle_minitest_timescaledb.sh | 4 +- 5 files changed, 77 insertions(+), 19 deletions(-) create mode 100644 load/loader_test_result.go diff --git a/load/loader.go b/load/loader.go index 424ed1652..3542b4e45 100644 --- a/load/loader.go +++ b/load/loader.go @@ -1,8 +1,10 @@ package load import ( + "encoding/json" "fmt" "github.com/timescale/tsbs/pkg/targets" + "io/ioutil" "log" "math/rand" "sync" @@ -29,21 +31,22 @@ var ( // BenchmarkRunnerConfig contains all the configuration information required for running BenchmarkRunner. type BenchmarkRunnerConfig struct { - DBName string `yaml:"db-name" mapstructure:"db-name"` - BatchSize uint `yaml:"batch-size" mapstructure:"batch-size"` - Workers uint `yaml:"workers" mapstructure:"workers"` - Limit uint64 `yaml:"limit" mapstructure:"limit"` - DoLoad bool `yaml:"do-load" mapstructure:"do-load"` - DoCreateDB bool `yaml:"do-create-db" mapstructure:"do-create-db"` - DoAbortOnExist bool `yaml:"do-abort-on-exist" mapstructure:"do-abort-on-exist"` - ReportingPeriod time.Duration `yaml:"reporting-period" mapstructure:"reporting-period"` - HashWorkers bool `yaml:"hash-workers" mapstructure:"hash-workers"` - NoFlowControl bool `yaml:"no-flow-control" mapstructure:"no-flow-control"` - ChannelCapacity uint `yaml:"channel-capacity" mapstructure:"channel-capacity"` - InsertIntervals string `yaml:"insert-intervals" mapstructure:"insert-intervals"` + DBName string `yaml:"db-name" mapstructure:"db-name" json:"db-name"` + BatchSize uint `yaml:"batch-size" mapstructure:"batch-size" json:"batch-sze"` + Workers uint `yaml:"workers" mapstructure:"workers" json:"workers"` + Limit uint64 `yaml:"limit" mapstructure:"limit" json:"limit"` + DoLoad bool `yaml:"do-load" mapstructure:"do-load" json:"do-load"` + DoCreateDB bool `yaml:"do-create-db" mapstructure:"do-create-db" json:"do-create-db"` + DoAbortOnExist bool `yaml:"do-abort-on-exist" mapstructure:"do-abort-on-exist" json:"do-abort-on-exist"` + ReportingPeriod time.Duration `yaml:"reporting-period" mapstructure:"reporting-period" json:"reporting-period"` + HashWorkers bool `yaml:"hash-workers" mapstructure:"hash-workers" json:"hash-workers"` + NoFlowControl bool `yaml:"no-flow-control" mapstructure:"no-flow-control" json:"no-flow-control"` + ChannelCapacity uint `yaml:"channel-capacity" mapstructure:"channel-capacity" json:"channel-capacity"` + InsertIntervals string `yaml:"insert-intervals" mapstructure:"insert-intervals" json:"insert-intervals"` + ResultsFile string `yaml:"results-file" mapstructure:"results-file" json:"results-file"` // deprecated, should not be used in other places other than tsbs_load_xx commands - FileName string `yaml:"file" mapstructure:"file"` - Seed int64 `yaml:"seed" mapstructure:"seed"` + FileName string `yaml:"file" mapstructure:"file" json:"file"` + Seed int64 `yaml:"seed" mapstructure:"seed" json:"seed"` } // AddToFlagSet adds command line flags needed by the BenchmarkRunnerConfig to the flag set. @@ -60,6 +63,7 @@ func (c BenchmarkRunnerConfig) AddToFlagSet(fs *pflag.FlagSet) { fs.Int64("seed", 0, "PRNG seed (default: 0, which uses the current timestamp)") fs.String("insert-intervals", "", "Time to wait between each insert, default '' => all workers insert ASAP. '1,2' = worker 1 waits 1s between inserts, worker 2 and others wait 2s") fs.Bool("hash-workers", false, "Whether to consistently hash insert data to the same workers (i.e., the data for a particular host always goes to the same worker)") + fs.String("results-file", "", "Write the test results summary json to this file") } type BenchmarkRunner interface { @@ -138,7 +142,41 @@ func (l *CommonBenchmarkRunner) postRun(wg *sync.WaitGroup, start *time.Time) { // Wait for all workers to finish wg.Wait() end := time.Now() - l.summary(end.Sub(*start)) + took := end.Sub(*start) + l.summary(took) + if len(l.BenchmarkRunnerConfig.ResultsFile) > 0 { + l.saveTestResult(took, *start, end) + } +} + +func (l *CommonBenchmarkRunner) saveTestResult(took time.Duration, start time.Time, end time.Time) { + totals := make(map[string]interface{}) + metricRate := float64(l.metricCnt) / took.Seconds() + totals["metricRate"] = metricRate + if l.rowCnt > 0 { + rowRate := float64(l.rowCnt) / float64(took.Seconds()) + totals["rowRate"] = rowRate + } + + testResult := LoaderTestResult{ + ResultFormatVersion: LoaderTestResultVersion, + RunnerConfig: l.BenchmarkRunnerConfig, + StartTime: start.Unix(), + EndTime: end.Unix(), + DurationMillis: took.Milliseconds(), + Totals: totals, + } + + _, _ = fmt.Printf("Saving results json file to %s\n", l.BenchmarkRunnerConfig.ResultsFile) + file, err := json.MarshalIndent(testResult, "", " ") + if err != nil { + log.Fatal(err) + } + + err = ioutil.WriteFile(l.BenchmarkRunnerConfig.ResultsFile, file, 0644) + if err != nil { + log.Fatal(err) + } } // RunBenchmark takes in a Benchmark b and uses it to run the load benchmark diff --git a/load/loader_test_result.go b/load/loader_test_result.go new file mode 100644 index 000000000..f2e57a98d --- /dev/null +++ b/load/loader_test_result.go @@ -0,0 +1,20 @@ +package load + +const LoaderTestResultVersion = "0.1" + +// LoaderTestResult aggregates the results of an insert or load benchmark in a common format across targets +type LoaderTestResult struct { + // Format Configs + ResultFormatVersion string `json:"ResultFormatVersion"` + + // RunnerConfig Configs + RunnerConfig BenchmarkRunnerConfig `json:"RunnerConfig"` + + // Run info + StartTime int64 `json:"StartTime` + EndTime int64 `json:"EndTime"` + DurationMillis int64 `json:"DurationMillis"` + + // Totals + Totals map[string]interface{} `json:"Totals"` +} diff --git a/scripts/full_cycle_minitest/full_cycle_minitest_clickhouse.sh b/scripts/full_cycle_minitest/full_cycle_minitest_clickhouse.sh index 6adf2bfd2..aebce964b 100755 --- a/scripts/full_cycle_minitest/full_cycle_minitest_clickhouse.sh +++ b/scripts/full_cycle_minitest/full_cycle_minitest_clickhouse.sh @@ -19,7 +19,7 @@ $GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format clickhouse - $GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format clickhouse --use-case cpu-only --scale 10 --seed 123 --query-type high-cpu-1 --file /tmp/bulk_data/clickhouse_query_high-cpu-1 # insert benchmark -$GOPATH/bin/tsbs_load_clickhouse --db-name=benchmark --host=127.0.0.1 --workers=1 --file=/tmp/bulk_data/clickhouse_data +$GOPATH/bin/tsbs_load_clickhouse --db-name=benchmark --host=127.0.0.1 --workers=1 --file=/tmp/bulk_data/clickhouse_data --results-file="clickhouse_load_results.json" # queries benchmark #last point query is broke diff --git a/scripts/full_cycle_minitest/full_cycle_minitest_cratedb.sh b/scripts/full_cycle_minitest/full_cycle_minitest_cratedb.sh index 7eb6ea877..5fb847925 100755 --- a/scripts/full_cycle_minitest/full_cycle_minitest_cratedb.sh +++ b/scripts/full_cycle_minitest/full_cycle_minitest_cratedb.sh @@ -16,7 +16,7 @@ $GOPATH/bin/tsbs_generate_queries --format cratedb --use-case cpu-only --scale 1 $GOPATH/bin/tsbs_generate_queries --format cratedb --use-case cpu-only --scale 10 --seed 123 --query-type groupby-orderby-limit --file /tmp/bulk_data/cratedb_query_groupby-orderby-limit # insert benchmark -$GOPATH/bin/tsbs_load_cratedb --db-name=benchmark --hosts=localhost --workers=1 --file=/tmp/bulk_data/cratedb_data +$GOPATH/bin/tsbs_load_cratedb --db-name=benchmark --hosts=localhost --workers=1 --file=/tmp/bulk_data/cratedb_data --results-file="cratedb_load_results.json" # queries benchmark $GOPATH/bin/tsbs_run_queries_cratedb --db-name=benchmark --hosts=localhost --workers=1 --max-queries=10 --file=/tmp/bulk_data/cratedb_query_lastpoint diff --git a/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh b/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh index c892180d4..7dfaacaa4 100755 --- a/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh +++ b/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh @@ -6,7 +6,7 @@ MAX_RPS=${MAX_RPS:-"0"} MAX_QUERIES=${MAX_QUERIES:-"1000"} -PASSWORD=${PASSWORD:-""} +PASSWORD=${PASSWORD:-"password"} mkdir -p /tmp/bulk_data @@ -19,7 +19,7 @@ $GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format timescaledb $GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format timescaledb --use-case cpu-only --scale 10 --seed 123 --query-type high-cpu-1 --file /tmp/bulk_data/timescaledb_query_high-cpu-1 # insert benchmark -$GOPATH/bin/tsbs_load_timescaledb --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --host=127.0.0.1 --user=postgres --workers=1 --file=/tmp/bulk_data/timescaledb_data +$GOPATH/bin/tsbs_load_timescaledb --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --results-file="timescaledb_load_results.json" --db-name=benchmark --host=127.0.0.1 --user=postgres --workers=1 --file=/tmp/bulk_data/timescaledb_data # queries benchmark $GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_lastpoint.hdr" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_lastpoint From 812e461b4da39ad7c289232ca5cbcc85501ce5bd Mon Sep 17 00:00:00 2001 From: Danni Moiseyev Date: Mon, 1 Mar 2021 17:11:29 +0200 Subject: [PATCH 03/72] WIP adujst to new APIs --- cmd/tsbs_load_redistimeseries/main.go | 43 +-- cmd/tsbs_load_redistimeseries/scan.go | 44 +-- cmd/tsbs_run_queries_redistimeseries/main.go | 2 +- .../initializers/target_initializers.go | 3 + .../redistimeseries/redistimeseries.go | 266 +++++++++--------- .../redistimeseries/redistimeseries_test.go | 4 +- ...ses_full_cycle_minitest_redistimeseries.sh | 5 +- 7 files changed, 191 insertions(+), 176 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 2fe564517..15ee68a72 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "crypto/md5" "encoding/binary" "fmt" @@ -16,6 +15,8 @@ import ( "github.com/gomodule/redigo/redis" "github.com/timescale/tsbs/load" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/data" ) // Program option vars: @@ -31,7 +32,7 @@ var ( // Global vars var ( - loader *load.BenchmarkRunner + loader load.BenchmarkRunner //bufPool sync.Pool ) @@ -70,11 +71,15 @@ type benchmark struct { dbc *dbCreator } +func (b *benchmark) GetDataSource() targets.DataSource { + panic("implement me") +} + type RedisIndexer struct { partitions uint } -func (i *RedisIndexer) GetIndex(p *load.Point) int { +func (i *RedisIndexer) GetIndex(p data.LoadedPoint) uint { row := p.Data.(string) key := strings.Split(row, " ")[1] start := strings.Index(key, "{") @@ -82,26 +87,26 @@ func (i *RedisIndexer) GetIndex(p *load.Point) int { _, _ = io.WriteString(md5h, key[start+1:end]) hash := binary.LittleEndian.Uint32(md5h.Sum(nil)) md5h.Reset() - return int(uint(hash) % i.partitions) + return uint(hash) % i.partitions } +// +//func (b *benchmark) GetPointDecoder(br *bufio.Reader) load.PointDecoder { +// return &decoder{scanner: bufio.NewScanner(br)} +//} -func (b *benchmark) GetPointDecoder(br *bufio.Reader) load.PointDecoder { - return &decoder{scanner: bufio.NewScanner(br)} -} - -func (b *benchmark) GetBatchFactory() load.BatchFactory { +func (b *benchmark) GetBatchFactory() targets.BatchFactory { return &factory{} } -func (b *benchmark) GetPointIndexer(maxPartitions uint) load.PointIndexer { +func (b *benchmark) GetPointIndexer(maxPartitions uint) targets.PointIndexer { return &RedisIndexer{partitions: maxPartitions} } -func (b *benchmark) GetProcessor() load.Processor { +func (b *benchmark) GetProcessor() targets.Processor { return &processor{b.dbc, nil, nil, nil} } -func (b *benchmark) GetDBCreator() load.DBCreator { +func (b *benchmark) GetDBCreator() targets.DBCreator { return b.dbc } @@ -146,10 +151,10 @@ func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint //fmt.Println(fmt.Sprintf("wg done for id %d\n",id)) } -func (p *processor) Init(_ int, _ bool) {} +func (p *processor) Init(_ int, _ bool, _ bool) {} // ProcessBatch reads eventsBatches which contain rows of data for TS.ADD redis command string -func (p *processor) ProcessBatch(b load.Batch, doLoad bool) (uint64, uint64) { +func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) { events := b.(*eventsBatch) rowCnt := uint64(len(events.rows)) metricCnt := uint64(0) @@ -227,11 +232,11 @@ func runCheckData() { } func main() { - workQueues := uint(load.WorkerPerQueue) - if singleQueue { - workQueues = load.SingleQueue - } - loader.RunBenchmark(&benchmark{dbc: &dbCreator{}}, workQueues) + //workQueues := uint(load.WorkerPerQueue) + //if singleQueue { + // workQueues = load.SingleQueue + //} + loader.RunBenchmark(&benchmark{dbc: &dbCreator{}}) if checkChunks > 0 { runCheckData() } diff --git a/cmd/tsbs_load_redistimeseries/scan.go b/cmd/tsbs_load_redistimeseries/scan.go index 818d325f0..0f87e6adf 100644 --- a/cmd/tsbs_load_redistimeseries/scan.go +++ b/cmd/tsbs_load_redistimeseries/scan.go @@ -1,32 +1,32 @@ package main import ( - "bufio" //"fmt" "log" "strings" "sync" "github.com/gomodule/redigo/redis" - "github.com/timescale/tsbs/load" + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/targets" ) -type decoder struct { - scanner *bufio.Scanner -} - -// Reads and returns a text line that encodes a data point for a specif field name. -// Since scanning happens in a single thread, we hold off on transforming it -// to an INSERT statement until it's being processed concurrently by a worker. -func (d *decoder) Decode(_ *bufio.Reader) *load.Point { - ok := d.scanner.Scan() - if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF - return nil - } else if !ok { - log.Fatalf("scan error: %v", d.scanner.Err()) - } - return load.NewPoint(d.scanner.Text()) -} +//type decoder struct { +// scanner *bufio.Scanner +//} +// +//// Reads and returns a text line that encodes a data point for a specif field name. +//// Since scanning happens in a single thread, we hold off on transforming it +//// to an INSERT statement until it's being processed concurrently by a worker. +//func (d *decoder) Decode(_ *bufio.Reader) *load.Point { +// ok := d.scanner.Scan() +// if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF +// return nil +// } else if !ok { +// log.Fatalf("scan error: %v", d.scanner.Err()) +// } +// return load.NewPoint(d.scanner.Text()) +//} func sendRedisCommand(conn redis.Conn, cmdName string, s redis.Args) (err error) { err = conn.Send(cmdName, s...) @@ -72,11 +72,11 @@ type eventsBatch struct { rows []string } -func (eb *eventsBatch) Len() int { - return len(eb.rows) +func (eb *eventsBatch) Len() uint { + return uint(len(eb.rows)) } -func (eb *eventsBatch) Append(item *load.Point) { +func (eb *eventsBatch) Append(item data.LoadedPoint) { that := item.Data.(string) eb.rows = append(eb.rows, that) } @@ -85,6 +85,6 @@ var ePool = &sync.Pool{New: func() interface{} { return &eventsBatch{rows: []str type factory struct{} -func (f *factory) New() load.Batch { +func (f *factory) New() targets.Batch { return ePool.Get().(*eventsBatch) } diff --git a/cmd/tsbs_run_queries_redistimeseries/main.go b/cmd/tsbs_run_queries_redistimeseries/main.go index 254f00e6a..99ba3dc98 100644 --- a/cmd/tsbs_run_queries_redistimeseries/main.go +++ b/cmd/tsbs_run_queries_redistimeseries/main.go @@ -15,7 +15,7 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/timescale/tsbs/internal/utils" - "github.com/timescale/tsbs/query" + "github.com/timescale/tsbs/pkg/query" "log" "sort" "strings" diff --git a/pkg/targets/initializers/target_initializers.go b/pkg/targets/initializers/target_initializers.go index 54ba801ff..4fe9bc335 100644 --- a/pkg/targets/initializers/target_initializers.go +++ b/pkg/targets/initializers/target_initializers.go @@ -11,6 +11,7 @@ import ( "github.com/timescale/tsbs/pkg/targets/influx" "github.com/timescale/tsbs/pkg/targets/mongo" "github.com/timescale/tsbs/pkg/targets/prometheus" + "github.com/timescale/tsbs/pkg/targets/redistimeseries" "github.com/timescale/tsbs/pkg/targets/siridb" "github.com/timescale/tsbs/pkg/targets/timescaledb" "github.com/timescale/tsbs/pkg/targets/timestream" @@ -42,6 +43,8 @@ func GetTarget(format string) targets.ImplementedTarget { return victoriametrics.NewTarget() case constants.FormatTimestream: return timestream.NewTarget() + case constants.FormatRedisTimeSeries: + return redistimeseries.NewTarget() } supportedFormatsStr := strings.Join(constants.SupportedFormats(), ",") diff --git a/pkg/targets/redistimeseries/redistimeseries.go b/pkg/targets/redistimeseries/redistimeseries.go index ef6c4e936..e36d1ff2c 100644 --- a/pkg/targets/redistimeseries/redistimeseries.go +++ b/pkg/targets/redistimeseries/redistimeseries.go @@ -1,130 +1,136 @@ -package serialize - -import ( - "crypto/md5" - "encoding/binary" - "fmt" - "io" -) - -// RedisTimeSeriesSerializer writes a Point in a serialized form for RedisTimeSeries -type RedisTimeSeriesSerializer struct{} - -var keysSoFar map[string]bool -var hashSoFar map[string][]byte - -// Serialize writes Point data to the given writer, in a format that will be easy to create a redis-timeseries command -// from. -// -// This function writes output that looks like: -//cpu_usage_user{md5(hostname=host_0|region=eu-central-1...)} 1451606400 58 LABELS hostname host_0 region eu-central-1 ... measurement cpu fieldname usage_user -// -// Which the loader will decode into a set of TS.ADD commands for each fieldKey. Once labels have been created for a each fieldKey, -// subsequent rows are ommitted with them and are ingested with TS.MADD for a row's metrics. -func (s *RedisTimeSeriesSerializer) Serialize(p *Point, w io.Writer) (err error) { - if keysSoFar == nil { - keysSoFar = make(map[string]bool) - } - - if hashSoFar == nil { - hashSoFar = make(map[string][]byte) - } - - var hashBytes []byte - //var hashExists bool - hostname := p.tagValues[0] - - for fieldID := 0; fieldID < len(p.fieldKeys); fieldID++ { - fieldName := p.fieldKeys[fieldID] - keyName := fmt.Sprintf("%s%s", hostname, fieldName) - //fmt.Errorf("%s\n",fieldName) - //if hashBytes, hashExists = hashSoFar[keyName]; hashExists == false { - //do something here - labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) - hashBytes = fastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) - //hashSoFar[keyName] = hashBytes - //} - - // if this key was already inserted and created, we don't to specify the labels again - if keysSoFar[keyName] == false { - w.Write([]byte("TS.CREATE ")) - writeKeyName(w, p, fieldName, hashBytes) - w.Write([]byte("LABELS")) - for i, v := range p.tagValues { - w.Write([]byte(" ")) - w.Write(p.tagKeys[i]) - w.Write([]byte(" ")) - w.Write(fastFormatAppend(v, []byte{})) - } - w.Write([]byte(" measurement ")) - // add measurement name as additional label to be used in queries - w.Write(p.measurementName) - - // additional label of fieldname - w.Write([]byte(" fieldname ")) - w.Write(fieldName) - w.Write([]byte("\n")) - keysSoFar[keyName] = true - } - } - w.Write([]byte("TS.MADD ")) - - for fieldID := 0; fieldID < len(p.fieldKeys); fieldID++ { - fieldName := p.fieldKeys[fieldID] - - //keyName := fmt.Sprintf("%s%s", hostname, fieldName) - //fmt.Fprint(os.Stderr, fmt.Sprintf("%s\n", keyName)) - - labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) - hashBytes = fastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) - - fieldValue := p.fieldValues[fieldID] - writeKeyName(w, p, fieldName, hashBytes) - writeTS_and_Value(w, p, fieldValue) - if fieldID < len(p.fieldKeys)-1 { - w.Write([]byte(" ")) - } - } - w.Write([]byte("\n")) - - return err -} - -func appendTS_and_Value(lbuf []byte, p *Point, fieldValue interface{}) []byte { - // write timestamp in ms - lbuf = fastFormatAppend(p.timestamp.UTC().Unix()*1000, lbuf) - lbuf = append(lbuf, ' ') - // write value - lbuf = fastFormatAppend(fieldValue, lbuf) - return lbuf -} - -func writeTS_and_Value(w io.Writer, p *Point, fieldValue interface{}) (err error) { - // write timestamp in ms - w.Write(fastFormatAppend(p.timestamp.UTC().Unix()*1000, []byte{})) - w.Write([]byte(" ")) - // write value - _, err = w.Write(fastFormatAppend(fieldValue, []byte{})) - return -} - -func appendKeyName(lbuf []byte, p *Point, fieldName []byte, hashBytes []byte) []byte { - lbuf = append(lbuf, p.measurementName...) - lbuf = append(lbuf, '_') - lbuf = append(lbuf, fieldName...) - - lbuf = append(lbuf, '{') - lbuf = append(lbuf, hashBytes...) - lbuf = append(lbuf, '}', ' ') - return lbuf -} - -func writeKeyName(w io.Writer, p *Point, fieldName []byte, hashBytes []byte) (err error) { - w.Write(p.measurementName) - w.Write([]byte("_")) - w.Write(fieldName) - w.Write([]byte("{")) - w.Write(hashBytes) - _, err = w.Write([]byte("} ")) - return -} +package redistimeseries +// +//import ( +// "crypto/md5" +// "encoding/binary" +// "fmt" +// "io" +// +// "github.com/timescale/tsbs/pkg/data" +// "github.com/timescale/tsbs/pkg/data/serialize" +//) +// +//// RedisTimeSeriesSerializer writes a Point in a serialized form for RedisTimeSeries +//type RedisTimeSeriesSerializer struct{} +// +//var keysSoFar map[string]bool +//var hashSoFar map[string][]byte +// +//// Serialize writes Point data to the given writer, in a format that will be easy to create a redis-timeseries command +//// from. +//// +//// This function writes output that looks like: +////cpu_usage_user{md5(hostname=host_0|region=eu-central-1...)} 1451606400 58 LABELS hostname host_0 region eu-central-1 ... measurement cpu fieldname usage_user +//// +//// Which the loader will decode into a set of TS.ADD commands for each fieldKey. Once labels have been created for a each fieldKey, +//// subsequent rows are ommitted with them and are ingested with TS.MADD for a row's metrics. +//func (s *RedisTimeSeriesSerializer) Serialize(p *data.Point, w io.Writer) (err error) { +// if keysSoFar == nil { +// keysSoFar = make(map[string]bool) +// } +// +// if hashSoFar == nil { +// hashSoFar = make(map[string][]byte) +// } +// +// var hashBytes []byte +// //var hashExists bool +// hostname := p.TagValues()[0] +// fieldKeys := p.FieldKeys() +// fieldValues := p.FieldValues() +// +// for fieldID := 0; fieldID < len(fieldKeys); fieldID++ { +// fieldName := fieldKeys[fieldID] +// keyName := fmt.Sprintf("%s%s", hostname, fieldName) +// //fmt.Errorf("%s\n",fieldName) +// //if hashBytes, hashExists = hashSoFar[keyName]; hashExists == false { +// //do something here +// labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) +// hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) +// //hashSoFar[keyName] = hashBytes +// //} +// +// // if this key was already inserted and created, we don't to specify the labels again +// if keysSoFar[keyName] == false { +// w.Write([]byte("TS.CREATE ")) +// writeKeyName(w, p, fieldName, hashBytes) +// w.Write([]byte("LABELS")) +// tagValues := p.TagValues() +// for i, v := range tagValues { +// w.Write([]byte(" ")) +// w.Write([]byte(tagValues[i].(string))) +// w.Write([]byte(" ")) +// w.Write(serialize.FastFormatAppend(v, []byte{})) +// } +// w.Write([]byte(" measurement ")) +// // add measurement name as additional label to be used in queries +// w.Write(p.MeasurementName()) +// +// // additional label of fieldname +// w.Write([]byte(" fieldname ")) +// w.Write(fieldName) +// w.Write([]byte("\n")) +// keysSoFar[keyName] = true +// } +// } +// w.Write([]byte("TS.MADD ")) +// +// for fieldID := 0; fieldID < len(fieldKeys); fieldID++ { +// fieldName := fieldKeys[fieldID] +// +// //keyName := fmt.Sprintf("%s%s", hostname, fieldName) +// //fmt.Fprint(os.Stderr, fmt.Sprintf("%s\n", keyName)) +// +// labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) +// hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) +// +// fieldValue := fieldValues[fieldID] +// writeKeyName(w, p, fieldName, hashBytes) +// writeTS_and_Value(w, p, fieldValue) +// if fieldID < len(fieldKeys)-1 { +// w.Write([]byte(" ")) +// } +// } +// w.Write([]byte("\n")) +// +// return err +//} +// +//func appendTS_and_Value(lbuf []byte, p *data.Point, fieldValue interface{}) []byte { +// // write timestamp in ms +// lbuf = serialize.FastFormatAppend(p.timestamp.UTC().Unix()*1000, lbuf) +// lbuf = append(lbuf, ' ') +// // write value +// lbuf = serialize.FastFormatAppend(fieldValue, lbuf) +// return lbuf +//} +// +////func writeTS_and_Value(w io.Writer, p *Point, fieldValue interface{}) (err error) { +//// // write timestamp in ms +//// w.Write(serialize.FastFormatAppend(p.timestamp.UTC().Unix()*1000, []byte{})) +//// w.Write([]byte(" ")) +//// // write value +//// _, err = w.Write(serialize.FastFormatAppend(fieldValue, []byte{})) +//// return +////} +// +////func appendKeyName(lbuf []byte, p *data.Point, fieldName []byte, hashBytes []byte) []byte { +//// lbuf = append(lbuf, p.MeasurementName()...) +//// lbuf = append(lbuf, '_') +//// lbuf = append(lbuf, fieldName...) +//// +//// lbuf = append(lbuf, '{') +//// lbuf = append(lbuf, hashBytes...) +//// lbuf = append(lbuf, '}', ' ') +//// return lbuf +////} +// +////func writeKeyName(w io.Writer, p *data.Point, fieldName []byte, hashBytes []byte) (err error) { +//// w.Write(p.MeasurementName()) +//// w.Write([]byte("_")) +//// w.Write(fieldName) +//// w.Write([]byte("{")) +//// w.Write(hashBytes) +//// _, err = w.Write([]byte("} ")) +//// return +////} diff --git a/pkg/targets/redistimeseries/redistimeseries_test.go b/pkg/targets/redistimeseries/redistimeseries_test.go index 9aa2924f8..a0e0d04d8 100644 --- a/pkg/targets/redistimeseries/redistimeseries_test.go +++ b/pkg/targets/redistimeseries/redistimeseries_test.go @@ -1,4 +1,4 @@ -package serialize +package redistimeseries import ( "testing" @@ -23,7 +23,7 @@ func TestRedisTimeSeriesSerializer(t *testing.T) { //}, } - testSerializer(t, cases, &RedisTimeSeriesSerializer{}) + testSerializer(t, cases, &Serializer{}) } func TestRedisTimeSeriesSerializerErr(t *testing.T) { diff --git a/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh b/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh index 597a04bc3..128f27aec 100755 --- a/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh +++ b/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e # showcases the ftsb 3 phases for timescaledb # - 1) data and query generation # - 2) data loading/insertion @@ -31,8 +32,8 @@ QUERY_TYPES_ALL="\ QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} mkdir -p /tmp/bulk_data -rm /tmp/bulk_data/${FORMAT}_* -rm docs/responses/${FORMAT}_* +rm -f /tmp/bulk_data/${FORMAT}_* +rm -f docs/responses/${FORMAT}_* redis-cli flushall From c8a2b197a562035c576965be8a035b8015913c72 Mon Sep 17 00:00:00 2001 From: Danni Moiseyev Date: Tue, 2 Mar 2021 12:37:29 +0200 Subject: [PATCH 04/72] More work to align RedisTimeSeries code to new TSBS framework --- .../databases/redistimeseries/common.go | 5 +- .../databases/redistimeseries/devops.go | 2 +- cmd/tsbs_load_redistimeseries/main.go | 59 +++++++++++++++---- cmd/tsbs_run_queries_redistimeseries/main.go | 13 ++-- pkg/query/factories/init_factories.go | 2 + .../redistimeseries/implemented_target.go | 7 ++- ...ses_full_cycle_minitest_redistimeseries.sh | 7 ++- 7 files changed, 69 insertions(+), 26 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/common.go b/cmd/tsbs_generate_queries/databases/redistimeseries/common.go index 0f172d087..222755856 100644 --- a/cmd/tsbs_generate_queries/databases/redistimeseries/common.go +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/common.go @@ -1,10 +1,11 @@ package redistimeseries import ( + "time" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/utils" - "github.com/timescale/tsbs/query" - "time" + "github.com/timescale/tsbs/pkg/query" ) // BaseGenerator contains settings specific for RedisTimeSeries database. diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go index 092522fa7..7c8071c15 100644 --- a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go @@ -5,7 +5,7 @@ import ( "time" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" - "github.com/timescale/tsbs/query" + "github.com/timescale/tsbs/pkg/query" ) // TODO: Remove the need for this by continuing to bubble up errors diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 15ee68a72..015d794a7 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -1,22 +1,27 @@ package main import ( + "bufio" "crypto/md5" "encoding/binary" "fmt" - "github.com/spf13/pflag" - "github.com/spf13/viper" - "github.com/timescale/tsbs/internal/utils" "io" "log" "strconv" "strings" "sync" + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/internal/utils" + "github.com/timescale/tsbs/pkg/data/usecases/common" + "github.com/timescale/tsbs/pkg/targets/constants" + "github.com/timescale/tsbs/pkg/targets/initializers" + "github.com/gomodule/redigo/redis" "github.com/timescale/tsbs/load" - "github.com/timescale/tsbs/pkg/targets" "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/targets" ) // Program option vars: @@ -33,6 +38,8 @@ var ( // Global vars var ( loader load.BenchmarkRunner + config load.BenchmarkRunnerConfig + target targets.ImplementedTarget //bufPool sync.Pool ) @@ -42,16 +49,16 @@ var md5h = md5.New() // Parse args: func init() { - var config load.BenchmarkRunnerConfig + target = initializers.GetTarget(constants.FormatRedisTimeSeries) + config = load.BenchmarkRunnerConfig{} config.AddToFlagSet(pflag.CommandLine) + target.TargetSpecificFlags("", pflag.CommandLine) - pflag.StringVar(&host, "host", "localhost:6379", "The host:port for Redis connection") - pflag.Uint64Var(&connections, "connections", 10, "The number of connections per worker") - pflag.Uint64Var(&pipeline, "pipeline", 50, "The pipeline's size") - pflag.BoolVar(&singleQueue, "single-queue", true, "Whether to use a single queue") - pflag.BoolVar(&compressionEnabled, "compression-enabled", true, "Whether to use compressed time series") - pflag.Uint64Var(&checkChunks, "check-chunks", 0, "Whether to perform post ingestion chunck count") - pflag.StringVar(&dataModel, "data-model", "redistimeseries", "Data model (redistimeseries, rediszsetdevice, rediszsetmetric, redisstream)") + //pflag.Uint64Var(&pipeline, "pipeline", 50, "The pipeline's size") + //pflag.BoolVar(&singleQueue, "single-queue", true, "Whether to use a single queue") + //pflag.BoolVar(&compressionEnabled, "compression-enabled", true, "Whether to use compressed time series") + //pflag.Uint64Var(&checkChunks, "check-chunks", 0, "Whether to perform post ingestion chunck count") + //pflag.StringVar(&dataModel, "data-model", "redistimeseries", "Data model (redistimeseries, rediszsetdevice, rediszsetmetric, redisstream)") pflag.Parse() err := utils.SetupConfigFile() @@ -63,6 +70,12 @@ func init() { if err := viper.Unmarshal(&config); err != nil { panic(fmt.Errorf("unable to decode config: %s", err)) } + host = viper.GetString("host") + connections = viper.GetUint64("connections") + pipeline = viper.GetUint64("pipeline") + dataModel = "redistimeseries" + compressionEnabled = true + loader = load.GetBenchmarkRunner(config) } @@ -72,7 +85,8 @@ type benchmark struct { } func (b *benchmark) GetDataSource() targets.DataSource { - panic("implement me") + log.Printf("creating DS from %s", config.FileName); + return &fileDataSource{scanner: bufio.NewScanner(load.GetBufferedReader(config.FileName))} } type RedisIndexer struct { @@ -93,6 +107,22 @@ func (i *RedisIndexer) GetIndex(p data.LoadedPoint) uint { //func (b *benchmark) GetPointDecoder(br *bufio.Reader) load.PointDecoder { // return &decoder{scanner: bufio.NewScanner(br)} //} +type fileDataSource struct { + scanner *bufio.Scanner +} + +func (d *fileDataSource) NextItem() data.LoadedPoint { + ok := d.scanner.Scan() + if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF + return data.LoadedPoint{} + } else if !ok { + fatal("scan error: %v", d.scanner.Err()) + return data.LoadedPoint{} + } + return data.NewLoadedPoint(d.scanner.Text()) +} + +func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { return nil } func (b *benchmark) GetBatchFactory() targets.BatchFactory { return &factory{} @@ -236,7 +266,10 @@ func main() { //if singleQueue { // workQueues = load.SingleQueue //} + log.Println("Starting benchmark") + config.NoFlowControl = true; loader.RunBenchmark(&benchmark{dbc: &dbCreator{}}) + log.Println("finished benchmark") if checkChunks > 0 { runCheckData() } diff --git a/cmd/tsbs_run_queries_redistimeseries/main.go b/cmd/tsbs_run_queries_redistimeseries/main.go index 99ba3dc98..d0e921ef0 100644 --- a/cmd/tsbs_run_queries_redistimeseries/main.go +++ b/cmd/tsbs_run_queries_redistimeseries/main.go @@ -9,17 +9,18 @@ import ( "database/sql" "encoding/json" "fmt" + "log" + "sort" + "strings" + "time" + redistimeseries "github.com/RedisTimeSeries/redistimeseries-go" + "github.com/blagojts/viper" _ "github.com/lib/pq" "github.com/pkg/errors" "github.com/spf13/pflag" - "github.com/spf13/viper" "github.com/timescale/tsbs/internal/utils" "github.com/timescale/tsbs/pkg/query" - "log" - "sort" - "strings" - "time" ) // Program option vars: @@ -141,7 +142,7 @@ func prettyPrintResponseRange(responses []interface{}, q *query.RedisTimeSeries) rows = append(rows, row) } resp["results"] = rows - case redistimeseries.Range: + case redistimeseries.Rangqe: resp["client_side_work"] = q.ApplyFunctor resp["results"] = res.(redistimeseries.Range) case []query.MultiRange: diff --git a/pkg/query/factories/init_factories.go b/pkg/query/factories/init_factories.go index 7675b0316..c5325719a 100644 --- a/pkg/query/factories/init_factories.go +++ b/pkg/query/factories/init_factories.go @@ -7,6 +7,7 @@ import ( "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/cratedb" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/influx" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/mongo" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/redistimeseries" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/siridb" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/timescaledb" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/timestream" @@ -37,5 +38,6 @@ func InitQueryFactories(config *config.QueryGeneratorConfig) map[string]interfac factories[constants.FormatTimestream] = ×tream.BaseGenerator{ DBName: config.DbName, } + factories[constants.FormatRedisTimeSeries] = &redistimeseries.BaseGenerator{} return factories } diff --git a/pkg/targets/redistimeseries/implemented_target.go b/pkg/targets/redistimeseries/implemented_target.go index 2ec34ebfd..d28c1c19d 100644 --- a/pkg/targets/redistimeseries/implemented_target.go +++ b/pkg/targets/redistimeseries/implemented_target.go @@ -18,9 +18,10 @@ type redistimeseriesTarget struct { func (t *redistimeseriesTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.FlagSet) { flagSet.String(flagPrefix+"host", "localhost:6379", "The host:port for Redis connection") - pflag.Uint64(flagPrefix+"pipeline", 50, "The pipeline's size") - pflag.Bool(flagPrefix+ "compression-enabled", true, "Whether to use compressed time series") - pflag.Bool(flagPrefix+ "cluster", false, "Whether to use OSS cluster API") + flagSet.Uint64(flagPrefix+"pipeline", 50, "The pipeline's size") + flagSet.Uint64(flagPrefix+"connections", 10, "The number of connections per worker") + flagSet.Bool(flagPrefix+ "compression-enabled", true, "Whether to use compressed time series") + flagSet.Bool(flagPrefix+ "cluster", false, "Whether to use OSS cluster API") } diff --git a/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh b/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh index 128f27aec..5057517e5 100755 --- a/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh +++ b/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh @@ -9,6 +9,9 @@ SCALE=${SCALE:-"10"} SEED=${SEED:-"123"} PASSWORD=${PASSWORD:-"password"} FORMAT="redistimeseries" +REDIS_HOST=${REDIS_HOST:-"localhost"} +REDIS_PORT=${REDIS_PORT:-"6379"} +REDIS=${REDIS_HOST}":"${REDIS_PORT} # All available query types (sorted alphabetically) QUERY_TYPES_ALL="\ @@ -35,7 +38,7 @@ mkdir -p /tmp/bulk_data rm -f /tmp/bulk_data/${FORMAT}_* rm -f docs/responses/${FORMAT}_* -redis-cli flushall +redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} flushall # generate data $GOPATH/bin/tsbs_generate_data --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} --file /tmp/bulk_data/${FORMAT}_data @@ -50,6 +53,7 @@ done # insert benchmark $GOPATH/bin/tsbs_load_${FORMAT} \ --workers=1 \ + --host=${REDIS} \ --file=/tmp/bulk_data/${FORMAT}_data # queries benchmark @@ -58,5 +62,6 @@ for queryName in $QUERY_TYPES; do $GOPATH/bin/tsbs_run_queries_${FORMAT} --print-responses \ --workers=1 \ --debug=3 \ + --host=${REDIS} \ --file /tmp/bulk_data/${FORMAT}_query_$queryName >docs/responses/${FORMAT}_$queryName.json done From d09e8ab1a0f2ed5a24f4d67a77c82fc462ef84b6 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 2 Mar 2021 22:07:01 +0000 Subject: [PATCH 05/72] [fix] Minor fixes on functional RTS script. showcasing the simulated datapoints progress... --- Makefile | 2 ++ cmd/tsbs_run_queries_redistimeseries/main.go | 2 +- go.mod | 1 + go.sum | 10 ++++++++++ internal/inputs/generator_data.go | 5 +++++ pkg/data/usecases/common/simulator.go | 1 + pkg/data/usecases/devops/generate_data.go | 4 ++++ .../devops/generic_metrics_generate_data.go | 4 ++++ ...responses_full_cycle_minitest_redistimeseries.sh | 13 ++++++++----- 9 files changed, 36 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 3969e095b..ca50b4e39 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ all: generators loaders runners generators: tsbs_generate_data tsbs_generate_queries +redistimeseries: tsbs_generate_data tsbs_generate_queries tsbs_load_redistimeseries tsbs_run_queries_redistimeseries + loaders: tsbs_load_redistimeseries tsbs_load_cassandra tsbs_load_clickhouse tsbs_load_influx tsbs_load_mongo tsbs_load_siridb tsbs_load_timescaledb runners: tsbs_run_queries_redistimeseries tsbs_run_queries_cassandra tsbs_run_queries_clickhouse tsbs_run_queries_influx tsbs_run_queries_mongo tsbs_run_queries_siridb tsbs_run_queries_timescaledb diff --git a/cmd/tsbs_run_queries_redistimeseries/main.go b/cmd/tsbs_run_queries_redistimeseries/main.go index d0e921ef0..1c034f2ca 100644 --- a/cmd/tsbs_run_queries_redistimeseries/main.go +++ b/cmd/tsbs_run_queries_redistimeseries/main.go @@ -142,7 +142,7 @@ func prettyPrintResponseRange(responses []interface{}, q *query.RedisTimeSeries) rows = append(rows, row) } resp["results"] = rows - case redistimeseries.Rangqe: + case redistimeseries.Range: resp["client_side_work"] = q.ApplyFunctor resp["results"] = res.(redistimeseries.Range) case []query.MultiRange: diff --git a/go.mod b/go.mod index f49b9b9a8..f2c6f226e 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 github.com/aws/aws-sdk-go v1.35.13 github.com/blagojts/viper v1.6.3-0.20200313094124-068f44cf5e69 + github.com/cheggaaa/pb/v3 v3.0.6 github.com/filipecosta90/hdrhistogram v0.0.0-20191025144016-6360d1757d33 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index e4c29703d..36ed86dd9 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921 h1:GIWN github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921/go.mod h1:s0x47OhsrJKfg9Iq5orGCVJQjwKklC3jZMFlgLe6Zew= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= @@ -143,6 +145,9 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= +github.com/cheggaaa/pb/v3 v3.0.6 h1:ULPm1wpzvj60FvmCrX7bIaB80UgbhI+zSaQJKRfCbAs= +github.com/cheggaaa/pb/v3 v3.0.6/go.mod h1:X1L61/+36nz9bjIsrDU52qHKOQukUQe2Ge+YvGuquCw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -214,6 +219,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/filipecosta90/hdrhistogram v0.0.0-20191025144016-6360d1757d33 h1:KURw4yhtighHMtV5MeHqI1GYvC8MAfAatn2K5J5INvI= github.com/filipecosta90/hdrhistogram v0.0.0-20191025144016-6360d1757d33/go.mod h1:Ws7v8qrWA96aL9o9Hl6X334iRA+l61XXODY/HY6JdvU= @@ -647,6 +653,7 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -656,9 +663,12 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= diff --git a/internal/inputs/generator_data.go b/internal/inputs/generator_data.go index bca8cb755..6fa2dd7c1 100644 --- a/internal/inputs/generator_data.go +++ b/internal/inputs/generator_data.go @@ -14,6 +14,7 @@ import ( "github.com/timescale/tsbs/pkg/data/usecases/common" "github.com/timescale/tsbs/pkg/targets" "github.com/timescale/tsbs/pkg/targets/constants" + "github.com/cheggaaa/pb/v3" ) // Error messages when using a DataGenerator @@ -105,6 +106,8 @@ func (g *DataGenerator) runSimulator(sim common.Simulator, serializer serialize. defer g.bufOut.Flush() currGroupID := uint(0) + fmt.Println(int(g.config.Limit)) + bar := pb.StartNew(int(sim.MaxPoints())) point := data.NewPoint() for !sim.Finished() { write := sim.Next(point) @@ -112,6 +115,7 @@ func (g *DataGenerator) runSimulator(sim common.Simulator, serializer serialize. point.Reset() continue } + bar.Increment() // in the default case this is always true if currGroupID == dgc.InterleavedGroupID { @@ -123,6 +127,7 @@ func (g *DataGenerator) runSimulator(sim common.Simulator, serializer serialize. point.Reset() currGroupID = (currGroupID + 1) % dgc.InterleavedNumGroups } + bar.Finish() return nil } diff --git a/pkg/data/usecases/common/simulator.go b/pkg/data/usecases/common/simulator.go index 48fbe69ad..7b0de2d6c 100644 --- a/pkg/data/usecases/common/simulator.go +++ b/pkg/data/usecases/common/simulator.go @@ -78,6 +78,7 @@ type Simulator interface { TagKeys() []string TagTypes() []string Headers() *GeneratedDataHeaders + MaxPoints() uint64 } // BaseSimulator generates data similar to truck readings. diff --git a/pkg/data/usecases/devops/generate_data.go b/pkg/data/usecases/devops/generate_data.go index 6a5db40e2..040117872 100644 --- a/pkg/data/usecases/devops/generate_data.go +++ b/pkg/data/usecases/devops/generate_data.go @@ -42,6 +42,10 @@ func (s *DevopsSimulator) TagKeys() []string { return tagKeysAsStr } +func (s *DevopsSimulator) MaxPoints() uint64 { + return s.maxPoints +} + func (s *DevopsSimulator) TagTypes() []string { types := make([]string, len(MachineTagKeys)) for i := 0; i < len(MachineTagKeys); i++ { diff --git a/pkg/data/usecases/devops/generic_metrics_generate_data.go b/pkg/data/usecases/devops/generic_metrics_generate_data.go index 30e23f18e..2479bafbe 100644 --- a/pkg/data/usecases/devops/generic_metrics_generate_data.go +++ b/pkg/data/usecases/devops/generic_metrics_generate_data.go @@ -57,6 +57,10 @@ func (c *GenericMetricsSimulatorConfig) NewSimulator(interval time.Duration, lim return dg } +func (gms *GenericMetricsSimulator) MaxPoints() uint64 { + return gms.maxPoints +} + // Fields returns a map of subsystems to metrics collected // Since each host has different number of fields (we use zipf distribution to assign # fields) we search // for the host with the max number of fields diff --git a/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh b/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh index 5057517e5..2b662468b 100755 --- a/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh +++ b/scripts/functional/debug_responses_full_cycle_minitest_redistimeseries.sh @@ -1,11 +1,12 @@ #!/bin/bash set -e +set -x # showcases the ftsb 3 phases for timescaledb # - 1) data and query generation # - 2) data loading/insertion # - 3) query execution -SCALE=${SCALE:-"10"} +SCALE=${SCALE:-"100"} SEED=${SEED:-"123"} PASSWORD=${PASSWORD:-"password"} FORMAT="redistimeseries" @@ -13,6 +14,8 @@ REDIS_HOST=${REDIS_HOST:-"localhost"} REDIS_PORT=${REDIS_PORT:-"6379"} REDIS=${REDIS_HOST}":"${REDIS_PORT} +mkdir -p docs/responses + # All available query types (sorted alphabetically) QUERY_TYPES_ALL="\ cpu-max-all-1 \ @@ -41,17 +44,17 @@ rm -f docs/responses/${FORMAT}_* redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} flushall # generate data -$GOPATH/bin/tsbs_generate_data --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} --file /tmp/bulk_data/${FORMAT}_data +./bin/tsbs_generate_data --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} --file /tmp/bulk_data/${FORMAT}_data for queryName in $QUERY_TYPES; do - $GOPATH/bin/tsbs_generate_queries --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} \ + ./bin/tsbs_generate_queries --format ${FORMAT} --use-case cpu-only --scale=${SCALE} --seed=${SEED} \ --queries=1 \ --query-type $queryName \ --file /tmp/bulk_data/${FORMAT}_query_$queryName done # insert benchmark -$GOPATH/bin/tsbs_load_${FORMAT} \ +./bin/tsbs_load_${FORMAT} \ --workers=1 \ --host=${REDIS} \ --file=/tmp/bulk_data/${FORMAT}_data @@ -59,7 +62,7 @@ $GOPATH/bin/tsbs_load_${FORMAT} \ # queries benchmark for queryName in $QUERY_TYPES; do echo "running query: $queryName" - $GOPATH/bin/tsbs_run_queries_${FORMAT} --print-responses \ + ./bin/tsbs_run_queries_${FORMAT} --print-responses \ --workers=1 \ --debug=3 \ --host=${REDIS} \ From eb5e4d46c3f6c21570f324d1294ec28acf796cb4 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 2 Mar 2021 23:32:36 +0000 Subject: [PATCH 06/72] [wip] using radix to ingest. OSS cluster and standalone working. --- cmd/tsbs_load_redistimeseries/cluster_conn.go | 33 +++++ cmd/tsbs_load_redistimeseries/creator.go | 9 +- cmd/tsbs_load_redistimeseries/main.go | 121 ++++++---------- cmd/tsbs_load_redistimeseries/main_test.go | 11 -- cmd/tsbs_load_redistimeseries/scan.go | 64 ++------- .../standalone_conn.go | 22 +++ go.mod | 1 + go.sum | 2 + internal/inputs/generator_data.go | 1 - .../redistimeseries/implemented_target.go | 4 +- .../redistimeseries/redistimeseries.go | 136 ------------------ .../redistimeseries/redistimeseries_test.go | 74 +++++----- pkg/targets/redistimeseries/serializer.go | 27 ---- 13 files changed, 152 insertions(+), 353 deletions(-) create mode 100644 cmd/tsbs_load_redistimeseries/cluster_conn.go delete mode 100644 cmd/tsbs_load_redistimeseries/main_test.go create mode 100644 cmd/tsbs_load_redistimeseries/standalone_conn.go delete mode 100644 pkg/targets/redistimeseries/redistimeseries.go diff --git a/cmd/tsbs_load_redistimeseries/cluster_conn.go b/cmd/tsbs_load_redistimeseries/cluster_conn.go new file mode 100644 index 000000000..f320038ef --- /dev/null +++ b/cmd/tsbs_load_redistimeseries/cluster_conn.go @@ -0,0 +1,33 @@ +package main + +import ( + "github.com/mediocregopher/radix/v3" + "log" +) + +func getOSSClusterConn(addr string, opts []radix.DialOpt, clients uint64) *radix.Cluster { + var vanillaCluster *radix.Cluster + var err error + + customConnFunc := func(network, addr string) (radix.Conn, error) { + return radix.Dial(network, addr, opts..., + ) + } + + // this cluster will use the ClientFunc to create a pool to each node in the + // cluster. + poolFunc := func(network, addr string) (radix.Client, error) { + return radix.NewPool(network, addr, int(clients), radix.PoolConnFunc(customConnFunc), radix.PoolPipelineWindow(0, 0)) + } + + vanillaCluster, err = radix.NewCluster([]string{addr}, radix.ClusterPoolFunc(poolFunc)) + if err != nil { + log.Fatalf("Error preparing for benchmark, while creating new connection. error = %v", err) + } + // Issue CLUSTER SLOTS command + err = vanillaCluster.Sync() + if err != nil { + log.Fatalf("Error preparing for benchmark, while issuing CLUSTER SLOTS. error = %v", err) + } + return vanillaCluster +} diff --git a/cmd/tsbs_load_redistimeseries/creator.go b/cmd/tsbs_load_redistimeseries/creator.go index 6d7bf6e32..9e20ca0f1 100644 --- a/cmd/tsbs_load_redistimeseries/creator.go +++ b/cmd/tsbs_load_redistimeseries/creator.go @@ -1,15 +1,9 @@ package main -import ( - redistimeseries "github.com/RedisTimeSeries/redistimeseries-go" -) - type dbCreator struct { - client *redistimeseries.Client } func (d *dbCreator) Init() { - d.client = redistimeseries.NewClient(host, "test_client", nil) } func (d *dbCreator) DBExists(dbName string) bool { @@ -25,6 +19,5 @@ func (d *dbCreator) CreateDB(dbName string) error { } func (d *dbCreator) Close() { - conn := d.client.Pool.Get() - conn.Close() + } diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 015d794a7..bd01c96e7 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -5,6 +5,7 @@ import ( "crypto/md5" "encoding/binary" "fmt" + "github.com/mediocregopher/radix/v3" "io" "log" "strconv" @@ -18,7 +19,6 @@ import ( "github.com/timescale/tsbs/pkg/targets/constants" "github.com/timescale/tsbs/pkg/targets/initializers" - "github.com/gomodule/redigo/redis" "github.com/timescale/tsbs/load" "github.com/timescale/tsbs/pkg/data" "github.com/timescale/tsbs/pkg/targets" @@ -33,6 +33,7 @@ var ( singleQueue bool dataModel string compressionEnabled bool + clusterMode bool ) // Global vars @@ -53,12 +54,6 @@ func init() { config = load.BenchmarkRunnerConfig{} config.AddToFlagSet(pflag.CommandLine) target.TargetSpecificFlags("", pflag.CommandLine) - - //pflag.Uint64Var(&pipeline, "pipeline", 50, "The pipeline's size") - //pflag.BoolVar(&singleQueue, "single-queue", true, "Whether to use a single queue") - //pflag.BoolVar(&compressionEnabled, "compression-enabled", true, "Whether to use compressed time series") - //pflag.Uint64Var(&checkChunks, "check-chunks", 0, "Whether to perform post ingestion chunck count") - //pflag.StringVar(&dataModel, "data-model", "redistimeseries", "Data model (redistimeseries, rediszsetdevice, rediszsetmetric, redisstream)") pflag.Parse() err := utils.SetupConfigFile() @@ -75,6 +70,7 @@ func init() { pipeline = viper.GetUint64("pipeline") dataModel = "redistimeseries" compressionEnabled = true + clusterMode = viper.GetBool("cluster") loader = load.GetBenchmarkRunner(config) @@ -85,7 +81,7 @@ type benchmark struct { } func (b *benchmark) GetDataSource() targets.DataSource { - log.Printf("creating DS from %s", config.FileName); + log.Printf("creating DS from %s", config.FileName) return &fileDataSource{scanner: bufio.NewScanner(load.GetBufferedReader(config.FileName))} } @@ -103,10 +99,7 @@ func (i *RedisIndexer) GetIndex(p data.LoadedPoint) uint { md5h.Reset() return uint(hash) % i.partitions } -// -//func (b *benchmark) GetPointDecoder(br *bufio.Reader) load.PointDecoder { -// return &decoder{scanner: bufio.NewScanner(br)} -//} + type fileDataSource struct { scanner *bufio.Scanner } @@ -147,38 +140,42 @@ type processor struct { wg *sync.WaitGroup } -func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint64, conn redis.Conn, id uint64) { +func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint64, conn radix.Client) { curPipe := uint64(0) - //fmt.Println(fmt.Sprintf("wg started for id %d\n",id)) + cmds := make([]radix.CmdAction, 0, 0) for row := range rows { - cmdname, s := buildCommand(row, compressionEnabled == false) - var err error - - if curPipe == pipeline { - cnt, err := sendRedisFlush(curPipe, conn) + cmd, tscreate := buildCommand(row, compressionEnabled == false) + if tscreate { + err := conn.Do(cmd) if err != nil { log.Fatalf("Flush failed with %v", err) } - metrics <- cnt - curPipe = 0 - } - err = sendRedisCommand(conn, cmdname, s) - if err != nil { - log.Fatalf("sendRedisCommand failed with %v", err) + } else { + cmds = append(cmds, cmd) + curPipe++ + + if curPipe == pipeline { + err := conn.Do(radix.Pipeline(cmds...)) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- curPipe + cmds = make([]radix.CmdAction, 0, 0) + curPipe = 0 + } } - curPipe++ - } if curPipe > 0 { - cnt, err := sendRedisFlush(curPipe, conn) + err := conn.Do(radix.Pipeline(cmds...)) if err != nil { log.Fatalf("Flush failed with %v", err) } - metrics <- cnt + metrics <- curPipe + cmds = make([]radix.CmdAction, 0, 0) + curPipe = 0 } wg.Done() - //fmt.Println(fmt.Sprintf("wg done for id %d\n",id)) } func (p *processor) Init(_ int, _ bool, _ bool) {} @@ -188,18 +185,29 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) events := b.(*eventsBatch) rowCnt := uint64(len(events.rows)) metricCnt := uint64(0) - // indexer := &RedisIndexer{partitions: uint(connections)} + opts := make([]radix.DialOpt, 0) + if doLoad { buflen := rowCnt + 1 p.rows = make([]chan string, connections) p.metrics = make(chan uint64, buflen) p.wg = &sync.WaitGroup{} + var cluster *radix.Cluster + var standalone *radix.Pool + if clusterMode { + cluster = getOSSClusterConn(host, opts, connections) + } else { + standalone = getStandaloneConn(host, opts, connections) + } + for i := uint64(0); i < connections; i++ { - conn := p.dbc.client.Pool.Get() - defer conn.Close() p.rows[i] = make(chan string, buflen) p.wg.Add(1) - go connectionProcessor(p.wg, p.rows[i], p.metrics, conn, i) + if clusterMode { + go connectionProcessor(p.wg, p.rows[i], p.metrics, cluster) + } else { + go connectionProcessor(p.wg, p.rows[i], p.metrics, standalone) + } } for _, row := range events.rows { key := strings.Split(row, " ")[1] @@ -215,8 +223,6 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) } p.wg.Wait() close(p.metrics) - //fmt.Println("out\n") - for val := range p.metrics { metricCnt += val } @@ -229,48 +235,11 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) func (p *processor) Close(_ bool) { } -func runCheckData() { - log.Println("Running post ingestion data check") - conn, err := redis.DialURL(fmt.Sprintf("redis://%s", host)) - if err != nil { - log.Fatalf("Error while dialing %v", err) - } - _, err = conn.Do("PING") - if err != nil { - log.Fatalf("Error while PING %v", err) - } - - cursor := 0 - total := 0 - for { - rep, _ := redis.Values(conn.Do("SCAN", cursor)) - cursor, _ = redis.Int(rep[0], nil) - keys, _ := redis.Strings(rep[1], nil) - for _, key := range keys { - total++ - info, _ := redis.Values(conn.Do("TS.INFO", key)) - chunks, _ := redis.Int(info[5], nil) - if chunks != int(checkChunks) { - log.Printf("Verification error: key %v has %v chunks", key, chunks) - } - } - if cursor == 0 { - break - } - } - log.Printf("Verified %v keys", total) -} - func main() { - //workQueues := uint(load.WorkerPerQueue) - //if singleQueue { - // workQueues = load.SingleQueue - //} log.Println("Starting benchmark") - config.NoFlowControl = true; + config.NoFlowControl = true + config.HashWorkers = true + loader.RunBenchmark(&benchmark{dbc: &dbCreator{}}) log.Println("finished benchmark") - if checkChunks > 0 { - runCheckData() - } } diff --git a/cmd/tsbs_load_redistimeseries/main_test.go b/cmd/tsbs_load_redistimeseries/main_test.go deleted file mode 100644 index 3dbd6f12e..000000000 --- a/cmd/tsbs_load_redistimeseries/main_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package main - -import ( - "testing" - - "github.com/timescale/tsbs/load" -) - -func TestRedisTimeSeriesLoader(t *testing.T) { - loader.RunBenchmark(&benchmark{dbc: &dbCreator{}}, load.WorkerPerQueue) -} diff --git a/cmd/tsbs_load_redistimeseries/scan.go b/cmd/tsbs_load_redistimeseries/scan.go index 0f87e6adf..cd4362229 100644 --- a/cmd/tsbs_load_redistimeseries/scan.go +++ b/cmd/tsbs_load_redistimeseries/scan.go @@ -1,73 +1,27 @@ package main import ( - //"fmt" - "log" + "github.com/mediocregopher/radix/v3" + "strings" "sync" - "github.com/gomodule/redigo/redis" "github.com/timescale/tsbs/pkg/data" "github.com/timescale/tsbs/pkg/targets" ) -//type decoder struct { -// scanner *bufio.Scanner -//} -// -//// Reads and returns a text line that encodes a data point for a specif field name. -//// Since scanning happens in a single thread, we hold off on transforming it -//// to an INSERT statement until it's being processed concurrently by a worker. -//func (d *decoder) Decode(_ *bufio.Reader) *load.Point { -// ok := d.scanner.Scan() -// if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF -// return nil -// } else if !ok { -// log.Fatalf("scan error: %v", d.scanner.Err()) -// } -// return load.NewPoint(d.scanner.Text()) -//} - -func sendRedisCommand(conn redis.Conn, cmdName string, s redis.Args) (err error) { - err = conn.Send(cmdName, s...) - if err != nil { - log.Fatalf("sendRedisCommand %s failed: %s\n", cmdName, err) - } - return -} - -func buildCommand(line string, forceUncompressed bool) (cmdname string, s redis.Args) { +func buildCommand(line string, forceUncompressed bool) (cmdA radix.CmdAction, tscreate bool) { t := strings.Split(line, " ") - cmdname = t[0] - if cmdname == "TS.CREATE" && forceUncompressed { - t = append(t, "UNCOMPRESSED") - s = s.Add(t[1]) - s = s.Add("UNCOMPRESSED") - s = s.AddFlat(t[2:]) - } else { - s = s.AddFlat(t[1:]) + tscreate = false + cmdname := t[0] + if cmdname == "TS.CREATE" { + tscreate = true } + key := t[1] + cmdA = radix.FlatCmd(nil, cmdname, key, t[2:]) return } -func sendRedisFlush(count uint64, conn redis.Conn) (metrics uint64, err error) { - metrics = uint64(0) - err = conn.Flush() - if err != nil { - return - } - for i := uint64(0); i < count; i++ { - _, err := conn.Receive() - //fmt.Println(r) - if err != nil { - log.Fatalf("Flush failed with %v", err) - } else { - metrics += 10 // ts.madd - } - } - return metrics, err -} - type eventsBatch struct { rows []string } diff --git a/cmd/tsbs_load_redistimeseries/standalone_conn.go b/cmd/tsbs_load_redistimeseries/standalone_conn.go new file mode 100644 index 000000000..01f34959d --- /dev/null +++ b/cmd/tsbs_load_redistimeseries/standalone_conn.go @@ -0,0 +1,22 @@ +package main + +import ( + "github.com/mediocregopher/radix/v3" + "log" +) + +func getStandaloneConn(addr string, opts []radix.DialOpt, clients uint64) *radix.Pool { + var pool *radix.Pool + var err error + + customConnFunc := func(network, addr string) (radix.Conn, error) { + return radix.Dial(network, addr, opts..., + ) + } + network := "tcp" + pool, err = radix.NewPool(network, addr, int(clients), radix.PoolConnFunc(customConnFunc), radix.PoolPipelineWindow(0, 0)) + if err != nil { + log.Fatalf("Error preparing for benchmark, while creating new connection. error = %v", err) + } + return pool +} diff --git a/go.mod b/go.mod index f2c6f226e..7c994d1c0 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/jmoiron/sqlx v1.2.0 github.com/kshvakov/clickhouse v1.3.11 github.com/lib/pq v1.3.0 + github.com/mediocregopher/radix/v3 v3.7.0 github.com/pkg/errors v0.9.1 github.com/prometheus/common v0.13.0 github.com/shirou/gopsutil v2.18.12+incompatible diff --git a/go.sum b/go.sum index 36ed86dd9..86c54de22 100644 --- a/go.sum +++ b/go.sum @@ -676,6 +676,8 @@ github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mediocregopher/radix/v3 v3.7.0 h1:SM9zJdme5pYGEVvh1HttjBjDmIaNBDKy+oDCv5w81Wo= +github.com/mediocregopher/radix/v3 v3.7.0/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= diff --git a/internal/inputs/generator_data.go b/internal/inputs/generator_data.go index 6fa2dd7c1..153c32345 100644 --- a/internal/inputs/generator_data.go +++ b/internal/inputs/generator_data.go @@ -106,7 +106,6 @@ func (g *DataGenerator) runSimulator(sim common.Simulator, serializer serialize. defer g.bufOut.Flush() currGroupID := uint(0) - fmt.Println(int(g.config.Limit)) bar := pb.StartNew(int(sim.MaxPoints())) point := data.NewPoint() for !sim.Finished() { diff --git a/pkg/targets/redistimeseries/implemented_target.go b/pkg/targets/redistimeseries/implemented_target.go index d28c1c19d..a051702dc 100644 --- a/pkg/targets/redistimeseries/implemented_target.go +++ b/pkg/targets/redistimeseries/implemented_target.go @@ -18,8 +18,8 @@ type redistimeseriesTarget struct { func (t *redistimeseriesTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.FlagSet) { flagSet.String(flagPrefix+"host", "localhost:6379", "The host:port for Redis connection") - flagSet.Uint64(flagPrefix+"pipeline", 50, "The pipeline's size") - flagSet.Uint64(flagPrefix+"connections", 10, "The number of connections per worker") + flagSet.Uint64(flagPrefix+"pipeline", 10, "The pipeline's size") + flagSet.Uint64(flagPrefix+"connections", 1, "The number of connections per worker") flagSet.Bool(flagPrefix+ "compression-enabled", true, "Whether to use compressed time series") flagSet.Bool(flagPrefix+ "cluster", false, "Whether to use OSS cluster API") diff --git a/pkg/targets/redistimeseries/redistimeseries.go b/pkg/targets/redistimeseries/redistimeseries.go deleted file mode 100644 index e36d1ff2c..000000000 --- a/pkg/targets/redistimeseries/redistimeseries.go +++ /dev/null @@ -1,136 +0,0 @@ -package redistimeseries -// -//import ( -// "crypto/md5" -// "encoding/binary" -// "fmt" -// "io" -// -// "github.com/timescale/tsbs/pkg/data" -// "github.com/timescale/tsbs/pkg/data/serialize" -//) -// -//// RedisTimeSeriesSerializer writes a Point in a serialized form for RedisTimeSeries -//type RedisTimeSeriesSerializer struct{} -// -//var keysSoFar map[string]bool -//var hashSoFar map[string][]byte -// -//// Serialize writes Point data to the given writer, in a format that will be easy to create a redis-timeseries command -//// from. -//// -//// This function writes output that looks like: -////cpu_usage_user{md5(hostname=host_0|region=eu-central-1...)} 1451606400 58 LABELS hostname host_0 region eu-central-1 ... measurement cpu fieldname usage_user -//// -//// Which the loader will decode into a set of TS.ADD commands for each fieldKey. Once labels have been created for a each fieldKey, -//// subsequent rows are ommitted with them and are ingested with TS.MADD for a row's metrics. -//func (s *RedisTimeSeriesSerializer) Serialize(p *data.Point, w io.Writer) (err error) { -// if keysSoFar == nil { -// keysSoFar = make(map[string]bool) -// } -// -// if hashSoFar == nil { -// hashSoFar = make(map[string][]byte) -// } -// -// var hashBytes []byte -// //var hashExists bool -// hostname := p.TagValues()[0] -// fieldKeys := p.FieldKeys() -// fieldValues := p.FieldValues() -// -// for fieldID := 0; fieldID < len(fieldKeys); fieldID++ { -// fieldName := fieldKeys[fieldID] -// keyName := fmt.Sprintf("%s%s", hostname, fieldName) -// //fmt.Errorf("%s\n",fieldName) -// //if hashBytes, hashExists = hashSoFar[keyName]; hashExists == false { -// //do something here -// labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) -// hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) -// //hashSoFar[keyName] = hashBytes -// //} -// -// // if this key was already inserted and created, we don't to specify the labels again -// if keysSoFar[keyName] == false { -// w.Write([]byte("TS.CREATE ")) -// writeKeyName(w, p, fieldName, hashBytes) -// w.Write([]byte("LABELS")) -// tagValues := p.TagValues() -// for i, v := range tagValues { -// w.Write([]byte(" ")) -// w.Write([]byte(tagValues[i].(string))) -// w.Write([]byte(" ")) -// w.Write(serialize.FastFormatAppend(v, []byte{})) -// } -// w.Write([]byte(" measurement ")) -// // add measurement name as additional label to be used in queries -// w.Write(p.MeasurementName()) -// -// // additional label of fieldname -// w.Write([]byte(" fieldname ")) -// w.Write(fieldName) -// w.Write([]byte("\n")) -// keysSoFar[keyName] = true -// } -// } -// w.Write([]byte("TS.MADD ")) -// -// for fieldID := 0; fieldID < len(fieldKeys); fieldID++ { -// fieldName := fieldKeys[fieldID] -// -// //keyName := fmt.Sprintf("%s%s", hostname, fieldName) -// //fmt.Fprint(os.Stderr, fmt.Sprintf("%s\n", keyName)) -// -// labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) -// hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) -// -// fieldValue := fieldValues[fieldID] -// writeKeyName(w, p, fieldName, hashBytes) -// writeTS_and_Value(w, p, fieldValue) -// if fieldID < len(fieldKeys)-1 { -// w.Write([]byte(" ")) -// } -// } -// w.Write([]byte("\n")) -// -// return err -//} -// -//func appendTS_and_Value(lbuf []byte, p *data.Point, fieldValue interface{}) []byte { -// // write timestamp in ms -// lbuf = serialize.FastFormatAppend(p.timestamp.UTC().Unix()*1000, lbuf) -// lbuf = append(lbuf, ' ') -// // write value -// lbuf = serialize.FastFormatAppend(fieldValue, lbuf) -// return lbuf -//} -// -////func writeTS_and_Value(w io.Writer, p *Point, fieldValue interface{}) (err error) { -//// // write timestamp in ms -//// w.Write(serialize.FastFormatAppend(p.timestamp.UTC().Unix()*1000, []byte{})) -//// w.Write([]byte(" ")) -//// // write value -//// _, err = w.Write(serialize.FastFormatAppend(fieldValue, []byte{})) -//// return -////} -// -////func appendKeyName(lbuf []byte, p *data.Point, fieldName []byte, hashBytes []byte) []byte { -//// lbuf = append(lbuf, p.MeasurementName()...) -//// lbuf = append(lbuf, '_') -//// lbuf = append(lbuf, fieldName...) -//// -//// lbuf = append(lbuf, '{') -//// lbuf = append(lbuf, hashBytes...) -//// lbuf = append(lbuf, '}', ' ') -//// return lbuf -////} -// -////func writeKeyName(w io.Writer, p *data.Point, fieldName []byte, hashBytes []byte) (err error) { -//// w.Write(p.MeasurementName()) -//// w.Write([]byte("_")) -//// w.Write(fieldName) -//// w.Write([]byte("{")) -//// w.Write(hashBytes) -//// _, err = w.Write([]byte("} ")) -//// return -////} diff --git a/pkg/targets/redistimeseries/redistimeseries_test.go b/pkg/targets/redistimeseries/redistimeseries_test.go index a0e0d04d8..7da4c31ef 100644 --- a/pkg/targets/redistimeseries/redistimeseries_test.go +++ b/pkg/targets/redistimeseries/redistimeseries_test.go @@ -1,38 +1,38 @@ package redistimeseries - -import ( - "testing" -) - -func TestRedisTimeSeriesSerializer(t *testing.T) { - cases := []serializeCase{ - { - desc: "a regular Point", - inputPoint: testPointDefault, - output: "TS.CREATE cpu_usage_guest_nice{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest_nice\nTS.MADD cpu_usage_guest_nice{1998426147} 1451606400000 38.24311829\n", - }, - { - desc: "a regular Point using int as value", - inputPoint: testPointInt, - output: "TS.CREATE cpu_usage_guest{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest\nTS.MADD cpu_usage_guest{1998426147} 1451606400000 38\n", - }, - //{ - // desc: "a Point with no tags", - // inputPoint: testPointNoTags, - // output: "TS.ADD cpu_usage_guest_nice{3558706393} 1451606400000 38.24311829 LABELS measurement cpu fieldname usage_guest_nice\n", - //}, - } - - testSerializer(t, cases, &Serializer{}) -} - -func TestRedisTimeSeriesSerializerErr(t *testing.T) { - p := testPointMultiField - s := &RedisTimeSeriesSerializer{} - err := s.Serialize(p, &errWriter{}) - if err == nil { - t.Errorf("no error returned when expected") - } else if err.Error() != errWriterAlwaysErr { - t.Errorf("unexpected writer error: %v", err) - } -} +// +//import ( +// "testing" +//) +// +//func TestRedisTimeSeriesSerializer(t *testing.T) { +// cases := []serializeCase{ +// { +// desc: "a regular Point", +// inputPoint: testPointDefault, +// output: "TS.CREATE cpu_usage_guest_nice{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest_nice\nTS.MADD cpu_usage_guest_nice{1998426147} 1451606400000 38.24311829\n", +// }, +// { +// desc: "a regular Point using int as value", +// inputPoint: testPointInt, +// output: "TS.CREATE cpu_usage_guest{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest\nTS.MADD cpu_usage_guest{1998426147} 1451606400000 38\n", +// }, +// //{ +// // desc: "a Point with no tags", +// // inputPoint: testPointNoTags, +// // output: "TS.ADD cpu_usage_guest_nice{3558706393} 1451606400000 38.24311829 LABELS measurement cpu fieldname usage_guest_nice\n", +// //}, +// } +// +// testSerializer(t, cases, &Serializer{}) +//} +// +//func TestRedisTimeSeriesSerializerErr(t *testing.T) { +// p := testPointMultiField +// s := &RedisTimeSeriesSerializer{} +// err := s.Serialize(p, &errWriter{}) +// if err == nil { +// t.Errorf("no error returned when expected") +// } else if err.Error() != errWriterAlwaysErr { +// t.Errorf("unexpected writer error: %v", err) +// } +//} diff --git a/pkg/targets/redistimeseries/serializer.go b/pkg/targets/redistimeseries/serializer.go index 3d1857821..4929a4515 100644 --- a/pkg/targets/redistimeseries/serializer.go +++ b/pkg/targets/redistimeseries/serializer.go @@ -40,13 +40,9 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) (err error) { for fieldID := 0; fieldID < len(p.FieldKeys()); fieldID++ { fieldName := p.FieldKeys()[fieldID] keyName := fmt.Sprintf("%s%s", hostname, fieldName) - //fmt.Errorf("%s\n",fieldName) - //if hashBytes, hashExists = hashSoFar[keyName]; hashExists == false { //do something here labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) - //hashSoFar[keyName] = hashBytes - //} // if this key was already inserted and created, we don't to specify the labels again if keysSoFar[keyName] == false { @@ -75,9 +71,6 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) (err error) { for fieldID := 0; fieldID < len(p.FieldKeys()); fieldID++ { fieldName := p.FieldKeys()[fieldID] - //keyName := fmt.Sprintf("%s%s", hostname, fieldName) - //fmt.Fprint(os.Stderr, fmt.Sprintf("%s\n", keyName)) - labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) @@ -93,15 +86,6 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) (err error) { return err } -func appendTS_and_Value(lbuf []byte, p *data.Point, fieldValue interface{}) []byte { - // write timestamp in ms - lbuf = serialize.FastFormatAppend(p.Timestamp().UTC().Unix()*1000, lbuf) - lbuf = append(lbuf, ' ') - // write value - lbuf = serialize.FastFormatAppend(fieldValue, lbuf) - return lbuf -} - func writeTS_and_Value(w io.Writer, p *data.Point, fieldValue interface{}) (err error) { // write timestamp in ms w.Write(serialize.FastFormatAppend(p.Timestamp().UTC().Unix()*1000, []byte{})) @@ -111,17 +95,6 @@ func writeTS_and_Value(w io.Writer, p *data.Point, fieldValue interface{}) (err return } -func appendKeyName(lbuf []byte, p *data.Point, fieldName []byte, hashBytes []byte) []byte { - lbuf = append(lbuf, p.MeasurementName()...) - lbuf = append(lbuf, '_') - lbuf = append(lbuf, fieldName...) - - lbuf = append(lbuf, '{') - lbuf = append(lbuf, hashBytes...) - lbuf = append(lbuf, '}', ' ') - return lbuf -} - func writeKeyName(w io.Writer, p *data.Point, fieldName []byte, hashBytes []byte) (err error) { w.Write(p.MeasurementName()) w.Write([]byte("_")) From e9bf8f03fd47dad08f1dda7ed36e80bf5011bee8 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Wed, 3 Mar 2021 16:43:31 +0000 Subject: [PATCH 07/72] [wip] working on merging tsbs with upstream changes --- .circleci/config.yml | 43 ++++++++++--------- Makefile | 7 ++- cmd/tsbs_load_redistimeseries/main.go | 1 - .../redistimeseries/redistimeseries_test.go | 38 ---------------- .../redistimeseries/serializer_test.go | 2 +- 5 files changed, 28 insertions(+), 63 deletions(-) delete mode 100644 pkg/targets/redistimeseries/redistimeseries_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index e0af09b3a..116e98632 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,24 +8,25 @@ jobs: - image: circleci/golang:1.13 working_directory: /go/src/github.com/RedisTimeSeries/tsbs - steps: - - checkout - - run: go get -t -v ./... - - run: GO111MODULE=on go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... - - run: bash <(curl -s https://codecov.io/bash) -t ${CODECOV_TOKEN} - - build-multiarch-docker: - machine: - enabled: true steps: - checkout - run: | - echo "$DOCKER_REDISBENCH_PWD" | base64 --decode | docker login --username $DOCKER_REDISBENCH_USER --password-stdin - - run: - name: Build - command: | - make docker-release - no_output_timeout: 20m + make redistimeseries + make test-rts + +# +# build-multiarch-docker: +# machine: +# enabled: true +# steps: +# - checkout +# - run: | +# echo "$DOCKER_REDISBENCH_PWD" | base64 --decode | docker login --username $DOCKER_REDISBENCH_USER --password-stdin +# - run: +# name: Build +# command: | +# make docker-release +# no_output_timeout: 20m workflows: version: 2 @@ -35,9 +36,9 @@ workflows: filters: tags: only: /.*/ - - build-multiarch-docker: - filters: - tags: - only: /.*/ - branches: - only: master +# - build-multiarch-docker: +# filters: +# tags: +# only: /.*/ +# branches: +# only: master diff --git a/Makefile b/Makefile index ca50b4e39..7b16ce703 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Go parameters -GOCMD=go +GOCMD=GO111MODULE=on go GOBUILD=$(GOCMD) build GOINSTALL=$(GOCMD) install GOCLEAN=$(GOCMD) clean @@ -26,8 +26,11 @@ loaders: tsbs_load_redistimeseries tsbs_load_cassandra tsbs_load_clickhouse tsbs runners: tsbs_run_queries_redistimeseries tsbs_run_queries_cassandra tsbs_run_queries_clickhouse tsbs_run_queries_influx tsbs_run_queries_mongo tsbs_run_queries_siridb tsbs_run_queries_timescaledb +test-rts: + $(GOTEST) -count=1 ./pkg/targets/redistimeseries/. + test: - GO111MODULE=on $(GOTEST) -v -race -coverprofile=coverage.txt -covermode=atomic ./... + $(GOTEST) -v -race -coverprofile=coverage.txt -covermode=atomic ./... tsbs_%: $(wildcard ./cmd/$@/*.go) $(GOGET) ./cmd/$@ diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index bd01c96e7..3bd64302e 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -41,7 +41,6 @@ var ( loader load.BenchmarkRunner config load.BenchmarkRunnerConfig target targets.ImplementedTarget - //bufPool sync.Pool ) // allows for testing diff --git a/pkg/targets/redistimeseries/redistimeseries_test.go b/pkg/targets/redistimeseries/redistimeseries_test.go deleted file mode 100644 index 7da4c31ef..000000000 --- a/pkg/targets/redistimeseries/redistimeseries_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package redistimeseries -// -//import ( -// "testing" -//) -// -//func TestRedisTimeSeriesSerializer(t *testing.T) { -// cases := []serializeCase{ -// { -// desc: "a regular Point", -// inputPoint: testPointDefault, -// output: "TS.CREATE cpu_usage_guest_nice{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest_nice\nTS.MADD cpu_usage_guest_nice{1998426147} 1451606400000 38.24311829\n", -// }, -// { -// desc: "a regular Point using int as value", -// inputPoint: testPointInt, -// output: "TS.CREATE cpu_usage_guest{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest\nTS.MADD cpu_usage_guest{1998426147} 1451606400000 38\n", -// }, -// //{ -// // desc: "a Point with no tags", -// // inputPoint: testPointNoTags, -// // output: "TS.ADD cpu_usage_guest_nice{3558706393} 1451606400000 38.24311829 LABELS measurement cpu fieldname usage_guest_nice\n", -// //}, -// } -// -// testSerializer(t, cases, &Serializer{}) -//} -// -//func TestRedisTimeSeriesSerializerErr(t *testing.T) { -// p := testPointMultiField -// s := &RedisTimeSeriesSerializer{} -// err := s.Serialize(p, &errWriter{}) -// if err == nil { -// t.Errorf("no error returned when expected") -// } else if err.Error() != errWriterAlwaysErr { -// t.Errorf("unexpected writer error: %v", err) -// } -//} diff --git a/pkg/targets/redistimeseries/serializer_test.go b/pkg/targets/redistimeseries/serializer_test.go index fd2a402c6..072151bd9 100644 --- a/pkg/targets/redistimeseries/serializer_test.go +++ b/pkg/targets/redistimeseries/serializer_test.go @@ -10,7 +10,7 @@ func TestInfluxSerializerSerialize(t *testing.T) { { Desc: "a regular Point", InputPoint: serialize.TestPointDefault(), - Output: "TS.CREATE cpu_usage_guest_nice{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest_nice\\nTS.MADD cpu_usage_guest_nice{1998426147} 1451606400000 38.24311829\\n", + Output: "TS.CREATE cpu_usage_guest_nice{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest_nice\nTS.MADD cpu_usage_guest_nice{1998426147} 1451606400000 38.24311829\n", }, //{ // Desc: "a regular Point using int as value", From 85cb7eb0f3640cd1af1cd7315da03b9af6b0cbae Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 4 Mar 2021 16:39:50 +0000 Subject: [PATCH 08/72] [fix] fixed metric ingestion value for RTS --- cmd/tsbs_load_redistimeseries/main.go | 23 ++++++++++++++++------- cmd/tsbs_load_redistimeseries/scan.go | 7 ++++++- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 3bd64302e..69de6b2fd 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -142,13 +142,15 @@ type processor struct { func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint64, conn radix.Client) { curPipe := uint64(0) cmds := make([]radix.CmdAction, 0, 0) + currMetricCount := 0 for row := range rows { - cmd, tscreate := buildCommand(row, compressionEnabled == false) + cmd, tscreate,metricCount := buildCommand(row, compressionEnabled == false) + currMetricCount += metricCount if tscreate { err := conn.Do(cmd) if err != nil { - log.Fatalf("Flush failed with %v", err) + log.Fatalf("TS.CREATE failed with %v", err) } } else { cmds = append(cmds, cmd) @@ -159,7 +161,8 @@ func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint if err != nil { log.Fatalf("Flush failed with %v", err) } - metrics <- curPipe + metrics <- uint64(currMetricCount) + currMetricCount = 0 cmds = make([]radix.CmdAction, 0, 0) curPipe = 0 } @@ -170,9 +173,10 @@ func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint if err != nil { log.Fatalf("Flush failed with %v", err) } - metrics <- curPipe + metrics <- uint64(currMetricCount) cmds = make([]radix.CmdAction, 0, 0) curPipe = 0 + currMetricCount = 0 } wg.Done() } @@ -195,8 +199,10 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) var standalone *radix.Pool if clusterMode { cluster = getOSSClusterConn(host, opts, connections) + defer cluster.Close() } else { standalone = getStandaloneConn(host, opts, connections) + defer standalone.Close() } for i := uint64(0); i < connections; i++ { @@ -237,8 +243,11 @@ func (p *processor) Close(_ bool) { func main() { log.Println("Starting benchmark") config.NoFlowControl = true - config.HashWorkers = true - - loader.RunBenchmark(&benchmark{dbc: &dbCreator{}}) + config.HashWorkers = false + b := benchmark{dbc: &dbCreator{}} + if config.Workers > 1 { + panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) + } + loader.RunBenchmark(&b) log.Println("finished benchmark") } diff --git a/cmd/tsbs_load_redistimeseries/scan.go b/cmd/tsbs_load_redistimeseries/scan.go index cd4362229..161fba994 100644 --- a/cmd/tsbs_load_redistimeseries/scan.go +++ b/cmd/tsbs_load_redistimeseries/scan.go @@ -10,12 +10,17 @@ import ( "github.com/timescale/tsbs/pkg/targets" ) -func buildCommand(line string, forceUncompressed bool) (cmdA radix.CmdAction, tscreate bool) { +func buildCommand(line string, forceUncompressed bool) (cmdA radix.CmdAction, tscreate bool, metricCount int) { t := strings.Split(line, " ") + metricCount = 1 tscreate = false cmdname := t[0] if cmdname == "TS.CREATE" { tscreate = true + metricCount = 0 + } + if cmdname == "TS.MADD" { + metricCount = (len(t)-1)/3 } key := t[1] cmdA = radix.FlatCmd(nil, cmdname, key, t[2:]) From 5c2bdd2ba4cec6d62711283382275cf546f451ce Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 4 Mar 2021 22:14:20 +0000 Subject: [PATCH 09/72] [wip] wip on reducing using client work --- Makefile | 3 + cmd/tsbs_load_redistimeseries/main.go | 169 ++++++++++++------ cmd/tsbs_load_redistimeseries/scan.go | 13 +- pkg/query/redistimeseries_functors.go | 40 ----- pkg/targets/redistimeseries/serializer.go | 40 ++--- .../redistimeseries/serializer_test.go | 2 +- scripts/generate_queries.sh | 19 +- scripts/run_queries_redistimeseries.sh | 7 +- 8 files changed, 159 insertions(+), 134 deletions(-) diff --git a/Makefile b/Makefile index 7b16ce703..8d0a1a9e4 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,9 @@ loaders: tsbs_load_redistimeseries tsbs_load_cassandra tsbs_load_clickhouse tsbs runners: tsbs_run_queries_redistimeseries tsbs_run_queries_cassandra tsbs_run_queries_clickhouse tsbs_run_queries_influx tsbs_run_queries_mongo tsbs_run_queries_siridb tsbs_run_queries_timescaledb +datagen-rts: + ./scripts/generate_queries.sh + test-rts: $(GOTEST) -count=1 ./pkg/targets/redistimeseries/. diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 69de6b2fd..4f3530b96 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -3,10 +3,8 @@ package main import ( "bufio" "crypto/md5" - "encoding/binary" "fmt" "github.com/mediocregopher/radix/v3" - "io" "log" "strconv" "strings" @@ -38,9 +36,14 @@ var ( // Global vars var ( - loader load.BenchmarkRunner - config load.BenchmarkRunnerConfig - target targets.ImplementedTarget + loader load.BenchmarkRunner + config load.BenchmarkRunnerConfig + target targets.ImplementedTarget + cluster *radix.Cluster + standalone *radix.Pool + addresses []string + slots [][][2]uint16 + conns []radix.Client ) // allows for testing @@ -73,6 +76,26 @@ func init() { loader = load.GetBenchmarkRunner(config) + opts := make([]radix.DialOpt, 0) + if clusterMode { + cluster = getOSSClusterConn(host, opts, connections) + cluster.Sync() + topology := cluster.Topo().Primaries().Map() + addresses = make([]string, 0) + slots = make([][][2]uint16, 0) + conns = make([]radix.Client, 0) + for nodeAddress, node := range topology { + addresses = append(addresses, nodeAddress) + slots = append(slots, node.Slots) + conn, _ := cluster.Client(nodeAddress) + conns = append(conns, conn) + } + fmt.Println(addresses) + fmt.Println(slots) + fmt.Println(conns) + } else { + standalone = getStandaloneConn(host, opts, connections) + } } type benchmark struct { @@ -90,13 +113,9 @@ type RedisIndexer struct { func (i *RedisIndexer) GetIndex(p data.LoadedPoint) uint { row := p.Data.(string) - key := strings.Split(row, " ")[1] - start := strings.Index(key, "{") - end := strings.Index(key, "}") - _, _ = io.WriteString(md5h, key[start+1:end]) - hash := binary.LittleEndian.Uint32(md5h.Sum(nil)) - md5h.Reset() - return uint(hash) % i.partitions + slotS := strings.Split(row, " ")[0] + clusterSlot, _ := strconv.ParseInt(slotS, 10, 0) + return uint(clusterSlot) % i.partitions } type fileDataSource struct { @@ -139,44 +158,93 @@ type processor struct { wg *sync.WaitGroup } +func nodeThatContainsSlot(slots [][][2]uint16, slot uint16) (result int) { + result = -1 + for nodePos, slotGroup := range slots { + for _, i2 := range slotGroup { + if slot >= i2[0] && slot <= i2[1] { + result = nodePos + return + } + } + } + return +} + +func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics chan uint64, cluster *radix.Cluster, clusterNodes int, addresses []string, slots [][][2]uint16, conns []radix.Client) { + cmds := make([][]radix.CmdAction, clusterNodes, clusterNodes) + curPipe := make([]uint64, clusterNodes, clusterNodes) + currMetricCount := make([]int, clusterNodes, clusterNodes) + for i := 0; i < clusterNodes; i++ { + cmds[i] = make([]radix.CmdAction, 0, 0) + curPipe[i] = 0 + currMetricCount[i] = 0 + } + + for row := range rows { + slot, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) + comdPos := nodeThatContainsSlot(slots, slot) + currMetricCount[comdPos] += metricCount + cmds[comdPos] = append(cmds[comdPos], cmd) + curPipe[comdPos]++ + + if curPipe[comdPos] == pipeline { + err := conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- uint64(currMetricCount[comdPos]) + currMetricCount[comdPos] = 0 + cmds[comdPos] = make([]radix.CmdAction, 0, 0) + curPipe[comdPos] = 0 + } + + } + for comdPos, u := range curPipe { + if u > 0 { + err := conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- uint64(currMetricCount[comdPos]) + } + } + wg.Done() +} + func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint64, conn radix.Client) { - curPipe := uint64(0) - cmds := make([]radix.CmdAction, 0, 0) + cmds := make([][]radix.CmdAction, 1, 1) + cmds[0] = make([]radix.CmdAction, 0, 0) + curPipe := make([]uint64, 1, 1) + curPipe[0] = 0 currMetricCount := 0 + comdPos := 0 for row := range rows { - cmd, tscreate,metricCount := buildCommand(row, compressionEnabled == false) + _, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) currMetricCount += metricCount - if tscreate { - err := conn.Do(cmd) + cmds[comdPos] = append(cmds[comdPos], cmd) + curPipe[comdPos]++ + + if curPipe[comdPos] == pipeline { + err := conn.Do(radix.Pipeline(cmds[comdPos]...)) if err != nil { - log.Fatalf("TS.CREATE failed with %v", err) - } - } else { - cmds = append(cmds, cmd) - curPipe++ - - if curPipe == pipeline { - err := conn.Do(radix.Pipeline(cmds...)) - if err != nil { - log.Fatalf("Flush failed with %v", err) - } - metrics <- uint64(currMetricCount) - currMetricCount = 0 - cmds = make([]radix.CmdAction, 0, 0) - curPipe = 0 + log.Fatalf("Flush failed with %v", err) } + metrics <- uint64(currMetricCount) + currMetricCount = 0 + cmds[comdPos] = make([]radix.CmdAction, 0, 0) + curPipe[comdPos] = 0 } } - if curPipe > 0 { - err := conn.Do(radix.Pipeline(cmds...)) - if err != nil { - log.Fatalf("Flush failed with %v", err) + for comdPos, u := range curPipe { + if u > 0 { + err := conn.Do(radix.Pipeline(cmds[comdPos]...)) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- uint64(currMetricCount) } - metrics <- uint64(currMetricCount) - cmds = make([]radix.CmdAction, 0, 0) - curPipe = 0 - currMetricCount = 0 } wg.Done() } @@ -188,38 +256,26 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) events := b.(*eventsBatch) rowCnt := uint64(len(events.rows)) metricCnt := uint64(0) - opts := make([]radix.DialOpt, 0) if doLoad { buflen := rowCnt + 1 p.rows = make([]chan string, connections) p.metrics = make(chan uint64, buflen) p.wg = &sync.WaitGroup{} - var cluster *radix.Cluster - var standalone *radix.Pool - if clusterMode { - cluster = getOSSClusterConn(host, opts, connections) - defer cluster.Close() - } else { - standalone = getStandaloneConn(host, opts, connections) - defer standalone.Close() - } for i := uint64(0); i < connections; i++ { p.rows[i] = make(chan string, buflen) p.wg.Add(1) if clusterMode { - go connectionProcessor(p.wg, p.rows[i], p.metrics, cluster) + go connectionProcessorCluster(p.wg, p.rows[i], p.metrics, cluster, len(addresses), addresses, slots, conns) } else { go connectionProcessor(p.wg, p.rows[i], p.metrics, standalone) } } for _, row := range events.rows { - key := strings.Split(row, " ")[1] - start := strings.Index(key, "{") - end := strings.Index(key, "}") - tag, _ := strconv.ParseUint(key[start+1:end], 10, 64) - i := tag % connections + slotS := strings.Split(row, " ")[0] + clusterSlot, _ := strconv.ParseInt(slotS, 10, 0) + i := uint64(clusterSlot) % connections p.rows[i] <- row } @@ -248,6 +304,7 @@ func main() { if config.Workers > 1 { panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) } + loader.RunBenchmark(&b) log.Println("finished benchmark") } diff --git a/cmd/tsbs_load_redistimeseries/scan.go b/cmd/tsbs_load_redistimeseries/scan.go index 161fba994..4db9f1800 100644 --- a/cmd/tsbs_load_redistimeseries/scan.go +++ b/cmd/tsbs_load_redistimeseries/scan.go @@ -2,6 +2,7 @@ package main import ( "github.com/mediocregopher/radix/v3" + "strconv" "strings" "sync" @@ -10,20 +11,22 @@ import ( "github.com/timescale/tsbs/pkg/targets" ) -func buildCommand(line string, forceUncompressed bool) (cmdA radix.CmdAction, tscreate bool, metricCount int) { +func buildCommand(line string, forceUncompressed bool) (clusterSlot uint16, cmdA radix.CmdAction, tscreate bool, metricCount int) { t := strings.Split(line, " ") metricCount = 1 tscreate = false - cmdname := t[0] + v, _ := strconv.ParseInt(t[0], 10, 0) + clusterSlot = uint16(v) + cmdname := t[1] if cmdname == "TS.CREATE" { tscreate = true metricCount = 0 } if cmdname == "TS.MADD" { - metricCount = (len(t)-1)/3 + metricCount = (len(t) - 2) / 3 } - key := t[1] - cmdA = radix.FlatCmd(nil, cmdname, key, t[2:]) + key := t[2] + cmdA = radix.FlatCmd(nil, cmdname, key, t[3:]) return } diff --git a/pkg/query/redistimeseries_functors.go b/pkg/query/redistimeseries_functors.go index 41c1bea74..953cb01b2 100644 --- a/pkg/query/redistimeseries_functors.go +++ b/pkg/query/redistimeseries_functors.go @@ -205,46 +205,6 @@ func MergeSeriesOnTimestamp(series []redistimeseries.Range) MultiRange { return MultiRange{names, labels, datapoints} } -func AvgReducerSeriesDatapoints(series []redistimeseries.Range) (c redistimeseries.Range, err error) { - allNames := make([]string, 0, len(series)) - for _, serie := range series { - allNames = append(allNames, serie.Name) - } - var vPoints = make(map[int64]float64) - var fPoints = make(map[int64]float64) - var cPoints = make(map[int64]int64) - pos := 0 - for pos < len(series) { - serie := series[pos] - for _, v := range serie.DataPoints { - _, found := cPoints[v.Timestamp] - if found == true { - cPoints[v.Timestamp] = cPoints[v.Timestamp] + 1 - vPoints[v.Timestamp] = vPoints[v.Timestamp] + v.Value - fPoints[v.Timestamp] = vPoints[v.Timestamp] / float64(cPoints[v.Timestamp]) - } else { - cPoints[v.Timestamp] = 1 - vPoints[v.Timestamp] = v.Value - fPoints[v.Timestamp] = v.Value - } - } - pos = pos + 1 - } - var keys []int - for k := range cPoints { - keys = append(keys, int(k)) - } - sort.Ints(keys) - datapoints := make([]redistimeseries.DataPoint, 0, len(keys)) - for _, k := range keys { - dp := fPoints[int64(k)] - datapoints = append(datapoints, redistimeseries.DataPoint{int64(k), dp}) - } - name := fmt.Sprintf("avg reduction over %s", strings.Join(allNames, " ")) - c = redistimeseries.Range{name, nil, datapoints} - return -} - func MaxReducerSeriesDatapoints(series []redistimeseries.Range) (c redistimeseries.Range, err error) { allNames := make([]string, 0, len(series)) for _, serie := range series { diff --git a/pkg/targets/redistimeseries/serializer.go b/pkg/targets/redistimeseries/serializer.go index 4929a4515..61c026ab6 100644 --- a/pkg/targets/redistimeseries/serializer.go +++ b/pkg/targets/redistimeseries/serializer.go @@ -1,9 +1,8 @@ package redistimeseries import ( - "crypto/md5" - "encoding/binary" "fmt" + "github.com/mediocregopher/radix/v3" "github.com/timescale/tsbs/pkg/data" "github.com/timescale/tsbs/pkg/data/serialize" "io" @@ -19,10 +18,10 @@ type Serializer struct{} // from. // // This function writes output that looks like: -//cpu_usage_user{md5(hostname=host_0|region=eu-central-1...)} 1451606400 58 LABELS hostname host_0 region eu-central-1 ... measurement cpu fieldname usage_user +// [CLUSTER SLOT of {host_0}] cpu_{host_0}_usage_user 1451606400 58 LABELS hostname host_0 region eu-central-1 ... measurement cpu fieldname usage_user // -// Which the loader will decode into a set of TS.ADD commands for each fieldKey. Once labels have been created for a each fieldKey, -// subsequent rows are ommitted with them and are ingested with TS.MADD for a row's metrics. +// Which the loader will decode into a set of TS.CREATE commands for each fieldKey. Once labels have been created for a each fieldKey, +// subsequent rows are omitted with them and are ingested with TS.MADD for a row's metrics. func (s *Serializer) Serialize(p *data.Point, w io.Writer) (err error) { if keysSoFar == nil { keysSoFar = make(map[string]bool) @@ -35,19 +34,19 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) (err error) { var hashBytes []byte //var hashExists bool p.TagValues() - hostname := p.TagValues()[0] + hostname := []byte(fmt.Sprintf("%s", p.TagValues()[0])) + labelsHash := int(radix.ClusterSlot(hostname)) + hashBytes = serialize.FastFormatAppend( labelsHash, []byte{}) for fieldID := 0; fieldID < len(p.FieldKeys()); fieldID++ { fieldName := p.FieldKeys()[fieldID] keyName := fmt.Sprintf("%s%s", hostname, fieldName) - //do something here - labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) - hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) // if this key was already inserted and created, we don't to specify the labels again if keysSoFar[keyName] == false { - w.Write([]byte("TS.CREATE ")) - writeKeyName(w, p, fieldName, hashBytes) + w.Write(hashBytes) + w.Write([]byte(" TS.CREATE ")) + writeKeyName(w, p, hostname, fieldName) w.Write([]byte("LABELS")) for i, v := range p.TagValues() { w.Write([]byte(" ")) @@ -66,16 +65,13 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) (err error) { keysSoFar[keyName] = true } } - w.Write([]byte("TS.MADD ")) + w.Write(hashBytes) + w.Write([]byte(" TS.MADD ")) for fieldID := 0; fieldID < len(p.FieldKeys()); fieldID++ { fieldName := p.FieldKeys()[fieldID] - - labelsHash := md5.Sum([]byte(fmt.Sprintf("%s", hostname))) - hashBytes = serialize.FastFormatAppend(int(binary.BigEndian.Uint32(labelsHash[:])), []byte{}) - fieldValue := p.FieldValues()[fieldID] - writeKeyName(w, p, fieldName, hashBytes) + writeKeyName(w, p, hostname, fieldName) writeTS_and_Value(w, p, fieldValue) if fieldID < len(p.FieldKeys())-1 { w.Write([]byte(" ")) @@ -95,12 +91,12 @@ func writeTS_and_Value(w io.Writer, p *data.Point, fieldValue interface{}) (err return } -func writeKeyName(w io.Writer, p *data.Point, fieldName []byte, hashBytes []byte) (err error) { +func writeKeyName(w io.Writer, p *data.Point, hostname []byte, fieldName []byte) (err error) { w.Write(p.MeasurementName()) - w.Write([]byte("_")) + w.Write([]byte("_{")) + w.Write(hostname) + w.Write([]byte("}_")) w.Write(fieldName) - w.Write([]byte("{")) - w.Write(hashBytes) - _, err = w.Write([]byte("} ")) + w.Write([]byte(" ")) return } diff --git a/pkg/targets/redistimeseries/serializer_test.go b/pkg/targets/redistimeseries/serializer_test.go index 072151bd9..e7d88cda6 100644 --- a/pkg/targets/redistimeseries/serializer_test.go +++ b/pkg/targets/redistimeseries/serializer_test.go @@ -10,7 +10,7 @@ func TestInfluxSerializerSerialize(t *testing.T) { { Desc: "a regular Point", InputPoint: serialize.TestPointDefault(), - Output: "TS.CREATE cpu_usage_guest_nice{1998426147} LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest_nice\nTS.MADD cpu_usage_guest_nice{1998426147} 1451606400000 38.24311829\n", + Output: "7116 TS.CREATE cpu_{host_0}_usage_guest_nice LABELS hostname host_0 region eu-west-1 datacenter eu-west-1b measurement cpu fieldname usage_guest_nice\n7116 TS.MADD cpu_{host_0}_usage_guest_nice 1451606400000 38.24311829\n", }, //{ // Desc: "a regular Point using int as value", diff --git a/scripts/generate_queries.sh b/scripts/generate_queries.sh index da8183e24..531580dad 100755 --- a/scripts/generate_queries.sh +++ b/scripts/generate_queries.sh @@ -37,13 +37,21 @@ FORMATS=${FORMATS:-"redistimeseries"} #single-groupby-5-1-12 \ #single-groupby-5-8-1" -# All available for generation query types (sorted alphabetically) +# redistimeseries supported query types (sorted alphabetically) QUERY_TYPES_ALL="\ cpu-max-all-1 \ +cpu-max-all-8 \ +double-groupby-1 \ +double-groupby-5 \ +double-groupby-all \ +groupby-orderby-limit \ +lastpoint \ single-groupby-1-1-1 \ single-groupby-1-1-12 \ +single-groupby-1-8-1 \ single-groupby-5-1-1 \ -single-groupby-5-1-12" +single-groupby-5-1-12 \ +single-groupby-5-8-1" # What query types to generate QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} @@ -52,7 +60,7 @@ QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} SCALE=${SCALE:-"100"} # Number of queries to generate -QUERIES=${QUERIES:-"1000000"} +QUERIES=${QUERIES:-"10000"} # Rand seed SEED=${SEED:-"123"} @@ -77,7 +85,7 @@ set -eo pipefail # Loop over all requested queries types and generate data for QUERY_TYPE in ${QUERY_TYPES}; do for FORMAT in ${FORMATS}; do - DATA_FILE_NAME="queries_${FORMAT}_${QUERY_TYPE}_${EXE_FILE_VERSION}_${QUERIES}_${SCALE}_${SEED}_${TS_START}_${TS_END}_${USE_CASE}.dat.gz" + DATA_FILE_NAME="queries_${FORMAT}_${QUERY_TYPE}_${EXE_FILE_VERSION}_${QUERIES}_${SCALE}_${SEED}_${TS_START}_${TS_END}_${USE_CASE}.dat" if [ -f "${DATA_FILE_NAME}" ] && [ "${SKIP_IF_EXISTS}" == "TRUE" ]; then echo "WARNING: file ${DATA_FILE_NAME} already exists, skip generating new data" else @@ -100,8 +108,7 @@ for QUERY_TYPE in ${QUERY_TYPES}; do --timescale-use-json=${USE_JSON} \ --timescale-use-tags=${USE_TAGS} \ --timescale-use-time-bucket=${USE_TIME_BUCKET} \ - --clickhouse-use-tags=${USE_TAGS} \ - | gzip > ${DATA_FILE_NAME} + --clickhouse-use-tags=${USE_TAGS} > ${DATA_FILE_NAME} trap - EXIT # Make short symlink for convenience diff --git a/scripts/run_queries_redistimeseries.sh b/scripts/run_queries_redistimeseries.sh index 92269f0ca..bbeccd90c 100755 --- a/scripts/run_queries_redistimeseries.sh +++ b/scripts/run_queries_redistimeseries.sh @@ -16,7 +16,7 @@ source ${EXE_DIR}/query_common.sh DATABASE_PORT=${DATABASE_PORT:-6379} # Print timing stats to stderr after this many queries (0 to disable) -QUERIES_PRINT_INTERVAL=${QUERIES_PRINT_INTERVAL:-"0"} +QUERIES_PRINT_INTERVAL=${QUERIES_PRINT_INTERVAL:-"1000"} # How many queries would be run MAX_QUERIES=${MAX_QUERIES:-"0"} @@ -24,7 +24,7 @@ REPETITIONS=${REPETITIONS:-3} PREFIX=${PREFIX:-""} # How many queries would be run -SLEEP_BETWEEN_RUNS=${SLEEP_BETWEEN_RUNS:-"60"} +SLEEP_BETWEEN_RUNS=${SLEEP_BETWEEN_RUNS:-"0"} # Ensure DATA DIR available mkdir -p ${RESULTS_DIR} @@ -59,9 +59,8 @@ for run in $(seq ${REPETITIONS}); do echo "Saving output to ${OUT_FULL_FILE_NAME}" echo "Saving HDR Latencies to ${HDR_FULL_FILE_NAME}" - cat $FULL_DATA_FILE_NAME | - $GUNZIP | $EXE_FILE_NAME \ + --file $FULL_DATA_FILE_NAME \ --max-queries=${MAX_QUERIES} \ --workers=${NUM_WORKERS} \ --print-interval=${QUERIES_PRINT_INTERVAL} \ From e338b962b853644ca1f4dfa434c99d3953591ef5 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 4 Mar 2021 23:52:43 +0000 Subject: [PATCH 10/72] [add] moved from client side work to server-side --- .../databases/redistimeseries/devops.go | 39 ++++++++----------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go index 7c8071c15..4c8a72ccb 100644 --- a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go @@ -87,22 +87,19 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t } redisQuery = append(redisQuery, []byte(redisArg)) + if nHosts > 1 && numMetrics == 1 { + redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("hostname"), []byte("REDUCE"), []byte("max") ) + } + if numMetrics > 1 { + redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("fieldname"), []byte("REDUCE"), []byte("max") ) + } + humanLabel := devops.GetSingleGroupByLabel("RedisTimeSeries", numMetrics, nHosts, string(timeRange)) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) d.fillInQueryStrings(qi, humanLabel, humanDesc) d.AddQuery(qi, redisQuery, []byte("TS.MRANGE")) - if numMetrics > 1 && nHosts == 1 { - functorName := query.GetFunctionName(query.SingleGroupByTime) - d.SetApplyFunctor(qi, true, functorName) - } - if nHosts > 1 && numMetrics == 1 { - functorName := query.GetFunctionName(query.GroupByTimeAndMax) - d.SetApplyFunctor(qi, true, functorName) - } - if nHosts > 1 && numMetrics > 1 { - functorName := query.GetFunctionName(query.GroupByTimeAndTagMax) - d.SetApplyFunctor(qi, true, functorName) - } + + } // GroupByTimeAndPrimaryTag selects the AVG of numMetrics metrics under 'cpu' per device per hour for a day @@ -140,6 +137,9 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { } redisQuery = append(redisQuery, []byte(redisArg)) } + if numMetrics > 1 { + redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("hostname"), []byte("REDUCE"), []byte("max") ) + } humanLabel := devops.GetDoubleGroupByLabel("RedisTimeSeries", numMetrics) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) @@ -181,18 +181,14 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) { redisArg += ")" } redisQuery = append(redisQuery, []byte(redisArg)) - + if nHosts > 1 { + redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("fieldname"), []byte("REDUCE"), []byte("max") ) + } humanLabel := devops.GetMaxAllLabel("RedisTimeSeries", nHosts) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) d.fillInQueryStrings(qi, humanLabel, humanDesc) + d.AddQuery(qi, redisQuery, []byte("TS.MRANGE")) - if nHosts == 1 { - functorName := query.GetFunctionName(query.SingleGroupByTime) - d.SetApplyFunctor(qi, true, functorName) - } else { - functorName := query.GetFunctionName(query.GroupByTimeAndTagMax) - d.SetApplyFunctor(qi, true, functorName) - } } // LastPointPerHost finds the last row for every host in the dataset @@ -213,9 +209,6 @@ func (d *Devops) LastPointPerHost(qi query.Query) { humanDesc := fmt.Sprintf("%s", humanLabel) d.fillInQueryStrings(qi, humanLabel, humanDesc) d.AddQuery(qi, redisQuery, []byte("TS.MREVRANGE")) - functorName := query.GetFunctionName(query.GroupByTimeAndTagHostname) - d.SetApplyFunctor(qi, true, functorName) - } func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { From 243ebdbbe7df55b63d5e66fb9ca59931a7003484 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 5 Mar 2021 12:18:49 +0000 Subject: [PATCH 11/72] [wip] improved scripting around redistimeseries tests --- .gitignore | 4 +- scripts/generate_data_redistimeseries.sh | 28 +++++++ scripts/generate_queries_redistimeseries.sh | 59 +++++++++++++++ scripts/load_redistimeseries.sh | 47 +++--------- scripts/query_common.sh | 4 +- scripts/redistimeseries_common.sh | 59 +++++++++++++++ scripts/run_queries_redistimeseries.sh | 84 ++++++--------------- 7 files changed, 183 insertions(+), 102 deletions(-) create mode 100755 scripts/generate_data_redistimeseries.sh create mode 100755 scripts/generate_queries_redistimeseries.sh create mode 100644 scripts/redistimeseries_common.sh diff --git a/.gitignore b/.gitignore index 8af262ac9..12ace57ab 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ bin /docs/responses coverage.txt -bin/ \ No newline at end of file +bin/ + +results/ \ No newline at end of file diff --git a/scripts/generate_data_redistimeseries.sh b/scripts/generate_data_redistimeseries.sh new file mode 100755 index 000000000..82a461361 --- /dev/null +++ b/scripts/generate_data_redistimeseries.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Ensure generator is available +EXE_FILE_NAME=./bin/tsbs_generate_data + +set -x + +EXE_DIR=${EXE_DIR:-$(dirname $0)} +source ${EXE_DIR}/query_common.sh +source ${EXE_DIR}/redistimeseries_common.sh + +INTERLEAVED_GENERATION_GROUPS=${INTERLEAVED_GENERATION_GROUPS:-"1"} + +# Ensure DATA DIR available +mkdir -p ${BULK_DATA_DIR} + +echo "Generating ${DATA_FILE_NAME}:" +${EXE_FILE_NAME} \ + --format=${FORMAT} \ + --use-case=${USE_CASE} \ + --scale=${SCALE} \ + --timestamp-start=${TS_START} \ + --timestamp-end=${TS_END} \ + --debug=${DEBUG} \ + --seed=${SEED} \ + --log-interval=${LOG_INTERVAL} \ + --interleaved-generation-groups=${INTERLEAVED_GENERATION_GROUPS} \ + --max-data-points=${MAX_DATA_POINTS} >${DATA_FILE_NAME} diff --git a/scripts/generate_queries_redistimeseries.sh b/scripts/generate_queries_redistimeseries.sh new file mode 100755 index 000000000..03079333b --- /dev/null +++ b/scripts/generate_queries_redistimeseries.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Ensure generator is available +EXE_FILE_NAME=./bin/tsbs_generate_queries + +set -x + +EXE_DIR=${EXE_DIR:-$(dirname $0)} +source ${EXE_DIR}/query_common.sh +source ${EXE_DIR}/redistimeseries_common.sh +# redistimeseries supported query types (sorted alphabetically) +QUERY_TYPES_ALL="\ +cpu-max-all-1 \ +cpu-max-all-8 \ +double-groupby-1 \ +double-groupby-5 \ +double-groupby-all \ +groupby-orderby-limit \ +lastpoint \ +single-groupby-1-1-1 \ +single-groupby-1-1-12 \ +single-groupby-1-8-1 \ +single-groupby-5-1-1 \ +single-groupby-5-1-12 \ +single-groupby-5-8-1" +#high-cpu-1 \ +#high-cpu-all \ + +# What query types to generate +QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} + +# Number of queries to generate +QUERIES=${QUERIES:-"10000"} + +# Whether to skip data generation if it already exists +SKIP_IF_EXISTS=${SKIP_IF_EXISTS:-"TRUE"} + +# Ensure DATA DIR available +mkdir -p ${BULK_DATA_DIR} + +# Ensure queries dir is clean +rm -rf ${BULK_DATA_DIR}/queries_${USE_CASE}_${FORMAT}_${SCALE}* + +# Loop over all requested queries types and generate data +for QUERY_TYPE in ${QUERY_TYPES}; do + QUERY_DATA_FILE_NAME="queries_${USE_CASE}_${FORMAT}_${SCALE}_${QUERY_TYPE}_${QUERIES}_${SEED}_${TS_START}_${TS_END}.dat" + echo "Generating ${QUERY_DATA_FILE_NAME}:" + ${EXE_FILE_NAME} \ + --format=${FORMAT} \ + --queries=${QUERIES} \ + --query-type=${QUERY_TYPE} \ + --scale=${SCALE} \ + --seed=${SEED} \ + --timestamp-start=${TS_START} \ + --timestamp-end=${TS_END} \ + --use-case=${USE_CASE} \ + > ${BULK_DATA_DIR}/${QUERY_DATA_FILE_NAME} + +done diff --git a/scripts/load_redistimeseries.sh b/scripts/load_redistimeseries.sh index e92329990..29e734308 100755 --- a/scripts/load_redistimeseries.sh +++ b/scripts/load_redistimeseries.sh @@ -1,57 +1,32 @@ #!/bin/bash -# Ensure loader is available -EXE_FILE_NAME=${EXE_FILE_NAME:-$(which tsbs_load_redistimeseries)} -if [[ -z "$EXE_FILE_NAME" ]]; then - echo "tsbs_load_redistimeseries not available. It is not specified explicitly and not found in \$PATH" - exit 1 -fi -FORMAT="redistimeseries" - -DATA_FILE_NAME=${DATA_FILE_NAME:-${FORMAT}-data.gz} -DATABASE_PORT=${DATABASE_PORT:-6379} -CONNECTIONS=${CONNECTIONS:-10} -REPETITIONS=${REPETITIONS:-3} -PIPELINE=${PIPELINE:-100} -EXTENSION="${DATA_FILE_NAME##*.}" -DIR=$(dirname "${DATA_FILE_NAME}") -NO_EXT_DATA_FILE_NAME="${DATA_FILE_NAME%.*}" -PREFIX=${PREFIX:-""} +EXE_FILE_NAME=./bin/tsbs_load_redistimeseries + +set -x EXE_DIR=${EXE_DIR:-$(dirname $0)} -COMPRESSION_ENABLED=${COMPRESSION_ENABLED:-true} -SLEEP_BETWEEN_RUNS=${SLEEP_BETWEEN_RUNS:-"60"} +source ${EXE_DIR}/query_common.sh +source ${EXE_DIR}/redistimeseries_common.sh -# Load parameters - common -source ${EXE_DIR}/load_common.sh +# Ensure RESULTS DIR available +mkdir -p ${RESULTS_DIR} for run in $(seq ${REPETITIONS}); do echo "Running RUN $run" - OUT_FULL_FILE_NAME="${DIR}/${PREFIX}_load_result_${NO_EXT_DATA_FILE_NAME}_run_${run}.out" + OUT_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}_load_result_run_${run}.out" echo "Using only 1 worker" echo "Saving results to ${OUT_FULL_FILE_NAME}" - # Remove previous database - redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} flushall - - # Retrieve command stats output - redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} config resetstat - - # Load new data - cat ${DATA_FILE} | $EXE_FILE_NAME \ + # Load new data + $EXE_FILE_NAME \ + --file ${DATA_FILE_NAME} \ --workers=1 \ --batch-size=${BATCH_SIZE} \ --reporting-period=${REPORTING_PERIOD} \ --host=${DATABASE_HOST}:${DATABASE_PORT} \ - --compression-enabled=${COMPRESSION_ENABLED} \ --connections=${CONNECTIONS} --pipeline=${PIPELINE} | tee ${OUT_FULL_FILE_NAME} - # Retrieve command stats output - redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info commandstats >> ${OUT_FULL_FILE_NAME} - redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info >> ${OUT_FULL_FILE_NAME} - redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info commandstats - echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" sleep ${SLEEP_BETWEEN_RUNS} diff --git a/scripts/query_common.sh b/scripts/query_common.sh index 1a812ee87..c9a478bba 100644 --- a/scripts/query_common.sh +++ b/scripts/query_common.sh @@ -15,8 +15,8 @@ BATCH_SIZE=${BATCH_SIZE:-10000} # Debug DEBUG=${DEBUG:-0} # How many concurrent worker would load data - match num of cores, or default to 8 -NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 8)} +NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 8)} BACKOFF_SECS=${BACKOFF_SECS:-1s} -REPORTING_PERIOD=${REPORTING_PERIOD:-10s} +REPORTING_PERIOD=${REPORTING_PERIOD:-1s} # #set -x diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh new file mode 100644 index 000000000..64ba05662 --- /dev/null +++ b/scripts/redistimeseries_common.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Database credentials +DATABASE_HOST=${DATABASE_HOST:-"localhost"} +DATABASE_PORT=${DATABASE_PORT:-6379} +PIPELINE=${PIPELINE:-100} +CONNECTIONS=${CONNECTIONS:-50} + +# Data folder +BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_data_redistimeseries"} + +# Data folder +RESULTS_DIR=${RESULTS_DIR:-"./results"} + +# Load parameters +BATCH_SIZE=${BATCH_SIZE:-10000} +# Debug +DEBUG=${DEBUG:-0} + +FORMAT="redistimeseries" + +SCALE=${SCALE:-"100"} + +# How many concurrent worker would load data - match num of cores, or default to 8 +NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 8)} +REPORTING_PERIOD=${REPORTING_PERIOD:-1s} + +DATA_FILE_NAME=${DATA_FILE_NAME:-${FORMAT}-data.gz} +REPETITIONS=${REPETITIONS:-3} + +# Start and stop time for generated timeseries +TS_START=${TS_START:-"2016-01-01T00:00:00Z"} +TS_END=${TS_END:-"2016-01-04T00:00:01Z"} + +# Rand seed +SEED=${SEED:-"123"} + +# Print timing stats to stderr after this many queries (0 to disable) +QUERIES_PRINT_INTERVAL=${QUERIES_PRINT_INTERVAL:-"1000"} + +# How many queries would be run +MAX_QUERIES=${MAX_QUERIES:-"0"} +REPETITIONS=${REPETITIONS:-1} +PREFIX=${PREFIX:-""} + +# How many queries would be run +SLEEP_BETWEEN_RUNS=${SLEEP_BETWEEN_RUNS:-"0"} + +# What set of data to generate: devops (multiple data), cpu-only (cpu-usage data) +USE_CASE=${USE_CASE:-"cpu-only"} + +# Step to generate data +LOG_INTERVAL=${LOG_INTERVAL:-"10s"} + +# Max number of points to generate data. 0 means "use TS_START TS_END with LOG_INTERVAL" +MAX_DATA_POINTS=${MAX_DATA_POINTS:-"0"} + +FORMAT=redistimeseries +DATA_FILE_NAME="${BULK_DATA_DIR}/data_${FORMAT}_${USE_CASE}_${SCALE}_${TS_START}_${TS_END}_${LOG_INTERVAL}_${SEED}.dat" diff --git a/scripts/run_queries_redistimeseries.sh b/scripts/run_queries_redistimeseries.sh index bbeccd90c..2bf9eff16 100755 --- a/scripts/run_queries_redistimeseries.sh +++ b/scripts/run_queries_redistimeseries.sh @@ -3,80 +3,38 @@ # Exit immediately if a command exits with a non-zero status. set -e -# Ensure runner is available -EXE_FILE_NAME=${EXE_FILE_NAME:-$(which tsbs_run_queries_redistimeseries)} -if [[ -z "$EXE_FILE_NAME" ]]; then - echo "tsbs_run_queries_redistimeseries not available. It is not specified explicitly and not found in \$PATH" - exit 1 -fi +EXE_FILE_NAME=./bin/tsbs_run_queries_redistimeseries + +set -x EXE_DIR=${EXE_DIR:-$(dirname $0)} source ${EXE_DIR}/query_common.sh +source ${EXE_DIR}/redistimeseries_common.sh -DATABASE_PORT=${DATABASE_PORT:-6379} - -# Print timing stats to stderr after this many queries (0 to disable) -QUERIES_PRINT_INTERVAL=${QUERIES_PRINT_INTERVAL:-"1000"} - -# How many queries would be run -MAX_QUERIES=${MAX_QUERIES:-"0"} -REPETITIONS=${REPETITIONS:-3} -PREFIX=${PREFIX:-""} - -# How many queries would be run -SLEEP_BETWEEN_RUNS=${SLEEP_BETWEEN_RUNS:-"0"} - -# Ensure DATA DIR available +# Ensure RESULTS DIR available mkdir -p ${RESULTS_DIR} -chmod a+rwx ${RESULTS_DIR} -for run in $(seq ${REPETITIONS}); do - for FULL_DATA_FILE_NAME in ${BULK_DATA_DIR}/queries_redistimeseries*; do - # $FULL_DATA_FILE_NAME: /full/path/to/file_with.ext - # $DATA_FILE_NAME: file_with.ext - # $DIR: /full/path/to - # $EXTENSION: ext - # NO_EXT_DATA_FILE_NAME: file_with +for FULL_DATA_FILE_NAME in ${BULK_DATA_DIR}/queries_${USE_CASE}_${FORMAT}_${SCALE}_*; do + for run in $(seq ${REPETITIONS}); do DATA_FILE_NAME=$(basename -- "${FULL_DATA_FILE_NAME}") - DIR=$(dirname "${FULL_DATA_FILE_NAME}") - EXTENSION="${DATA_FILE_NAME##*.}" - NO_EXT_DATA_FILE_NAME="${DATA_FILE_NAME%.*}" - - OUT_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}_result_${NO_EXT_DATA_FILE_NAME}_${run}.out" - HDR_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}_HDR_TXT_result_${NO_EXT_DATA_FILE_NAME}_${run}.out" - - if [ "${EXTENSION}" == "gz" ]; then - GUNZIP="gunzip" - else - GUNZIP="cat" - fi - - echo "Reseting Redis command stats" - redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} config resetstat - - echo "Running ${DATA_FILE_NAME}" - echo "Saving output to ${OUT_FULL_FILE_NAME}" - echo "Saving HDR Latencies to ${HDR_FULL_FILE_NAME}" - - $EXE_FILE_NAME \ - --file $FULL_DATA_FILE_NAME \ - --max-queries=${MAX_QUERIES} \ - --workers=${NUM_WORKERS} \ - --print-interval=${QUERIES_PRINT_INTERVAL} \ - --debug=${DEBUG} \ - --hdr-latencies=${HDR_FULL_FILE_NAME} \ - --host=${DATABASE_HOST}:${DATABASE_PORT} | + OUT_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}_result_${DATA_FILE_NAME}_${run}.out" + HDR_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}_HDR_TXT_result_${DATA_FILE_NAME}_${run}.out" + + $EXE_FILE_NAME \ + --file $FULL_DATA_FILE_NAME \ + --max-queries=${MAX_QUERIES} \ + --workers=${NUM_WORKERS} \ + --print-interval=${QUERIES_PRINT_INTERVAL} \ + --debug=${DEBUG} \ + --hdr-latencies=${HDR_FULL_FILE_NAME} \ + --host=${DATABASE_HOST}:${DATABASE_PORT} | tee $OUT_FULL_FILE_NAME - # Retrieve command stats output - redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info commandstats >> $OUT_FULL_FILE_NAME - redis-cli -h ${DATABASE_HOST} -p ${DATABASE_PORT} info commandstats - - echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" - sleep ${SLEEP_BETWEEN_RUNS} + echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" + sleep ${SLEEP_BETWEEN_RUNS} done echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" sleep ${SLEEP_BETWEEN_RUNS} -done \ No newline at end of file +done From 04661285acf6ba97abf98ef9db68e46621078911 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 5 Mar 2021 12:32:02 +0000 Subject: [PATCH 12/72] [add] Improved load script --- scripts/load_redistimeseries.sh | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/scripts/load_redistimeseries.sh b/scripts/load_redistimeseries.sh index 29e734308..5c16cfa6d 100755 --- a/scripts/load_redistimeseries.sh +++ b/scripts/load_redistimeseries.sh @@ -11,23 +11,16 @@ source ${EXE_DIR}/redistimeseries_common.sh # Ensure RESULTS DIR available mkdir -p ${RESULTS_DIR} -for run in $(seq ${REPETITIONS}); do - echo "Running RUN $run" - OUT_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}_load_result_run_${run}.out" - echo "Using only 1 worker" - echo "Saving results to ${OUT_FULL_FILE_NAME}" - - # Load new data - $EXE_FILE_NAME \ - --file ${DATA_FILE_NAME} \ - --workers=1 \ - --batch-size=${BATCH_SIZE} \ - --reporting-period=${REPORTING_PERIOD} \ - --host=${DATABASE_HOST}:${DATABASE_PORT} \ - --connections=${CONNECTIONS} --pipeline=${PIPELINE} | - tee ${OUT_FULL_FILE_NAME} - - echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" - sleep ${SLEEP_BETWEEN_RUNS} - -done \ No newline at end of file +OUT_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}_load_result.out" +echo "Using only 1 worker" +echo "Saving results to ${OUT_FULL_FILE_NAME}" + +# Load new data +$EXE_FILE_NAME \ + --file ${DATA_FILE_NAME} \ + --workers=1 \ + --batch-size=${BATCH_SIZE} \ + --reporting-period=${REPORTING_PERIOD} \ + --host=${DATABASE_HOST}:${DATABASE_PORT} \ + --connections=${CONNECTIONS} --pipeline=${PIPELINE} | + tee ${OUT_FULL_FILE_NAME} From 2ec34e6db4054f5e36b8499497c9f0476217098a Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 5 Mar 2021 19:50:59 +0000 Subject: [PATCH 13/72] [add] updated default timeframe for 4 full days --- scripts/redistimeseries_common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh index 64ba05662..aa78e551e 100644 --- a/scripts/redistimeseries_common.sh +++ b/scripts/redistimeseries_common.sh @@ -29,8 +29,8 @@ DATA_FILE_NAME=${DATA_FILE_NAME:-${FORMAT}-data.gz} REPETITIONS=${REPETITIONS:-3} # Start and stop time for generated timeseries -TS_START=${TS_START:-"2016-01-01T00:00:00Z"} -TS_END=${TS_END:-"2016-01-04T00:00:01Z"} +TS_START=${TS_START:-"2021-01-01T00:00:00Z"} +TS_END=${TS_END:-"2021-01-05T00:00:01Z"} # Rand seed SEED=${SEED:-"123"} From bfd57ee6bd2e344635467c7dae1b5d34509f4914 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 8 Mar 2021 09:06:27 +0000 Subject: [PATCH 14/72] [add] added cluster flag to loader script --- scripts/load_redistimeseries.sh | 2 +- scripts/redistimeseries_common.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/load_redistimeseries.sh b/scripts/load_redistimeseries.sh index 5c16cfa6d..9b8a81ae1 100755 --- a/scripts/load_redistimeseries.sh +++ b/scripts/load_redistimeseries.sh @@ -21,6 +21,6 @@ $EXE_FILE_NAME \ --workers=1 \ --batch-size=${BATCH_SIZE} \ --reporting-period=${REPORTING_PERIOD} \ - --host=${DATABASE_HOST}:${DATABASE_PORT} \ + --host=${DATABASE_HOST}:${DATABASE_PORT} ${CLUSTER_FLAG} \ --connections=${CONNECTIONS} --pipeline=${PIPELINE} | tee ${OUT_FULL_FILE_NAME} diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh index aa78e551e..e87358528 100644 --- a/scripts/redistimeseries_common.sh +++ b/scripts/redistimeseries_common.sh @@ -20,6 +20,7 @@ DEBUG=${DEBUG:-0} FORMAT="redistimeseries" SCALE=${SCALE:-"100"} +CLUSTER_FLAG=${CLUSTER_FLAG:-""} # How many concurrent worker would load data - match num of cores, or default to 8 NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 8)} From ef4a248bc02bfeb258eee0526ad2288d834b41b6 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 8 Mar 2021 09:57:42 +0000 Subject: [PATCH 15/72] [wip] Enabling cluster mode on rts query runner --- .../cluster_conn.go | 33 +++ cmd/tsbs_run_queries_redistimeseries/main.go | 250 ++++-------------- .../standalone_conn.go | 22 ++ scripts/run_queries_redistimeseries.sh | 2 +- 4 files changed, 105 insertions(+), 202 deletions(-) create mode 100644 cmd/tsbs_run_queries_redistimeseries/cluster_conn.go create mode 100644 cmd/tsbs_run_queries_redistimeseries/standalone_conn.go diff --git a/cmd/tsbs_run_queries_redistimeseries/cluster_conn.go b/cmd/tsbs_run_queries_redistimeseries/cluster_conn.go new file mode 100644 index 000000000..f320038ef --- /dev/null +++ b/cmd/tsbs_run_queries_redistimeseries/cluster_conn.go @@ -0,0 +1,33 @@ +package main + +import ( + "github.com/mediocregopher/radix/v3" + "log" +) + +func getOSSClusterConn(addr string, opts []radix.DialOpt, clients uint64) *radix.Cluster { + var vanillaCluster *radix.Cluster + var err error + + customConnFunc := func(network, addr string) (radix.Conn, error) { + return radix.Dial(network, addr, opts..., + ) + } + + // this cluster will use the ClientFunc to create a pool to each node in the + // cluster. + poolFunc := func(network, addr string) (radix.Client, error) { + return radix.NewPool(network, addr, int(clients), radix.PoolConnFunc(customConnFunc), radix.PoolPipelineWindow(0, 0)) + } + + vanillaCluster, err = radix.NewCluster([]string{addr}, radix.ClusterPoolFunc(poolFunc)) + if err != nil { + log.Fatalf("Error preparing for benchmark, while creating new connection. error = %v", err) + } + // Issue CLUSTER SLOTS command + err = vanillaCluster.Sync() + if err != nil { + log.Fatalf("Error preparing for benchmark, while issuing CLUSTER SLOTS. error = %v", err) + } + return vanillaCluster +} diff --git a/cmd/tsbs_run_queries_redistimeseries/main.go b/cmd/tsbs_run_queries_redistimeseries/main.go index 1c034f2ca..42cbf9da5 100644 --- a/cmd/tsbs_run_queries_redistimeseries/main.go +++ b/cmd/tsbs_run_queries_redistimeseries/main.go @@ -5,19 +5,15 @@ package main import ( - "bytes" - "database/sql" - "encoding/json" "fmt" + "github.com/mediocregopher/radix/v3" "log" - "sort" + "math/rand" "strings" "time" - redistimeseries "github.com/RedisTimeSeries/redistimeseries-go" "github.com/blagojts/viper" _ "github.com/lib/pq" - "github.com/pkg/errors" "github.com/spf13/pflag" "github.com/timescale/tsbs/internal/utils" "github.com/timescale/tsbs/pkg/query" @@ -27,7 +23,13 @@ import ( var ( host string showExplain bool - // scale uint64 + clusterMode bool + cluster *radix.Cluster + standalone *radix.Pool + addresses []string + slots [][][2]uint16 + conns []radix.Client + r *rand.Rand ) // Global vars: @@ -43,17 +45,13 @@ var ( reflect_HighCpu = query.GetFunctionName(query.HighCpu) ) -var ( - redisConnector *redistimeseries.Client -) - // Parse args: func init() { var config query.BenchmarkRunnerConfig config.AddToFlagSet(pflag.CommandLine) pflag.StringVar(&host, "host", "localhost:6379", "Redis host address and port") - + pflag.BoolVar(&clusterMode, "cluster", false, "Whether to use OSS cluster API") pflag.Parse() err := utils.SetupConfigFile() @@ -65,12 +63,32 @@ func init() { if err := viper.Unmarshal(&config); err != nil { panic(fmt.Errorf("unable to decode config: %s", err)) } - runner = query.NewBenchmarkRunner(config) - redisConnector = redistimeseries.NewClient( - host, runner.DatabaseName(), nil) + s := rand.NewSource(time.Now().Unix()) + r = rand.New(s) // initialize local pseudorandom generator + + opts := make([]radix.DialOpt, 0) + if clusterMode { + cluster = getOSSClusterConn(host, opts, uint64(config.Workers)) + cluster.Sync() + topology := cluster.Topo().Primaries().Map() + addresses = make([]string, 0) + slots = make([][][2]uint16, 0) + conns = make([]radix.Client, 0) + for nodeAddress, node := range topology { + addresses = append(addresses, nodeAddress) + slots = append(slots, node.Slots) + conn, _ := cluster.Client(nodeAddress) + conns = append(conns, conn) + } + fmt.Println(addresses) + fmt.Println(slots) + fmt.Println(conns) + } else { + standalone = getStandaloneConn(host, opts, uint64(config.Workers)) + } + runner = query.NewBenchmarkRunner(config) } - func main() { runner.Run(&query.RedisTimeSeriesPool, newProcessor) } @@ -95,115 +113,6 @@ func (p *processor) Init(numWorker int) { } } -func mapRows(r *sql.Rows) []map[string]interface{} { - rows := []map[string]interface{}{} - cols, _ := r.Columns() - for r.Next() { - row := make(map[string]interface{}) - values := make([]interface{}, len(cols)) - for i := range values { - values[i] = new(interface{}) - } - - err := r.Scan(values...) - if err != nil { - panic(errors.Wrap(err, "error while reading values")) - } - - for i, column := range cols { - row[column] = *values[i].(*interface{}) - } - rows = append(rows, row) - } - return rows -} - -// prettyPrintResponseRange prints a Query and its response in JSON format with two -// keys: 'query' which has a value of the RedisTimeseries query used to generate the second key -// 'results' which is an array of each element in the return set. -func prettyPrintResponseRange(responses []interface{}, q *query.RedisTimeSeries) { - full := make(map[string]interface{}) - for idx, qry := range q.RedisQueries { - resp := make(map[string]interface{}) - fullcmd := append([][]byte{q.CommandNames[idx]}, qry...) - resp["query"] = strings.Join(ByteArrayToStringArray(fullcmd), " ") - - res := responses[idx] - switch v := res.(type) { - case []redistimeseries.Range: - resp["client_side_work"] = q.ApplyFunctor - rows := []map[string]interface{}{} - for _, r := range res.([]redistimeseries.Range) { - row := make(map[string]interface{}) - values := make(map[string]interface{}) - values["datapoints"] = r.DataPoints - values["labels"] = r.Labels - row[r.Name] = values - rows = append(rows, row) - } - resp["results"] = rows - case redistimeseries.Range: - resp["client_side_work"] = q.ApplyFunctor - resp["results"] = res.(redistimeseries.Range) - case []query.MultiRange: - resp["client_side_work"] = q.ApplyFunctor - rows := []map[string]interface{}{} - for _, converted := range res.([]query.MultiRange) { - query_result := map[string]interface{}{} - //converted := r.(query.MultiRange) - query_result["names"] = converted.Names - query_result["labels"] = converted.Labels - datapoints := make([]query.MultiDataPoint, 0, len(converted.DataPoints)) - var keys []int - for k := range converted.DataPoints { - keys = append(keys, int(k)) - } - sort.Ints(keys) - for _, k := range keys { - dp := converted.DataPoints[int64(k)] - time_str := time.Unix(dp.Timestamp/1000, 0).Format(time.RFC3339) - dp.HumanReadbleTime = &time_str - datapoints = append(datapoints, dp) - } - query_result["datapoints"] = datapoints - rows = append(rows, query_result) - } - resp["results"] = rows - case query.MultiRange: - resp["client_side_work"] = q.ApplyFunctor - query_result := map[string]interface{}{} - converted := res.(query.MultiRange) - query_result["names"] = converted.Names - query_result["labels"] = converted.Labels - datapoints := make([]query.MultiDataPoint, 0, len(converted.DataPoints)) - var keys []int - for k := range converted.DataPoints { - keys = append(keys, int(k)) - } - sort.Ints(keys) - for _, k := range keys { - dp := converted.DataPoints[int64(k)] - time_str := time.Unix(dp.Timestamp/1000, 0).Format(time.RFC3339) - dp.HumanReadbleTime = &time_str - datapoints = append(datapoints, dp) - } - query_result["datapoints"] = datapoints - resp["results"] = query_result - default: - fmt.Printf("I don't know about type %T!\n", v) - } - - full[fmt.Sprintf("query %d", idx+1)] = resp - } - - line, err := json.MarshalIndent(full, "", " ") - if err != nil { - panic(err) - } - - fmt.Println(string(line) + "\n") -} - func (p *processor) ProcessQuery(q query.Query, isWarm bool) (queryStats []*query.Stat, err error) { // No need to run again for EXPLAIN @@ -211,101 +120,40 @@ func (p *processor) ProcessQuery(q query.Query, isWarm bool) (queryStats []*quer return nil, nil } tq := q.(*query.RedisTimeSeries) - var parsedResponses = make([]interface{}, 0, 0) - var cmds = make([][]interface{}, 0, 0) + var cmds = make([][]string, 0, 0) for _, qry := range tq.RedisQueries { - cmds = append(cmds, ByteArrayToInterfaceArray(qry)) + cmds = append(cmds, ByteArrayToStringArray(qry)) } - conn := redisConnector.Pool.Get() - start := time.Now() for idx, commandArgs := range cmds { - var result interface{} + var err error = nil if p.opts.debug { fmt.Println(fmt.Sprintf("Issuing command (%s %s)", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "))) } - res, err := conn.Do(string(tq.CommandNames[idx]), commandArgs...) + if clusterMode { + if string(tq.CommandNames[idx]) == "TS.MRANGE" || string(tq.CommandNames[idx]) == "TS.QUERYINDEX" || string(tq.CommandNames[idx]) == "TS.MGET" || string(tq.CommandNames[idx]) == "TS.MREVRANGE" { + rPos := r.Intn(len(conns)) + conn := conns[rPos] + err = conn.Do(radix.Cmd(nil, string(tq.CommandNames[idx]), commandArgs...)) + } else { + err = cluster.Do(radix.Cmd(nil, string(tq.CommandNames[idx]), commandArgs...)) + } + } else { + err = standalone.Do(radix.Cmd(nil, string(tq.CommandNames[idx]), commandArgs...)) + } if err != nil { log.Fatalf("Command (%s %s) failed with error: %v\n", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "), err) } if err != nil { return nil, err } - if bytes.Compare(tq.CommandNames[idx], cmdMrange) == 0 || bytes.Compare(tq.CommandNames[idx], cmdMRevRange) == 0 { - - if err != nil { - return nil, err - } - if tq.ApplyFunctor { - if p.opts.debug { - fmt.Println(fmt.Sprintf("Applying functor %s on %s", tq.Functor, tq.HumanLabel)) - } - switch tq.Functor { - case reflect_SingleGroupByTime: - if p.opts.debug { - fmt.Println(fmt.Sprintf("Applying functor reflect_SingleGroupByTime %s", reflect_SingleGroupByTime)) - } - result, err = query.SingleGroupByTime(res) - if err != nil { - return nil, err - } - case reflect_GroupByTimeAndMax: - if p.opts.debug { - fmt.Println(fmt.Sprintf("Applying functor reflect_GroupByTimeAndMax %s", reflect_GroupByTimeAndMax)) - } - result, err = query.GroupByTimeAndMax(res) - if err != nil { - return nil, err - } - case reflect_GroupByTimeAndTagMax: - if p.opts.debug { - fmt.Println(fmt.Sprintf("Applying functor reflect_GroupByTimeAndTagMax %s", reflect_GroupByTimeAndTagMax)) - } - result, err = query.GroupByTimeAndTagMax(res) - if err != nil { - return nil, err - } - case reflect_GroupByTimeAndTagHostname: - if p.opts.debug { - fmt.Println(fmt.Sprintf("Applying functor reflect_GroupByTimeAndTagHostname %s", reflect_GroupByTimeAndTagHostname)) - } - result, err = query.GroupByTimeAndTagHostname(res) - if err != nil { - return nil, err - } - case reflect_HighCpu: - if p.opts.debug { - fmt.Println(fmt.Sprintf("Applying functor reflect_HighCpu %s", reflect_HighCpu)) - } - result, err = query.HighCpu(res) - if err != nil { - return nil, err - } - default: - errors.Errorf("The selected functor %s is not known!\n", tq.Functor) - } - } else { - result, err = redistimeseries.ParseRanges(res) - if err != nil { - return nil, err - } - } - - } else if bytes.Compare(tq.CommandNames[idx], cmdQueryIndex) == 0 { - var parsedRes = make([]redistimeseries.Range, 0, 0) - parsedResponses = append(parsedResponses, parsedRes) - } - parsedResponses = append(parsedResponses, result) } took := float64(time.Since(start).Nanoseconds()) / 1e6 - if p.opts.printResponse { - prettyPrintResponseRange(parsedResponses, tq) - } + stat := query.GetStat() stat.Init(q.HumanLabelName(), took) queryStats = []*query.Stat{stat} - return queryStats, err } diff --git a/cmd/tsbs_run_queries_redistimeseries/standalone_conn.go b/cmd/tsbs_run_queries_redistimeseries/standalone_conn.go new file mode 100644 index 000000000..01f34959d --- /dev/null +++ b/cmd/tsbs_run_queries_redistimeseries/standalone_conn.go @@ -0,0 +1,22 @@ +package main + +import ( + "github.com/mediocregopher/radix/v3" + "log" +) + +func getStandaloneConn(addr string, opts []radix.DialOpt, clients uint64) *radix.Pool { + var pool *radix.Pool + var err error + + customConnFunc := func(network, addr string) (radix.Conn, error) { + return radix.Dial(network, addr, opts..., + ) + } + network := "tcp" + pool, err = radix.NewPool(network, addr, int(clients), radix.PoolConnFunc(customConnFunc), radix.PoolPipelineWindow(0, 0)) + if err != nil { + log.Fatalf("Error preparing for benchmark, while creating new connection. error = %v", err) + } + return pool +} diff --git a/scripts/run_queries_redistimeseries.sh b/scripts/run_queries_redistimeseries.sh index 2bf9eff16..76e48b65a 100755 --- a/scripts/run_queries_redistimeseries.sh +++ b/scripts/run_queries_redistimeseries.sh @@ -28,7 +28,7 @@ for FULL_DATA_FILE_NAME in ${BULK_DATA_DIR}/queries_${USE_CASE}_${FORMAT}_${SCAL --print-interval=${QUERIES_PRINT_INTERVAL} \ --debug=${DEBUG} \ --hdr-latencies=${HDR_FULL_FILE_NAME} \ - --host=${DATABASE_HOST}:${DATABASE_PORT} | + --host=${DATABASE_HOST}:${DATABASE_PORT} ${CLUSTER_FLAG} | tee $OUT_FULL_FILE_NAME echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" From 30d943e266ce367fb2d0b921af3674acd7c958a3 Mon Sep 17 00:00:00 2001 From: Danni Moiseyev Date: Tue, 9 Mar 2021 10:08:45 +0200 Subject: [PATCH 16/72] use `MGET` for lastpoint query --- .../databases/redistimeseries/devops.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go index 4c8a72ccb..80011a3d5 100644 --- a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go @@ -194,11 +194,6 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) { // LastPointPerHost finds the last row for every host in the dataset func (d *Devops) LastPointPerHost(qi query.Query) { redisQuery := [][]byte{ - //[]byte("TS.MREVRANGE"), Just to help understanding - []byte("-"), - []byte("+"), - []byte("COUNT"), - []byte("1"), []byte("WITHLABELS"), []byte("FILTER"), []byte("measurement=cpu"), @@ -208,7 +203,7 @@ func (d *Devops) LastPointPerHost(qi query.Query) { humanLabel := "RedisTimeSeries last row per host" humanDesc := fmt.Sprintf("%s", humanLabel) d.fillInQueryStrings(qi, humanLabel, humanDesc) - d.AddQuery(qi, redisQuery, []byte("TS.MREVRANGE")) + d.AddQuery(qi, redisQuery, []byte("TS.MGET")) } func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { From 9bb285c21dbb3fcd8443887a1485f5a1ea127a33 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 21 Mar 2021 22:18:53 +0000 Subject: [PATCH 17/72] [add] avoiding pipeline when we only have one command on the slice --- cmd/tsbs_load_redistimeseries/main.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 4f3530b96..ded90e8a3 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -189,7 +189,12 @@ func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics ch curPipe[comdPos]++ if curPipe[comdPos] == pipeline { - err := conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) + var err error = nil + if len(cmds[comdPos])==1{ + err = conns[comdPos].Do(cmds[comdPos][0]) + } else { + err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) + } if err != nil { log.Fatalf("Flush failed with %v", err) } @@ -202,7 +207,13 @@ func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics ch } for comdPos, u := range curPipe { if u > 0 { - err := conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) + var err error = nil + if len(cmds[comdPos])==1{ + err = conns[comdPos].Do(cmds[comdPos][0]) + } else { + err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) + } + err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) if err != nil { log.Fatalf("Flush failed with %v", err) } From 5b05cfc4663c9d3ec7f48070f2e0d460a70f947e Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 21 Mar 2021 22:30:22 +0000 Subject: [PATCH 18/72] [wip] wip on cluster slot issue --- cmd/tsbs_load_redistimeseries/main.go | 4 ++-- cmd/tsbs_load_redistimeseries/scan.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index ded90e8a3..77a7d42b2 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -158,11 +158,11 @@ type processor struct { wg *sync.WaitGroup } -func nodeThatContainsSlot(slots [][][2]uint16, slot uint16) (result int) { +func nodeThatContainsSlot(slots [][][2]uint16, slot int) (result int) { result = -1 for nodePos, slotGroup := range slots { for _, i2 := range slotGroup { - if slot >= i2[0] && slot <= i2[1] { + if slot >= int(i2[0]) && slot <= int(i2[1]) { result = nodePos return } diff --git a/cmd/tsbs_load_redistimeseries/scan.go b/cmd/tsbs_load_redistimeseries/scan.go index 4db9f1800..6eb27b4b5 100644 --- a/cmd/tsbs_load_redistimeseries/scan.go +++ b/cmd/tsbs_load_redistimeseries/scan.go @@ -11,12 +11,12 @@ import ( "github.com/timescale/tsbs/pkg/targets" ) -func buildCommand(line string, forceUncompressed bool) (clusterSlot uint16, cmdA radix.CmdAction, tscreate bool, metricCount int) { +func buildCommand(line string, forceUncompressed bool) (clusterSlot int, cmdA radix.CmdAction, tscreate bool, metricCount int) { t := strings.Split(line, " ") metricCount = 1 tscreate = false v, _ := strconv.ParseInt(t[0], 10, 0) - clusterSlot = uint16(v) + clusterSlot = int(v) cmdname := t[1] if cmdname == "TS.CREATE" { tscreate = true From 60044c769bca7081e217baabfa136cc070fe8ea0 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 21 Mar 2021 22:37:43 +0000 Subject: [PATCH 19/72] [fix] cluster slots upper limit on radix is exclusivegit add cmd/tsbs_load_redistimeseries/main.go ( different from redis spec ) --- cmd/tsbs_load_redistimeseries/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 77a7d42b2..7519f9b16 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -162,7 +162,7 @@ func nodeThatContainsSlot(slots [][][2]uint16, slot int) (result int) { result = -1 for nodePos, slotGroup := range slots { for _, i2 := range slotGroup { - if slot >= int(i2[0]) && slot <= int(i2[1]) { + if slot >= int(i2[0]) && slot < int(i2[1]) { result = nodePos return } From 5b15ccba93efae63ea6da36e44936d55708afee4 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 21 Mar 2021 22:41:46 +0000 Subject: [PATCH 20/72] [fix] cluster slots upper limit on radix is exclusivegit add cmd/tsbs_load_redistimeseries/main.go ( different from redis spec ) --- cmd/tsbs_load_redistimeseries/main.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 7519f9b16..e54b5fc43 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -190,11 +190,7 @@ func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics ch if curPipe[comdPos] == pipeline { var err error = nil - if len(cmds[comdPos])==1{ - err = conns[comdPos].Do(cmds[comdPos][0]) - } else { - err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) - } + err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) if err != nil { log.Fatalf("Flush failed with %v", err) } @@ -208,11 +204,6 @@ func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics ch for comdPos, u := range curPipe { if u > 0 { var err error = nil - if len(cmds[comdPos])==1{ - err = conns[comdPos].Do(cmds[comdPos][0]) - } else { - err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) - } err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) if err != nil { log.Fatalf("Flush failed with %v", err) From bf41a794dd454427315f8167e54672a575522c19 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 13:58:01 +0000 Subject: [PATCH 21/72] [add] small update on RTS scripts --- scripts/generate_data.sh | 2 +- scripts/run_queries_redistimeseries.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_data.sh b/scripts/generate_data.sh index a16608a9a..4b9b43ae4 100755 --- a/scripts/generate_data.sh +++ b/scripts/generate_data.sh @@ -72,7 +72,7 @@ for FORMAT in ${FORMATS}; do --seed=${SEED} \ --log-interval=${LOG_INTERVAL} \ --interleaved-generation-groups=${INTERLEAVED_GENERATION_GROUPS} \ - --max-data-points=${MAX_DATA_POINTS} > ${DATA_FILE_NAME} + --max-data-points=${MAX_DATA_POINTS} --file ${DATA_FILE_NAME} # | gzip > ${DATA_FILE_NAME} trap - EXIT diff --git a/scripts/run_queries_redistimeseries.sh b/scripts/run_queries_redistimeseries.sh index 76e48b65a..53ca34992 100755 --- a/scripts/run_queries_redistimeseries.sh +++ b/scripts/run_queries_redistimeseries.sh @@ -5,7 +5,7 @@ set -e EXE_FILE_NAME=./bin/tsbs_run_queries_redistimeseries -set -x +#set -x EXE_DIR=${EXE_DIR:-$(dirname $0)} source ${EXE_DIR}/query_common.sh From 14112b2bef181f3516ede8aea480c19c7265ab48 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 14:20:20 +0000 Subject: [PATCH 22/72] [wip] Improved scripts for RTS --- README.md | 11 +++----- scripts/generate_data.sh | 4 +-- scripts/generate_data_redistimeseries.sh | 2 -- scripts/{ => load}/load_redistimeseries.sh | 1 + scripts/redistimeseries_common.sh | 27 ++++++++++--------- .../run_queries_redistimeseries.sh | 0 6 files changed, 21 insertions(+), 24 deletions(-) rename scripts/{ => load}/load_redistimeseries.sh (98%) rename scripts/{ => run_queries}/run_queries_redistimeseries.sh (100%) diff --git a/README.md b/README.md index 95eac7e30..0b1c48c94 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Current databases supported: + CrateDB [(supplemental docs)](docs/cratedb.md) + InfluxDB [(supplemental docs)](docs/influx.md) + MongoDB [(supplemental docs)](docs/mongo.md) ++ RedisTimeSeries [(supplemental docs)](docs/redistimeseries.md) + SiriDB [(supplemental docs)](docs/siridb.md) + TimescaleDB [(supplemental docs)](docs/timescaledb.md) + Timestream [(supplemental docs)](docs/timestream.md) @@ -75,6 +76,7 @@ cases are implemented for each database: |CrateDB|X|| |InfluxDB|X|X| |MongoDB|X| +|RedisTimeSeries|X| |SiriDB|X| |TimescaleDB|X|X| |Timestream|X|| @@ -105,13 +107,8 @@ scripts). The easiest way to get and install the Go programs is to use `go get` and then `go install`: ```bash # Fetch TSBS and its dependencies -$ go get github.com/timescale/tsbs -$ cd $GOPATH/src/github.com/timescale/tsbs/cmd -$ go get ./... -$ git remote add redistimeseries https://github.com/RedisTimeSeries/tsbs.git -$ git fetch -$ git checkout redistimeseries -$ cd $GOPATH/src/github.com/timescale/tsbs +$ git clone https://github.com/RedisTimeSeries/tsbs.git --branch redistimeseries-v1.4 +$ cd tsbs $ make $ FORMATS="redistimeseries" SCALE=100 SEED=123 TS_END="2016-01-31T00:00:00Z" ./scripts/generate_data.sh # Benchmark part. in PREFIX add a text that will be prefixed on the output results file diff --git a/scripts/generate_data.sh b/scripts/generate_data.sh index 4b9b43ae4..bcad41a37 100755 --- a/scripts/generate_data.sh +++ b/scripts/generate_data.sh @@ -72,8 +72,8 @@ for FORMAT in ${FORMATS}; do --seed=${SEED} \ --log-interval=${LOG_INTERVAL} \ --interleaved-generation-groups=${INTERLEAVED_GENERATION_GROUPS} \ - --max-data-points=${MAX_DATA_POINTS} --file ${DATA_FILE_NAME} -# | gzip > ${DATA_FILE_NAME} + --max-data-points=${MAX_DATA_POINTS} \ + | gzip > ${DATA_FILE_NAME} trap - EXIT # Make short symlink for convenience diff --git a/scripts/generate_data_redistimeseries.sh b/scripts/generate_data_redistimeseries.sh index 82a461361..f5b8bcb1b 100755 --- a/scripts/generate_data_redistimeseries.sh +++ b/scripts/generate_data_redistimeseries.sh @@ -9,8 +9,6 @@ EXE_DIR=${EXE_DIR:-$(dirname $0)} source ${EXE_DIR}/query_common.sh source ${EXE_DIR}/redistimeseries_common.sh -INTERLEAVED_GENERATION_GROUPS=${INTERLEAVED_GENERATION_GROUPS:-"1"} - # Ensure DATA DIR available mkdir -p ${BULK_DATA_DIR} diff --git a/scripts/load_redistimeseries.sh b/scripts/load/load_redistimeseries.sh similarity index 98% rename from scripts/load_redistimeseries.sh rename to scripts/load/load_redistimeseries.sh index 9b8a81ae1..247f49c59 100755 --- a/scripts/load_redistimeseries.sh +++ b/scripts/load/load_redistimeseries.sh @@ -3,6 +3,7 @@ EXE_FILE_NAME=./bin/tsbs_load_redistimeseries set -x +set -e EXE_DIR=${EXE_DIR:-$(dirname $0)} source ${EXE_DIR}/query_common.sh diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh index e87358528..ed53c4646 100644 --- a/scripts/redistimeseries_common.sh +++ b/scripts/redistimeseries_common.sh @@ -6,10 +6,7 @@ DATABASE_PORT=${DATABASE_PORT:-6379} PIPELINE=${PIPELINE:-100} CONNECTIONS=${CONNECTIONS:-50} -# Data folder -BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_data_redistimeseries"} - -# Data folder +# Results folder RESULTS_DIR=${RESULTS_DIR:-"./results"} # Load parameters @@ -17,8 +14,6 @@ BATCH_SIZE=${BATCH_SIZE:-10000} # Debug DEBUG=${DEBUG:-0} -FORMAT="redistimeseries" - SCALE=${SCALE:-"100"} CLUSTER_FLAG=${CLUSTER_FLAG:-""} @@ -26,13 +21,8 @@ CLUSTER_FLAG=${CLUSTER_FLAG:-""} NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 8)} REPORTING_PERIOD=${REPORTING_PERIOD:-1s} -DATA_FILE_NAME=${DATA_FILE_NAME:-${FORMAT}-data.gz} REPETITIONS=${REPETITIONS:-3} -# Start and stop time for generated timeseries -TS_START=${TS_START:-"2021-01-01T00:00:00Z"} -TS_END=${TS_END:-"2021-01-05T00:00:01Z"} - # Rand seed SEED=${SEED:-"123"} @@ -50,11 +40,22 @@ SLEEP_BETWEEN_RUNS=${SLEEP_BETWEEN_RUNS:-"0"} # What set of data to generate: devops (multiple data), cpu-only (cpu-usage data) USE_CASE=${USE_CASE:-"cpu-only"} -# Step to generate data +########################## +# Data generation related +# For benchmarking read latency, we used the following setup for each database (the machine configuration is the same as the one used in the Insert comparison): +# Dataset: 100–4,000 simulated devices generated 1–10 CPU metrics every 10 seconds for 4 full days (100M+ reading intervals, 1B+ metrics) +# 10,000 batch size should be used + +# Start and stop time for generated timeseries +TS_START=${TS_START:-"2016-01-01T00:00:00Z"} +TS_END=${TS_END:-"2016-01-04T00:00:00Z"} + LOG_INTERVAL=${LOG_INTERVAL:-"10s"} # Max number of points to generate data. 0 means "use TS_START TS_END with LOG_INTERVAL" MAX_DATA_POINTS=${MAX_DATA_POINTS:-"0"} -FORMAT=redistimeseries +FORMAT=${FORMAT:-"redistimeseries"} +INTERLEAVED_GENERATION_GROUPS=${INTERLEAVED_GENERATION_GROUPS:-"1"} +BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_data_${BULK_DATA_DIR}"} DATA_FILE_NAME="${BULK_DATA_DIR}/data_${FORMAT}_${USE_CASE}_${SCALE}_${TS_START}_${TS_END}_${LOG_INTERVAL}_${SEED}.dat" diff --git a/scripts/run_queries_redistimeseries.sh b/scripts/run_queries/run_queries_redistimeseries.sh similarity index 100% rename from scripts/run_queries_redistimeseries.sh rename to scripts/run_queries/run_queries_redistimeseries.sh From f65e197c40526c7afae71ca219d6e53b4868c26d Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 14:37:06 +0000 Subject: [PATCH 23/72] [fix] Fixed influxdb runner --- scripts/run_queries/run_queries_influx.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/run_queries/run_queries_influx.sh b/scripts/run_queries/run_queries_influx.sh index 749902e63..14cd48209 100755 --- a/scripts/run_queries/run_queries_influx.sh +++ b/scripts/run_queries/run_queries_influx.sh @@ -44,8 +44,11 @@ function run_file() cat $FULL_DATA_FILE_NAME \ | $GUNZIP \ | $EXE_FILE_NAME \ - --max-queries $MAX_QUERIES \ - --workers $NUM_WORKERS \ + --max-queries=${MAX_QUERIES} \ + --db-name=${DATABASE_NAME} \ + --workers=${NUM_WORKERS} \ + --batch-size=${BATCH_SIZE} \ + --urls=http://${DATABASE_HOST}:${DATABASE_PORT} \ | tee $OUT_FULL_FILE_NAME } From f81978b9f4cc8e7d09742f1fcd36cbaaaa698589 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 14:37:39 +0000 Subject: [PATCH 24/72] [fix] Fixed influxdb runner --- scripts/run_queries/run_queries_influx.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/run_queries/run_queries_influx.sh b/scripts/run_queries/run_queries_influx.sh index 14cd48209..d4557bc89 100755 --- a/scripts/run_queries/run_queries_influx.sh +++ b/scripts/run_queries/run_queries_influx.sh @@ -47,7 +47,6 @@ function run_file() --max-queries=${MAX_QUERIES} \ --db-name=${DATABASE_NAME} \ --workers=${NUM_WORKERS} \ - --batch-size=${BATCH_SIZE} \ --urls=http://${DATABASE_HOST}:${DATABASE_PORT} \ | tee $OUT_FULL_FILE_NAME } From f4a41064566a82b17e6e63842badc845edf05e84 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 14:38:49 +0000 Subject: [PATCH 25/72] [fix] Fixed influxdb runner --- scripts/run_queries/run_queries_influx.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/run_queries/run_queries_influx.sh b/scripts/run_queries/run_queries_influx.sh index d4557bc89..8e107a75f 100755 --- a/scripts/run_queries/run_queries_influx.sh +++ b/scripts/run_queries/run_queries_influx.sh @@ -7,12 +7,22 @@ if [[ -z "$EXE_FILE_NAME" ]]; then exit 1 fi +DATABASE_PORT=${DATABASE_PORT:-8086} + +EXE_DIR=${EXE_DIR:-$(dirname $0)} +source ${EXE_DIR}/load_common.sh + # Default queries folder BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"} MAX_QUERIES=${MAX_QUERIES:-"0"} # How many concurrent worker would run queries - match num of cores, or default to 4 NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 4)} +until curl http://${DATABASE_HOST}:${DATABASE_PORT}/ping 2>/dev/null; do + echo "Waiting for InfluxDB" + sleep 1 +done + # # Run test for one file # From 7c6fbba4aa9788db076506418686b789acf77fd5 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 14:54:48 +0000 Subject: [PATCH 26/72] [wip] Improved influx runner --- scripts/run_queries/run_queries_influx.sh | 49 +++++++++++++++-------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/scripts/run_queries/run_queries_influx.sh b/scripts/run_queries/run_queries_influx.sh index 8e107a75f..cbc98f17f 100755 --- a/scripts/run_queries/run_queries_influx.sh +++ b/scripts/run_queries/run_queries_influx.sh @@ -14,9 +14,17 @@ source ${EXE_DIR}/load_common.sh # Default queries folder BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"} + +# How many queries would be run MAX_QUERIES=${MAX_QUERIES:-"0"} + # How many concurrent worker would run queries - match num of cores, or default to 4 -NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 4)} +NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 4)} + +# Print timing stats to stderr after this many queries (0 to disable) +QUERIES_PRINT_INTERVAL=${QUERIES_PRINT_INTERVAL:-"0"} + +REPETITIONS=${REPETITIONS:-3} until curl http://${DATABASE_HOST}:${DATABASE_PORT}/ping 2>/dev/null; do echo "Waiting for InfluxDB" @@ -26,8 +34,7 @@ done # # Run test for one file # -function run_file() -{ +function run_file() { # $FULL_DATA_FILE_NAME: /full/path/to/file_with.ext # $DATA_FILE_NAME: file_with.ext # $DIR: /full/path/to @@ -39,26 +46,34 @@ function run_file() EXTENSION="${DATA_FILE_NAME##*.}" NO_EXT_DATA_FILE_NAME="${DATA_FILE_NAME%.*}" - # Several options on how to name results file - #OUT_FULL_FILE_NAME="${DIR}/result_${DATA_FILE_NAME}" - OUT_FULL_FILE_NAME="${DIR}/result_${NO_EXT_DATA_FILE_NAME}.out" - #OUT_FULL_FILE_NAME="${DIR}/${NO_EXT_DATA_FILE_NAME}.out" - if [ "${EXTENSION}" == "gz" ]; then GUNZIP="gunzip" else GUNZIP="cat" fi - echo "Running ${DATA_FILE_NAME}" - cat $FULL_DATA_FILE_NAME \ - | $GUNZIP \ - | $EXE_FILE_NAME \ - --max-queries=${MAX_QUERIES} \ - --db-name=${DATABASE_NAME} \ - --workers=${NUM_WORKERS} \ - --urls=http://${DATABASE_HOST}:${DATABASE_PORT} \ - | tee $OUT_FULL_FILE_NAME + for run in $(seq ${REPETITIONS}); do + # Several options on how to name results file + #OUT_FULL_FILE_NAME="${DIR}/result_${DATA_FILE_NAME}" + OUT_FULL_FILE_NAME="${DIR}/result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + #OUT_FULL_FILE_NAME="${DIR}/${NO_EXT_DATA_FILE_NAME}.out" + HDR_FULL_FILE_NAME="${DIR}/HDR_TXT_result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + + echo "Running ${DATA_FILE_NAME}" + echo " Saving results to ${OUT_FULL_FILE_NAME}" + echo " Saving HDR results to ${HDR_FULL_FILE_NAME}" + + cat $FULL_DATA_FILE_NAME | + $GUNZIP | + $EXE_FILE_NAME \ + --max-queries=${MAX_QUERIES} \ + --db-name=${DATABASE_NAME} \ + --workers=${NUM_WORKERS} \ + --print-interval=${QUERIES_PRINT_INTERVAL} \ + --hdr-latencies=${HDR_FULL_FILE_NAME} \ + --urls=http://${DATABASE_HOST}:${DATABASE_PORT} | + tee $OUT_FULL_FILE_NAME + done } if [ "$#" -gt 0 ]; then From 18dc6e2f6038258f8d62ac107fbfef55f8197135 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 14:58:50 +0000 Subject: [PATCH 27/72] [wip] Improved influx runner --- scripts/run_queries/run_queries_influx.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/run_queries/run_queries_influx.sh b/scripts/run_queries/run_queries_influx.sh index cbc98f17f..e30fd4ccf 100755 --- a/scripts/run_queries/run_queries_influx.sh +++ b/scripts/run_queries/run_queries_influx.sh @@ -12,6 +12,12 @@ DATABASE_PORT=${DATABASE_PORT:-8086} EXE_DIR=${EXE_DIR:-$(dirname $0)} source ${EXE_DIR}/load_common.sh +# Results folder +RESULTS_DIR=${RESULTS_DIR:-"./results"} + +# Debug +DEBUG=${DEBUG:-0} + # Default queries folder BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"} @@ -31,6 +37,9 @@ until curl http://${DATABASE_HOST}:${DATABASE_PORT}/ping 2>/dev/null; do sleep 1 done +# Ensure RESULTS DIR available +mkdir -p ${RESULTS_DIR} + # # Run test for one file # @@ -55,9 +64,9 @@ function run_file() { for run in $(seq ${REPETITIONS}); do # Several options on how to name results file #OUT_FULL_FILE_NAME="${DIR}/result_${DATA_FILE_NAME}" - OUT_FULL_FILE_NAME="${DIR}/result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + OUT_FULL_FILE_NAME="${RESULTS_DIR}/result_${NO_EXT_DATA_FILE_NAME}_${run}.out" #OUT_FULL_FILE_NAME="${DIR}/${NO_EXT_DATA_FILE_NAME}.out" - HDR_FULL_FILE_NAME="${DIR}/HDR_TXT_result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + HDR_FULL_FILE_NAME="${RESULTS_DIR}/HDR_TXT_result_${NO_EXT_DATA_FILE_NAME}_${run}.out" echo "Running ${DATA_FILE_NAME}" echo " Saving results to ${OUT_FULL_FILE_NAME}" @@ -71,6 +80,7 @@ function run_file() { --workers=${NUM_WORKERS} \ --print-interval=${QUERIES_PRINT_INTERVAL} \ --hdr-latencies=${HDR_FULL_FILE_NAME} \ + --debug=${DEBUG} \ --urls=http://${DATABASE_HOST}:${DATABASE_PORT} | tee $OUT_FULL_FILE_NAME done From 7ea41c288b8fc04a9038bfa1f773eb47ba99f05d Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 15:09:07 +0000 Subject: [PATCH 28/72] [wip] Improved influx runner --- scripts/generate_queries.sh | 40 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/generate_queries.sh b/scripts/generate_queries.sh index 531580dad..2335db1eb 100755 --- a/scripts/generate_queries.sh +++ b/scripts/generate_queries.sh @@ -19,25 +19,7 @@ USE_TIME_BUCKET=${USE_TIME_BUCKET:-true} # Space-separated list of target DB formats to generate FORMATS=${FORMATS:-"redistimeseries"} -# All available for generation query types (sorted alphabetically) -#QUERY_TYPES_ALL="\ -#cpu-max-all-1 \ -#cpu-max-all-8 \ -#double-groupby-1 \ -#double-groupby-5 \ -#double-groupby-all \ -#groupby-orderby-limit \ -#high-cpu-1 \ -#high-cpu-all \ -#lastpoint \ -#single-groupby-1-1-1 \ -#single-groupby-1-1-12 \ -#single-groupby-1-8-1 \ -#single-groupby-5-1-1 \ -#single-groupby-5-1-12 \ -#single-groupby-5-8-1" - -# redistimeseries supported query types (sorted alphabetically) + All available for generation query types (sorted alphabetically) QUERY_TYPES_ALL="\ cpu-max-all-1 \ cpu-max-all-8 \ @@ -45,6 +27,8 @@ double-groupby-1 \ double-groupby-5 \ double-groupby-all \ groupby-orderby-limit \ +high-cpu-1 \ +high-cpu-all \ lastpoint \ single-groupby-1-1-1 \ single-groupby-1-1-12 \ @@ -52,6 +36,22 @@ single-groupby-1-8-1 \ single-groupby-5-1-1 \ single-groupby-5-1-12 \ single-groupby-5-8-1" +# +## redistimeseries supported query types (sorted alphabetically) +#QUERY_TYPES_ALL="\ +#cpu-max-all-1 \ +#cpu-max-all-8 \ +#double-groupby-1 \ +#double-groupby-5 \ +#double-groupby-all \ +#groupby-orderby-limit \ +#lastpoint \ +#single-groupby-1-1-1 \ +#single-groupby-1-1-12 \ +#single-groupby-1-8-1 \ +#single-groupby-5-1-1 \ +#single-groupby-5-1-12 \ +#single-groupby-5-8-1" # What query types to generate QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} @@ -67,7 +67,7 @@ SEED=${SEED:-"123"} # Start and stop time for generated timeseries TS_START=${TS_START:-"2016-01-01T00:00:00Z"} -TS_END=${TS_END:-"2016-01-31T00:00:01Z"} +TS_END=${TS_END:-"2016-01-04T00:00:00Z"} # What set of data to generate: devops (multiple data), cpu-only (cpu-usage data) USE_CASE=${USE_CASE:-"cpu-only"} From 8da87fd5bef7daf9c5fdfc400202171c252d0810 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 15:09:49 +0000 Subject: [PATCH 29/72] [wip] Improved influx runner --- scripts/generate_queries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_queries.sh b/scripts/generate_queries.sh index 2335db1eb..252954fe6 100755 --- a/scripts/generate_queries.sh +++ b/scripts/generate_queries.sh @@ -19,7 +19,7 @@ USE_TIME_BUCKET=${USE_TIME_BUCKET:-true} # Space-separated list of target DB formats to generate FORMATS=${FORMATS:-"redistimeseries"} - All available for generation query types (sorted alphabetically) +# All available for generation query types (sorted alphabetically) QUERY_TYPES_ALL="\ cpu-max-all-1 \ cpu-max-all-8 \ From a4343e63d86a6b802ea54d96ea8909773fde03e7 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 15:34:52 +0000 Subject: [PATCH 30/72] [wip] Fixing influxdb query runner --- Makefile | 2 ++ cmd/tsbs_generate_queries/databases/influx/common.go | 2 ++ pkg/query/factories/init_factories.go | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8d0a1a9e4..5fa8b2e71 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ all: generators loaders runners generators: tsbs_generate_data tsbs_generate_queries +influx: tsbs_generate_data tsbs_generate_queries tsbs_load_influx tsbs_run_queries_influx + redistimeseries: tsbs_generate_data tsbs_generate_queries tsbs_load_redistimeseries tsbs_run_queries_redistimeseries loaders: tsbs_load_redistimeseries tsbs_load_cassandra tsbs_load_clickhouse tsbs_load_influx tsbs_load_mongo tsbs_load_siridb tsbs_load_timescaledb diff --git a/cmd/tsbs_generate_queries/databases/influx/common.go b/cmd/tsbs_generate_queries/databases/influx/common.go index 19558fb07..65cf012ab 100644 --- a/cmd/tsbs_generate_queries/databases/influx/common.go +++ b/cmd/tsbs_generate_queries/databases/influx/common.go @@ -13,6 +13,7 @@ import ( // BaseGenerator contains settings specific for Influx database. type BaseGenerator struct { + DBName string } // GenerateEmptyQuery returns an empty query.HTTP. @@ -23,6 +24,7 @@ func (g *BaseGenerator) GenerateEmptyQuery() query.Query { // fillInQuery fills the query struct with data. func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, influxql string) { v := url.Values{} + v.Set("db", g.DBName) v.Set("q", influxql) q := qi.(*query.HTTP) q.HumanLabel = []byte(humanLabel) diff --git a/pkg/query/factories/init_factories.go b/pkg/query/factories/init_factories.go index c5325719a..8220c4631 100644 --- a/pkg/query/factories/init_factories.go +++ b/pkg/query/factories/init_factories.go @@ -23,7 +23,9 @@ func InitQueryFactories(config *config.QueryGeneratorConfig) map[string]interfac UseTags: config.ClickhouseUseTags, } factories[constants.FormatCrateDB] = &cratedb.BaseGenerator{} - factories[constants.FormatInflux] = &influx.BaseGenerator{} + factories[constants.FormatInflux] = &influx.BaseGenerator{ + DBName: config.DbName, + } factories[constants.FormatTimescaleDB] = ×caledb.BaseGenerator{ UseJSON: config.TimescaleUseJSON, UseTags: config.TimescaleUseTags, From afc3e863fcb556f33c0a34253df9311475bab0e0 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 15:40:36 +0000 Subject: [PATCH 31/72] [wip] Fixing influxdb query runner --- cmd/tsbs_generate_queries/databases/influx/common.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/influx/common.go b/cmd/tsbs_generate_queries/databases/influx/common.go index 65cf012ab..41e68d5ca 100644 --- a/cmd/tsbs_generate_queries/databases/influx/common.go +++ b/cmd/tsbs_generate_queries/databases/influx/common.go @@ -24,14 +24,13 @@ func (g *BaseGenerator) GenerateEmptyQuery() query.Query { // fillInQuery fills the query struct with data. func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, influxql string) { v := url.Values{} - v.Set("db", g.DBName) v.Set("q", influxql) q := qi.(*query.HTTP) q.HumanLabel = []byte(humanLabel) q.RawQuery = []byte(influxql) q.HumanDescription = []byte(humanDesc) q.Method = []byte("POST") - q.Path = []byte(fmt.Sprintf("/query?%s", v.Encode())) + q.Path = []byte(fmt.Sprintf("/query?db=%s&%s", g.DBName, v.Encode())) q.Body = nil } From 1b8f021f08292863ed5bae51d34e25b301c6493b Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 15:55:05 +0000 Subject: [PATCH 32/72] [fix] Fixed influx runner --- cmd/tsbs_generate_queries/databases/influx/common.go | 3 +-- pkg/query/factories/init_factories.go | 4 +--- scripts/run_queries/run_queries_influx.sh | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/influx/common.go b/cmd/tsbs_generate_queries/databases/influx/common.go index 41e68d5ca..19558fb07 100644 --- a/cmd/tsbs_generate_queries/databases/influx/common.go +++ b/cmd/tsbs_generate_queries/databases/influx/common.go @@ -13,7 +13,6 @@ import ( // BaseGenerator contains settings specific for Influx database. type BaseGenerator struct { - DBName string } // GenerateEmptyQuery returns an empty query.HTTP. @@ -30,7 +29,7 @@ func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, influ q.RawQuery = []byte(influxql) q.HumanDescription = []byte(humanDesc) q.Method = []byte("POST") - q.Path = []byte(fmt.Sprintf("/query?db=%s&%s", g.DBName, v.Encode())) + q.Path = []byte(fmt.Sprintf("/query?%s", v.Encode())) q.Body = nil } diff --git a/pkg/query/factories/init_factories.go b/pkg/query/factories/init_factories.go index 8220c4631..c5325719a 100644 --- a/pkg/query/factories/init_factories.go +++ b/pkg/query/factories/init_factories.go @@ -23,9 +23,7 @@ func InitQueryFactories(config *config.QueryGeneratorConfig) map[string]interfac UseTags: config.ClickhouseUseTags, } factories[constants.FormatCrateDB] = &cratedb.BaseGenerator{} - factories[constants.FormatInflux] = &influx.BaseGenerator{ - DBName: config.DbName, - } + factories[constants.FormatInflux] = &influx.BaseGenerator{} factories[constants.FormatTimescaleDB] = ×caledb.BaseGenerator{ UseJSON: config.TimescaleUseJSON, UseTags: config.TimescaleUseTags, diff --git a/scripts/run_queries/run_queries_influx.sh b/scripts/run_queries/run_queries_influx.sh index e30fd4ccf..e0aa4be43 100755 --- a/scripts/run_queries/run_queries_influx.sh +++ b/scripts/run_queries/run_queries_influx.sh @@ -10,7 +10,7 @@ fi DATABASE_PORT=${DATABASE_PORT:-8086} EXE_DIR=${EXE_DIR:-$(dirname $0)} -source ${EXE_DIR}/load_common.sh +source ${EXE_DIR}/../load/load_common.sh # Results folder RESULTS_DIR=${RESULTS_DIR:-"./results"} From 4e576f12be8ad1e4e6378e37d6ab67788e4f9296 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 15:58:30 +0000 Subject: [PATCH 33/72] [fix] Fixed influx runner --- scripts/run_queries/run_common.sh | 26 +++++++++++++++++++++++ scripts/run_queries/run_queries_influx.sh | 21 +----------------- 2 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 scripts/run_queries/run_common.sh diff --git a/scripts/run_queries/run_common.sh b/scripts/run_queries/run_common.sh new file mode 100644 index 000000000..5445a6b49 --- /dev/null +++ b/scripts/run_queries/run_common.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Database credentials +DATABASE_HOST=${DATABASE_HOST:-"localhost"} +DATABASE_NAME=${DATABASE_NAME:-"benchmark"} + + +# Default queries folder +BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"} + +# Results folder +RESULTS_DIR=${RESULTS_DIR:-"./results"} + +# Debug +DEBUG=${DEBUG:-0} + +# How many queries would be run +MAX_QUERIES=${MAX_QUERIES:-"0"} + +# How many concurrent worker would run queries - match num of cores, or default to 4 +NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 4)} + +# Print timing stats to stderr after this many queries (0 to disable) +QUERIES_PRINT_INTERVAL=${QUERIES_PRINT_INTERVAL:-"0"} + +REPETITIONS=${REPETITIONS:-3} \ No newline at end of file diff --git a/scripts/run_queries/run_queries_influx.sh b/scripts/run_queries/run_queries_influx.sh index e0aa4be43..37b86c353 100755 --- a/scripts/run_queries/run_queries_influx.sh +++ b/scripts/run_queries/run_queries_influx.sh @@ -10,27 +10,8 @@ fi DATABASE_PORT=${DATABASE_PORT:-8086} EXE_DIR=${EXE_DIR:-$(dirname $0)} -source ${EXE_DIR}/../load/load_common.sh +source ${EXE_DIR}/run_common.sh -# Results folder -RESULTS_DIR=${RESULTS_DIR:-"./results"} - -# Debug -DEBUG=${DEBUG:-0} - -# Default queries folder -BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"} - -# How many queries would be run -MAX_QUERIES=${MAX_QUERIES:-"0"} - -# How many concurrent worker would run queries - match num of cores, or default to 4 -NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 4)} - -# Print timing stats to stderr after this many queries (0 to disable) -QUERIES_PRINT_INTERVAL=${QUERIES_PRINT_INTERVAL:-"0"} - -REPETITIONS=${REPETITIONS:-3} until curl http://${DATABASE_HOST}:${DATABASE_PORT}/ping 2>/dev/null; do echo "Waiting for InfluxDB" From 836aa7354bfd708dd7d7301b75962fbb18dfba58 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 18:09:36 +0000 Subject: [PATCH 34/72] [add] Added helper to run rate-limited queries on RTS --- scripts/load/load_redistimeseries.sh | 4 +- scripts/produce_query_results_table.py | 108 ++++++++++++++++++ scripts/query_common.sh | 2 +- scripts/redistimeseries_common.sh | 3 - .../run_queries_redistimeseries.sh | 4 +- ...ies_redistimeseries_rate_limited_influx.sh | 54 +++++++++ 6 files changed, 167 insertions(+), 8 deletions(-) create mode 100755 scripts/produce_query_results_table.py create mode 100755 scripts/run_queries/run_queries_redistimeseries_rate_limited_influx.sh diff --git a/scripts/load/load_redistimeseries.sh b/scripts/load/load_redistimeseries.sh index 247f49c59..0fd13cd94 100755 --- a/scripts/load/load_redistimeseries.sh +++ b/scripts/load/load_redistimeseries.sh @@ -6,8 +6,8 @@ set -x set -e EXE_DIR=${EXE_DIR:-$(dirname $0)} -source ${EXE_DIR}/query_common.sh -source ${EXE_DIR}/redistimeseries_common.sh +source ${EXE_DIR}/../query_common.sh +source ${EXE_DIR}/../redistimeseries_common.sh # Ensure RESULTS DIR available mkdir -p ${RESULTS_DIR} diff --git a/scripts/produce_query_results_table.py b/scripts/produce_query_results_table.py new file mode 100755 index 000000000..e88e3ab8a --- /dev/null +++ b/scripts/produce_query_results_table.py @@ -0,0 +1,108 @@ +import argparse +import json +import os +import re + + +def process_txt_files(dirname: str, database: str, queries: [str], prefix: str = ""): + files_list = os.listdir(dirname) + p = re.compile('Run complete after (\d+) queries with (\d) workers \(Overall query rate (\d+\.\d*) queries\/sec\):\n(.+)\:\n.+\nall queries.+:\n.+med:\s+(\d+.\d*)ms,.+, count: (\d+)\n') + results_table = {} + for query in queries: + query_table = {} + for fname in files_list: + if fname.startswith("result_queries") and database in fname and query in fname and ((prefix != "" and prefix in fname) or (prefix == "")): + if database not in query_table: + query_table[database] = [] + full_path = "{}/{}".format(dirname, fname) + with open(full_path) as txt_file: + result = txt_file.read() + match = p.match(result) + if match is not None: + query_description = match.group(4) + number_workers = match.group(2) + total_queries = match.group(1) + overall_query_rate = match.group(3) + overall_query_p50 = match.group(5) + result_line = { + "query_name": query, + "query_description": query_description, + "number_workers": number_workers, + "total_queries": total_queries, + "overall_query_rate": overall_query_rate, + "overall_query_p50": overall_query_p50, + } + query_table[database].append(result_line) + results_table[query] = query_table + return results_table + + +def get_best_results_for(query_results, metric_name, database, query, functor=min): + metric_value = None + best_pos = -1 + distinct_results = 0 + if database in query_results: + database_results = query_results[database] + total_results_arr = [] + for result in database_results: + total_results_arr.append(result[metric_name]) + distinct_results = len(total_results_arr) + if distinct_results: + metric_value = functor(total_results_arr) + best_pos = total_results_arr.index(metric_value) + return metric_value, best_pos, distinct_results + + +def get_result_at_pos(query_results, metric_name, database, query, pos): + metric_value = None + if database in query_results: + database_results = query_results[database] + metric_value = database_results[pos][metric_name] + return metric_value + + +parser = argparse.ArgumentParser( + description="Simple script to process TSBS query runners results and output overall metrics", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) +parser.add_argument("--dir", type=str, required=True) +parser.add_argument("--prefix", type=str, default="", + help="prefix to filter the result files by") +parser.add_argument("--database", type=str, required=True, + help="database to filter the result files by") +parser.add_argument("--query", type=str, required=True, + help="comma separated query types search for results files") + +args = parser.parse_args() + +results_table = process_txt_files(args.dir, args.database, args.query.split(","), args.prefix) +print("-------------------") +print("Database, Query Type, Distinct Runs, Workers, Total queries, p50 latency @rps, rps") +database = args.database +for query, query_results in results_table.items(): + overall_query_p50_str = "n/a" + overall_query_rate_str = "n/a" + number_workers_str = "n/a" + total_queries_str = "n/a" + distinct_runs_str = "n/a" + + overall_query_p50, best_pos, distinct_runs = get_best_results_for( + query_results, "overall_query_p50", database, query, min) + if overall_query_p50 is not None: + overall_query_p50_str = "{}".format(overall_query_p50) + distinct_runs_str = "{}".format(distinct_runs) + overall_query_rate = get_result_at_pos( + query_results, "overall_query_rate", database, query, best_pos) + if overall_query_rate is not None: + overall_query_rate_str = "{}".format(overall_query_rate) + number_workers = get_result_at_pos( + query_results, "number_workers", database, query, best_pos) + if number_workers is not None: + number_workers_str = "{}".format(number_workers) + total_queries = get_result_at_pos( + query_results, "total_queries", database, query, best_pos) + if total_queries is not None: + total_queries_str = "{}".format(total_queries) + + print("{}, {}, {}, {}, {}, {}, {}".format(database, query, distinct_runs_str, + number_workers_str, total_queries_str, overall_query_p50_str, overall_query_rate_str)) diff --git a/scripts/query_common.sh b/scripts/query_common.sh index c9a478bba..63c690139 100644 --- a/scripts/query_common.sh +++ b/scripts/query_common.sh @@ -8,7 +8,7 @@ DATABASE_NAME=${DATABASE_NAME:-"benchmark"} BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"} # Data folder -RESULTS_DIR=${RESULTS_DIR:-"/tmp/results_queries"} +RESULTS_DIR=${RESULTS_DIR:-"./results"} # Load parameters BATCH_SIZE=${BATCH_SIZE:-10000} diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh index ed53c4646..db81e6ca1 100644 --- a/scripts/redistimeseries_common.sh +++ b/scripts/redistimeseries_common.sh @@ -6,9 +6,6 @@ DATABASE_PORT=${DATABASE_PORT:-6379} PIPELINE=${PIPELINE:-100} CONNECTIONS=${CONNECTIONS:-50} -# Results folder -RESULTS_DIR=${RESULTS_DIR:-"./results"} - # Load parameters BATCH_SIZE=${BATCH_SIZE:-10000} # Debug diff --git a/scripts/run_queries/run_queries_redistimeseries.sh b/scripts/run_queries/run_queries_redistimeseries.sh index 53ca34992..4c003861e 100755 --- a/scripts/run_queries/run_queries_redistimeseries.sh +++ b/scripts/run_queries/run_queries_redistimeseries.sh @@ -8,8 +8,8 @@ EXE_FILE_NAME=./bin/tsbs_run_queries_redistimeseries #set -x EXE_DIR=${EXE_DIR:-$(dirname $0)} -source ${EXE_DIR}/query_common.sh -source ${EXE_DIR}/redistimeseries_common.sh +source ${EXE_DIR}/../query_common.sh +source ${EXE_DIR}/../redistimeseries_common.sh # Ensure RESULTS DIR available mkdir -p ${RESULTS_DIR} diff --git a/scripts/run_queries/run_queries_redistimeseries_rate_limited_influx.sh b/scripts/run_queries/run_queries_redistimeseries_rate_limited_influx.sh new file mode 100755 index 000000000..69ba39d3a --- /dev/null +++ b/scripts/run_queries/run_queries_redistimeseries_rate_limited_influx.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +set -e + +EXE_FILE_NAME=./bin/tsbs_run_queries_redistimeseries + +# set -x + +EXE_DIR=${EXE_DIR:-$(dirname $0)} +source ${EXE_DIR}/../query_common.sh +source ${EXE_DIR}/../redistimeseries_common.sh + +# Ensure RESULTS DIR available +mkdir -p ${RESULTS_DIR} +RPS_ARRAY=("cpu-max-all-1":"4139" "cpu-max-all-8":"588" "double-groupby-1":"324" "double-groupby-5":"70" "double-groupby-all":"35" "groupby-orderby-limit":"28" "high-cpu-1":"1816" "high-cpu-all":"18" "lastpoint":"488" "single-groupby-1-1-1":"2320" "single-groupby-1-1-12":"2320" "single-groupby-1-8-1":"4458" "single-groupby-5-1-1":"648" "single-groupby-5-1-12":"648" "single-groupby-5-8-1":"1158") + +for FULL_DATA_FILE_NAME in ${BULK_DATA_DIR}/queries_${USE_CASE}_${FORMAT}_${SCALE}_*; do + for run in $(seq ${REPETITIONS}); do + + DATA_FILE_NAME=$(basename -- "${FULL_DATA_FILE_NAME}") + OUT_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}result_${DATA_FILE_NAME}_${run}.out" + HDR_FULL_FILE_NAME="${RESULTS_DIR}/${PREFIX}HDR_TXT_result_${DATA_FILE_NAME}_${run}.out" + MAX_QUERY_RPS="0" + for rps_settings in "${RPS_ARRAY[@]}"; do + KEY="${rps_settings%%:*}" + VALUE="${rps_settings##*:}" + if [[ $DATA_FILE_NAME == *"$KEY"* ]]; then + echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" + echo "Using rate-limit info to run query $KEY. Input file data $DATA_FILE_NAME" + echo "Rate limit: $VALUE" + echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" + MAX_QUERY_RPS=$VALUE + fi + done + + $EXE_FILE_NAME \ + --file $FULL_DATA_FILE_NAME \ + --max-queries=${MAX_QUERIES} \ + --max-rps=${MAX_QUERY_RPS} \ + --workers=${NUM_WORKERS} \ + --print-interval=${QUERIES_PRINT_INTERVAL} \ + --debug=${DEBUG} \ + --hdr-latencies=${HDR_FULL_FILE_NAME} \ + --host=${DATABASE_HOST}:${DATABASE_PORT} ${CLUSTER_FLAG} | + tee $OUT_FULL_FILE_NAME + + echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" + sleep ${SLEEP_BETWEEN_RUNS} + done + echo "Sleeping for ${SLEEP_BETWEEN_RUNS} seconds" + sleep ${SLEEP_BETWEEN_RUNS} + +done From 219304919ce8f6e16073b08c951fda455737df86 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 18:10:31 +0000 Subject: [PATCH 35/72] [add] updated rate-limit script --- scripts/produce_query_results_table.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/produce_query_results_table.py b/scripts/produce_query_results_table.py index e88e3ab8a..5b1f60cbf 100755 --- a/scripts/produce_query_results_table.py +++ b/scripts/produce_query_results_table.py @@ -3,10 +3,12 @@ import os import re +all_query_types = "cpu-max-all-1,cpu-max-all-8,double-groupby-1,double-groupby-5,double-groupby-all,groupby-orderby-limit,high-cpu-1,high-cpu-all,lastpoint,single-groupby-1-1-1,single-groupby-1-1-12,single-groupby-1-8-1,single-groupby-5-1-1,single-groupby-5-1-12,single-groupby-5-8-1" + def process_txt_files(dirname: str, database: str, queries: [str], prefix: str = ""): files_list = os.listdir(dirname) - p = re.compile('Run complete after (\d+) queries with (\d) workers \(Overall query rate (\d+\.\d*) queries\/sec\):\n(.+)\:\n.+\nall queries.+:\n.+med:\s+(\d+.\d*)ms,.+, count: (\d+)\n') + p = re.compile('Run complete after (\d+) queries with (\d+) workers.+\(Overall query rate (\d+\.\d*) queries\/sec\):\n(.+)\:\n.+\nall queries.+:\n.+med:\s+(\d+.\d*)ms,.+, count: (\d+)\n') results_table = {} for query in queries: query_table = {} @@ -15,10 +17,12 @@ def process_txt_files(dirname: str, database: str, queries: [str], prefix: str = if database not in query_table: query_table[database] = [] full_path = "{}/{}".format(dirname, fname) + with open(full_path) as txt_file: result = txt_file.read() match = p.match(result) if match is not None: + print(full_path) query_description = match.group(4) number_workers = match.group(2) total_queries = match.group(1) @@ -70,21 +74,26 @@ def get_result_at_pos(query_results, metric_name, database, query, pos): help="prefix to filter the result files by") parser.add_argument("--database", type=str, required=True, help="database to filter the result files by") -parser.add_argument("--query", type=str, required=True, +parser.add_argument("--query", type=str, default=all_query_types, help="comma separated query types search for results files") args = parser.parse_args() -results_table = process_txt_files(args.dir, args.database, args.query.split(","), args.prefix) +results_table = process_txt_files( + args.dir, args.database, args.query.split(","), args.prefix) print("-------------------") print("Database, Query Type, Distinct Runs, Workers, Total queries, p50 latency @rps, rps") database = args.database +bash_hash_table_of_rps = {} + for query, query_results in results_table.items(): overall_query_p50_str = "n/a" overall_query_rate_str = "n/a" number_workers_str = "n/a" total_queries_str = "n/a" distinct_runs_str = "n/a" + bash_max_rps = "0" + overall_query_p50, best_pos, distinct_runs = get_best_results_for( query_results, "overall_query_p50", database, query, min) @@ -95,6 +104,7 @@ def get_result_at_pos(query_results, metric_name, database, query, pos): query_results, "overall_query_rate", database, query, best_pos) if overall_query_rate is not None: overall_query_rate_str = "{}".format(overall_query_rate) + bash_max_rps = "{}".format(int(float(overall_query_rate))) number_workers = get_result_at_pos( query_results, "number_workers", database, query, best_pos) if number_workers is not None: @@ -103,6 +113,11 @@ def get_result_at_pos(query_results, metric_name, database, query, pos): query_results, "total_queries", database, query, best_pos) if total_queries is not None: total_queries_str = "{}".format(total_queries) - + bash_hash_table_of_rps[query] = bash_max_rps print("{}, {}, {}, {}, {}, {}, {}".format(database, query, distinct_runs_str, - number_workers_str, total_queries_str, overall_query_p50_str, overall_query_rate_str)) + number_workers_str, total_queries_str, overall_query_p50_str, overall_query_rate_str)) +print("-------------------") +print("\n") +print("To run a same-rate competitor benchmark use the following bash RPS_ARRAY:") +bash_array = " ".join( ["\"{}\":\"{}\"".format(k,v) for k,v in bash_hash_table_of_rps.items() ] ) +print("RPS_ARRAY=({})\n".format(bash_array)) From 667c48555421970fcb3fd1fb83e5c9687041fbbe Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 20:11:57 +0000 Subject: [PATCH 36/72] [add] updated redistimeseries loader --- scripts/load/load_redistimeseries.sh | 1 - scripts/redistimeseries_common.sh | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/load/load_redistimeseries.sh b/scripts/load/load_redistimeseries.sh index 0fd13cd94..27a4f65ba 100755 --- a/scripts/load/load_redistimeseries.sh +++ b/scripts/load/load_redistimeseries.sh @@ -6,7 +6,6 @@ set -x set -e EXE_DIR=${EXE_DIR:-$(dirname $0)} -source ${EXE_DIR}/../query_common.sh source ${EXE_DIR}/../redistimeseries_common.sh # Ensure RESULTS DIR available diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh index db81e6ca1..ac2a4a600 100644 --- a/scripts/redistimeseries_common.sh +++ b/scripts/redistimeseries_common.sh @@ -56,3 +56,6 @@ FORMAT=${FORMAT:-"redistimeseries"} INTERLEAVED_GENERATION_GROUPS=${INTERLEAVED_GENERATION_GROUPS:-"1"} BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_data_${BULK_DATA_DIR}"} DATA_FILE_NAME="${BULK_DATA_DIR}/data_${FORMAT}_${USE_CASE}_${SCALE}_${TS_START}_${TS_END}_${LOG_INTERVAL}_${SEED}.dat" + +# Results folder +RESULTS_DIR=${RESULTS_DIR:-"./results"} From 2ca4e3595dcdfbdddcf213e6fcdbbebcd1e5cf8c Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 20:13:20 +0000 Subject: [PATCH 37/72] [add] updated redistimeseries loader --- scripts/generate_data_redistimeseries.sh | 1 - scripts/redistimeseries_common.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/generate_data_redistimeseries.sh b/scripts/generate_data_redistimeseries.sh index f5b8bcb1b..dec264ab0 100755 --- a/scripts/generate_data_redistimeseries.sh +++ b/scripts/generate_data_redistimeseries.sh @@ -6,7 +6,6 @@ EXE_FILE_NAME=./bin/tsbs_generate_data set -x EXE_DIR=${EXE_DIR:-$(dirname $0)} -source ${EXE_DIR}/query_common.sh source ${EXE_DIR}/redistimeseries_common.sh # Ensure DATA DIR available diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh index ac2a4a600..ba5a63b26 100644 --- a/scripts/redistimeseries_common.sh +++ b/scripts/redistimeseries_common.sh @@ -54,7 +54,7 @@ MAX_DATA_POINTS=${MAX_DATA_POINTS:-"0"} FORMAT=${FORMAT:-"redistimeseries"} INTERLEAVED_GENERATION_GROUPS=${INTERLEAVED_GENERATION_GROUPS:-"1"} -BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_data_${BULK_DATA_DIR}"} +BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_data_${FORMAT}"} DATA_FILE_NAME="${BULK_DATA_DIR}/data_${FORMAT}_${USE_CASE}_${SCALE}_${TS_START}_${TS_END}_${LOG_INTERVAL}_${SEED}.dat" # Results folder From b08b09ca5181f3dddb4e66421de4ae51c154335d Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 20:24:19 +0000 Subject: [PATCH 38/72] [add] updated redistimeseries loader --- cmd/tsbs_load_redistimeseries/scan.go | 4 ++-- pkg/targets/redistimeseries/serializer.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/scan.go b/cmd/tsbs_load_redistimeseries/scan.go index 6eb27b4b5..cc1176d94 100644 --- a/cmd/tsbs_load_redistimeseries/scan.go +++ b/cmd/tsbs_load_redistimeseries/scan.go @@ -25,8 +25,8 @@ func buildCommand(line string, forceUncompressed bool) (clusterSlot int, cmdA ra if cmdname == "TS.MADD" { metricCount = (len(t) - 2) / 3 } - key := t[2] - cmdA = radix.FlatCmd(nil, cmdname, key, t[3:]) + //key := t[2] + cmdA = radix.Cmd(nil, cmdname, t[2:]...) return } diff --git a/pkg/targets/redistimeseries/serializer.go b/pkg/targets/redistimeseries/serializer.go index 61c026ab6..10af949e2 100644 --- a/pkg/targets/redistimeseries/serializer.go +++ b/pkg/targets/redistimeseries/serializer.go @@ -92,10 +92,11 @@ func writeTS_and_Value(w io.Writer, p *data.Point, fieldValue interface{}) (err } func writeKeyName(w io.Writer, p *data.Point, hostname []byte, fieldName []byte) (err error) { - w.Write(p.MeasurementName()) - w.Write([]byte("_{")) + w.Write([]byte("{")) w.Write(hostname) w.Write([]byte("}_")) + w.Write(p.MeasurementName()) + w.Write([]byte("_")) w.Write(fieldName) w.Write([]byte(" ")) return From 4cb66c6619738bc3455bd445b66e4719097aa171 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 26 Mar 2021 23:15:20 +0000 Subject: [PATCH 39/72] [fix] Removed spurius prints from RTS query runner --- cmd/tsbs_run_queries_redistimeseries/main.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/tsbs_run_queries_redistimeseries/main.go b/cmd/tsbs_run_queries_redistimeseries/main.go index 42cbf9da5..f20405024 100644 --- a/cmd/tsbs_run_queries_redistimeseries/main.go +++ b/cmd/tsbs_run_queries_redistimeseries/main.go @@ -81,9 +81,12 @@ func init() { conn, _ := cluster.Client(nodeAddress) conns = append(conns, conn) } - fmt.Println(addresses) - fmt.Println(slots) - fmt.Println(conns) + //if p.opts.debug { + // fmt.Println(addresses) + // fmt.Println(slots) + // fmt.Println(conns) + //} + } else { standalone = getStandaloneConn(host, opts, uint64(config.Workers)) } From 5e067d5eb46f4f94eb5d8e62f0b2134fa046d60f Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sat, 27 Mar 2021 10:09:07 +0000 Subject: [PATCH 40/72] [wip] enabling run_queries_timescale to be run on remote host --- .../run_queries/run_queries_timescaledb.sh | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/scripts/run_queries/run_queries_timescaledb.sh b/scripts/run_queries/run_queries_timescaledb.sh index 1e7347d6b..8a3a9124f 100755 --- a/scripts/run_queries/run_queries_timescaledb.sh +++ b/scripts/run_queries/run_queries_timescaledb.sh @@ -10,14 +10,14 @@ if [[ -z "$EXE_FILE_NAME" ]]; then exit 1 fi -# Queries folder -BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"} +EXE_DIR=${EXE_DIR:-$(dirname $0)} -# How many queries would be run -MAX_QUERIES=${MAX_QUERIES:-"0"} +DATABASE_PORT=${DATABASE_PORT:-5433} -# How many concurrent worker would run queries - match num of cores, or default to 4 -NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 4)} +source ${EXE_DIR}/run_common.sh + +# Ensure RESULTS DIR available +mkdir -p ${RESULTS_DIR} for FULL_DATA_FILE_NAME in ${BULK_DATA_DIR}/queries_timescaledb*; do # $FULL_DATA_FILE_NAME: /full/path/to/file_with.ext @@ -30,23 +30,31 @@ for FULL_DATA_FILE_NAME in ${BULK_DATA_DIR}/queries_timescaledb*; do DIR=$(dirname "${FULL_DATA_FILE_NAME}") EXTENSION="${DATA_FILE_NAME##*.}" NO_EXT_DATA_FILE_NAME="${DATA_FILE_NAME%.*}" - - # Several options on how to name results file - #OUT_FULL_FILE_NAME="${DIR}/result_${DATA_FILE_NAME}" - OUT_FULL_FILE_NAME="${DIR}/result_${NO_EXT_DATA_FILE_NAME}.out" - #OUT_FULL_FILE_NAME="${DIR}/${NO_EXT_DATA_FILE_NAME}.out" - - if [ "${EXTENSION}" == "gz" ]; then - GUNZIP="gunzip" - else - GUNZIP="cat" - fi - - echo "Running ${DATA_FILE_NAME}" - cat $FULL_DATA_FILE_NAME \ - | $GUNZIP \ - | $EXE_FILE_NAME \ - --max-queries $MAX_QUERIES \ - --workers $NUM_WORKERS \ - | tee $OUT_FULL_FILE_NAME + for run in $(seq ${REPETITIONS}); do + + # Several options on how to name results file + #OUT_FULL_FILE_NAME="${DIR}/result_${DATA_FILE_NAME}" + OUT_FULL_FILE_NAME="${RESULTS_DIR}/result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + #OUT_FULL_FILE_NAME="${DIR}/${NO_EXT_DATA_FILE_NAME}.out" + HDR_FULL_FILE_NAME="${RESULTS_DIR}/HDR_TXT_result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + + if [ "${EXTENSION}" == "gz" ]; then + GUNZIP="gunzip" + else + GUNZIP="cat" + fi + + echo "Running ${DATA_FILE_NAME}" + cat $FULL_DATA_FILE_NAME | + $GUNZIP | + $EXE_FILE_NAME \ + --max-queries=$MAX_QUERIES \ + --workers=$NUM_WORKERS \ + --print-interval=${QUERIES_PRINT_INTERVAL} \ + --hdr-latencies=${HDR_FULL_FILE_NAME} \ + --debug=${DEBUG} \ + --hosts=${DATABASE_HOST}:${DATABASE_PORT} \ + --user=postgres | + tee $OUT_FULL_FILE_NAME + done done From 1064d12fc5ecaa95996614c592222d0214ca1152 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sat, 27 Mar 2021 10:16:15 +0000 Subject: [PATCH 41/72] [fix] Removed spurius prints from RTS query runner --- scripts/run_queries/run_queries_timescaledb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_queries/run_queries_timescaledb.sh b/scripts/run_queries/run_queries_timescaledb.sh index 8a3a9124f..3ce438403 100755 --- a/scripts/run_queries/run_queries_timescaledb.sh +++ b/scripts/run_queries/run_queries_timescaledb.sh @@ -12,7 +12,7 @@ fi EXE_DIR=${EXE_DIR:-$(dirname $0)} -DATABASE_PORT=${DATABASE_PORT:-5433} +DATABASE_PORT=${DATABASE_PORT:-5432} source ${EXE_DIR}/run_common.sh From d5003b1d95176cc880acb1e5f5d1fe73a36e5886 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sat, 27 Mar 2021 10:19:29 +0000 Subject: [PATCH 42/72] [fix] Removed spurius prints from RTS query runner --- scripts/run_queries/run_queries_timescaledb.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/run_queries/run_queries_timescaledb.sh b/scripts/run_queries/run_queries_timescaledb.sh index 3ce438403..0e0e0f2cc 100755 --- a/scripts/run_queries/run_queries_timescaledb.sh +++ b/scripts/run_queries/run_queries_timescaledb.sh @@ -12,8 +12,6 @@ fi EXE_DIR=${EXE_DIR:-$(dirname $0)} -DATABASE_PORT=${DATABASE_PORT:-5432} - source ${EXE_DIR}/run_common.sh # Ensure RESULTS DIR available @@ -53,7 +51,7 @@ for FULL_DATA_FILE_NAME in ${BULK_DATA_DIR}/queries_timescaledb*; do --print-interval=${QUERIES_PRINT_INTERVAL} \ --hdr-latencies=${HDR_FULL_FILE_NAME} \ --debug=${DEBUG} \ - --hosts=${DATABASE_HOST}:${DATABASE_PORT} \ + --hosts=${DATABASE_HOST} \ --user=postgres | tee $OUT_FULL_FILE_NAME done From 9a1efdbef99585e15a2fcd2924c0f1d920092844 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sat, 27 Mar 2021 10:51:25 +0000 Subject: [PATCH 43/72] [add] added rate limited timescaledb runner matching influx rps --- ...queries_timescaledb_rate_limited_influx.sh | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 scripts/run_queries/run_queries_timescaledb_rate_limited_influx.sh diff --git a/scripts/run_queries/run_queries_timescaledb_rate_limited_influx.sh b/scripts/run_queries/run_queries_timescaledb_rate_limited_influx.sh new file mode 100755 index 000000000..3ebee3c0f --- /dev/null +++ b/scripts/run_queries/run_queries_timescaledb_rate_limited_influx.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +set -e + +# Ensure runner is available +EXE_FILE_NAME=${EXE_FILE_NAME:-$(which tsbs_run_queries_timescaledb)} +if [[ -z "$EXE_FILE_NAME" ]]; then + echo "tsbs_run_queries_timescaledb not available. It is not specified explicitly and not found in \$PATH" + exit 1 +fi + +EXE_DIR=${EXE_DIR:-$(dirname $0)} + +source ${EXE_DIR}/run_common.sh + +# Ensure RESULTS DIR available +mkdir -p ${RESULTS_DIR} +RPS_ARRAY=("cpu-max-all-1":"4139" "cpu-max-all-8":"588" "double-groupby-1":"324" "double-groupby-5":"70" "double-groupby-all":"35" "groupby-orderby-limit":"28" "high-cpu-1":"1816" "high-cpu-all":"18" "lastpoint":"488" "single-groupby-1-1-1":"2320" "single-groupby-1-1-12":"2320" "single-groupby-1-8-1":"4458" "single-groupby-5-1-1":"648" "single-groupby-5-1-12":"648" "single-groupby-5-8-1":"1158") + +for FULL_DATA_FILE_NAME in ${BULK_DATA_DIR}/queries_timescaledb*; do + # $FULL_DATA_FILE_NAME: /full/path/to/file_with.ext + # $DATA_FILE_NAME: file_with.ext + # $DIR: /full/path/to + # $EXTENSION: ext + # NO_EXT_DATA_FILE_NAME: file_with + + DATA_FILE_NAME=$(basename -- "${FULL_DATA_FILE_NAME}") + DIR=$(dirname "${FULL_DATA_FILE_NAME}") + EXTENSION="${DATA_FILE_NAME##*.}" + NO_EXT_DATA_FILE_NAME="${DATA_FILE_NAME%.*}" + for run in $(seq ${REPETITIONS}); do + + # Several options on how to name results file + #OUT_FULL_FILE_NAME="${DIR}/result_${DATA_FILE_NAME}" + OUT_FULL_FILE_NAME="${RESULTS_DIR}/result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + #OUT_FULL_FILE_NAME="${DIR}/${NO_EXT_DATA_FILE_NAME}.out" + HDR_FULL_FILE_NAME="${RESULTS_DIR}/HDR_TXT_result_${NO_EXT_DATA_FILE_NAME}_${run}.out" + + if [ "${EXTENSION}" == "gz" ]; then + GUNZIP="gunzip" + else + GUNZIP="cat" + fi + + MAX_QUERY_RPS="0" + for rps_settings in "${RPS_ARRAY[@]}"; do + KEY="${rps_settings%%:*}" + VALUE="${rps_settings##*:}" + if [[ $DATA_FILE_NAME == *"$KEY"* ]]; then + echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" + echo "Using rate-limit info to run query $KEY. Input file data $DATA_FILE_NAME" + echo "Rate limit: $VALUE" + echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" + MAX_QUERY_RPS=$VALUE + fi + done + + echo "Running ${DATA_FILE_NAME}" + cat $FULL_DATA_FILE_NAME | + $GUNZIP | + $EXE_FILE_NAME \ + --max-queries=$MAX_QUERIES \ + --max-rps=${MAX_QUERY_RPS} \ + --workers=$NUM_WORKERS \ + --print-interval=${QUERIES_PRINT_INTERVAL} \ + --hdr-latencies=${HDR_FULL_FILE_NAME} \ + --debug=${DEBUG} \ + --hosts=${DATABASE_HOST} \ + --user=postgres | + tee $OUT_FULL_FILE_NAME + done +done From 49cd3cc2664d23189cf400bb1e388327e585b26c Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 28 Mar 2021 22:57:00 +0100 Subject: [PATCH 44/72] [wip] testing --hash-workers features on rts --- cmd/tsbs_load_redistimeseries/main.go | 6 +++--- scripts/load/load_redistimeseries.sh | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index e54b5fc43..0257ec219 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -303,9 +303,9 @@ func main() { config.NoFlowControl = true config.HashWorkers = false b := benchmark{dbc: &dbCreator{}} - if config.Workers > 1 { - panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) - } + //if config.Workers > 1 { + // panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) + //} loader.RunBenchmark(&b) log.Println("finished benchmark") diff --git a/scripts/load/load_redistimeseries.sh b/scripts/load/load_redistimeseries.sh index 27a4f65ba..89f97b3c5 100755 --- a/scripts/load/load_redistimeseries.sh +++ b/scripts/load/load_redistimeseries.sh @@ -18,7 +18,8 @@ echo "Saving results to ${OUT_FULL_FILE_NAME}" # Load new data $EXE_FILE_NAME \ --file ${DATA_FILE_NAME} \ - --workers=1 \ + --workers=${NUM_WORKERS} \ + --hash-workers=true \ --batch-size=${BATCH_SIZE} \ --reporting-period=${REPORTING_PERIOD} \ --host=${DATABASE_HOST}:${DATABASE_PORT} ${CLUSTER_FLAG} \ From 5492d9887da9af4090c36b9f0681d13c12ad34da Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 28 Mar 2021 23:19:19 +0100 Subject: [PATCH 45/72] [wip] testing --hash-workers features on rts --- cmd/tsbs_load_redistimeseries/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 0257ec219..dd2ab04c3 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -301,7 +301,7 @@ func (p *processor) Close(_ bool) { func main() { log.Println("Starting benchmark") config.NoFlowControl = true - config.HashWorkers = false + //config.HashWorkers = false b := benchmark{dbc: &dbCreator{}} //if config.Workers > 1 { // panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) From bd8e96656109b123eed08ee01ef3a11f96cb8202 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 1 Apr 2021 22:51:32 +0100 Subject: [PATCH 46/72] [wip] working with interleaved group ids for using multiple ingestion workers --- scripts/generate_data_redistimeseries.sh | 3 ++- scripts/load/load_redistimeseries.sh | 3 +-- scripts/redistimeseries_common.sh | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/generate_data_redistimeseries.sh b/scripts/generate_data_redistimeseries.sh index dec264ab0..aa8169c6e 100755 --- a/scripts/generate_data_redistimeseries.sh +++ b/scripts/generate_data_redistimeseries.sh @@ -21,5 +21,6 @@ ${EXE_FILE_NAME} \ --debug=${DEBUG} \ --seed=${SEED} \ --log-interval=${LOG_INTERVAL} \ - --interleaved-generation-groups=${INTERLEAVED_GENERATION_GROUPS} \ + --interleaved-generation-groups=${IG_GROUPS} \ + --interleaved-generation-group-id=${IG_GROUPS_ID} \ --max-data-points=${MAX_DATA_POINTS} >${DATA_FILE_NAME} diff --git a/scripts/load/load_redistimeseries.sh b/scripts/load/load_redistimeseries.sh index 89f97b3c5..27a4f65ba 100755 --- a/scripts/load/load_redistimeseries.sh +++ b/scripts/load/load_redistimeseries.sh @@ -18,8 +18,7 @@ echo "Saving results to ${OUT_FULL_FILE_NAME}" # Load new data $EXE_FILE_NAME \ --file ${DATA_FILE_NAME} \ - --workers=${NUM_WORKERS} \ - --hash-workers=true \ + --workers=1 \ --batch-size=${BATCH_SIZE} \ --reporting-period=${REPORTING_PERIOD} \ --host=${DATABASE_HOST}:${DATABASE_PORT} ${CLUSTER_FLAG} \ diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh index ba5a63b26..79e31d7ba 100644 --- a/scripts/redistimeseries_common.sh +++ b/scripts/redistimeseries_common.sh @@ -54,6 +54,8 @@ MAX_DATA_POINTS=${MAX_DATA_POINTS:-"0"} FORMAT=${FORMAT:-"redistimeseries"} INTERLEAVED_GENERATION_GROUPS=${INTERLEAVED_GENERATION_GROUPS:-"1"} +IG_GROUPS=${IG_GROUPS:-"1"} +IG_GROUPS_ID=${IG_GROUPS_ID:-"0"} BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_data_${FORMAT}"} DATA_FILE_NAME="${BULK_DATA_DIR}/data_${FORMAT}_${USE_CASE}_${SCALE}_${TS_START}_${TS_END}_${LOG_INTERVAL}_${SEED}.dat" From 147a289b5b41e290051cc78731088bfa3d7e6a18 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 1 Apr 2021 23:10:40 +0100 Subject: [PATCH 47/72] [wip] Ignoring ts.create on multi group ingest --- cmd/tsbs_load_redistimeseries/main.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index dd2ab04c3..bf9e25dc4 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -5,6 +5,7 @@ import ( "crypto/md5" "fmt" "github.com/mediocregopher/radix/v3" + "github.com/pkg/errors" "log" "strconv" "strings" @@ -49,6 +50,7 @@ var ( // allows for testing var fatal = log.Fatal var md5h = md5.New() +var errorTsCreate = errors.New("ERR TSDB: key already exists") // Parse args: func init() { @@ -182,14 +184,26 @@ func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics ch } for row := range rows { - slot, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) + slot, cmd, tscreate, metricCount := buildCommand(row, compressionEnabled == false) comdPos := nodeThatContainsSlot(slots, slot) + var err error = nil + + if tscreate { + err = conns[comdPos].Do(cmd) + if err != nil { + if strings.Compare(err.Error(), "ERR TSDB: key already exists") == 0 { + log.Println("Ignoring TS.CREATE given key already existed. Error message %v", err) + } else { + log.Fatalf("Flush failed with %v", err) + } + } + continue + } currMetricCount[comdPos] += metricCount cmds[comdPos] = append(cmds[comdPos], cmd) curPipe[comdPos]++ if curPipe[comdPos] == pipeline { - var err error = nil err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) if err != nil { log.Fatalf("Flush failed with %v", err) From 5fb5d2f673efcd53f934e55a0f40054550019221 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 1 Apr 2021 23:14:31 +0100 Subject: [PATCH 48/72] [wip] Ignoring ts.create on multi group ingest --- cmd/tsbs_load_redistimeseries/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index bf9e25dc4..fed3e1709 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -315,11 +315,11 @@ func (p *processor) Close(_ bool) { func main() { log.Println("Starting benchmark") config.NoFlowControl = true - //config.HashWorkers = false + config.HashWorkers = false b := benchmark{dbc: &dbCreator{}} - //if config.Workers > 1 { - // panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) - //} + if config.Workers > 1 { + panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) + } loader.RunBenchmark(&b) log.Println("finished benchmark") From ad66b4c652a894c102904927191c7b0833133d2b Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 2 Apr 2021 00:21:57 +0100 Subject: [PATCH 49/72] [wip] Ignoring ts.create on multi group ingest --- cmd/tsbs_load_redistimeseries/main.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index fed3e1709..e39397c7e 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -184,21 +184,21 @@ func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics ch } for row := range rows { - slot, cmd, tscreate, metricCount := buildCommand(row, compressionEnabled == false) + slot, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) comdPos := nodeThatContainsSlot(slots, slot) var err error = nil - if tscreate { - err = conns[comdPos].Do(cmd) - if err != nil { - if strings.Compare(err.Error(), "ERR TSDB: key already exists") == 0 { - log.Println("Ignoring TS.CREATE given key already existed. Error message %v", err) - } else { - log.Fatalf("Flush failed with %v", err) - } - } - continue - } + //if tscreate { + // err = conns[comdPos].Do(cmd) + // if err != nil { + // if strings.Compare(err.Error(), "ERR TSDB: key already exists") == 0 { + // log.Println("Ignoring TS.CREATE given key already existed. Error message %v", err) + // } else { + // log.Fatalf("Flush failed with %v", err) + // } + // } + // continue + //} currMetricCount[comdPos] += metricCount cmds[comdPos] = append(cmds[comdPos], cmd) curPipe[comdPos]++ From 2e0de5a4badae9ccaa8e5aa3bd0391865806d6a1 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 5 Apr 2021 00:02:09 +0100 Subject: [PATCH 50/72] [wip] Improving loader logic --- Makefile | 4 + cmd/tsbs_load_redistimeseries/benchmark.go | 33 +++ cmd/tsbs_load_redistimeseries/cluster_conn.go | 72 ++++++ .../file_datasource.go | 24 ++ cmd/tsbs_load_redistimeseries/main.go | 226 +----------------- cmd/tsbs_load_redistimeseries/processor.go | 62 +++++ .../standalone_conn.go | 38 +++ 7 files changed, 235 insertions(+), 224 deletions(-) create mode 100644 cmd/tsbs_load_redistimeseries/benchmark.go create mode 100644 cmd/tsbs_load_redistimeseries/file_datasource.go create mode 100644 cmd/tsbs_load_redistimeseries/processor.go diff --git a/Makefile b/Makefile index 5fa8b2e71..c8cc3852c 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,10 @@ DOCKER_LATEST:="${DOCKER_REPO}:latest" .PHONY: all generators loaders runners all: generators loaders runners +fmt: + $(GOFMT) ./cmd/tsbs_load_redistimeseries/*.go + $(GOFMT) ./cmd/tsbs_run_queries_redistimeseries/*.go + generators: tsbs_generate_data tsbs_generate_queries influx: tsbs_generate_data tsbs_generate_queries tsbs_load_influx tsbs_run_queries_influx diff --git a/cmd/tsbs_load_redistimeseries/benchmark.go b/cmd/tsbs_load_redistimeseries/benchmark.go new file mode 100644 index 000000000..fb70fe0fe --- /dev/null +++ b/cmd/tsbs_load_redistimeseries/benchmark.go @@ -0,0 +1,33 @@ +package main + +import ( + "bufio" + "github.com/timescale/tsbs/load" + "github.com/timescale/tsbs/pkg/targets" + "log" +) + +func (b *benchmark) GetBatchFactory() targets.BatchFactory { + return &factory{} +} + +func (b *benchmark) GetPointIndexer(maxPartitions uint) targets.PointIndexer { + return &RedisIndexer{partitions: maxPartitions} +} + +func (b *benchmark) GetProcessor() targets.Processor { + return &processor{b.dbc, nil, nil, nil} +} + +func (b *benchmark) GetDBCreator() targets.DBCreator { + return b.dbc +} + +type benchmark struct { + dbc *dbCreator +} + +func (b *benchmark) GetDataSource() targets.DataSource { + log.Printf("creating DS from %s", config.FileName) + return &fileDataSource{scanner: bufio.NewScanner(load.GetBufferedReader(config.FileName))} +} diff --git a/cmd/tsbs_load_redistimeseries/cluster_conn.go b/cmd/tsbs_load_redistimeseries/cluster_conn.go index f320038ef..bb7be1c1a 100644 --- a/cmd/tsbs_load_redistimeseries/cluster_conn.go +++ b/cmd/tsbs_load_redistimeseries/cluster_conn.go @@ -2,7 +2,11 @@ package main import ( "github.com/mediocregopher/radix/v3" + "github.com/timescale/tsbs/pkg/data" "log" + "strconv" + "strings" + "sync" ) func getOSSClusterConn(addr string, opts []radix.DialOpt, clients uint64) *radix.Cluster { @@ -31,3 +35,71 @@ func getOSSClusterConn(addr string, opts []radix.DialOpt, clients uint64) *radix } return vanillaCluster } + +func nodeThatContainsSlot(slots [][][2]uint16, slot int) (result int) { + result = -1 + for nodePos, slotGroup := range slots { + for _, i2 := range slotGroup { + if slot >= int(i2[0]) && slot < int(i2[1]) { + result = nodePos + return + } + } + } + return +} + +func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics chan uint64, cluster *radix.Cluster, clusterNodes int, addresses []string, slots [][][2]uint16, conns []radix.Client) { + cmds := make([][]radix.CmdAction, clusterNodes, clusterNodes) + curPipe := make([]uint64, clusterNodes, clusterNodes) + currMetricCount := make([]int, clusterNodes, clusterNodes) + for i := 0; i < clusterNodes; i++ { + cmds[i] = make([]radix.CmdAction, 0, 0) + curPipe[i] = 0 + currMetricCount[i] = 0 + } + + for row := range rows { + slot, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) + comdPos := nodeThatContainsSlot(slots, slot) + var err error = nil + + currMetricCount[comdPos] += metricCount + cmds[comdPos] = append(cmds[comdPos], cmd) + curPipe[comdPos]++ + + if curPipe[comdPos] == pipeline { + err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- uint64(currMetricCount[comdPos]) + currMetricCount[comdPos] = 0 + cmds[comdPos] = make([]radix.CmdAction, 0, 0) + curPipe[comdPos] = 0 + } + + } + for comdPos, u := range curPipe { + if u > 0 { + var err error = nil + err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- uint64(currMetricCount[comdPos]) + } + } + wg.Done() +} + +type RedisIndexer struct { + partitions uint +} + +func (i *RedisIndexer) GetIndex(p data.LoadedPoint) uint { + row := p.Data.(string) + slotS := strings.Split(row, " ")[0] + clusterSlot, _ := strconv.ParseInt(slotS, 10, 0) + return uint(clusterSlot) % i.partitions +} diff --git a/cmd/tsbs_load_redistimeseries/file_datasource.go b/cmd/tsbs_load_redistimeseries/file_datasource.go new file mode 100644 index 000000000..4ad3fd15b --- /dev/null +++ b/cmd/tsbs_load_redistimeseries/file_datasource.go @@ -0,0 +1,24 @@ +package main + +import ( + "bufio" + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/usecases/common" +) + +type fileDataSource struct { + scanner *bufio.Scanner +} + +func (d *fileDataSource) NextItem() data.LoadedPoint { + ok := d.scanner.Scan() + if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF + return data.LoadedPoint{} + } else if !ok { + fatal("scan error: %v", d.scanner.Err()) + return data.LoadedPoint{} + } + return data.NewLoadedPoint(d.scanner.Text()) +} + +func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { return nil } diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index e39397c7e..37e800b2d 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -1,25 +1,18 @@ package main import ( - "bufio" "crypto/md5" "fmt" + "github.com/blagojts/viper" "github.com/mediocregopher/radix/v3" "github.com/pkg/errors" - "log" - "strconv" - "strings" - "sync" - - "github.com/blagojts/viper" "github.com/spf13/pflag" "github.com/timescale/tsbs/internal/utils" - "github.com/timescale/tsbs/pkg/data/usecases/common" "github.com/timescale/tsbs/pkg/targets/constants" "github.com/timescale/tsbs/pkg/targets/initializers" + "log" "github.com/timescale/tsbs/load" - "github.com/timescale/tsbs/pkg/data" "github.com/timescale/tsbs/pkg/targets" ) @@ -92,226 +85,11 @@ func init() { conn, _ := cluster.Client(nodeAddress) conns = append(conns, conn) } - fmt.Println(addresses) - fmt.Println(slots) - fmt.Println(conns) } else { standalone = getStandaloneConn(host, opts, connections) } } -type benchmark struct { - dbc *dbCreator -} - -func (b *benchmark) GetDataSource() targets.DataSource { - log.Printf("creating DS from %s", config.FileName) - return &fileDataSource{scanner: bufio.NewScanner(load.GetBufferedReader(config.FileName))} -} - -type RedisIndexer struct { - partitions uint -} - -func (i *RedisIndexer) GetIndex(p data.LoadedPoint) uint { - row := p.Data.(string) - slotS := strings.Split(row, " ")[0] - clusterSlot, _ := strconv.ParseInt(slotS, 10, 0) - return uint(clusterSlot) % i.partitions -} - -type fileDataSource struct { - scanner *bufio.Scanner -} - -func (d *fileDataSource) NextItem() data.LoadedPoint { - ok := d.scanner.Scan() - if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF - return data.LoadedPoint{} - } else if !ok { - fatal("scan error: %v", d.scanner.Err()) - return data.LoadedPoint{} - } - return data.NewLoadedPoint(d.scanner.Text()) -} - -func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { return nil } - -func (b *benchmark) GetBatchFactory() targets.BatchFactory { - return &factory{} -} - -func (b *benchmark) GetPointIndexer(maxPartitions uint) targets.PointIndexer { - return &RedisIndexer{partitions: maxPartitions} -} - -func (b *benchmark) GetProcessor() targets.Processor { - return &processor{b.dbc, nil, nil, nil} -} - -func (b *benchmark) GetDBCreator() targets.DBCreator { - return b.dbc -} - -type processor struct { - dbc *dbCreator - rows []chan string - metrics chan uint64 - wg *sync.WaitGroup -} - -func nodeThatContainsSlot(slots [][][2]uint16, slot int) (result int) { - result = -1 - for nodePos, slotGroup := range slots { - for _, i2 := range slotGroup { - if slot >= int(i2[0]) && slot < int(i2[1]) { - result = nodePos - return - } - } - } - return -} - -func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics chan uint64, cluster *radix.Cluster, clusterNodes int, addresses []string, slots [][][2]uint16, conns []radix.Client) { - cmds := make([][]radix.CmdAction, clusterNodes, clusterNodes) - curPipe := make([]uint64, clusterNodes, clusterNodes) - currMetricCount := make([]int, clusterNodes, clusterNodes) - for i := 0; i < clusterNodes; i++ { - cmds[i] = make([]radix.CmdAction, 0, 0) - curPipe[i] = 0 - currMetricCount[i] = 0 - } - - for row := range rows { - slot, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) - comdPos := nodeThatContainsSlot(slots, slot) - var err error = nil - - //if tscreate { - // err = conns[comdPos].Do(cmd) - // if err != nil { - // if strings.Compare(err.Error(), "ERR TSDB: key already exists") == 0 { - // log.Println("Ignoring TS.CREATE given key already existed. Error message %v", err) - // } else { - // log.Fatalf("Flush failed with %v", err) - // } - // } - // continue - //} - currMetricCount[comdPos] += metricCount - cmds[comdPos] = append(cmds[comdPos], cmd) - curPipe[comdPos]++ - - if curPipe[comdPos] == pipeline { - err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) - if err != nil { - log.Fatalf("Flush failed with %v", err) - } - metrics <- uint64(currMetricCount[comdPos]) - currMetricCount[comdPos] = 0 - cmds[comdPos] = make([]radix.CmdAction, 0, 0) - curPipe[comdPos] = 0 - } - - } - for comdPos, u := range curPipe { - if u > 0 { - var err error = nil - err = conns[comdPos].Do(radix.Pipeline(cmds[comdPos]...)) - if err != nil { - log.Fatalf("Flush failed with %v", err) - } - metrics <- uint64(currMetricCount[comdPos]) - } - } - wg.Done() -} - -func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint64, conn radix.Client) { - cmds := make([][]radix.CmdAction, 1, 1) - cmds[0] = make([]radix.CmdAction, 0, 0) - curPipe := make([]uint64, 1, 1) - curPipe[0] = 0 - currMetricCount := 0 - comdPos := 0 - - for row := range rows { - _, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) - currMetricCount += metricCount - cmds[comdPos] = append(cmds[comdPos], cmd) - curPipe[comdPos]++ - - if curPipe[comdPos] == pipeline { - err := conn.Do(radix.Pipeline(cmds[comdPos]...)) - if err != nil { - log.Fatalf("Flush failed with %v", err) - } - metrics <- uint64(currMetricCount) - currMetricCount = 0 - cmds[comdPos] = make([]radix.CmdAction, 0, 0) - curPipe[comdPos] = 0 - } - } - for comdPos, u := range curPipe { - if u > 0 { - err := conn.Do(radix.Pipeline(cmds[comdPos]...)) - if err != nil { - log.Fatalf("Flush failed with %v", err) - } - metrics <- uint64(currMetricCount) - } - } - wg.Done() -} - -func (p *processor) Init(_ int, _ bool, _ bool) {} - -// ProcessBatch reads eventsBatches which contain rows of data for TS.ADD redis command string -func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) { - events := b.(*eventsBatch) - rowCnt := uint64(len(events.rows)) - metricCnt := uint64(0) - - if doLoad { - buflen := rowCnt + 1 - p.rows = make([]chan string, connections) - p.metrics = make(chan uint64, buflen) - p.wg = &sync.WaitGroup{} - - for i := uint64(0); i < connections; i++ { - p.rows[i] = make(chan string, buflen) - p.wg.Add(1) - if clusterMode { - go connectionProcessorCluster(p.wg, p.rows[i], p.metrics, cluster, len(addresses), addresses, slots, conns) - } else { - go connectionProcessor(p.wg, p.rows[i], p.metrics, standalone) - } - } - for _, row := range events.rows { - slotS := strings.Split(row, " ")[0] - clusterSlot, _ := strconv.ParseInt(slotS, 10, 0) - i := uint64(clusterSlot) % connections - p.rows[i] <- row - } - - for i := uint64(0); i < connections; i++ { - close(p.rows[i]) - } - p.wg.Wait() - close(p.metrics) - for val := range p.metrics { - metricCnt += val - } - } - events.rows = events.rows[:0] - ePool.Put(events) - return metricCnt, rowCnt -} - -func (p *processor) Close(_ bool) { -} - func main() { log.Println("Starting benchmark") config.NoFlowControl = true diff --git a/cmd/tsbs_load_redistimeseries/processor.go b/cmd/tsbs_load_redistimeseries/processor.go new file mode 100644 index 000000000..866f3e0c1 --- /dev/null +++ b/cmd/tsbs_load_redistimeseries/processor.go @@ -0,0 +1,62 @@ +package main + +import ( + "github.com/timescale/tsbs/pkg/targets" + "strconv" + "strings" + "sync" +) + +type processor struct { + dbc *dbCreator + rows []chan string + metrics chan uint64 + wg *sync.WaitGroup +} + +func (p *processor) Init(_ int, _ bool, _ bool) {} + +// ProcessBatch reads eventsBatches which contain rows of data for TS.ADD redis command string +func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) { + events := b.(*eventsBatch) + rowCnt := uint64(len(events.rows)) + metricCnt := uint64(0) + + if doLoad { + buflen := rowCnt + 1 + p.rows = make([]chan string, connections) + p.metrics = make(chan uint64, buflen) + p.wg = &sync.WaitGroup{} + + for i := uint64(0); i < connections; i++ { + p.rows[i] = make(chan string, buflen) + p.wg.Add(1) + if clusterMode { + go connectionProcessorCluster(p.wg, p.rows[i], p.metrics, cluster, len(addresses), addresses, slots, conns) + } else { + go connectionProcessor(p.wg, p.rows[i], p.metrics, standalone) + } + } + for _, row := range events.rows { + slotS := strings.Split(row, " ")[0] + clusterSlot, _ := strconv.ParseInt(slotS, 10, 0) + i := uint64(clusterSlot) % connections + p.rows[i] <- row + } + + for i := uint64(0); i < connections; i++ { + close(p.rows[i]) + } + p.wg.Wait() + close(p.metrics) + for val := range p.metrics { + metricCnt += val + } + } + events.rows = events.rows[:0] + ePool.Put(events) + return metricCnt, rowCnt +} + +func (p *processor) Close(_ bool) { +} diff --git a/cmd/tsbs_load_redistimeseries/standalone_conn.go b/cmd/tsbs_load_redistimeseries/standalone_conn.go index 01f34959d..b9ef5a9a3 100644 --- a/cmd/tsbs_load_redistimeseries/standalone_conn.go +++ b/cmd/tsbs_load_redistimeseries/standalone_conn.go @@ -3,6 +3,7 @@ package main import ( "github.com/mediocregopher/radix/v3" "log" + "sync" ) func getStandaloneConn(addr string, opts []radix.DialOpt, clients uint64) *radix.Pool { @@ -20,3 +21,40 @@ func getStandaloneConn(addr string, opts []radix.DialOpt, clients uint64) *radix } return pool } + +func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint64, conn radix.Client) { + cmds := make([][]radix.CmdAction, 1, 1) + cmds[0] = make([]radix.CmdAction, 0, 0) + curPipe := make([]uint64, 1, 1) + curPipe[0] = 0 + currMetricCount := 0 + comdPos := 0 + + for row := range rows { + _, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) + currMetricCount += metricCount + cmds[comdPos] = append(cmds[comdPos], cmd) + curPipe[comdPos]++ + + if curPipe[comdPos] == pipeline { + err := conn.Do(radix.Pipeline(cmds[comdPos]...)) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- uint64(currMetricCount) + currMetricCount = 0 + cmds[comdPos] = make([]radix.CmdAction, 0, 0) + curPipe[comdPos] = 0 + } + } + for comdPos, u := range curPipe { + if u > 0 { + err := conn.Do(radix.Pipeline(cmds[comdPos]...)) + if err != nil { + log.Fatalf("Flush failed with %v", err) + } + metrics <- uint64(currMetricCount) + } + } + wg.Done() +} From 5d491902835824d1e4c40bdba4167756c73bc09c Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 5 Apr 2021 00:20:10 +0100 Subject: [PATCH 51/72] [add] wip on hashing series per worker --- cmd/tsbs_load_redistimeseries/main.go | 12 ++++++------ scripts/load/load_redistimeseries.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 37e800b2d..1a9bd4920 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -68,7 +68,8 @@ func init() { dataModel = "redistimeseries" compressionEnabled = true clusterMode = viper.GetBool("cluster") - + config.NoFlowControl = true + config.HashWorkers = true loader = load.GetBenchmarkRunner(config) opts := make([]radix.DialOpt, 0) @@ -92,12 +93,11 @@ func init() { func main() { log.Println("Starting benchmark") - config.NoFlowControl = true - config.HashWorkers = false + b := benchmark{dbc: &dbCreator{}} - if config.Workers > 1 { - panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) - } + //if config.Workers > 1 { + // panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) + //} loader.RunBenchmark(&b) log.Println("finished benchmark") diff --git a/scripts/load/load_redistimeseries.sh b/scripts/load/load_redistimeseries.sh index 27a4f65ba..39c741bb2 100755 --- a/scripts/load/load_redistimeseries.sh +++ b/scripts/load/load_redistimeseries.sh @@ -18,7 +18,7 @@ echo "Saving results to ${OUT_FULL_FILE_NAME}" # Load new data $EXE_FILE_NAME \ --file ${DATA_FILE_NAME} \ - --workers=1 \ + --workers=${NUM_WORKERS} \ --batch-size=${BATCH_SIZE} \ --reporting-period=${REPORTING_PERIOD} \ --host=${DATABASE_HOST}:${DATABASE_PORT} ${CLUSTER_FLAG} \ From 71519d047cb76b814e5e0032639a4980b28d6dae Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 23 Apr 2021 14:18:23 +0100 Subject: [PATCH 52/72] [add] included release utils for redistimeseries runners --- .gitignore | 3 ++- Makefile | 6 ++++++ go.mod | 2 ++ go.sum | 22 ++++++++++++++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 12ace57ab..bc1bc0759 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ coverage.txt bin/ -results/ \ No newline at end of file +results/ +dist \ No newline at end of file diff --git a/Makefile b/Makefile index c8cc3852c..9c7102479 100644 --- a/Makefile +++ b/Makefile @@ -77,3 +77,9 @@ docker-tag: docker-tag-latest docker-tag-latest: @echo 'create tag latest' docker tag $(DOCKER_APP_NAME) $(DOCKER_LATEST) + +release-redistimeseries: + $(GOGET) github.com/mitchellh/gox + $(GOGET) github.com/tcnksm/ghr + GO111MODULE=on gox -osarch "linux/amd64 darwin/amd64" -output "dist/tsbs_run_queries_redistimeseries_{{.OS}}_{{.Arch}}" ./cmd/tsbs_run_queries_redistimeseries + GO111MODULE=on gox -osarch "linux/amd64 darwin/amd64" -output "dist/tsbs_load_redistimeseries_{{.OS}}_{{.Arch}}" ./cmd/tsbs_load_redistimeseries diff --git a/go.mod b/go.mod index fda29c444..1949730b4 100644 --- a/go.mod +++ b/go.mod @@ -23,11 +23,13 @@ require ( github.com/kshvakov/clickhouse v1.3.11 github.com/lib/pq v1.3.0 github.com/mediocregopher/radix/v3 v3.7.0 + github.com/mitchellh/gox v1.0.1 // indirect github.com/pkg/errors v0.9.1 github.com/prometheus/common v0.13.0 github.com/shirou/gopsutil v2.18.12+incompatible github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 + github.com/tcnksm/ghr v0.13.0 // indirect github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 github.com/transceptor-technology/go-qpack v0.0.0-20190116123619-49a14b216a45 github.com/valyala/fasthttp v1.15.1 diff --git a/go.sum b/go.sum index d16ade842..f4010c97b 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921 h1:GIWNb0z3t/YKr7xcGNhFgxasaTpnsX91Z0Zt4CeLk+c= github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921/go.mod h1:s0x47OhsrJKfg9Iq5orGCVJQjwKklC3jZMFlgLe6Zew= +github.com/Songmu/retry v0.1.0 h1:hPA5xybQsksLR/ry/+t/7cFajPW+dqjmjhzZhioBILA= +github.com/Songmu/retry v0.1.0/go.mod h1:7sXIW7eseB9fq0FUvigRcQMVLR9tuHI0Scok+rkpAuA= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= @@ -413,6 +415,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -479,6 +484,8 @@ github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjG github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -680,12 +687,17 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/gox v1.0.1 h1:x0jD3dcHk9a9xPSDN6YEL4xL6Qz0dvNYm8yZqui5chI= +github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4= +github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -906,6 +918,12 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tcnksm/ghr v0.13.0 h1:a5ZbaUAfiaiw6rEDJVUEDYA9YreZOkh3XAfXHWn8zu8= +github.com/tcnksm/ghr v0.13.0/go.mod h1:tcp6tzbRYE0LqFSG7ykXP/BVG1/2BkX6aIn9FFV1mIQ= +github.com/tcnksm/go-gitconfig v0.1.2 h1:iiDhRitByXAEyjgBqsKi9QU4o2TNtv9kPP3RgPgXBPw= +github.com/tcnksm/go-gitconfig v0.1.2/go.mod h1:/8EhP4H7oJZdIPyT+/UIsG87kTzrzM4UsLGSItWYCpE= +github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e h1:IWllFTiDjjLIf2oeKxpIUmtiDV5sn71VgeQgg6vcE7k= +github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e/go.mod h1:d7u6HkTYKSv5m6MCKkOQlHwaShTMl3HjqSGW3XtVhXM= github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= github.com/testcontainers/testcontainers-go v0.5.1/go.mod h1:Oc/G02bjZiX0p3lzyh6b1GCELP0e4/6Cg3ciU/LnFvU= github.com/tetafro/godot v0.4.8/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= @@ -1078,8 +1096,10 @@ golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOL golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1089,6 +1109,7 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1270,6 +1291,7 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= From 648ae7fee3985b8b78023d346b856bcaf568fb66 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Fri, 23 Apr 2021 16:38:53 +0100 Subject: [PATCH 53/72] [add] Added publish redistimeseries make rules --- Makefile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9c7102479..591ce0435 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ GOTEST=$(GOCMD) test GOGET=$(GOCMD) get GOMOD=$(GOCMD) mod GOFMT=$(GOCMD) fmt +DISTDIR = ./dist # DOCKER DOCKER_APP_NAME=tsbs @@ -81,5 +82,12 @@ docker-tag-latest: release-redistimeseries: $(GOGET) github.com/mitchellh/gox $(GOGET) github.com/tcnksm/ghr - GO111MODULE=on gox -osarch "linux/amd64 darwin/amd64" -output "dist/tsbs_run_queries_redistimeseries_{{.OS}}_{{.Arch}}" ./cmd/tsbs_run_queries_redistimeseries - GO111MODULE=on gox -osarch "linux/amd64 darwin/amd64" -output "dist/tsbs_load_redistimeseries_{{.OS}}_{{.Arch}}" ./cmd/tsbs_load_redistimeseries + GO111MODULE=on gox -osarch "linux/amd64 darwin/amd64" -output "${DISTDIR}/tsbs_run_queries_redistimeseries_{{.OS}}_{{.Arch}}" ./cmd/tsbs_run_queries_redistimeseries + GO111MODULE=on gox -osarch "linux/amd64 darwin/amd64" -output "${DISTDIR}/tsbs_load_redistimeseries_{{.OS}}_{{.Arch}}" ./cmd/tsbs_load_redistimeseries + +publish-redistimeseries: release-redistimeseries + @for f in $(shell ls ${DISTDIR}); \ + do \ + echo "copying ${DISTDIR}/$${f}"; \ + aws s3 cp ${DISTDIR}/$${f} s3://benchmarks.redislabs/redistimeseries/tools/tsbs/$${f} --acl public-read; \ + done From 30c2770747430cda9b8448032f6a73f78ef185b0 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 25 Apr 2021 18:19:01 +0100 Subject: [PATCH 54/72] [wip] wip results for benchmark run --- go.mod | 3 +- go.sum | 7 +++++ pkg/query/benchmark_result.go | 22 ++++++++++++++ pkg/query/benchmarker.go | 31 ++++++++++++++++++++ pkg/query/stat_processor.go | 54 ++++++++++++++++++++++++----------- 5 files changed, 100 insertions(+), 17 deletions(-) create mode 100644 pkg/query/benchmark_result.go diff --git a/go.mod b/go.mod index bd883ae0d..1106116ed 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,11 @@ require ( github.com/lib/pq v1.3.0 github.com/pkg/errors v0.9.1 github.com/prometheus/common v0.13.0 - github.com/shirou/gopsutil v2.18.12+incompatible + github.com/shirou/gopsutil v3.21.3+incompatible github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 + github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/transceptor-technology/go-qpack v0.0.0-20190116123619-49a14b216a45 github.com/valyala/fasthttp v1.15.1 go.uber.org/atomic v1.6.0 diff --git a/go.sum b/go.sum index ac238c6d1..6b3ac40ef 100644 --- a/go.sum +++ b/go.sum @@ -830,6 +830,8 @@ github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAx github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= +github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -898,6 +900,9 @@ github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiff github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 h1:jdJdzLyz0SNBuvt5rYyBxDqhgZ2EcbA7eWVBMqcyEHc= github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84/go.mod h1:rkhy9b91Qv9zTCqZh5kPUezLIRhghkZSwnmwRCXK8C0= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= @@ -1133,6 +1138,8 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200821140526-fda516888d29/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96 h1:gJciq3lOg0eS9fSZJcoHfv7q1BfC6cJfnmSSKL1yu3Q= golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa h1:ZYxPR6aca/uhfRJyaOAtflSHjJYiktO7QnJC5ut7iY4= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/query/benchmark_result.go b/pkg/query/benchmark_result.go new file mode 100644 index 000000000..4e6fc6b6e --- /dev/null +++ b/pkg/query/benchmark_result.go @@ -0,0 +1,22 @@ +package query + + +const BenchmarkTestResultVersion = "0.1" + +// LoaderTestResult aggregates the results of an query benchmark in a common format across targets +type LoaderTestResult struct { + // Format Configs + ResultFormatVersion string `json:"ResultFormatVersion"` + + // RunnerConfig Configs + RunnerConfig BenchmarkRunnerConfig `json:"RunnerConfig"` + + // Run info + StartTime int64 `json:"StartTime` + EndTime int64 `json:"EndTime"` + DurationMillis int64 `json:"DurationMillis"` + + // Totals + Totals map[string]interface{} `json:"Totals"` +} + diff --git a/pkg/query/benchmarker.go b/pkg/query/benchmarker.go index 3b82d4d59..b557c2f4e 100644 --- a/pkg/query/benchmarker.go +++ b/pkg/query/benchmarker.go @@ -2,7 +2,9 @@ package query import ( "bufio" + "encoding/json" "fmt" + "io/ioutil" "log" "os" "runtime/pprof" @@ -35,6 +37,7 @@ type BenchmarkRunnerConfig struct { BurnIn uint64 `mapstructure:"burn-in"` PrintInterval uint64 `mapstructure:"print-interval"` PrewarmQueries bool `mapstructure:"prewarm-queries"` + ResultsFile string `mapstructure:"results-file"` } // AddToFlagSet adds command line flags needed by the BenchmarkRunnerConfig to the flag set. @@ -51,6 +54,7 @@ func (c BenchmarkRunnerConfig) AddToFlagSet(fs *pflag.FlagSet) { fs.Bool("print-responses", false, "Pretty print response bodies for correctness checking (default false).") fs.Int("debug", 0, "Whether to print debug messages.") fs.String("file", "", "File name to read queries from") + fs.String("results-file", "", "Write the test results summary json to this file") } // BenchmarkRunner contains the common components for running a query benchmarking @@ -183,6 +187,33 @@ func (b *BenchmarkRunner) Run(queryPool *sync.Pool, processorCreateFn ProcessorC pprof.WriteHeapProfile(f) f.Close() } + + // (Optional) save the results file: + if len(b.BenchmarkRunnerConfig.ResultsFile) > 0 { + b.saveTestResult(wallTook, wallStart, wallEnd) + } +} + +func (b *BenchmarkRunner) saveTestResult(took time.Duration, start time.Time, end time.Time) { + testResult := LoaderTestResult{ + ResultFormatVersion: BenchmarkTestResultVersion, + RunnerConfig: b.BenchmarkRunnerConfig, + StartTime: start.Unix(), + EndTime: end.Unix(), + DurationMillis: took.Milliseconds(), + Totals: b.sp.GetTotalsMap(), + } + + _, _ = fmt.Printf("Saving results json file to %s\n", b.BenchmarkRunnerConfig.ResultsFile) + file, err := json.MarshalIndent(testResult, "", " ") + if err != nil { + log.Fatal(err) + } + + err = ioutil.WriteFile(b.BenchmarkRunnerConfig.ResultsFile, file, 0644) + if err != nil { + log.Fatal(err) + } } func (b *BenchmarkRunner) processorHandler(wg *sync.WaitGroup, rateLimiter *rate.Limiter, queryPool *sync.Pool, processor Processor, workerNum int) { diff --git a/pkg/query/stat_processor.go b/pkg/query/stat_processor.go index 1e7a136a1..14717fb8e 100644 --- a/pkg/query/stat_processor.go +++ b/pkg/query/stat_processor.go @@ -19,6 +19,7 @@ type statProcessor interface { sendWarm(stats []*Stat) process(workers uint) CloseAndWait() + GetTotalsMap() map[string]interface{} } type statProcessorArgs struct { @@ -36,6 +37,9 @@ type defaultStatProcessor struct { wg sync.WaitGroup c chan *Stat // c is the channel for Stats to be sent for processing opsCount uint64 + startTime time.Time + endTime time.Time + statMapping map[string]*statGroup } func newStatProcessor(args *statProcessorArgs) statProcessor { @@ -76,18 +80,18 @@ func (sp *defaultStatProcessor) process(workers uint) { sp.c = make(chan *Stat, workers) sp.wg.Add(1) const allQueriesLabel = labelAllQueries - statMapping := map[string]*statGroup{ + sp.statMapping = map[string]*statGroup{ allQueriesLabel: newStatGroup(*sp.args.limit), } // Only needed when differentiating between cold & warm if sp.args.prewarmQueries { - statMapping[labelColdQueries] = newStatGroup(*sp.args.limit) - statMapping[labelWarmQueries] = newStatGroup(*sp.args.limit) + sp.statMapping[labelColdQueries] = newStatGroup(*sp.args.limit) + sp.statMapping[labelWarmQueries] = newStatGroup(*sp.args.limit) } i := uint64(0) - start := time.Now() - prevTime := start + sp.startTime = time.Now() + prevTime := sp.startTime prevRequestCount := uint64(0) for stat := range sp.c { @@ -102,21 +106,21 @@ func (sp *defaultStatProcessor) process(workers uint) { log.Fatal(err) } } - if _, ok := statMapping[string(stat.label)]; !ok { - statMapping[string(stat.label)] = newStatGroup(*sp.args.limit) + if _, ok := sp.statMapping[string(stat.label)]; !ok { + sp.statMapping[string(stat.label)] = newStatGroup(*sp.args.limit) } - statMapping[string(stat.label)].push(stat.value) + sp.statMapping[string(stat.label)].push(stat.value) if !stat.isPartial { - statMapping[allQueriesLabel].push(stat.value) + sp.statMapping[allQueriesLabel].push(stat.value) // Only needed when differentiating between cold & warm if sp.args.prewarmQueries { if stat.isWarm { - statMapping[labelWarmQueries].push(stat.value) + sp.statMapping[labelWarmQueries].push(stat.value) } else { - statMapping[labelColdQueries].push(stat.value) + sp.statMapping[labelColdQueries].push(stat.value) } } @@ -133,7 +137,7 @@ func (sp *defaultStatProcessor) process(workers uint) { // print stats to stderr (if printInterval is greater than zero): if sp.args.printInterval > 0 && i > 0 && i%sp.args.printInterval == 0 && (i < *sp.args.limit || *sp.args.limit == 0) { now := time.Now() - sinceStart := now.Sub(start) + sinceStart := now.Sub(sp.startTime) took := now.Sub(prevTime) intervalQueryRate := float64(sp.opsCount-prevRequestCount) / float64(took.Seconds()) overallQueryRate := float64(sp.opsCount) / float64(sinceStart.Seconds()) @@ -146,7 +150,7 @@ func (sp *defaultStatProcessor) process(workers uint) { if err != nil { log.Fatal(err) } - err = writeStatGroupMap(os.Stderr, statMapping) + err = writeStatGroupMap(os.Stderr, sp.statMapping) if err != nil { log.Fatal(err) } @@ -158,14 +162,14 @@ func (sp *defaultStatProcessor) process(workers uint) { prevTime = now } } - sinceStart := time.Now().Sub(start) + sinceStart := time.Now().Sub(sp.startTime) overallQueryRate := float64(sp.opsCount) / float64(sinceStart.Seconds()) // the final stats output goes to stdout: _, err := fmt.Printf("Run complete after %d queries with %d workers (Overall query rate %0.2f queries/sec):\n", i-sp.args.burnIn, workers, overallQueryRate) if err != nil { log.Fatal(err) } - err = writeStatGroupMap(os.Stdout, statMapping) + err = writeStatGroupMap(os.Stdout, sp.statMapping) if err != nil { log.Fatal(err) } @@ -174,7 +178,7 @@ func (sp *defaultStatProcessor) process(workers uint) { _, _ = fmt.Printf("Saving High Dynamic Range (HDR) Histogram of Response Latencies to %s\n", sp.args.hdrLatenciesFile) var b bytes.Buffer bw := bufio.NewWriter(&b) - _, err = statMapping[allQueriesLabel].latencyHDRHistogram.PercentilesPrint(bw, 10, 1000.0) + _, err = sp.statMapping[labelAllQueries].latencyHDRHistogram.PercentilesPrint(bw, 10, 1000.0) if err != nil { log.Fatal(err) } @@ -188,6 +192,24 @@ func (sp *defaultStatProcessor) process(workers uint) { sp.wg.Done() } +func (sp *defaultStatProcessor) GetTotalsMap() map[string]interface{}{ + totals := make(map[string]interface{}) + // PrewarmQueries tells the StatProcessor whether we're running each query twice to prewarm the cache + totals["prewarmQueries"] = sp.args.prewarmQueries + // limit is the number of statistics to analyze before stopping + totals["limit"] = sp.args.limit + // burnIn is the number of statistics to ignore before analyzing + totals["burnIn"] = sp.args.burnIn + sinceStart := time.Now().Sub(sp.startTime) + overallQueryRate := float64(sp.opsCount) / float64(sinceStart.Seconds()) + totals["overallQueryRate"] = overallQueryRate + //quantiles := make(map[string]interface{}) + //for i, i := range sp. { + // + //} + return totals +} + // CloseAndWait closes the stats channel and blocks until the StatProcessor has finished all the stats on its channel. func (sp *defaultStatProcessor) CloseAndWait() { close(sp.c) From f5d15138eec46710f55372c1705908eb63d80ba8 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 25 Apr 2021 18:21:17 +0100 Subject: [PATCH 55/72] [fix] Fixed build deps(gopsutil) on darwin arm64 --- go.mod | 3 ++- go.sum | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bd883ae0d..1106116ed 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,11 @@ require ( github.com/lib/pq v1.3.0 github.com/pkg/errors v0.9.1 github.com/prometheus/common v0.13.0 - github.com/shirou/gopsutil v2.18.12+incompatible + github.com/shirou/gopsutil v3.21.3+incompatible github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 + github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/transceptor-technology/go-qpack v0.0.0-20190116123619-49a14b216a45 github.com/valyala/fasthttp v1.15.1 go.uber.org/atomic v1.6.0 diff --git a/go.sum b/go.sum index ac238c6d1..6b3ac40ef 100644 --- a/go.sum +++ b/go.sum @@ -830,6 +830,8 @@ github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAx github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= +github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -898,6 +900,9 @@ github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiff github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 h1:jdJdzLyz0SNBuvt5rYyBxDqhgZ2EcbA7eWVBMqcyEHc= github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84/go.mod h1:rkhy9b91Qv9zTCqZh5kPUezLIRhghkZSwnmwRCXK8C0= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= @@ -1133,6 +1138,8 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200821140526-fda516888d29/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96 h1:gJciq3lOg0eS9fSZJcoHfv7q1BfC6cJfnmSSKL1yu3Q= golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa h1:ZYxPR6aca/uhfRJyaOAtflSHjJYiktO7QnJC5ut7iY4= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From e3f6b85415c82a7b5ae2e1bf18b3f069016400ea Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 25 Apr 2021 18:58:57 +0100 Subject: [PATCH 56/72] [add] Enabled results output for query runners --- pkg/query/stat_processor.go | 54 ++++++++++++++++--- .../full_cycle_minitest_timescaledb.sh | 6 +-- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/pkg/query/stat_processor.go b/pkg/query/stat_processor.go index 14717fb8e..85347136a 100644 --- a/pkg/query/stat_processor.go +++ b/pkg/query/stat_processor.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "github.com/HdrHistogram/hdrhistogram-go" "io/ioutil" "log" "os" @@ -192,6 +193,39 @@ func (sp *defaultStatProcessor) process(workers uint) { sp.wg.Done() } +func generateQuantileMap(hist *hdrhistogram.Histogram) (int64, map[string]float64) { + ops := hist.TotalCount() + q0 := 0.0 + q50 := 0.0 + q95 := 0.0 + q99 := 0.0 + q999 := 0.0 + q100 := 0.0 + if ops > 0 { + q0 = float64(hist.ValueAtQuantile(0.0)) / 10e2 + q50 = float64(hist.ValueAtQuantile(50.0)) / 10e2 + q95 = float64(hist.ValueAtQuantile(95.0)) / 10e2 + q99 = float64(hist.ValueAtQuantile(99.0)) / 10e2 + q999 = float64(hist.ValueAtQuantile(99.90)) / 10e2 + q100 = float64(hist.ValueAtQuantile(100.0)) / 10e2 + } + + mp := map[string]float64{"q0": q0, "q50": q50, "q95": q95, "q99": q99, "q999": q999, "q100": q100} + return ops, mp +} + +func (b *BenchmarkRunner) GetOverallQuantiles(label string,histogram *hdrhistogram.Histogram) map[string]interface{} { + configs := map[string]interface{}{} + _, all := generateQuantileMap(histogram) + configs[label] = all + configs["EncodedHistogram"] = nil + encodedHist, err := histogram.Encode(hdrhistogram.V2CompressedEncodingCookieBase) + if err == nil { + configs["EncodedHistogram"] = encodedHist + } + return configs +} + func (sp *defaultStatProcessor) GetTotalsMap() map[string]interface{}{ totals := make(map[string]interface{}) // PrewarmQueries tells the StatProcessor whether we're running each query twice to prewarm the cache @@ -201,12 +235,20 @@ func (sp *defaultStatProcessor) GetTotalsMap() map[string]interface{}{ // burnIn is the number of statistics to ignore before analyzing totals["burnIn"] = sp.args.burnIn sinceStart := time.Now().Sub(sp.startTime) - overallQueryRate := float64(sp.opsCount) / float64(sinceStart.Seconds()) - totals["overallQueryRate"] = overallQueryRate - //quantiles := make(map[string]interface{}) - //for i, i := range sp. { - // - //} + // calculate overall query rates + queryRates := make(map[string]interface{}) + for label, statGroup := range sp.statMapping { + overallQueryRate := float64(statGroup.count) / float64(sinceStart.Seconds()) + queryRates[label] = overallQueryRate + } + totals["overallQueryRates"] = queryRates + // calculate overall quantiles + quantiles := make(map[string]interface{}) + for label, statGroup := range sp.statMapping { + _, all := generateQuantileMap(statGroup.latencyHDRHistogram) + quantiles[label] = all + } + totals["overallQuantiles"] = quantiles return totals } diff --git a/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh b/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh index 7dfaacaa4..178363d8f 100755 --- a/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh +++ b/scripts/full_cycle_minitest/full_cycle_minitest_timescaledb.sh @@ -22,6 +22,6 @@ $GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format timescaledb $GOPATH/bin/tsbs_load_timescaledb --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --results-file="timescaledb_load_results.json" --db-name=benchmark --host=127.0.0.1 --user=postgres --workers=1 --file=/tmp/bulk_data/timescaledb_data # queries benchmark -$GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_lastpoint.hdr" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_lastpoint -$GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_cpu-max-all-1.hdr" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_cpu-max-all-1 -$GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_high-cpu-1.hdr" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_high-cpu-1 +$GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_lastpoint.hdr" --results-file="timescaledb_query_lastpoint_results.json" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_lastpoint +$GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_cpu-max-all-1.hdr" --results-file="timescaledb_query_cpu-max-all-1_results.json" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_cpu-max-all-1 +$GOPATH/bin/tsbs_run_queries_timescaledb --max-rps=${MAX_RPS} --hdr-latencies="${MAX_RPS}rps_timescaledb_query_high-cpu-1.hdr" --results-file="timescaledb_query_high-cpu-1_results.json" --pass=${PASSWORD} --postgres="sslmode=disable port=5433" --db-name=benchmark --hosts=127.0.0.1 --user=postgres --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/timescaledb_query_high-cpu-1 From 842092108d221e9f693738a21984c7e6b1460dbe Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 26 Apr 2021 00:55:26 +0100 Subject: [PATCH 57/72] [add] Fixed query exporting --- Makefile | 7 +++++++ go.sum | 1 + pkg/query/benchmarker.go | 4 ++-- pkg/query/stat_processor.go | 10 ++++++++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 591ce0435..90aa16603 100644 --- a/Makefile +++ b/Makefile @@ -91,3 +91,10 @@ publish-redistimeseries: release-redistimeseries echo "copying ${DISTDIR}/$${f}"; \ aws s3 cp ${DISTDIR}/$${f} s3://benchmarks.redislabs/redistimeseries/tools/tsbs/$${f} --acl public-read; \ done + +publish-redistimeseries-queries: + @for f in $(shell ls /tmp/bulk_queries); \ + do \ + echo "copying $${f}"; \ + aws s3 cp /tmp/bulk_queries/$${f} s3://benchmarks.redislabs/redistimeseries/tsbs/queries/devops/scale100/devops-scale100-4days/$${f} --acl public-read; \ + done diff --git a/go.sum b/go.sum index 7214fd2e8..74134fc82 100644 --- a/go.sum +++ b/go.sum @@ -936,6 +936,7 @@ github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84/go.mod h1:rkhy github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= diff --git a/pkg/query/benchmarker.go b/pkg/query/benchmarker.go index b557c2f4e..f3956d4e2 100644 --- a/pkg/query/benchmarker.go +++ b/pkg/query/benchmarker.go @@ -198,8 +198,8 @@ func (b *BenchmarkRunner) saveTestResult(took time.Duration, start time.Time, en testResult := LoaderTestResult{ ResultFormatVersion: BenchmarkTestResultVersion, RunnerConfig: b.BenchmarkRunnerConfig, - StartTime: start.Unix(), - EndTime: end.Unix(), + StartTime: start.UTC().Unix()*1000, + EndTime: end.UTC().Unix()*1000, DurationMillis: took.Milliseconds(), Totals: b.sp.GetTotalsMap(), } diff --git a/pkg/query/stat_processor.go b/pkg/query/stat_processor.go index 85347136a..b85231ad9 100644 --- a/pkg/query/stat_processor.go +++ b/pkg/query/stat_processor.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "log" "os" + "regexp" "sync" "sync/atomic" "time" @@ -239,19 +240,24 @@ func (sp *defaultStatProcessor) GetTotalsMap() map[string]interface{}{ queryRates := make(map[string]interface{}) for label, statGroup := range sp.statMapping { overallQueryRate := float64(statGroup.count) / float64(sinceStart.Seconds()) - queryRates[label] = overallQueryRate + queryRates[stripRegex(label)] = overallQueryRate } totals["overallQueryRates"] = queryRates // calculate overall quantiles quantiles := make(map[string]interface{}) for label, statGroup := range sp.statMapping { _, all := generateQuantileMap(statGroup.latencyHDRHistogram) - quantiles[label] = all + quantiles[stripRegex(label)] = all } totals["overallQuantiles"] = quantiles return totals } +func stripRegex(in string) string { + reg, _ := regexp.Compile("[^a-zA-Z0-9]+") + return reg.ReplaceAllString(in, "_") +} + // CloseAndWait closes the stats channel and blocks until the StatProcessor has finished all the stats on its channel. func (sp *defaultStatProcessor) CloseAndWait() { close(sp.c) From e016c16db577db8cc0d1e55dd2d55a1a61706afb Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 17 May 2021 23:37:08 +0100 Subject: [PATCH 58/72] [fix] Increased dial timeout to 120secs due to long running queries --- cmd/tsbs_run_queries_redistimeseries/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/tsbs_run_queries_redistimeseries/main.go b/cmd/tsbs_run_queries_redistimeseries/main.go index f20405024..1a8999942 100644 --- a/cmd/tsbs_run_queries_redistimeseries/main.go +++ b/cmd/tsbs_run_queries_redistimeseries/main.go @@ -68,6 +68,7 @@ func init() { r = rand.New(s) // initialize local pseudorandom generator opts := make([]radix.DialOpt, 0) + opts = append(opts, radix.DialReadTimeout(120*time.Second)) if clusterMode { cluster = getOSSClusterConn(host, opts, uint64(config.Workers)) cluster.Sync() From 80e708da5a5081e413d42ef733ed3b0f904e267a Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 5 Jul 2021 14:34:42 +0100 Subject: [PATCH 59/72] [add] Enabled to load with different timeseries compressions --- cmd/tsbs_load_redistimeseries/cluster_conn.go | 4 ++-- cmd/tsbs_load_redistimeseries/main.go | 22 +++++++++---------- cmd/tsbs_load_redistimeseries/processor.go | 4 ++-- cmd/tsbs_load_redistimeseries/scan.go | 4 ++-- .../standalone_conn.go | 4 ++-- .../redistimeseries/implemented_target.go | 3 +-- scripts/load/load_redistimeseries.sh | 1 + scripts/redistimeseries_common.sh | 2 ++ 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cmd/tsbs_load_redistimeseries/cluster_conn.go b/cmd/tsbs_load_redistimeseries/cluster_conn.go index bb7be1c1a..e19a46643 100644 --- a/cmd/tsbs_load_redistimeseries/cluster_conn.go +++ b/cmd/tsbs_load_redistimeseries/cluster_conn.go @@ -49,7 +49,7 @@ func nodeThatContainsSlot(slots [][][2]uint16, slot int) (result int) { return } -func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics chan uint64, cluster *radix.Cluster, clusterNodes int, addresses []string, slots [][][2]uint16, conns []radix.Client) { +func connectionProcessorCluster(wg *sync.WaitGroup, compressionType string, rows chan string, metrics chan uint64, cluster *radix.Cluster, clusterNodes int, addresses []string, slots [][][2]uint16, conns []radix.Client) { cmds := make([][]radix.CmdAction, clusterNodes, clusterNodes) curPipe := make([]uint64, clusterNodes, clusterNodes) currMetricCount := make([]int, clusterNodes, clusterNodes) @@ -60,7 +60,7 @@ func connectionProcessorCluster(wg *sync.WaitGroup, rows chan string, metrics ch } for row := range rows { - slot, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) + slot, cmd, _, metricCount := buildCommand(row, compressionType) comdPos := nodeThatContainsSlot(slots, slot) var err error = nil diff --git a/cmd/tsbs_load_redistimeseries/main.go b/cmd/tsbs_load_redistimeseries/main.go index 1a9bd4920..fd8ed4208 100644 --- a/cmd/tsbs_load_redistimeseries/main.go +++ b/cmd/tsbs_load_redistimeseries/main.go @@ -18,14 +18,14 @@ import ( // Program option vars: var ( - host string - connections uint64 - pipeline uint64 - checkChunks uint64 - singleQueue bool - dataModel string - compressionEnabled bool - clusterMode bool + host string + connections uint64 + pipeline uint64 + checkChunks uint64 + singleQueue bool + dataModel string + compressionType string + clusterMode bool ) // Global vars @@ -66,7 +66,7 @@ func init() { connections = viper.GetUint64("connections") pipeline = viper.GetUint64("pipeline") dataModel = "redistimeseries" - compressionEnabled = true + compressionType = viper.GetString("compression") clusterMode = viper.GetBool("cluster") config.NoFlowControl = true config.HashWorkers = true @@ -95,9 +95,7 @@ func main() { log.Println("Starting benchmark") b := benchmark{dbc: &dbCreator{}} - //if config.Workers > 1 { - // panic(fmt.Errorf("You should only use 1 worker and multiple connections per worker (set via --connections)")) - //} + log.Println("Using compression: ", compressionType) loader.RunBenchmark(&b) log.Println("finished benchmark") diff --git a/cmd/tsbs_load_redistimeseries/processor.go b/cmd/tsbs_load_redistimeseries/processor.go index 866f3e0c1..e259dda5e 100644 --- a/cmd/tsbs_load_redistimeseries/processor.go +++ b/cmd/tsbs_load_redistimeseries/processor.go @@ -32,9 +32,9 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) p.rows[i] = make(chan string, buflen) p.wg.Add(1) if clusterMode { - go connectionProcessorCluster(p.wg, p.rows[i], p.metrics, cluster, len(addresses), addresses, slots, conns) + go connectionProcessorCluster(p.wg, compressionType, p.rows[i], p.metrics, cluster, len(addresses), addresses, slots, conns) } else { - go connectionProcessor(p.wg, p.rows[i], p.metrics, standalone) + go connectionProcessor(p.wg, compressionType, p.rows[i], p.metrics, standalone) } } for _, row := range events.rows { diff --git a/cmd/tsbs_load_redistimeseries/scan.go b/cmd/tsbs_load_redistimeseries/scan.go index cc1176d94..55c0367b3 100644 --- a/cmd/tsbs_load_redistimeseries/scan.go +++ b/cmd/tsbs_load_redistimeseries/scan.go @@ -11,7 +11,7 @@ import ( "github.com/timescale/tsbs/pkg/targets" ) -func buildCommand(line string, forceUncompressed bool) (clusterSlot int, cmdA radix.CmdAction, tscreate bool, metricCount int) { +func buildCommand(line string, compression_type string) (clusterSlot int, cmdA radix.CmdAction, tscreate bool, metricCount int) { t := strings.Split(line, " ") metricCount = 1 tscreate = false @@ -21,11 +21,11 @@ func buildCommand(line string, forceUncompressed bool) (clusterSlot int, cmdA ra if cmdname == "TS.CREATE" { tscreate = true metricCount = 0 + t = append([]string{t[0], t[1], t[2], compression_type}, t[3:]...) } if cmdname == "TS.MADD" { metricCount = (len(t) - 2) / 3 } - //key := t[2] cmdA = radix.Cmd(nil, cmdname, t[2:]...) return } diff --git a/cmd/tsbs_load_redistimeseries/standalone_conn.go b/cmd/tsbs_load_redistimeseries/standalone_conn.go index b9ef5a9a3..bafc03e81 100644 --- a/cmd/tsbs_load_redistimeseries/standalone_conn.go +++ b/cmd/tsbs_load_redistimeseries/standalone_conn.go @@ -22,7 +22,7 @@ func getStandaloneConn(addr string, opts []radix.DialOpt, clients uint64) *radix return pool } -func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint64, conn radix.Client) { +func connectionProcessor(wg *sync.WaitGroup, compressionType string, rows chan string, metrics chan uint64, conn radix.Client) { cmds := make([][]radix.CmdAction, 1, 1) cmds[0] = make([]radix.CmdAction, 0, 0) curPipe := make([]uint64, 1, 1) @@ -31,7 +31,7 @@ func connectionProcessor(wg *sync.WaitGroup, rows chan string, metrics chan uint comdPos := 0 for row := range rows { - _, cmd, _, metricCount := buildCommand(row, compressionEnabled == false) + _, cmd, _, metricCount := buildCommand(row, compressionType) currMetricCount += metricCount cmds[comdPos] = append(cmds[comdPos], cmd) curPipe[comdPos]++ diff --git a/pkg/targets/redistimeseries/implemented_target.go b/pkg/targets/redistimeseries/implemented_target.go index a051702dc..7ca1106ae 100644 --- a/pkg/targets/redistimeseries/implemented_target.go +++ b/pkg/targets/redistimeseries/implemented_target.go @@ -20,9 +20,8 @@ func (t *redistimeseriesTarget) TargetSpecificFlags(flagPrefix string, flagSet * flagSet.String(flagPrefix+"host", "localhost:6379", "The host:port for Redis connection") flagSet.Uint64(flagPrefix+"pipeline", 10, "The pipeline's size") flagSet.Uint64(flagPrefix+"connections", 1, "The number of connections per worker") - flagSet.Bool(flagPrefix+ "compression-enabled", true, "Whether to use compressed time series") + flagSet.String(flagPrefix+ "compression", "compressed:turbo_gorilla", "Type of compression used for each serie ( one of: 'compressed','compressed:turbo_gorilla','uncompressed' )") flagSet.Bool(flagPrefix+ "cluster", false, "Whether to use OSS cluster API") - } func (t *redistimeseriesTarget) TargetName() string { diff --git a/scripts/load/load_redistimeseries.sh b/scripts/load/load_redistimeseries.sh index 39c741bb2..546557d9f 100755 --- a/scripts/load/load_redistimeseries.sh +++ b/scripts/load/load_redistimeseries.sh @@ -19,6 +19,7 @@ echo "Saving results to ${OUT_FULL_FILE_NAME}" $EXE_FILE_NAME \ --file ${DATA_FILE_NAME} \ --workers=${NUM_WORKERS} \ + --compression=${COMPRESSION_TYPE} \ --batch-size=${BATCH_SIZE} \ --reporting-period=${REPORTING_PERIOD} \ --host=${DATABASE_HOST}:${DATABASE_PORT} ${CLUSTER_FLAG} \ diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh index 79e31d7ba..2ad1a10f9 100644 --- a/scripts/redistimeseries_common.sh +++ b/scripts/redistimeseries_common.sh @@ -18,6 +18,8 @@ CLUSTER_FLAG=${CLUSTER_FLAG:-""} NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 8)} REPORTING_PERIOD=${REPORTING_PERIOD:-1s} +COMPRESSION_TYPE=${COMPRESSION_TYPE:-"compressed:turbo_gorilla"} + REPETITIONS=${REPETITIONS:-3} # Rand seed From 1c77d5b860763dd3a3d1f8724973312644a4efb3 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 6 Jul 2021 15:54:31 +0100 Subject: [PATCH 60/72] [add] re-enabled debug print --- .../databases/redistimeseries/devops.go | 9 +++-- cmd/tsbs_run_queries_redistimeseries/main.go | 40 +++++++++++++++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go index 80011a3d5..695599663 100644 --- a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go @@ -39,7 +39,6 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t //[]byte("TS.MRANGE"), Just to help understanding []byte(fmt.Sprintf("%d", interval.StartUnixMillis())), []byte(fmt.Sprintf("%d", interval.EndUnixMillis())), - []byte("WITHLABELS"), []byte("AGGREGATION"), []byte("MAX"), []byte(fmt.Sprintf("%d", oneMinuteMillis)), @@ -88,7 +87,7 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t redisQuery = append(redisQuery, []byte(redisArg)) if nHosts > 1 && numMetrics == 1 { - redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("hostname"), []byte("REDUCE"), []byte("max") ) + redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("fieldname"), []byte("REDUCE"), []byte("max") ) } if numMetrics > 1 { redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("fieldname"), []byte("REDUCE"), []byte("max") ) @@ -109,7 +108,6 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { //[]byte("TS.MRANGE"), Just to help understanding []byte(fmt.Sprintf("%d", interval.StartUnixMillis())), []byte(fmt.Sprintf("%d", interval.EndUnixMillis())), - []byte("WITHLABELS"), []byte("AGGREGATION"), []byte("AVG"), []byte(fmt.Sprintf("%d", oneHourMillis)), @@ -159,7 +157,6 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) { //[]byte("TS.MRANGE"), Just to help understanding []byte(fmt.Sprintf("%d", interval.StartUnixMillis())), []byte(fmt.Sprintf("%d", interval.EndUnixMillis())), - []byte("WITHLABELS"), []byte("AGGREGATION"), []byte("MAX"), []byte(fmt.Sprintf("%d", oneHourMillis)), @@ -258,6 +255,10 @@ func (d *Devops) GroupByOrderByLimit(qi query.Query) { []byte("FILTER"), []byte("measurement=cpu"), []byte("fieldname=usage_user"), + []byte("GROUPBY"), + []byte("fieldname"), + []byte("REDUCE"), + []byte("max"), } humanLabel := devops.GetGroupByOrderByLimitLabel("RedisTimeSeries") diff --git a/cmd/tsbs_run_queries_redistimeseries/main.go b/cmd/tsbs_run_queries_redistimeseries/main.go index 1a8999942..1c42e3d32 100644 --- a/cmd/tsbs_run_queries_redistimeseries/main.go +++ b/cmd/tsbs_run_queries_redistimeseries/main.go @@ -126,8 +126,10 @@ func (p *processor) ProcessQuery(q query.Query, isWarm bool) (queryStats []*quer tq := q.(*query.RedisTimeSeries) var cmds = make([][]string, 0, 0) + var replies = make([][]interface{}, 0, 0) for _, qry := range tq.RedisQueries { cmds = append(cmds, ByteArrayToStringArray(qry)) + replies = append(replies, []interface{}{}) } start := time.Now() for idx, commandArgs := range cmds { @@ -139,12 +141,12 @@ func (p *processor) ProcessQuery(q query.Query, isWarm bool) (queryStats []*quer if string(tq.CommandNames[idx]) == "TS.MRANGE" || string(tq.CommandNames[idx]) == "TS.QUERYINDEX" || string(tq.CommandNames[idx]) == "TS.MGET" || string(tq.CommandNames[idx]) == "TS.MREVRANGE" { rPos := r.Intn(len(conns)) conn := conns[rPos] - err = conn.Do(radix.Cmd(nil, string(tq.CommandNames[idx]), commandArgs...)) + err = conn.Do(radix.Cmd(&replies[idx], string(tq.CommandNames[idx]), commandArgs...)) } else { - err = cluster.Do(radix.Cmd(nil, string(tq.CommandNames[idx]), commandArgs...)) + err = cluster.Do(radix.Cmd(&replies[idx], string(tq.CommandNames[idx]), commandArgs...)) } } else { - err = standalone.Do(radix.Cmd(nil, string(tq.CommandNames[idx]), commandArgs...)) + err = standalone.Do(radix.Cmd(&replies[idx], string(tq.CommandNames[idx]), commandArgs...)) } if err != nil { log.Fatalf("Command (%s %s) failed with error: %v\n", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "), err) @@ -152,6 +154,38 @@ func (p *processor) ProcessQuery(q query.Query, isWarm bool) (queryStats []*quer if err != nil { return nil, err } + if p.opts.debug { + fmt.Println(fmt.Sprintf("Command reply. Total series %d", len(replies[idx]))) + for _, serie := range replies[idx] { + converted_serie := serie.([]interface{}) + serie_name := string(converted_serie[0].([]uint8)) + fmt.Println(fmt.Sprintf("\tSerie name: %s", serie_name)) + serie_labels := converted_serie[1].([]interface{}) + fmt.Println(fmt.Sprintf("\tSerie labels:")) + for _, kvpair := range serie_labels { + kvpairc := kvpair.([]interface{}) + k := string(kvpairc[0].([]uint8)) + v := string(kvpairc[1].([]uint8)) + fmt.Println(fmt.Sprintf("\t\t%s: %s", k, v)) + } + fmt.Println(fmt.Sprintf("\tSerie datapoints:")) + serie_datapoints := converted_serie[2].([]interface{}) + if string(tq.CommandNames[idx]) == "TS.MGET" { + ts := serie_datapoints[0].(int64) + v := serie_datapoints[1].(string) + fmt.Println(fmt.Sprintf("\t\tts: %d value: %s", ts, v)) + + } else { + for _, datapointpair := range serie_datapoints { + datapoint := datapointpair.([]interface{}) + ts := datapoint[0].(int64) + v := datapoint[1].(string) + fmt.Println(fmt.Sprintf("\t\tts: %d value: %s", ts, v)) + } + } + + } + } } took := float64(time.Since(start).Nanoseconds()) / 1e6 From 4e3cf280b297dad1b67885908d063c8989b63f1b Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 6 Jul 2021 17:41:07 +0100 Subject: [PATCH 61/72] [add] enabled high-cpu-1 and high-cpu-all queries --- .../databases/redistimeseries/devops.go | 22 +++- .../common.go | 31 +++++ cmd/tsbs_run_queries_redistimeseries/debug.go | 54 +++++++++ .../functors.go | 35 ++++++ cmd/tsbs_run_queries_redistimeseries/main.go | 76 +----------- go.mod | 11 +- go.sum | 109 +++++++++++++++--- scripts/generate_queries.sh | 16 --- scripts/generate_queries_redistimeseries.sh | 7 +- 9 files changed, 245 insertions(+), 116 deletions(-) create mode 100644 cmd/tsbs_run_queries_redistimeseries/common.go create mode 100644 cmd/tsbs_run_queries_redistimeseries/debug.go create mode 100644 cmd/tsbs_run_queries_redistimeseries/functors.go diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go index 695599663..43a7f8b2c 100644 --- a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go @@ -203,6 +203,20 @@ func (d *Devops) LastPointPerHost(qi query.Query) { d.AddQuery(qi, redisQuery, []byte("TS.MGET")) } + + +// HighCPUForHosts populates a query that gets CPU metrics when the CPU has high +// usage between a time period for a number of hosts (if 0, it will search all hosts), +// e.g. in pseudo-SQL: +// +// SELECT * FROM cpu +// WHERE usage_user > 90.0 +// AND time >= '$TIME_START' AND time < '$TIME_END' +// AND (hostname = '$HOST' OR hostname = '$HOST2'...) +// +// Resultsets: +// high-cpu-1 +// high-cpu-all func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { hostnames, err := d.GetRandomHosts(nHosts) interval := d.Interval.MustRandWindow(devops.HighCPUDuration) @@ -210,10 +224,11 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { //[]byte("TS.MRANGE"), Just to help understanding []byte(fmt.Sprintf("%d", interval.StartUnixMillis())), []byte(fmt.Sprintf("%d", interval.EndUnixMillis())), + []byte("FILTER_BY_VALUE"), []byte("90.0"),[]byte("1000"), []byte("FILTER"), + []byte("fieldname=usage_user"), []byte("measurement=cpu"), } - if nHosts > 0 { redisArg := "hostname=" if nHosts > 1 { @@ -230,13 +245,14 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { } redisQuery = append(redisQuery, []byte(redisArg)) } + redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("fieldname"), []byte("REDUCE"), []byte("max") ) + humanLabel, err := devops.GetHighCPULabel("RedisTimeSeries", nHosts) panicIfErr(err) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) d.fillInQueryStrings(qi, humanLabel, humanDesc) d.AddQuery(qi, redisQuery, []byte("TS.MRANGE")) - functorName := query.GetFunctionName(query.HighCpu) - d.SetApplyFunctor(qi, true, functorName) + d.SetApplyFunctor(qi, true, "FILTER_BY_TS") } // GroupByOrderByLimit populates a query.Query that has a time WHERE clause, that groups by a truncated date, orders by that date, and takes a limit: diff --git a/cmd/tsbs_run_queries_redistimeseries/common.go b/cmd/tsbs_run_queries_redistimeseries/common.go new file mode 100644 index 000000000..0ab0d1170 --- /dev/null +++ b/cmd/tsbs_run_queries_redistimeseries/common.go @@ -0,0 +1,31 @@ +package main + +import ( + "fmt" + "github.com/mediocregopher/radix/v3" + "github.com/timescale/tsbs/pkg/query" + "log" + "strings" +) + +func inner_cmd_logic(p *processor, tq *query.RedisTimeSeries, idx int, replies [][]interface{}, commandArgs []string) error { + var err error = nil + if p.opts.debug { + fmt.Println(fmt.Sprintf("Issuing command (%s %s)", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "))) + } + if clusterMode { + if string(tq.CommandNames[idx]) == "TS.MRANGE" || string(tq.CommandNames[idx]) == "TS.QUERYINDEX" || string(tq.CommandNames[idx]) == "TS.MGET" || string(tq.CommandNames[idx]) == "TS.MREVRANGE" { + rPos := r.Intn(len(conns)) + conn := conns[rPos] + err = conn.Do(radix.Cmd(&replies[idx], string(tq.CommandNames[idx]), commandArgs...)) + } else { + err = cluster.Do(radix.Cmd(&replies[idx], string(tq.CommandNames[idx]), commandArgs...)) + } + } else { + err = standalone.Do(radix.Cmd(&replies[idx], string(tq.CommandNames[idx]), commandArgs...)) + } + if err != nil { + log.Fatalf("Command (%s %s) failed with error: %v\n", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "), err) + } + return err +} diff --git a/cmd/tsbs_run_queries_redistimeseries/debug.go b/cmd/tsbs_run_queries_redistimeseries/debug.go new file mode 100644 index 000000000..648d909aa --- /dev/null +++ b/cmd/tsbs_run_queries_redistimeseries/debug.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "github.com/timescale/tsbs/pkg/query" +) + +func debug_print_redistimeseries_reply(reply [][]interface{}, idx int, tq *query.RedisTimeSeries) { + fmt.Println(fmt.Sprintf("Command reply. Total series %d", len(reply[idx]))) + for _, serie := range reply[idx] { + converted_serie := serie.([]interface{}) + serie_name := string(converted_serie[0].([]uint8)) + fmt.Println(fmt.Sprintf("\tSerie name: %s", serie_name)) + serie_labels := converted_serie[1].([]interface{}) + fmt.Println(fmt.Sprintf("\tSerie labels:")) + for _, kvpair := range serie_labels { + kvpairc := kvpair.([]interface{}) + k := string(kvpairc[0].([]uint8)) + v := string(kvpairc[1].([]uint8)) + fmt.Println(fmt.Sprintf("\t\t%s: %s", k, v)) + } + fmt.Println(fmt.Sprintf("\tSerie datapoints:")) + serie_datapoints := converted_serie[2].([]interface{}) + if string(tq.CommandNames[idx]) == "TS.MGET" { + ts := serie_datapoints[0].(int64) + v := serie_datapoints[1].(string) + fmt.Println(fmt.Sprintf("\t\tts: %d value: %s", ts, v)) + + } else { + for _, datapointpair := range serie_datapoints { + datapoint := datapointpair.([]interface{}) + ts := datapoint[0].(int64) + v := datapoint[1].(string) + fmt.Println(fmt.Sprintf("\t\tts: %d value: %s", ts, v)) + } + } + } +} + +func ByteArrayToInterfaceArray(qry [][]byte) []interface{} { + commandArgs := make([]interface{}, len(qry)) + for i := 0; i < len(qry); i++ { + commandArgs[i] = qry[i] + } + return commandArgs +} + +func ByteArrayToStringArray(qry [][]byte) []string { + commandArgs := make([]string, len(qry)) + for i := 0; i < len(qry); i++ { + commandArgs[i] = string(qry[i]) + } + return commandArgs +} diff --git a/cmd/tsbs_run_queries_redistimeseries/functors.go b/cmd/tsbs_run_queries_redistimeseries/functors.go new file mode 100644 index 000000000..be154e83c --- /dev/null +++ b/cmd/tsbs_run_queries_redistimeseries/functors.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + "github.com/timescale/tsbs/pkg/query" + "strings" +) + +// specifically for Resultsets: high-cpu-1, high-cpu-all +// we need to take the reply timestamps and re-issue the query now with +// FILTER_BY_TS given the timestamps that passed the first query condition ( the FILTER_BY_VALUE ) +func highCpuFilterByTsFunctor(tq *query.RedisTimeSeries, replies [][]interface{}, idx int, commandArgs []string, p *processor, err error) error { + if len(replies[idx]) > 0 { + new_query := []string{commandArgs[0], commandArgs[1], "FILTER_BY_TS"} + first_serie := replies[idx][0] + serie_datapoints := first_serie.([]interface{})[2].([]interface{}) + for _, datapointpair := range serie_datapoints { + datapoint := datapointpair.([]interface{}) + ts := datapoint[0].(int64) + new_query = append(new_query, fmt.Sprintf("%d", ts)) + } + for _, arg := range commandArgs[4:8] { + new_query = append(new_query, arg) + } + if p.opts.debug { + fmt.Println(fmt.Sprintf("Applying FILTER_BY_TS condition command (%s %s)", string(tq.CommandNames[idx]), strings.Join(new_query, " "))) + } + err = inner_cmd_logic(p, tq, idx, replies, new_query) + } else { + if p.opts.debug { + fmt.Println(fmt.Sprintf("Applying FILTER_BY_VALUE condition returned zero series")) + } + } + return err +} diff --git a/cmd/tsbs_run_queries_redistimeseries/main.go b/cmd/tsbs_run_queries_redistimeseries/main.go index 1c42e3d32..c909727b4 100644 --- a/cmd/tsbs_run_queries_redistimeseries/main.go +++ b/cmd/tsbs_run_queries_redistimeseries/main.go @@ -7,9 +7,7 @@ package main import ( "fmt" "github.com/mediocregopher/radix/v3" - "log" "math/rand" - "strings" "time" "github.com/blagojts/viper" @@ -82,12 +80,6 @@ func init() { conn, _ := cluster.Client(nodeAddress) conns = append(conns, conn) } - //if p.opts.debug { - // fmt.Println(addresses) - // fmt.Println(slots) - // fmt.Println(conns) - //} - } else { standalone = getStandaloneConn(host, opts, uint64(config.Workers)) } @@ -131,60 +123,18 @@ func (p *processor) ProcessQuery(q query.Query, isWarm bool) (queryStats []*quer cmds = append(cmds, ByteArrayToStringArray(qry)) replies = append(replies, []interface{}{}) } + start := time.Now() for idx, commandArgs := range cmds { - var err error = nil - if p.opts.debug { - fmt.Println(fmt.Sprintf("Issuing command (%s %s)", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "))) - } - if clusterMode { - if string(tq.CommandNames[idx]) == "TS.MRANGE" || string(tq.CommandNames[idx]) == "TS.QUERYINDEX" || string(tq.CommandNames[idx]) == "TS.MGET" || string(tq.CommandNames[idx]) == "TS.MREVRANGE" { - rPos := r.Intn(len(conns)) - conn := conns[rPos] - err = conn.Do(radix.Cmd(&replies[idx], string(tq.CommandNames[idx]), commandArgs...)) - } else { - err = cluster.Do(radix.Cmd(&replies[idx], string(tq.CommandNames[idx]), commandArgs...)) - } - } else { - err = standalone.Do(radix.Cmd(&replies[idx], string(tq.CommandNames[idx]), commandArgs...)) - } - if err != nil { - log.Fatalf("Command (%s %s) failed with error: %v\n", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "), err) + err := inner_cmd_logic(p, tq, idx, replies, commandArgs) + if tq.Functor == "FILTER_BY_TS" { + err = highCpuFilterByTsFunctor(tq, replies, idx, commandArgs, p, err) } if err != nil { return nil, err } if p.opts.debug { - fmt.Println(fmt.Sprintf("Command reply. Total series %d", len(replies[idx]))) - for _, serie := range replies[idx] { - converted_serie := serie.([]interface{}) - serie_name := string(converted_serie[0].([]uint8)) - fmt.Println(fmt.Sprintf("\tSerie name: %s", serie_name)) - serie_labels := converted_serie[1].([]interface{}) - fmt.Println(fmt.Sprintf("\tSerie labels:")) - for _, kvpair := range serie_labels { - kvpairc := kvpair.([]interface{}) - k := string(kvpairc[0].([]uint8)) - v := string(kvpairc[1].([]uint8)) - fmt.Println(fmt.Sprintf("\t\t%s: %s", k, v)) - } - fmt.Println(fmt.Sprintf("\tSerie datapoints:")) - serie_datapoints := converted_serie[2].([]interface{}) - if string(tq.CommandNames[idx]) == "TS.MGET" { - ts := serie_datapoints[0].(int64) - v := serie_datapoints[1].(string) - fmt.Println(fmt.Sprintf("\t\tts: %d value: %s", ts, v)) - - } else { - for _, datapointpair := range serie_datapoints { - datapoint := datapointpair.([]interface{}) - ts := datapoint[0].(int64) - v := datapoint[1].(string) - fmt.Println(fmt.Sprintf("\t\tts: %d value: %s", ts, v)) - } - } - - } + debug_print_redistimeseries_reply(replies, idx, tq) } } took := float64(time.Since(start).Nanoseconds()) / 1e6 @@ -194,19 +144,3 @@ func (p *processor) ProcessQuery(q query.Query, isWarm bool) (queryStats []*quer queryStats = []*query.Stat{stat} return queryStats, err } - -func ByteArrayToInterfaceArray(qry [][]byte) []interface{} { - commandArgs := make([]interface{}, len(qry)) - for i := 0; i < len(qry); i++ { - commandArgs[i] = qry[i] - } - return commandArgs -} - -func ByteArrayToStringArray(qry [][]byte) []string { - commandArgs := make([]string, len(qry)) - for i := 0; i < len(qry); i++ { - commandArgs[i] = string(qry[i]) - } - return commandArgs -} diff --git a/go.mod b/go.mod index 3cad983f7..9ea761b8c 100644 --- a/go.mod +++ b/go.mod @@ -14,14 +14,16 @@ require ( github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gocql/gocql v0.0.0-20190810123941-df4b9cc33030 - github.com/golang/protobuf v1.4.2 + github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.1 github.com/google/flatbuffers v1.11.0 - github.com/google/go-cmp v0.5.2 + github.com/google/go-cmp v0.5.5 github.com/jackc/pgx/v4 v4.8.0 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 github.com/kshvakov/clickhouse v1.3.11 github.com/lib/pq v1.3.0 + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.13 // indirect github.com/mediocregopher/radix/v3 v3.7.0 github.com/mitchellh/gox v1.0.1 // indirect github.com/pkg/errors v0.9.1 @@ -29,13 +31,14 @@ require ( github.com/shirou/gopsutil v3.21.3+incompatible github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 - github.com/tcnksm/ghr v0.13.0 // indirect + github.com/tcnksm/ghr v0.14.0 // indirect github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/transceptor-technology/go-qpack v0.0.0-20190116123619-49a14b216a45 github.com/valyala/fasthttp v1.15.1 go.uber.org/atomic v1.6.0 - golang.org/x/net v0.0.0-20200904194848-62affa334b73 + golang.org/x/net v0.0.0-20210614182718-04defd469f4e + golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 74134fc82..e80e80b27 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,17 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= @@ -21,9 +28,12 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v45.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -367,12 +377,15 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -380,8 +393,10 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -411,22 +426,27 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -487,6 +507,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw= +github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -658,8 +680,9 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -668,8 +691,9 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= +github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= @@ -856,11 +880,8 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= -github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= -github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= @@ -920,8 +941,8 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tcnksm/ghr v0.13.0 h1:a5ZbaUAfiaiw6rEDJVUEDYA9YreZOkh3XAfXHWn8zu8= -github.com/tcnksm/ghr v0.13.0/go.mod h1:tcp6tzbRYE0LqFSG7ykXP/BVG1/2BkX6aIn9FFV1mIQ= +github.com/tcnksm/ghr v0.14.0 h1:3iCanOmn2xDgLnbbTy3ifzHiJ8/gCzYv6K5YWW+1P1s= +github.com/tcnksm/ghr v0.14.0/go.mod h1:6FdZv7a1yCKqJGwA4srq5ZtqGzEUnSCtRQOfGuuzSd4= github.com/tcnksm/go-gitconfig v0.1.2 h1:iiDhRitByXAEyjgBqsKi9QU4o2TNtv9kPP3RgPgXBPw= github.com/tcnksm/go-gitconfig v0.1.2/go.mod h1:/8EhP4H7oJZdIPyT+/UIsG87kTzrzM4UsLGSItWYCpE= github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e h1:IWllFTiDjjLIf2oeKxpIUmtiDV5sn71VgeQgg6vcE7k= @@ -987,6 +1008,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1080,6 +1102,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1092,21 +1115,27 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c h1:pkQiBZBvdos9qq4wBAHqlzuZHEXo07pqV06ef90u1WI= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1115,8 +1144,9 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1166,25 +1196,35 @@ golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200821140526-fda516888d29/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96 h1:gJciq3lOg0eS9fSZJcoHfv7q1BfC6cJfnmSSKL1yu3Q= golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa h1:ZYxPR6aca/uhfRJyaOAtflSHjJYiktO7QnJC5ut7iY4= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1253,19 +1293,29 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200321224714-0d839f3cf2ed/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200519015757-0d0afa43d58a/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200701041122-1837592efa10/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200822203824-307de81be3f4/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200908211811-12e1bf57a112 h1:DmrRJy1qn9VDMf4+GSpRlwfZ51muIF7r96MFBFP4bPM= golang.org/x/tools v0.0.0-20200908211811-12e1bf57a112/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1291,16 +1341,22 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1321,10 +1377,20 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200815001618-f69a88009b70/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d h1:92D1fum1bJLKSdr11OJ+54YeCMCGYIygTA7R/YZxH5M= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1342,6 +1408,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0= google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= @@ -1354,8 +1422,10 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1398,6 +1468,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= k8s.io/apimachinery v0.18.8/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= diff --git a/scripts/generate_queries.sh b/scripts/generate_queries.sh index 252954fe6..627392644 100755 --- a/scripts/generate_queries.sh +++ b/scripts/generate_queries.sh @@ -36,22 +36,6 @@ single-groupby-1-8-1 \ single-groupby-5-1-1 \ single-groupby-5-1-12 \ single-groupby-5-8-1" -# -## redistimeseries supported query types (sorted alphabetically) -#QUERY_TYPES_ALL="\ -#cpu-max-all-1 \ -#cpu-max-all-8 \ -#double-groupby-1 \ -#double-groupby-5 \ -#double-groupby-all \ -#groupby-orderby-limit \ -#lastpoint \ -#single-groupby-1-1-1 \ -#single-groupby-1-1-12 \ -#single-groupby-1-8-1 \ -#single-groupby-5-1-1 \ -#single-groupby-5-1-12 \ -#single-groupby-5-8-1" # What query types to generate QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} diff --git a/scripts/generate_queries_redistimeseries.sh b/scripts/generate_queries_redistimeseries.sh index 03079333b..114925397 100755 --- a/scripts/generate_queries_redistimeseries.sh +++ b/scripts/generate_queries_redistimeseries.sh @@ -8,6 +8,7 @@ set -x EXE_DIR=${EXE_DIR:-$(dirname $0)} source ${EXE_DIR}/query_common.sh source ${EXE_DIR}/redistimeseries_common.sh + # redistimeseries supported query types (sorted alphabetically) QUERY_TYPES_ALL="\ cpu-max-all-1 \ @@ -22,9 +23,9 @@ single-groupby-1-1-12 \ single-groupby-1-8-1 \ single-groupby-5-1-1 \ single-groupby-5-1-12 \ -single-groupby-5-8-1" -#high-cpu-1 \ -#high-cpu-all \ +single-groupby-5-8-1 \ +high-cpu-1 \ +high-cpu-all" # What query types to generate QUERY_TYPES=${QUERY_TYPES:-$QUERY_TYPES_ALL} From 2f19e18425e21243596a7f155e83ccaf9ee0448e Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 6 Jul 2021 18:17:59 +0100 Subject: [PATCH 62/72] [fix] high-cpu-* queries --- .../databases/redistimeseries/devops.go | 6 +----- cmd/tsbs_run_queries_redistimeseries/common.go | 2 +- cmd/tsbs_run_queries_redistimeseries/functors.go | 3 ++- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go index 43a7f8b2c..113447e82 100644 --- a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go @@ -87,7 +87,7 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t redisQuery = append(redisQuery, []byte(redisArg)) if nHosts > 1 && numMetrics == 1 { - redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("fieldname"), []byte("REDUCE"), []byte("max") ) + redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("hostname"), []byte("REDUCE"), []byte("max") ) } if numMetrics > 1 { redisQuery = append(redisQuery,[]byte("GROUPBY"), []byte("fieldname"), []byte("REDUCE"), []byte("max") ) @@ -271,10 +271,6 @@ func (d *Devops) GroupByOrderByLimit(qi query.Query) { []byte("FILTER"), []byte("measurement=cpu"), []byte("fieldname=usage_user"), - []byte("GROUPBY"), - []byte("fieldname"), - []byte("REDUCE"), - []byte("max"), } humanLabel := devops.GetGroupByOrderByLimitLabel("RedisTimeSeries") diff --git a/cmd/tsbs_run_queries_redistimeseries/common.go b/cmd/tsbs_run_queries_redistimeseries/common.go index 0ab0d1170..d9b96e730 100644 --- a/cmd/tsbs_run_queries_redistimeseries/common.go +++ b/cmd/tsbs_run_queries_redistimeseries/common.go @@ -11,7 +11,7 @@ import ( func inner_cmd_logic(p *processor, tq *query.RedisTimeSeries, idx int, replies [][]interface{}, commandArgs []string) error { var err error = nil if p.opts.debug { - fmt.Println(fmt.Sprintf("Issuing command (%s %s)", string(tq.CommandNames[idx]), strings.Join(ByteArrayToStringArray(tq.RedisQueries[idx]), " "))) + fmt.Println(fmt.Sprintf("Issuing command (%s %s)", string(tq.CommandNames[idx]), strings.Join(commandArgs, " "))) } if clusterMode { if string(tq.CommandNames[idx]) == "TS.MRANGE" || string(tq.CommandNames[idx]) == "TS.QUERYINDEX" || string(tq.CommandNames[idx]) == "TS.MGET" || string(tq.CommandNames[idx]) == "TS.MREVRANGE" { diff --git a/cmd/tsbs_run_queries_redistimeseries/functors.go b/cmd/tsbs_run_queries_redistimeseries/functors.go index be154e83c..2ba6fb524 100644 --- a/cmd/tsbs_run_queries_redistimeseries/functors.go +++ b/cmd/tsbs_run_queries_redistimeseries/functors.go @@ -19,7 +19,8 @@ func highCpuFilterByTsFunctor(tq *query.RedisTimeSeries, replies [][]interface{} ts := datapoint[0].(int64) new_query = append(new_query, fmt.Sprintf("%d", ts)) } - for _, arg := range commandArgs[4:8] { + new_query = append(new_query, "FILTER") + for _, arg := range commandArgs[7 : len(commandArgs)-4] { new_query = append(new_query, arg) } if p.opts.debug { From 7fbf4ae1fec96e43d1d3bf80493e9104a7eb3460 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Tue, 6 Jul 2021 20:28:26 +0100 Subject: [PATCH 63/72] [wip] using selected_labels for lastpoint --- cmd/tsbs_generate_queries/databases/redistimeseries/devops.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go index 113447e82..ceb8966a8 100644 --- a/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go +++ b/cmd/tsbs_generate_queries/databases/redistimeseries/devops.go @@ -191,7 +191,9 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) { // LastPointPerHost finds the last row for every host in the dataset func (d *Devops) LastPointPerHost(qi query.Query) { redisQuery := [][]byte{ - []byte("WITHLABELS"), + []byte("SELECTED_LABELS"), + []byte("hostname"), + []byte("fieldname"), []byte("FILTER"), []byte("measurement=cpu"), []byte("hostname!="), From 8bcba2613e1d6c803cfdd602e10d0195d932d13e Mon Sep 17 00:00:00 2001 From: Ryan Booz Date: Tue, 13 Jul 2021 13:33:29 -0400 Subject: [PATCH 64/72] Create go.yaml for GitHub workflow (#171) * Create go.yaml for GitHub workflow * Update go.yml --- .github/workflows/go.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 000000000..d09213273 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,21 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: Test + run: go test -v -race ./... From 2a0fe05a0d9a0e83dc60f60470b22ce738fe505d Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 1 Aug 2021 22:14:35 +0100 Subject: [PATCH 65/72] [add] moving back from turbo_gorilla to gorilla --- scripts/redistimeseries_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/redistimeseries_common.sh b/scripts/redistimeseries_common.sh index 2ad1a10f9..8dbb4be52 100644 --- a/scripts/redistimeseries_common.sh +++ b/scripts/redistimeseries_common.sh @@ -18,7 +18,7 @@ CLUSTER_FLAG=${CLUSTER_FLAG:-""} NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 8)} REPORTING_PERIOD=${REPORTING_PERIOD:-1s} -COMPRESSION_TYPE=${COMPRESSION_TYPE:-"compressed:turbo_gorilla"} +COMPRESSION_TYPE=${COMPRESSION_TYPE:-"compressed"} REPETITIONS=${REPETITIONS:-3} From 161c39d88082b55e90d90c1aeb5d287e694d3913 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 16 Aug 2021 21:19:49 +0300 Subject: [PATCH 66/72] victoriametrics: follow-up after https://github.com/timescale/tsbs/pull/168 (#175) The interface change in https://github.com/timescale/tsbs/pull/168 caused unexpected panics for VictoriaMetrics mode. This commit updates the interface implementation. It also updates docs and load script with adding more details. --- .../databases/victoriametrics/devops.go | 4 +-- .../databases/victoriametrics/devops_test.go | 3 +- docs/victoriametrics.md | 30 ++++++++++++++----- scripts/load/load_victoriametrics.sh | 3 +- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/victoriametrics/devops.go b/cmd/tsbs_generate_queries/databases/victoriametrics/devops.go index 16551b1aa..f6b571547 100644 --- a/cmd/tsbs_generate_queries/databases/victoriametrics/devops.go +++ b/cmd/tsbs_generate_queries/databases/victoriametrics/devops.go @@ -91,13 +91,13 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qq query.Query, numMetrics int) { // {hostname=~"hostname1|hostname2...|hostnameN"}[1h] // ) // ) by (__name__) -func (d *Devops) MaxAllCPU(qq query.Query, nHosts int) { +func (d *Devops) MaxAllCPU(qq query.Query, nHosts int, duration time.Duration) { hosts := d.mustGetRandomHosts(nHosts) selectClause := getSelectClause(devops.GetAllCPUMetrics(), hosts) qi := &queryInfo{ query: fmt.Sprintf("max(max_over_time(%s[1h])) by (__name__)", selectClause), label: devops.GetMaxAllLabel("VictoriaMetrics", nHosts), - interval: d.Interval.MustRandWindow(devops.MaxAllDuration), + interval: d.Interval.MustRandWindow(duration), step: "3600", } d.fillInQuery(qq, qi) diff --git a/cmd/tsbs_generate_queries/databases/victoriametrics/devops_test.go b/cmd/tsbs_generate_queries/databases/victoriametrics/devops_test.go index 7496ddadc..809102375 100644 --- a/cmd/tsbs_generate_queries/databases/victoriametrics/devops_test.go +++ b/cmd/tsbs_generate_queries/databases/victoriametrics/devops_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" "github.com/timescale/tsbs/pkg/query" ) @@ -47,7 +48,7 @@ func Test_what(t *testing.T) { }, "MaxAllCPU": { fn: func(g *Devops, q *query.HTTP) { - g.MaxAllCPU(q, 5) + g.MaxAllCPU(q, 5, devops.MaxAllDuration) }, expQuery: "max(max_over_time({__name__=~'cpu_(usage_user|usage_system|usage_idle|usage_nice|usage_iowait|usage_irq|usage_softirq|usage_steal|usage_guest|usage_guest_nice)', hostname=~'host_5|host_9|host_3|host_1|host_7'}[1h])) by (__name__)", expStep: "3600", diff --git a/docs/victoriametrics.md b/docs/victoriametrics.md index 02de43a1c..75c54ef0c 100644 --- a/docs/victoriametrics.md +++ b/docs/victoriametrics.md @@ -43,9 +43,12 @@ All data out of retention period will be automatically deleted after insertion. One of the ways to generate data for insertion is to use `scripts/generate_data.sh`: ```text -FORMATS=victoriametrics SCALE=100 TS_START=2019-09-01T00:00:00Z TS_END=2019-09-03T00:00:00Z ./scripts/generate_data.sh +FORMATS=victoriametrics SCALE=100 TS_START=2021-08-01T00:00:00Z TS_END=2021-08-03T00:00:00Z ./scripts/generate_data.sh ``` +Important: please ensure that VictoriaMetrics retention setting covers the time range +set by TS_START and TS_END params. + --- ## `tsbs_load_victoriametrics` @@ -54,19 +57,23 @@ See recommendations for insertion in [InfluxDB protocol](https://github.com/Vict One of the ways to load data in VictoriaMetrics is to use `scripts/load_victoriametrics.sh`: ```text -./scripts/load_victoriametrics.sh +./scripts/load/load_victoriametrics.sh ``` > Assumed that VictoriaMetrics is already installed and ready for insertion on default port `8428`. If not - please set `DATABASE_PORT` variable accordingly. +> If you're using cluster version of VictoriaMetrics please specify `vminsert` port (`8480` by default) + and `DATABASE_PATH=insert/0/influx/write`, where `0` is tenant ID. + See more about URL format [here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). ### Additional Flags -#### `-urls` (type: `string`, default: `http://localhost:8428/write`) +#### `--urls` (type: `string`, default: `http://localhost:8428/write`) Comma-separated list of URLs to connect to for inserting data. It can be just a single-version URL or list of VMInsert URLs. Workers will be distributed in a round robin fashion across the URLs. +See more about URL format [here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). --- @@ -81,13 +88,13 @@ types for `devops` use-case: The `iot` use-case wasn't implemented yet. -Of of the ways to generate queries for VictoriaMetrics is to use `scripts/generate_queries.sh`: +One of the ways to generate queries for VictoriaMetrics is to use `scripts/generate_queries.sh`: ```text - FORMATS=victoriametrics SCALE=100 TS_START=2019-09-01T00:00:00Z TS_END=2019-09-03T00:00:00Z \ + FORMATS=victoriametrics SCALE=100 TS_START=2021-08-01T00:00:00Z TS_END=2021-08-03T00:00:00Z \ QUERY_TYPES="cpu-max-all-8 double-groupby-1" ./scripts/generate_queries.sh ``` -Consider to generate queries with same params as generated data. +Important: generate queries with same params as used for data loading on previous steps. --- @@ -98,11 +105,18 @@ To run generated queries follow examples in documentation: cat /tmp/bulk_queries/victoriametrics-cpu-max-all-8-queries.gz | gunzip | tsbs_run_queries_victoriametrics ``` +> By default, tsbs_run_queries_victoriametrics assumes that VictoriaMetrics is already installed and ready + for accepting queries on `http://localhost:8428`. To change the address, please specify `--urls` flags. +> If you're using cluster version of VictoriaMetrics please specify `--urls` flag as + `http://localhost:8481/select/0/prometheus`, where `localhost:8481` is vmselect address and port, + and `0` is tenant ID. See more about URL format [here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). + + ### Additional flags -#### `-urls` (type: `string`, default: `http://localhost:8428`) +#### `--urls` (type: `string`, default: `http://localhost:8428`) Comma-separated list of URLs to connect to for querying. It can be just a single-version URL or list of VMSelect URLs. Workers will be -distributed in a round robin fashion across the URLs. +distributed in a round robin fashion across the URLs. See help for additional info. diff --git a/scripts/load/load_victoriametrics.sh b/scripts/load/load_victoriametrics.sh index b56d769e6..792dc759c 100755 --- a/scripts/load/load_victoriametrics.sh +++ b/scripts/load/load_victoriametrics.sh @@ -10,10 +10,11 @@ fi # Load parameters - common DATA_FILE_NAME=${DATA_FILE_NAME:-victoriametrics-data.gz} DATABASE_PORT=${DATABASE_PORT:-8428} +DATABASE_PATH=${DATABASE_PATH:write} EXE_DIR=${EXE_DIR:-$(dirname $0)} source ${EXE_DIR}/load_common.sh # Load data cat ${DATA_FILE} | gunzip | $EXE_FILE_NAME \ - --urls=http://${DATABASE_HOST}:${DATABASE_PORT}/write + --urls=http://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_PATH} From 1eb7705ff921fd31784c09f193e38f2292e86df2 Mon Sep 17 00:00:00 2001 From: Patrick Mackinlay Date: Tue, 24 Aug 2021 18:53:28 +0100 Subject: [PATCH 67/72] Questdb benchmark support (#157) Co-authored-by: bsmth Co-authored-by: Vlad Ilyushchenko --- .gitignore | 3 +- README.md | 4 +- .../databases/questdb/common.go | 50 +++ .../databases/questdb/devops.go | 210 ++++++++++++ .../databases/questdb/devops_test.go | 298 ++++++++++++++++++ cmd/tsbs_load_questdb/creator.go | 86 +++++ cmd/tsbs_load_questdb/main.go | 106 +++++++ cmd/tsbs_load_questdb/process.go | 51 +++ cmd/tsbs_load_questdb/process_test.go | 124 ++++++++ cmd/tsbs_load_questdb/scan.go | 65 ++++ cmd/tsbs_load_questdb/scan_test.go | 105 ++++++ cmd/tsbs_run_queries_questdb/http_client.go | 133 ++++++++ cmd/tsbs_run_queries_questdb/main.go | 143 +++++++++ docs/questdb.md | 162 ++++++++++ internal/inputs/generator_data_test.go | 1 + internal/inputs/generator_queries_test.go | 8 + pkg/query/factories/init_factories.go | 2 + pkg/targets/constants/constants.go | 2 + .../initializers/target_initializers.go | 3 + pkg/targets/questdb/implemented_target.go | 34 ++ pkg/targets/questdb/serializer.go | 95 ++++++ pkg/targets/questdb/serializer_test.go | 41 +++ .../full_cycle_minitest_questdb.sh | 26 ++ scripts/load/load_questdb.sh | 36 +++ scripts/run_queries/run_queries_questdb.sh | 62 ++++ 25 files changed, 1848 insertions(+), 2 deletions(-) create mode 100644 cmd/tsbs_generate_queries/databases/questdb/common.go create mode 100644 cmd/tsbs_generate_queries/databases/questdb/devops.go create mode 100644 cmd/tsbs_generate_queries/databases/questdb/devops_test.go create mode 100644 cmd/tsbs_load_questdb/creator.go create mode 100644 cmd/tsbs_load_questdb/main.go create mode 100644 cmd/tsbs_load_questdb/process.go create mode 100644 cmd/tsbs_load_questdb/process_test.go create mode 100644 cmd/tsbs_load_questdb/scan.go create mode 100644 cmd/tsbs_load_questdb/scan_test.go create mode 100644 cmd/tsbs_run_queries_questdb/http_client.go create mode 100644 cmd/tsbs_run_queries_questdb/main.go create mode 100644 docs/questdb.md create mode 100644 pkg/targets/questdb/implemented_target.go create mode 100644 pkg/targets/questdb/serializer.go create mode 100644 pkg/targets/questdb/serializer_test.go create mode 100755 scripts/full_cycle_minitest/full_cycle_minitest_questdb.sh create mode 100755 scripts/load/load_questdb.sh create mode 100755 scripts/run_queries/run_queries_questdb.sh diff --git a/.gitignore b/.gitignore index 04dd8bed8..caf0afa9a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .DS_Store .idea .vscode +*~ # High Dynamic Range (HDR) Histogram files -*.hdr \ No newline at end of file +*.hdr diff --git a/README.md b/README.md index 27f0be5f2..bf78a77df 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Current databases supported: + CrateDB [(supplemental docs)](docs/cratedb.md) + InfluxDB [(supplemental docs)](docs/influx.md) + MongoDB [(supplemental docs)](docs/mongo.md) ++ QuestDB [(supplemental docs)](docs/questdb.md) + SiriDB [(supplemental docs)](docs/siridb.md) + TimescaleDB [(supplemental docs)](docs/timescaledb.md) + Timestream [(supplemental docs)](docs/timestream.md) @@ -75,6 +76,7 @@ cases are implemented for each database: |CrateDB|X|| |InfluxDB|X|X| |MongoDB|X| +|QuestDB|X|X |SiriDB|X| |TimescaleDB|X|X| |Timestream|X|| @@ -132,7 +134,7 @@ Variables needed: 1. an end time. E.g., `2016-01-04T00:00:00Z` 1. how much time should be between each reading per device, in seconds. E.g., `10s` 1. and which database(s) you want to generate for. E.g., `timescaledb` - (choose from `cassandra`, `clickhouse`, `cratedb`, `influx`, `mongo`, `siridb`, + (choose from `cassandra`, `clickhouse`, `cratedb`, `influx`, `mongo`, `questdb`, `siridb`, `timescaledb` or `victoriametrics`) Given the above steps you can now generate a dataset (or multiple diff --git a/cmd/tsbs_generate_queries/databases/questdb/common.go b/cmd/tsbs_generate_queries/databases/questdb/common.go new file mode 100644 index 000000000..407555dba --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/questdb/common.go @@ -0,0 +1,50 @@ +package questdb + +import ( + "fmt" + "net/url" + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/utils" + "github.com/timescale/tsbs/pkg/query" +) + +// BaseGenerator contains settings specific for QuestDB +type BaseGenerator struct { +} + +// GenerateEmptyQuery returns an empty query.QuestDB. +func (g *BaseGenerator) GenerateEmptyQuery() query.Query { + return query.NewHTTP() +} + +// fillInQuery fills the query struct with data. +func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, sql string) { + v := url.Values{} + v.Set("count", "false") + v.Set("query", sql) + q := qi.(*query.HTTP) + q.HumanLabel = []byte(humanLabel) + q.RawQuery = []byte(sql) + q.HumanDescription = []byte(humanDesc) + q.Method = []byte("GET") + q.Path = []byte(fmt.Sprintf("/exec?%s", v.Encode())) + q.Body = nil +} + +// NewDevops creates a new devops use case query generator. +func (g *BaseGenerator) NewDevops(start, end time.Time, scale int) (utils.QueryGenerator, error) { + core, err := devops.NewCore(start, end, scale) + + if err != nil { + return nil, err + } + + devops := &Devops{ + BaseGenerator: g, + Core: core, + } + + return devops, nil +} diff --git a/cmd/tsbs_generate_queries/databases/questdb/devops.go b/cmd/tsbs_generate_queries/databases/questdb/devops.go new file mode 100644 index 000000000..ca1b3f9cc --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/questdb/devops.go @@ -0,0 +1,210 @@ +package questdb + +import ( + "fmt" + "strings" + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" + "github.com/timescale/tsbs/pkg/query" +) + +// TODO: Remove the need for this by continuing to bubble up errors +func panicIfErr(err error) { + if err != nil { + panic(err.Error()) + } +} + +// Devops produces QuestDB-specific queries for all the devops query types. +type Devops struct { + *BaseGenerator + *devops.Core +} + +// getSelectAggClauses builds specified aggregate function clauses for +// a set of column idents. +// +// For instance: +// max(cpu_time) AS max_cpu_time +func (d *Devops) getSelectAggClauses(aggFunc string, idents []string) []string { + selectAggClauses := make([]string, len(idents)) + for i, ident := range idents { + selectAggClauses[i] = + fmt.Sprintf("%[1]s(%[2]s) AS %[1]s_%[2]s", aggFunc, ident) + } + return selectAggClauses +} + +// MaxAllCPU selects the MAX of all metrics under 'cpu' per hour for N random +// hosts +// +// Queries: +// cpu-max-all-1 +// cpu-max-all-8 +func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) { + interval := d.Interval.MustRandWindow(devops.MaxAllDuration) + selectClauses := d.getSelectAggClauses("max", devops.GetAllCPUMetrics()) + hosts, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + + sql := fmt.Sprintf(` + SELECT + hour(timestamp) AS hour, + %s + FROM cpu + WHERE hostname IN ('%s') + AND timestamp >= '%s' + AND timestamp < '%s' + SAMPLE BY 1h`, + strings.Join(selectClauses, ", "), + strings.Join(hosts, "', '"), + interval.StartString(), + interval.EndString()) + + humanLabel := devops.GetMaxAllLabel("QuestDB", nHosts) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} + +// GroupByTimeAndPrimaryTag selects the AVG of metrics in the group `cpu` per device +// per hour for a day +// +// Queries: +// double-groupby-1 +// double-groupby-5 +// double-groupby-all +func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { + metrics, err := devops.GetCPUMetricsSlice(numMetrics) + panicIfErr(err) + interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration) + selectClauses := d.getSelectAggClauses("avg", metrics) + + sql := fmt.Sprintf(` + SELECT timestamp, hostname, + %s + FROM cpu + WHERE timestamp >= '%s' + AND timestamp < '%s' + SAMPLE BY 1h + GROUP BY timestamp, hostname`, + strings.Join(selectClauses, ", "), + interval.StartString(), + interval.EndString()) + + humanLabel := devops.GetDoubleGroupByLabel("QuestDB", numMetrics) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} + +// GroupByOrderByLimit populates a query.Query that has a time WHERE clause, +// that groups by a truncated date, orders by that date, and takes a limit: +// +// Queries: +// groupby-orderby-limit +func (d *Devops) GroupByOrderByLimit(qi query.Query) { + interval := d.Interval.MustRandWindow(time.Hour) + sql := fmt.Sprintf(` + SELECT timestamp AS minute, + max(usage_user) + FROM cpu + WHERE timestamp < '%s' + SAMPLE BY 1m + LIMIT 5`, + interval.EndString()) + + humanLabel := "QuestDB max cpu over last 5 min-intervals (random end)" + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.EndString()) + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} + +// LastPointPerHost finds the last row for every host in the dataset +// +// Queries: +// lastpoint +func (d *Devops) LastPointPerHost(qi query.Query) { + sql := fmt.Sprintf(`SELECT * FROM cpu latest by hostname`) + + humanLabel := "QuestDB last row per host" + humanDesc := humanLabel + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} + +// HighCPUForHosts populates a query that gets CPU metrics when the CPU has +// high usage between a time period for a number of hosts (if 0, it will +// search all hosts) +// +// Queries: +// high-cpu-1 +// high-cpu-all +func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { + interval := d.Interval.MustRandWindow(devops.HighCPUDuration) + sql := "" + if nHosts > 0 { + hosts, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + + sql = fmt.Sprintf(` + SELECT * + FROM cpu + WHERE usage_user > 90.0 + AND hostname IN ('%s') + AND timestamp >= '%s' + AND timestamp < '%s'`, + strings.Join(hosts, "', '"), + interval.StartString(), + interval.EndString()) + } else { + sql = fmt.Sprintf(` + SELECT * + FROM cpu + WHERE usage_user > 90.0 + AND timestamp >= '%s' + AND timestamp < '%s'`, + interval.StartString(), + interval.EndString()) + } + + humanLabel, err := devops.GetHighCPULabel("QuestDB", nHosts) + panicIfErr(err) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} + +// GroupByTime selects the MAX for metrics under 'cpu', per minute for N random +// hosts +// +// Resultsets: +// single-groupby-1-1-12 +// single-groupby-1-1-1 +// single-groupby-1-8-1 +// single-groupby-5-1-12 +// single-groupby-5-1-1 +// single-groupby-5-8-1 +func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) { + interval := d.Interval.MustRandWindow(timeRange) + metrics, err := devops.GetCPUMetricsSlice(numMetrics) + panicIfErr(err) + selectClauses := d.getSelectAggClauses("max", metrics) + hosts, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + + sql := fmt.Sprintf(` + SELECT timestamp, + %s + FROM cpu + WHERE hostname IN ('%s') + AND timestamp >= '%s' + AND timestamp < '%s' + SAMPLE BY 1m`, + strings.Join(selectClauses, ", "), + strings.Join(hosts, "', '"), + interval.StartString(), + interval.EndString()) + + humanLabel := fmt.Sprintf( + "QuestDB %d cpu metric(s), random %4d hosts, random %s by 1m", + numMetrics, nHosts, timeRange) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} diff --git a/cmd/tsbs_generate_queries/databases/questdb/devops_test.go b/cmd/tsbs_generate_queries/databases/questdb/devops_test.go new file mode 100644 index 000000000..ae1f3792f --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/questdb/devops_test.go @@ -0,0 +1,298 @@ +package questdb + +import ( + "math/rand" + "net/url" + "regexp" + "testing" + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" + "github.com/timescale/tsbs/pkg/query" +) + +func TestDevopsGroupByTime(t *testing.T) { + expectedHumanLabel := "QuestDB 1 cpu metric(s), random 1 hosts, random 1s by 1m" + expectedHumanDesc := "QuestDB 1 cpu metric(s), random 1 hosts, random 1s by 1m: 1970-01-01T00:05:58Z" + expectedQuery := "SELECT timestamp, max(usage_user) AS max_usage_user FROM cpu " + + "WHERE hostname IN ('host_9') AND timestamp >= '1970-01-01T00:05:58Z' AND timestamp < '1970-01-01T00:05:59Z' SAMPLE BY 1m" + + rand.Seed(123) // Setting seed for testing purposes. + s := time.Unix(0, 0) + e := s.Add(time.Hour) + b := BaseGenerator{} + dq, err := b.NewDevops(s, e, 10) + if err != nil { + t.Fatalf("Error while creating devops generator") + } + d := dq.(*Devops) + + metrics := 1 + nHosts := 1 + duration := time.Second + + q := d.GenerateEmptyQuery() + d.GroupByTime(q, nHosts, metrics, duration) + + verifyQuery(t, q, expectedHumanLabel, expectedHumanDesc, expectedQuery) +} + +func TestDevopsGroupByOrderByLimit(t *testing.T) { + expectedHumanLabel := "QuestDB max cpu over last 5 min-intervals (random end)" + expectedHumanDesc := "QuestDB max cpu over last 5 min-intervals (random end): 1970-01-01T01:16:22Z" + expectedQuery := "SELECT timestamp AS minute, max(usage_user) FROM cpu " + + "WHERE timestamp < '1970-01-01T01:16:22Z' SAMPLE BY 1m LIMIT 5" + + rand.Seed(123) // Setting seed for testing purposes. + s := time.Unix(0, 0) + e := s.Add(2 * time.Hour) + b := BaseGenerator{} + dq, err := b.NewDevops(s, e, 10) + if err != nil { + t.Fatalf("Error while creating devops generator") + } + d := dq.(*Devops) + + q := d.GenerateEmptyQuery() + d.GroupByOrderByLimit(q) + + verifyQuery(t, q, expectedHumanLabel, expectedHumanDesc, expectedQuery) +} + +func TestDevopsGroupByTimeAndPrimaryTag(t *testing.T) { + cases := []testCase{ + { + desc: "zero metrics", + input: 0, + fail: true, + failMsg: "cannot get 0 metrics", + }, + { + desc: "1 metric", + input: 1, + expectedHumanLabel: "QuestDB mean of 1 metrics, all hosts, random 12h0m0s by 1h", + expectedHumanDesc: "QuestDB mean of 1 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:16:22Z", + expectedQuery: "SELECT timestamp, hostname, avg(usage_user) AS avg_usage_user FROM cpu " + + "WHERE timestamp >= '1970-01-01T00:16:22Z' AND timestamp < '1970-01-01T12:16:22Z' " + + "SAMPLE BY 1h GROUP BY timestamp, hostname", + }, + { + desc: "5 metrics", + input: 5, + expectedHumanLabel: "QuestDB mean of 5 metrics, all hosts, random 12h0m0s by 1h", + expectedHumanDesc: "QuestDB mean of 5 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:54:10Z", + expectedQuery: "SELECT timestamp, hostname, avg(usage_user) AS avg_usage_user, avg(usage_system) AS avg_usage_system, avg(usage_idle) AS avg_usage_idle, avg(usage_nice) AS avg_usage_nice, avg(usage_iowait) AS avg_usage_iowait FROM cpu " + + "WHERE timestamp >= '1970-01-01T00:54:10Z' AND timestamp < '1970-01-01T12:54:10Z' " + + "SAMPLE BY 1h GROUP BY timestamp, hostname", + }, + } + + testFunc := func(d *Devops, c testCase) query.Query { + q := d.GenerateEmptyQuery() + d.GroupByTimeAndPrimaryTag(q, c.input) + return q + } + + start := time.Unix(0, 0) + end := start.Add(devops.DoubleGroupByDuration).Add(time.Hour) + + runTestCases(t, testFunc, start, end, cases) +} + +func TestMaxAllCPU(t *testing.T) { + cases := []testCase{ + { + desc: "zero hosts", + input: 0, + fail: true, + failMsg: "number of hosts cannot be < 1; got 0", + }, + { + desc: "1 host", + input: 1, + expectedHumanLabel: "QuestDB max of all CPU metrics, random 1 hosts, random 8h0m0s by 1h", + expectedHumanDesc: "QuestDB max of all CPU metrics, random 1 hosts, random 8h0m0s by 1h: 1970-01-01T00:54:10Z", + expectedQuery: "SELECT hour(timestamp) AS hour, max(usage_user) AS max_usage_user, max(usage_system) AS max_usage_system, max(usage_idle) AS max_usage_idle, max(usage_nice) AS max_usage_nice, max(usage_iowait) AS max_usage_iowait, max(usage_irq) AS max_usage_irq, max(usage_softirq) AS max_usage_softirq, max(usage_steal) AS max_usage_steal, max(usage_guest) AS max_usage_guest, max(usage_guest_nice) AS max_usage_guest_nice FROM cpu " + + "WHERE hostname IN ('host_3') AND timestamp >= '1970-01-01T00:54:10Z' AND timestamp < '1970-01-01T08:54:10Z' " + + "SAMPLE BY 1h", + }, + { + desc: "5 hosts", + input: 5, + expectedHumanLabel: "QuestDB max of all CPU metrics, random 5 hosts, random 8h0m0s by 1h", + expectedHumanDesc: "QuestDB max of all CPU metrics, random 5 hosts, random 8h0m0s by 1h: 1970-01-01T00:37:12Z", + expectedQuery: "SELECT hour(timestamp) AS hour, max(usage_user) AS max_usage_user, max(usage_system) AS max_usage_system, max(usage_idle) AS max_usage_idle, max(usage_nice) AS max_usage_nice, max(usage_iowait) AS max_usage_iowait, max(usage_irq) AS max_usage_irq, max(usage_softirq) AS max_usage_softirq, max(usage_steal) AS max_usage_steal, max(usage_guest) AS max_usage_guest, max(usage_guest_nice) AS max_usage_guest_nice FROM cpu " + + "WHERE hostname IN ('host_9', 'host_5', 'host_1', 'host_7', 'host_2') AND timestamp >= '1970-01-01T00:37:12Z' AND timestamp < '1970-01-01T08:37:12Z' " + + "SAMPLE BY 1h", + }, + } + + testFunc := func(d *Devops, c testCase) query.Query { + q := d.GenerateEmptyQuery() + d.MaxAllCPU(q, c.input) + return q + } + + start := time.Unix(0, 0) + end := start.Add(devops.MaxAllDuration).Add(time.Hour) + + runTestCases(t, testFunc, start, end, cases) +} + +func TestLastPointPerHost(t *testing.T) { + expectedHumanLabel := "QuestDB last row per host" + expectedHumanDesc := "QuestDB last row per host" + expectedQuery := `SELECT * FROM cpu latest by hostname` + + rand.Seed(123) // Setting seed for testing purposes. + s := time.Unix(0, 0) + e := s.Add(2 * time.Hour) + b := BaseGenerator{} + dq, err := b.NewDevops(s, e, 10) + if err != nil { + t.Fatalf("Error while creating devops generator") + } + d := dq.(*Devops) + + q := d.GenerateEmptyQuery() + d.LastPointPerHost(q) + + verifyQuery(t, q, expectedHumanLabel, expectedHumanDesc, expectedQuery) +} + +func TestHighCPUForHosts(t *testing.T) { + cases := []testCase{ + { + desc: "negative hosts", + input: -1, + fail: true, + failMsg: "nHosts cannot be negative", + }, + { + desc: "zero hosts", + input: 0, + expectedHumanLabel: "QuestDB CPU over threshold, all hosts", + expectedHumanDesc: "QuestDB CPU over threshold, all hosts: 1970-01-01T00:54:10Z", + expectedQuery: "SELECT * FROM cpu " + + "WHERE usage_user > 90.0 AND " + + "timestamp >= '1970-01-01T00:54:10Z' AND timestamp < '1970-01-01T12:54:10Z'", + }, + { + desc: "1 host", + input: 1, + expectedHumanLabel: "QuestDB CPU over threshold, 1 host(s)", + expectedHumanDesc: "QuestDB CPU over threshold, 1 host(s): 1970-01-01T00:47:30Z", + expectedQuery: "SELECT * FROM cpu " + + "WHERE usage_user > 90.0 AND hostname IN ('host_5') AND " + + "timestamp >= '1970-01-01T00:47:30Z' AND timestamp < '1970-01-01T12:47:30Z'", + }, + { + desc: "5 hosts", + input: 5, + expectedHumanLabel: "QuestDB CPU over threshold, 5 host(s)", + expectedHumanDesc: "QuestDB CPU over threshold, 5 host(s): 1970-01-01T00:17:45Z", + expectedQuery: "SELECT * FROM cpu " + + "WHERE usage_user > 90.0 AND " + + "hostname IN ('host_9', 'host_5', 'host_1', 'host_7', 'host_2') AND " + + "timestamp >= '1970-01-01T00:17:45Z' AND timestamp < '1970-01-01T12:17:45Z'", + }, + } + + testFunc := func(d *Devops, c testCase) query.Query { + q := d.GenerateEmptyQuery() + d.HighCPUForHosts(q, c.input) + return q + } + + start := time.Unix(0, 0) + end := start.Add(devops.HighCPUDuration).Add(time.Hour) + + runTestCases(t, testFunc, start, end, cases) +} + +type testCase struct { + desc string + input int + fail bool + failMsg string + expectedHumanLabel string + expectedHumanDesc string + expectedQuery string +} + +func runTestCases(t *testing.T, testFunc func(*Devops, testCase) query.Query, s time.Time, e time.Time, cases []testCase) { + rand.Seed(123) // Setting seed for testing purposes. + + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + b := BaseGenerator{} + dq, err := b.NewDevops(s, e, 10) + if err != nil { + t.Fatalf("Error while creating devops generator") + } + d := dq.(*Devops) + + if c.fail { + func() { + defer func() { + r := recover() + if r == nil { + t.Errorf("did not panic when should") + } + + if r != c.failMsg { + t.Fatalf("incorrect fail message: got %s, want %s", r, c.failMsg) + } + }() + + testFunc(d, c) + }() + } else { + q := testFunc(d, c) + verifyQuery(t, q, c.expectedHumanLabel, c.expectedHumanDesc, c.expectedQuery) + } + + }) + } +} + +func verifyQuery(t *testing.T, q query.Query, humanLabel, humanDesc, expectedSql string) { + sql, ok := q.(*query.HTTP) + + if !ok { + t.Fatal("Filled query is not *query.HTTP type") + } + + if got := string(sql.HumanLabel); got != humanLabel { + t.Errorf("incorrect human label:\ngot\n%s\nwant\n%s", got, humanLabel) + } + + if got := string(sql.HumanDescription); got != humanDesc { + t.Errorf("incorrect human description:\ngot\n%s\nwant\n%s", got, humanDesc) + } + + if got := string(sql.Method); got != "GET" { + t.Errorf("incorrect method:\ngot\n%s\nwant GET", got) + } + + uri := string(sql.Path) + u, err := url.Parse(uri) + if err != nil { + t.Errorf("Failed to decode %s: %s", uri, err) + } + actualSql := normaliseField(u.Query()["query"][0]) + + if expectedSql != actualSql { + t.Errorf("expcted %s, actual %s", expectedSql, actualSql) + } +} + +func normaliseField(fieldValue string) string { + m1 := regexp.MustCompile("^\\s+") + m2 := regexp.MustCompile("\\s+$") + m3 := regexp.MustCompile("\\s+") + fieldValue = m1.ReplaceAllString(fieldValue, "") + fieldValue = m2.ReplaceAllString(fieldValue, "") + fieldValue = m3.ReplaceAllString(fieldValue, " ") + return fieldValue +} diff --git a/cmd/tsbs_load_questdb/creator.go b/cmd/tsbs_load_questdb/creator.go new file mode 100644 index 000000000..fc0df7634 --- /dev/null +++ b/cmd/tsbs_load_questdb/creator.go @@ -0,0 +1,86 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + "time" +) + +type dbCreator struct { + questdbRESTEndPoint string +} + +func (d *dbCreator) Init() { + d.questdbRESTEndPoint = questdbRESTEndPoint +} + +func (d *dbCreator) DBExists(dbName string) bool { + r, err := execQuery(questdbRESTEndPoint, "SHOW TABLES") + if err != nil { + panic(fmt.Errorf("fatal error, failed to query questdb: %s", err)) + } + for i, v := range r.Dataset { + if i >= 0 && v[0] == "cpu" { + panic(fmt.Errorf("fatal error, cpu table already exists")) + } + } + // Create minimal table with o3 params + // r, err = execQuery(questdbRESTEndPoint, "CREATE TABLE cpu (hostname SYMBOL, region SYMBOL, datacenter SYMBOL, rack SYMBOL, os SYMBOL, arch SYMBOL, team SYMBOL, service SYMBOL, service_version SYMBOL, service_environment SYMBOL, usage_user LONG, usage_system LONG, usage_idle LONG, usage_nice LONG, usage_iowait LONG, usage_irq LONG, usage_softirq LONG, usage_steal LONG, usage_guest LONG, usage_guest_nice LONG, timestamp TIMESTAMP) timestamp(timestamp) PARTITION BY DAY WITH o3MaxUncommittedRows=500000, o3CommitHysteresis=300s") + // if err != nil { + // panic(fmt.Errorf("fatal error, failed to create cpu table: %s", err)) + // } + + return false +} + +func (d *dbCreator) RemoveOldDB(dbName string) error { + return nil +} + +func (d *dbCreator) CreateDB(dbName string) error { + time.Sleep(time.Second) + return nil +} + +type QueryResponseColumns struct { + Name string + Type string +} + +type QueryResponse struct { + Query string + Columns []QueryResponseColumns + Dataset [][]interface{} + Count int + Error string +} + +func execQuery(uriRoot string, query string) (QueryResponse, error) { + var qr QueryResponse + if strings.HasSuffix(uriRoot, "/") { + uriRoot = uriRoot[:len(uriRoot)-1] + } + uriRoot = uriRoot + "/exec?query=" + url.QueryEscape(query) + resp, err := http.Get(uriRoot) + if err != nil { + return qr, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return qr, err + } + err = json.Unmarshal(body, &qr) + if err != nil { + return qr, err + } + if qr.Error != "" { + return qr, errors.New(qr.Error) + } + return qr, nil +} diff --git a/cmd/tsbs_load_questdb/main.go b/cmd/tsbs_load_questdb/main.go new file mode 100644 index 000000000..852700975 --- /dev/null +++ b/cmd/tsbs_load_questdb/main.go @@ -0,0 +1,106 @@ +// bulk_load_questdb loads an QuestDB daemon with data from stdin. +// +// The caller is responsible for assuring that the database is empty before +// bulk load. +package main + +import ( + "bufio" + "bytes" + "fmt" + "log" + "sync" + "time" + + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/internal/utils" + "github.com/timescale/tsbs/load" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/constants" + "github.com/timescale/tsbs/pkg/targets/initializers" +) + +// Program option vars: +var ( + questdbRESTEndPoint string + questdbILPBindTo string + doAbortOnExist bool +) + +// Global vars +var ( + loader load.BenchmarkRunner + config load.BenchmarkRunnerConfig + bufPool sync.Pool + target targets.ImplementedTarget +) + +// allows for testing +var fatal = log.Fatalf + +// Parse args: +func init() { + target = initializers.GetTarget(constants.FormatQuestDB) + config = load.BenchmarkRunnerConfig{} + // Not all the default flags apply to QuestDB + // config.AddToFlagSet(pflag.CommandLine) + pflag.CommandLine.Uint("batch-size", 10000, "Number of items to batch together in a single insert") + pflag.CommandLine.Uint("workers", 1, "Number of parallel clients inserting") + pflag.CommandLine.Uint64("limit", 0, "Number of items to insert (0 = all of them).") + pflag.CommandLine.Bool("do-load", true, "Whether to write data. Set this flag to false to check input read speed.") + pflag.CommandLine.Duration("reporting-period", 10*time.Second, "Period to report write stats") + pflag.CommandLine.String("file", "", "File name to read data from") + pflag.CommandLine.Int64("seed", 0, "PRNG seed (default: 0, which uses the current timestamp)") + pflag.CommandLine.String("insert-intervals", "", "Time to wait between each insert, default '' => all workers insert ASAP. '1,2' = worker 1 waits 1s between inserts, worker 2 and others wait 2s") + pflag.CommandLine.Bool("hash-workers", false, "Whether to consistently hash insert data to the same workers (i.e., the data for a particular host always goes to the same worker)") + target.TargetSpecificFlags("", pflag.CommandLine) + pflag.Parse() + + err := utils.SetupConfigFile() + + if err != nil { + panic(fmt.Errorf("fatal error config file: %s", err)) + } + + if err := viper.Unmarshal(&config); err != nil { + panic(fmt.Errorf("unable to decode config: %s", err)) + } + + questdbRESTEndPoint = viper.GetString("url") + questdbILPBindTo = viper.GetString("ilp-bind-to") + config.HashWorkers = false + loader = load.GetBenchmarkRunner(config) +} + +type benchmark struct{} + +func (b *benchmark) GetDataSource() targets.DataSource { + return &fileDataSource{scanner: bufio.NewScanner(load.GetBufferedReader(config.FileName))} +} + +func (b *benchmark) GetBatchFactory() targets.BatchFactory { + return &factory{} +} + +func (b *benchmark) GetPointIndexer(_ uint) targets.PointIndexer { + return &targets.ConstantIndexer{} +} + +func (b *benchmark) GetProcessor() targets.Processor { + return &processor{} +} + +func (b *benchmark) GetDBCreator() targets.DBCreator { + return &dbCreator{} +} + +func main() { + bufPool = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(make([]byte, 0, 4*1024*1024)) + }, + } + + loader.RunBenchmark(&benchmark{}) +} diff --git a/cmd/tsbs_load_questdb/process.go b/cmd/tsbs_load_questdb/process.go new file mode 100644 index 000000000..c7fe24f40 --- /dev/null +++ b/cmd/tsbs_load_questdb/process.go @@ -0,0 +1,51 @@ +package main + +import ( + "fmt" + "net" + + "github.com/timescale/tsbs/pkg/targets" +) + +// allows for testing +var printFn = fmt.Printf + +type processor struct { + ilpConn (*net.TCPConn) +} + +func (p *processor) Init(numWorker int, _, _ bool) { + tcpAddr, err := net.ResolveTCPAddr("tcp4", questdbILPBindTo) + if err != nil { + fatal("Failed to resolve %s: %s\n", questdbILPBindTo, err.Error()) + } + p.ilpConn, err = net.DialTCP("tcp", nil, tcpAddr) + if err != nil { + fatal("Failed connect to %s: %s\n", questdbILPBindTo, err.Error()) + } +} + +func (p *processor) Close(_ bool) { + defer p.ilpConn.Close() +} + +func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (uint64, uint64) { + batch := b.(*batch) + + // Write the batch: try until backoff is not needed. + if doLoad { + var err error + _, err = p.ilpConn.Write(batch.buf.Bytes()) + if err != nil { + fatal("Error writing: %s\n", err.Error()) + } + } + + metricCnt := batch.metrics + rowCnt := batch.rows + + // Return the batch buffer to the pool. + batch.buf.Reset() + bufPool.Put(batch.buf) + return metricCnt, uint64(rowCnt) +} diff --git a/cmd/tsbs_load_questdb/process_test.go b/cmd/tsbs_load_questdb/process_test.go new file mode 100644 index 000000000..16c9d4833 --- /dev/null +++ b/cmd/tsbs_load_questdb/process_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "net" + "sync" + "testing" + "time" + + "github.com/timescale/tsbs/pkg/data" +) + +func emptyLog(_ string, _ ...interface{}) (int, error) { + return 0, nil +} + +type mockServer struct { + ln net.Listener + listenPort int +} + +func mockServerStop(ms *mockServer) { + ms.ln.Close() +} + +func mockServerStart() *mockServer { + ln, err := net.Listen("tcp", ":0") + if err != nil { + fatal("Failed to start server listen socket: %s\n", err.Error()) + } + fmt.Println("Mock TCP server listening on port:", ln.Addr().(*net.TCPAddr).Port) + ms := &mockServer{ + ln: ln, + listenPort: ln.Addr().(*net.TCPAddr).Port, + } + go func() { + for { + + conn, err := ln.Accept() + if err != nil { + // listen socket is closed + return + } + go func() { + data := make([]byte, 512) + for { + rc, err := conn.Read(data) + if err != nil { + if err != io.EOF { + fatal("failed to read from connection: ", err.Error()) + } + return + } + fmt.Println(conn, " read ", rc) + } + }() + } + }() + return ms +} + +func TestProcessorInit(t *testing.T) { + ms := mockServerStart() + defer mockServerStop(ms) + questdbILPBindTo = fmt.Sprintf("127.0.0.1:%d", ms.listenPort) + printFn = emptyLog + p := &processor{} + p.Init(0, false, false) + p.Close(true) + + p = &processor{} + p.Init(1, false, false) + p.Close(true) +} + +func TestProcessorProcessBatch(t *testing.T) { + bufPool = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(make([]byte, 0, 4*1024*1024)) + }, + } + f := &factory{} + b := f.New().(*batch) + pt := data.LoadedPoint{ + Data: []byte("tag1=tag1val,tag2=tag2val col1=0.0,col2=0.0 140\n"), + } + b.Append(pt) + + cases := []struct { + doLoad bool + }{ + { + doLoad: false, + }, + { + doLoad: true, + }, + } + + for _, c := range cases { + fatal = func(format string, args ...interface{}) { + t.Errorf("fatal called for case %v unexpectedly\n", c) + fmt.Printf(format, args...) + } + + ms := mockServerStart() + questdbILPBindTo = fmt.Sprintf("127.0.0.1:%d", ms.listenPort) + + p := &processor{} + p.Init(0, true, true) + mCnt, rCnt := p.ProcessBatch(b, c.doLoad) + if mCnt != b.metrics { + t.Errorf("process batch returned less metrics than batch: got %d want %d", mCnt, b.metrics) + } + if rCnt != uint64(b.rows) { + t.Errorf("process batch returned less rows than batch: got %d want %d", rCnt, b.rows) + } + p.Close(true) + mockServerStop(ms) + time.Sleep(50 * time.Millisecond) + } +} diff --git a/cmd/tsbs_load_questdb/scan.go b/cmd/tsbs_load_questdb/scan.go new file mode 100644 index 000000000..c4605d7aa --- /dev/null +++ b/cmd/tsbs_load_questdb/scan.go @@ -0,0 +1,65 @@ +package main + +import ( + "bufio" + "bytes" + "strings" + + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/usecases/common" + "github.com/timescale/tsbs/pkg/targets" +) + +const errNotThreeTuplesFmt = "parse error: line does not have 3 tuples, has %d" + +var newLine = []byte("\n") + +type fileDataSource struct { + scanner *bufio.Scanner +} + +func (d *fileDataSource) NextItem() data.LoadedPoint { + ok := d.scanner.Scan() + if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF + return data.LoadedPoint{} + } else if !ok { + fatal("scan error: %v", d.scanner.Err()) + return data.LoadedPoint{} + } + return data.NewLoadedPoint(d.scanner.Bytes()) +} + +func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { return nil } + +type batch struct { + buf *bytes.Buffer + rows uint + metrics uint64 +} + +func (b *batch) Len() uint { + return b.rows +} + +func (b *batch) Append(item data.LoadedPoint) { + that := item.Data.([]byte) + thatStr := string(that) + b.rows++ + // Each influx line is format "csv-tags csv-fields timestamp", so we split by space + // and then on the middle element, we split by comma to count number of fields added + args := strings.Split(thatStr, " ") + if len(args) != 3 { + fatal(errNotThreeTuplesFmt, len(args)) + return + } + b.metrics += uint64(len(strings.Split(args[1], ","))) + + b.buf.Write(that) + b.buf.Write(newLine) +} + +type factory struct{} + +func (f *factory) New() targets.Batch { + return &batch{buf: bufPool.Get().(*bytes.Buffer)} +} diff --git a/cmd/tsbs_load_questdb/scan_test.go b/cmd/tsbs_load_questdb/scan_test.go new file mode 100644 index 000000000..a4144946f --- /dev/null +++ b/cmd/tsbs_load_questdb/scan_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "sync" + "testing" + + "github.com/timescale/tsbs/pkg/data" +) + +func TestBatch(t *testing.T) { + bufPool = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(make([]byte, 0, 4*1024*1024)) + }, + } + f := &factory{} + b := f.New().(*batch) + if b.Len() != 0 { + t.Errorf("batch not initialized with count 0") + } + p := data.LoadedPoint{ + Data: []byte("tag1=tag1val,tag2=tag2val col1=0.0,col2=0.0 140"), + } + b.Append(p) + if b.Len() != 1 { + t.Errorf("batch count is not 1 after first append") + } + if b.rows != 1 { + t.Errorf("batch row count is not 1 after first append") + } + if b.metrics != 2 { + t.Errorf("batch metric count is not 2 after first append") + } + + p = data.LoadedPoint{ + Data: []byte("tag1=tag1val,tag2=tag2val col1=1.0,col2=1.0 190"), + } + b.Append(p) + if b.Len() != 2 { + t.Errorf("batch count is not 1 after first append") + } + if b.rows != 2 { + t.Errorf("batch row count is not 1 after first append") + } + if b.metrics != 4 { + t.Errorf("batch metric count is not 2 after first append") + } + + p = data.LoadedPoint{ + Data: []byte("bad_point"), + } + errMsg := "" + fatal = func(f string, args ...interface{}) { + errMsg = fmt.Sprintf(f, args...) + } + b.Append(p) + if errMsg == "" { + t.Errorf("batch append did not error with ill-formed point") + } +} + +func TestFileDataSourceNextItem(t *testing.T) { + cases := []struct { + desc string + input string + result []byte + shouldFatal bool + }{ + { + desc: "correct input", + input: "cpu,tag1=tag1text,tag2=tag2text col1=0.0,col2=0.0 140\n", + result: []byte("cpu,tag1=tag1text,tag2=tag2text col1=0.0,col2=0.0 140"), + }, + { + desc: "correct input with extra", + input: "cpu,tag1=tag1text,tag2=tag2text col1=0.0,col2=0.0 140\nextra_is_ignored", + result: []byte("cpu,tag1=tag1text,tag2=tag2text col1=0.0,col2=0.0 140"), + }, + } + + for _, c := range cases { + br := bufio.NewReader(bytes.NewReader([]byte(c.input))) + ds := &fileDataSource{scanner: bufio.NewScanner(br)} + p := ds.NextItem() + data := p.Data.([]byte) + if !bytes.Equal(data, c.result) { + t.Errorf("%s: incorrect result: got\n%v\nwant\n%v", c.desc, data, c.result) + } + } +} + +func TestDecodeEOF(t *testing.T) { + input := []byte("cpu,tag1=tag1text,tag2=tag2text col1=0.0,col2=0.0 140") + br := bufio.NewReader(bytes.NewReader([]byte(input))) + ds := &fileDataSource{scanner: bufio.NewScanner(br)} + _ = ds.NextItem() + // nothing left, should be EOF + p := ds.NextItem() + if p.Data != nil { + t.Errorf("expected p to be nil, got %v", p) + } +} diff --git a/cmd/tsbs_run_queries_questdb/http_client.go b/cmd/tsbs_run_queries_questdb/http_client.go new file mode 100644 index 000000000..daa19d815 --- /dev/null +++ b/cmd/tsbs_run_queries_questdb/http_client.go @@ -0,0 +1,133 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "sync" + "time" + + "github.com/timescale/tsbs/pkg/query" +) + +var bytesSlash = []byte("/") // heap optimization + +// HTTPClient is a reusable HTTP Client. +type HTTPClient struct { + //client fasthttp.Client + client *http.Client + Host []byte + HostString string + uri []byte +} + +// HTTPClientDoOptions wraps options uses when calling `Do`. +type HTTPClientDoOptions struct { + Debug int + PrettyPrintResponses bool + chunkSize uint64 + database string +} + +var httpClientOnce = sync.Once{} +var httpClient *http.Client + +func getHttpClient() *http.Client { + httpClientOnce.Do(func() { + tr := &http.Transport{ + MaxIdleConnsPerHost: 1024, + } + httpClient = &http.Client{Transport: tr} + }) + return httpClient +} + +// NewHTTPClient creates a new HTTPClient. +func NewHTTPClient(host string) *HTTPClient { + if strings.HasSuffix(host, "/") { + host = host[:len(host)-1] + } + return &HTTPClient{ + client: getHttpClient(), + Host: []byte(host), + HostString: host, + uri: []byte{}, // heap optimization + } +} + +// Do performs the action specified by the given Query. It uses fasthttp, and +// tries to minimize heap allocations. +func (w *HTTPClient) Do(q *query.HTTP, opts *HTTPClientDoOptions) (lag float64, err error) { + // populate uri from the reusable byte slice: + w.uri = w.uri[:0] + w.uri = append(w.uri, w.Host...) + w.uri = append(w.uri, q.Path...) + + // populate a request with data from the Query: + req, err := http.NewRequest(string(q.Method), string(w.uri), nil) + if err != nil { + panic(err) + } + + // Perform the request while tracking latency: + start := time.Now() + resp, err := w.client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + panic("http request did not return status 200 OK") + } + + var body []byte + body, err = ioutil.ReadAll(resp.Body) + + if err != nil { + panic(err) + } + + lag = float64(time.Since(start).Nanoseconds()) / 1e6 // milliseconds + + if opts != nil { + // Print debug messages, if applicable: + switch opts.Debug { + case 1: + fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms\n", q.HumanLabel, lag) + case 2: + fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription) + case 3: + fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription) + fmt.Fprintf(os.Stderr, "debug: request: %s\n", string(q.String())) + case 4: + fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription) + fmt.Fprintf(os.Stderr, "debug: request: %s\n", string(q.String())) + fmt.Fprintf(os.Stderr, "debug: response: %s\n", string(body)) + default: + } + + // Pretty print JSON responses, if applicable: + if opts.PrettyPrintResponses { + // Assumes the response is JSON! This holds for Influx + // and Elastic. + + prefix := fmt.Sprintf("ID %d: ", q.GetID()) + var v interface{} + var line []byte + full := make(map[string]interface{}) + full["influxql"] = string(q.RawQuery) + json.Unmarshal(body, &v) + full["response"] = v + line, err = json.MarshalIndent(full, prefix, " ") + if err != nil { + return + } + fmt.Println(string(line) + "\n") + } + } + + return lag, err +} diff --git a/cmd/tsbs_run_queries_questdb/main.go b/cmd/tsbs_run_queries_questdb/main.go new file mode 100644 index 000000000..69db180e0 --- /dev/null +++ b/cmd/tsbs_run_queries_questdb/main.go @@ -0,0 +1,143 @@ +// tsbs_run_queries_influx speed tests InfluxDB using requests from stdin. +// +// It reads encoded Query objects from stdin, and makes concurrent requests +// to the provided HTTP endpoint. This program has no knowledge of the +// internals of the endpoint. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "strings" + + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/internal/utils" + "github.com/timescale/tsbs/pkg/query" +) + +// Program option vars: +var ( + daemonUrls []string +) + +// Global vars: +var ( + runner *query.BenchmarkRunner +) + +// Parse args: +func init() { + var config query.BenchmarkRunnerConfig + config.AddToFlagSet(pflag.CommandLine) + var csvDaemonUrls string + + pflag.String("urls", "http://localhost:9000/", "Daemon URLs, comma-separated. Will be used in a round-robin fashion.") + + pflag.Parse() + + err := utils.SetupConfigFile() + + if err != nil { + panic(fmt.Errorf("fatal error config file: %s", err)) + } + + if err := viper.Unmarshal(&config); err != nil { + panic(fmt.Errorf("unable to decode config: %s", err)) + } + + csvDaemonUrls = viper.GetString("urls") + + daemonUrls = strings.Split(csvDaemonUrls, ",") + if len(daemonUrls) == 0 { + log.Fatal("missing 'urls' flag") + } + + // Add an index to the hostname column in the cpu table + r, err := execQuery(daemonUrls[0], "show columns from cpu") + if err == nil && r.Count != 0 { + r, err := execQuery(daemonUrls[0], "ALTER TABLE cpu ALTER COLUMN hostname ADD INDEX") + _ = r + // fmt.Println("error:", err) + // fmt.Printf("%+v\n", r) + if err == nil { + fmt.Println("Added index to hostname column of cpu table") + } + } + + runner = query.NewBenchmarkRunner(config) +} + +func main() { + runner.Run(&query.HTTPPool, newProcessor) +} + +type processor struct { + w *HTTPClient + opts *HTTPClientDoOptions +} + +func newProcessor() query.Processor { return &processor{} } + +func (p *processor) Init(workerNumber int) { + p.opts = &HTTPClientDoOptions{ + Debug: runner.DebugLevel(), + PrettyPrintResponses: runner.DoPrintResponses(), + } + url := daemonUrls[workerNumber%len(daemonUrls)] + p.w = NewHTTPClient(url) +} + +func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { + hq := q.(*query.HTTP) + lag, err := p.w.Do(hq, p.opts) + if err != nil { + return nil, err + } + stat := query.GetStat() + stat.Init(q.HumanLabelName(), lag) + return []*query.Stat{stat}, nil +} + +type QueryResponseColumns struct { + Name string + Type string +} + +type QueryResponse struct { + Query string + Columns []QueryResponseColumns + Dataset []interface{} + Count int + Error string +} + +func execQuery(uriRoot string, query string) (QueryResponse, error) { + var qr QueryResponse + if strings.HasSuffix(uriRoot, "/") { + uriRoot = uriRoot[:len(uriRoot)-1] + } + uriRoot = uriRoot + "/exec?query=" + url.QueryEscape(query) + resp, err := http.Get(uriRoot) + if err != nil { + return qr, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return qr, err + } + err = json.Unmarshal(body, &qr) + if err != nil { + return qr, err + } + if qr.Error != "" { + return qr, errors.New(qr.Error) + } + return qr, nil +} diff --git a/docs/questdb.md b/docs/questdb.md new file mode 100644 index 000000000..affdb0e3f --- /dev/null +++ b/docs/questdb.md @@ -0,0 +1,162 @@ +# TSBS Supplemental Guide: QuestDB + +QuestDB is a high-performance open-source time series database with SQL as a +query language with time-oriented extensions. QuestDB implements PostgreSQL wire +protocol, a REST API, and supports ingestion using InfluxDB line protocol. + +This guide explains how the data for TSBS is generated along with additional +flags available when using the data importer (`tsbs_load_questdb`). +**This should be read _after_ the main README.** + +## Data format + +Data generated by `tsbs_generate_data` is in InfluxDB line protocol format where each +reading is composed of the following: + +- the table name followed by a comma +- several comma-separated items of tags in the format `