From 21617a1181034b9de88f862b1dc261956f7a9a55 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Tue, 19 Dec 2017 02:02:29 +0100 Subject: [PATCH 1/9] Improvements --- common/redis-latency-monitor.spec | 6 +++++- redis-latency-monitor.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/common/redis-latency-monitor.spec b/common/redis-latency-monitor.spec index d4b9013..ea3c759 100644 --- a/common/redis-latency-monitor.spec +++ b/common/redis-latency-monitor.spec @@ -10,7 +10,7 @@ Summary: Tiny Redis client for latency measurement Name: redis-latency-monitor -Version: 2.4.0 +Version: 2.4.1 Release: 0%{?dist} Group: Applications/System License: EKOL @@ -58,6 +58,10 @@ rm -rf %{buildroot} ############################################################################### %changelog +* Tue Dec 19 2017 Anton Novojilov - 2.4.1-0 +- Updated ek package to latest version +- Updated stats package to latest version + * Tue Oct 03 2017 Anton Novojilov - 2.4.0-0 - Added option -T/--timestamps for output time as unix timestamp diff --git a/redis-latency-monitor.go b/redis-latency-monitor.go index d21802a..9304a4f 100644 --- a/redis-latency-monitor.go +++ b/redis-latency-monitor.go @@ -33,7 +33,7 @@ import ( const ( APP = "Redis Latency Monitor" - VER = "2.4.0" + VER = "2.4.1" DESC = "Tiny Redis client for latency measurement" ) From 90d9eb0d843b7c58bdda3e8eae0a315fe1e276b4 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Tue, 19 Dec 2017 02:03:53 +0100 Subject: [PATCH 2/9] Regenerated Makefile --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index a6a511b..41163ab 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ -######################################################################################## +################################################################################ -# This Makefile generated by GoMakeGen 0.6.0 using next command: +# This Makefile generated by GoMakeGen 0.6.1 using next command: # gomakegen . -######################################################################################## +################################################################################ .PHONY = fmt all clean deps -######################################################################################## +################################################################################ all: redis-latency-monitor @@ -25,4 +25,4 @@ fmt: clean: rm -f redis-latency-monitor -######################################################################################## +################################################################################ From 6bc3e2da0014eff83278ab5e965fbf9a6a754473 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 20 Dec 2017 15:09:35 +0100 Subject: [PATCH 3/9] Improvements --- Makefile | 1 - common/redis-latency-monitor.spec | 11 +++-- redis-latency-monitor.go | 70 ++++++++++++++++--------------- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 41163ab..6e5a29d 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,6 @@ redis-latency-monitor: deps: git config --global http.https://pkg.re.followRedirects true - go get -d -v github.com/montanaflynn/stats go get -d -v pkg.re/essentialkaos/ek.v9 fmt: diff --git a/common/redis-latency-monitor.spec b/common/redis-latency-monitor.spec index ea3c759..74822e7 100644 --- a/common/redis-latency-monitor.spec +++ b/common/redis-latency-monitor.spec @@ -10,7 +10,7 @@ Summary: Tiny Redis client for latency measurement Name: redis-latency-monitor -Version: 2.4.1 +Version: 3.0.0 Release: 0%{?dist} Group: Applications/System License: EKOL @@ -58,9 +58,12 @@ rm -rf %{buildroot} ############################################################################### %changelog -* Tue Dec 19 2017 Anton Novojilov - 2.4.1-0 -- Updated ek package to latest version -- Updated stats package to latest version +* Tue Dec 19 2017 Anton Novojilov - 3.0.0-0 +- ek package updated to latest version +- stats package updated to latest version +- Improved stats calculation +- Improved UI +- Code refactoring * Tue Oct 03 2017 Anton Novojilov - 2.4.0-0 - Added option -T/--timestamps for output time as unix timestamp diff --git a/redis-latency-monitor.go b/redis-latency-monitor.go index 9304a4f..1a39590 100644 --- a/redis-latency-monitor.go +++ b/redis-latency-monitor.go @@ -11,7 +11,6 @@ import ( "bufio" "fmt" "io" - "math" "net" "os" "runtime" @@ -22,18 +21,19 @@ import ( "pkg.re/essentialkaos/ek.v9/fmtutil" "pkg.re/essentialkaos/ek.v9/fmtutil/table" "pkg.re/essentialkaos/ek.v9/log" + "pkg.re/essentialkaos/ek.v9/mathutil" "pkg.re/essentialkaos/ek.v9/options" "pkg.re/essentialkaos/ek.v9/timeutil" "pkg.re/essentialkaos/ek.v9/usage" - "github.com/montanaflynn/stats" + "github.com/essentialkaos/redis-latency-monitor/stats" ) // ////////////////////////////////////////////////////////////////////////////////// // const ( APP = "Redis Latency Monitor" - VER = "2.4.1" + VER = "3.0.0" DESC = "Tiny Redis client for latency measurement" ) @@ -196,7 +196,7 @@ func connectToRedis(reconnect bool) error { // measureLatency measure latency func measureLatency(interval time.Duration, prettyOutput bool) { var ( - measurements []float64 + measurements stats.Data count, pointer int t *table.Table sampleRate int @@ -232,7 +232,7 @@ func measureLatency(interval time.Duration, prettyOutput bool) { errors += execCommand(buf) } - dur := float64(time.Since(start)) / float64(time.Millisecond) + dur := uint64(time.Since(start) / time.Microsecond) measurements[pointer] = dur if time.Since(last) >= interval { @@ -319,15 +319,15 @@ func makeConnection() int { } // printMeasurements calculate and print measurements -func printMeasurements(t *table.Table, errors int, measurements []float64, prettyOutput bool) { - min, _ := stats.Min(measurements) - max, _ := stats.Max(measurements) - men, _ := stats.Mean(measurements) - med, _ := stats.Median(measurements) - mgh, _ := stats.Midhinge(measurements) - sdv, _ := stats.StandardDeviation(measurements) - p95, _ := stats.Percentile(measurements, 95.0) - p99, _ := stats.Percentile(measurements, 99.0) +func printMeasurements(t *table.Table, errors int, measurements stats.Data, prettyOutput bool) { + measurements.Sort() + + min := stats.Min(measurements) + max := stats.Max(measurements) + men := stats.Mean(measurements) + sdv := stats.StandardDeviation(measurements) + p95 := stats.Percentile(measurements, 95.0) + p99 := stats.Percentile(measurements, 99.0) if prettyOutput { t.Print( @@ -335,26 +335,25 @@ func printMeasurements(t *table.Table, errors int, measurements []float64, prett fmtutil.PrettyNum(len(measurements)), fmtutil.PrettyNum(errors), formatNumber(min), formatNumber(max), - formatNumber(men), formatNumber(med), - formatNumber(mgh), formatNumber(sdv), + formatNumber(men), formatNumber(sdv), formatNumber(p95), formatNumber(p99), ) } else { if options.GetB(OPT_TIMESTAMPS) { outputWriter.WriteString( fmt.Sprintf( - "%d;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", + "%d;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", time.Now().Unix(), len(measurements), errors, - min, max, men, med, mgh, sdv, p95, p99, + min, max, men, sdv, p95, p99, ), ) } else { outputWriter.WriteString( fmt.Sprintf( - "%s;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", + "%s;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", timeutil.Format(time.Now(), "%Y/%m/%d %H:%M:%S.%K"), len(measurements), errors, - min, max, men, med, mgh, sdv, p95, p99, + min, max, men, sdv, p95, p99, ), ) } @@ -363,27 +362,30 @@ func printMeasurements(t *table.Table, errors int, measurements []float64, prett } // formatNumber format floating number -func formatNumber(value float64) string { - if math.IsNaN(value) { - return "------" +func formatNumber(value uint64) string { + if value == 0 { + return "{s-}------{!}" } - if value == 0.0 { - return "0{s-}.001{!}" - } + fv := float64(value) / 1000.0 - if value > 1000.0 { - value = math.Floor(value) + switch { + case fv > 1000.0: + fv = mathutil.Round(fv, 0) + case fv > 10: + fv = mathutil.Round(fv, 1) + case fv > 1: + fv = mathutil.Round(fv, 2) } - return strings.Replace(fmtutil.PrettyNum(value), ".", "{s-}.", -1) + "{!}" + return strings.Replace(fmtutil.PrettyNum(fv), ".", "{s-}.", -1) + "{!}" } // createOutputTable create and configure output table struct func createOutputTable() *table.Table { t := table.NewTable( - "TIME", "SAMPLES", "ERRORS", "MIN", "MAX", "MEAN", - "MEDIAN", "STDDEV", "PERC 95", "PERC 99", + "TIME", "SAMPLES", "ERRORS", "MIN", "MAX", + "MEAN", "STDDEV", "PERC 95", "PERC 99", ) t.SetSizes(12, 8, 8, 8, 10, 8, 8, 8) @@ -391,7 +393,7 @@ func createOutputTable() *table.Table { t.SetAlignments( table.ALIGN_RIGHT, table.ALIGN_RIGHT, table.ALIGN_RIGHT, table.ALIGN_RIGHT, table.ALIGN_RIGHT, table.ALIGN_RIGHT, - table.ALIGN_RIGHT, table.ALIGN_RIGHT, + table.ALIGN_RIGHT, ) return t @@ -419,9 +421,9 @@ func alignTime() time.Time { } // createMeasurementsSlice create float64 slice for measurements -func createMeasurementsSlice(sampleRate int) []float64 { +func createMeasurementsSlice(sampleRate int) []uint64 { size := (options.GetI(OPT_INTERVAL) * 1000) / sampleRate - return make([]float64, size) + return make(stats.Data, size) } // flushOutput is function for flushing output From b0a9cb899efaa7b5898516daae69c2e4849a0db4 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 20 Dec 2017 15:09:45 +0100 Subject: [PATCH 4/9] Improvements --- stats/stats.go | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 stats/stats.go diff --git a/stats/stats.go b/stats/stats.go new file mode 100644 index 0000000..d8a08e4 --- /dev/null +++ b/stats/stats.go @@ -0,0 +1,99 @@ +package stats + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2009-2017 ESSENTIAL KAOS // +// Essential Kaos Open Source License // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "math" + "sort" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Data contains stats data +type Data []uint64 + +func (s Data) Len() int { return len(s) } +func (s Data) Less(i, j int) bool { return s[i] < s[j] } +func (s Data) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Sort sort measurements data +func (d Data) Sort() { + if len(d) == 0 { + return + } + + sort.Sort(d) +} + +// Sum calculate sum of all measurements +func (d Data) Sum() uint64 { + if len(d) == 0 { + return 0 + } + + var sum uint64 + + for _, v := range d { + sum += v + } + + return sum +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Min return minimum value in slice +func Min(d Data) uint64 { + if len(d) == 0 { + return 0 + } + + return d[0] +} + +// Max return maximum value in slice +func Max(d Data) uint64 { + return d[len(d)-1] +} + +// Mean return average value +func Mean(d Data) uint64 { + return d.Sum() / uint64(len(d)) +} + +// StandardDeviation return amount of variation in the dataset +func StandardDeviation(d Data) uint64 { + m := Mean(d) + + var variance int64 + + for _, v := range d { + variance += (int64(v) - int64(m)) * (int64(v) - int64(m)) + } + + vr := float64(variance / int64(len(d))) + + return uint64(math.Pow(vr, 0.5)) +} + +// Percentile calcualte percetile +func Percentile(d Data, percent float64) uint64 { + if percent > 100 { + return 0 + } + + index := (percent / 100.0) * float64(len(d)) + + if index == float64(int64(index)) { + return d[int(index-1)] + } + + return d[int(index)] +} From d7be7044179cde3ef9fd41ae11ffe6e5e6b7b201 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 20 Dec 2017 16:26:42 +0100 Subject: [PATCH 5/9] Added codebeat settings --- .codebeatsettings | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .codebeatsettings diff --git a/.codebeatsettings b/.codebeatsettings new file mode 100644 index 0000000..fbca893 --- /dev/null +++ b/.codebeatsettings @@ -0,0 +1,9 @@ +{ + "GOLANG": { + "ABC": [15, 25, 50, 70], + "LOC": [30, 45, 70, 100], + "TOO_MANY_IVARS": [12, 16, 20, 24], + "TOO_MANY_FUNCTIONS": [50, 70, 90, 120], + "TOTAL_LOC": [500, 750, 1000, 2000] + } +} From cf18fac4bd36f7fe8c65bd0f66fa60eacf061616 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 20 Dec 2017 16:58:08 +0100 Subject: [PATCH 6/9] Improved spec changelog --- common/redis-latency-monitor.spec | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/redis-latency-monitor.spec b/common/redis-latency-monitor.spec index 74822e7..1e58953 100644 --- a/common/redis-latency-monitor.spec +++ b/common/redis-latency-monitor.spec @@ -59,9 +59,10 @@ rm -rf %{buildroot} %changelog * Tue Dec 19 2017 Anton Novojilov - 3.0.0-0 +- Fixed bug with percentile calculation - ek package updated to latest version -- stats package updated to latest version -- Improved stats calculation +- More precise latency calculation +- Removed external packages - Improved UI - Code refactoring From a56388e9d60eafde89e7164c9cdfc1eecd92ea77 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 20 Dec 2017 22:27:59 +0100 Subject: [PATCH 7/9] Fixes --- redis-latency-monitor.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/redis-latency-monitor.go b/redis-latency-monitor.go index 1a39590..103d81a 100644 --- a/redis-latency-monitor.go +++ b/redis-latency-monitor.go @@ -31,17 +31,20 @@ import ( // ////////////////////////////////////////////////////////////////////////////////// // +// App info const ( APP = "Redis Latency Monitor" VER = "3.0.0" DESC = "Tiny Redis client for latency measurement" ) +// Main constatnts const ( LATENCY_SAMPLE_RATE int = 10 CONNECT_SAMPLE_RATE = 100 ) +// Options const ( OPT_HOST = "h:host" OPT_PORT = "p:port" @@ -342,7 +345,7 @@ func printMeasurements(t *table.Table, errors int, measurements stats.Data, pret if options.GetB(OPT_TIMESTAMPS) { outputWriter.WriteString( fmt.Sprintf( - "%d;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", + "%d;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", time.Now().Unix(), len(measurements), errors, min, max, men, sdv, p95, p99, ), @@ -350,7 +353,7 @@ func printMeasurements(t *table.Table, errors int, measurements stats.Data, pret } else { outputWriter.WriteString( fmt.Sprintf( - "%s;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", + "%s;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", timeutil.Format(time.Now(), "%Y/%m/%d %H:%M:%S.%K"), len(measurements), errors, min, max, men, sdv, p95, p99, From 3f864dbd22761a664a6624eb9c3d43680e86f756 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 20 Dec 2017 22:40:43 +0100 Subject: [PATCH 8/9] Raw output fixes --- redis-latency-monitor.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/redis-latency-monitor.go b/redis-latency-monitor.go index 103d81a..8ba3ec2 100644 --- a/redis-latency-monitor.go +++ b/redis-latency-monitor.go @@ -347,7 +347,8 @@ func printMeasurements(t *table.Table, errors int, measurements stats.Data, pret fmt.Sprintf( "%d;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", time.Now().Unix(), len(measurements), errors, - min, max, men, sdv, p95, p99, + usToMs(min), usToMs(max), usToMs(men), + usToMs(sdv), usToMs(p95), usToMs(p99), ), ) } else { @@ -356,7 +357,8 @@ func printMeasurements(t *table.Table, errors int, measurements stats.Data, pret "%s;%d;%d;%.03f;%.03f;%.03f;%.03f;%.03f;%.03f;\n", timeutil.Format(time.Now(), "%Y/%m/%d %H:%M:%S.%K"), len(measurements), errors, - min, max, men, sdv, p95, p99, + usToMs(min), usToMs(max), usToMs(men), + usToMs(sdv), usToMs(p95), usToMs(p99), ), ) } @@ -384,6 +386,11 @@ func formatNumber(value uint64) string { return strings.Replace(fmtutil.PrettyNum(fv), ".", "{s-}.", -1) + "{!}" } +// usToMs convert us in uint64 to ms in float64 +func usToMs(us uint64) float64 { + return float64(us) / 1000.0 +} + // createOutputTable create and configure output table struct func createOutputTable() *table.Table { t := table.NewTable( From 1078b9e5b387e06cb7ff562a5a9261bc2259aa28 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 20 Dec 2017 22:41:58 +0100 Subject: [PATCH 9/9] Improved github templates --- .github/CODE_OF_CONDUCT.md | 42 ++++++++++++++++++++++++++++++++++++++ .github/CONTRIBUTING.md | 40 +++++++++++++++++------------------- 2 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 .github/CODE_OF_CONDUCT.md diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..9bf5de2 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,42 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute through reporting issues, posting feature requests, updating +documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting a project maintainer at [INSERT EMAIL ADDRESS]. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 2a651a3..d7da3fb 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,21 +1,19 @@ -# Contributing Guidelines - -Contributing guidelines for open-source EK projects. - -**IMPORTANT! Contribute your code only if you have an excellent understanding of project idea and all existing code base. Otherwise, a nicely formatted issue will be more helpful to us.** - -### Issues - -1. Provide product version where the problem was found; -2. Provide info about your environment; -3. Provide detailed info about your problem; -4. Provide steps to reproduce the problem; -5. Provide actual and expected results. - -### Code - -1. Check your code **before** creating pull request; -2. If tests are present in a project, add tests for your code; -3. Add inline documentation for your code; -4. Apply code style used throughout the project; -5. Create your pull request to `develop` branch (_pull requests to other branches are not allowed_). +# Contributing Guidelines + +**IMPORTANT! Contribute your code only if you have an excellent understanding of project idea and all existing code base. Otherwise, a nicely formatted issue will be more helpful to us.** + +### Issues + +1. Provide product version where the problem was found; +2. Provide info about your environment; +3. Provide detailed info about your problem; +4. Provide steps to reproduce the problem; +5. Provide actual and expected results. + +### Code + +1. Check your code **before** creating pull request; +2. If tests are present in a project, add tests for your code; +3. Add inline documentation for your code; +4. Apply code style used throughout the project; +5. Create your pull request to `develop` branch (_pull requests to other branches are not allowed_).