From 3995bea4733ab92537a751f445e254c52a705b1f Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Mon, 18 Nov 2024 16:29:56 -0500 Subject: [PATCH 01/12] POC dedupe-logs --- components/motor/fake/motor.go | 28 +++++++++++-- logging/impl.go | 61 +++++++++++++++++++++++++++ logging/logging.go | 77 +++++++++++++++++++++++++--------- 3 files changed, 142 insertions(+), 24 deletions(-) diff --git a/components/motor/fake/motor.go b/components/motor/fake/motor.go index 68b082ca215..d90d733195c 100644 --- a/components/motor/fake/motor.go +++ b/components/motor/fake/motor.go @@ -18,6 +18,7 @@ import ( "go.viam.com/rdk/logging" "go.viam.com/rdk/operation" "go.viam.com/rdk/resource" + "go.viam.com/utils" ) var ( @@ -79,7 +80,6 @@ func init() { // direction. type Motor struct { resource.Named - resource.TriviallyCloseable mu sync.Mutex powerPct float64 @@ -93,14 +93,29 @@ type Motor struct { OpMgr *operation.SingleOperationManager Logger logging.Logger + + logWorkers *utils.StoppableWorkers } // NewMotor creates a new fake motor. func NewMotor(ctx context.Context, deps resource.Dependencies, conf resource.Config, logger logging.Logger) (motor.Motor, error) { + logWorker := func(ctx context.Context) { + for { + if ctx.Err() != nil { + logger.Info("stopping log worker") + return + } + + logger.Info("here is an annoying spammy message") + time.Sleep(10 * time.Millisecond) + } + } + m := &Motor{ - Named: conf.ResourceName().AsNamed(), - Logger: logger, - OpMgr: operation.NewSingleOperationManager(), + Named: conf.ResourceName().AsNamed(), + Logger: logger, + OpMgr: operation.NewSingleOperationManager(), + logWorkers: utils.NewBackgroundStoppableWorkers(logWorker), } if err := m.Reconfigure(ctx, deps, conf); err != nil { return nil, err @@ -431,3 +446,8 @@ func (m *Motor) IsMoving(ctx context.Context) (bool, error) { defer m.mu.Unlock() return math.Abs(m.powerPct) >= 0.005, nil } + +func (m *Motor) Close(ctx context.Context) error { + m.logWorkers.Stop() + return nil +} diff --git a/logging/impl.go b/logging/impl.go index 5492386699f..41ee35803ee 100644 --- a/logging/impl.go +++ b/logging/impl.go @@ -15,6 +15,14 @@ import ( "go.uber.org/zap/zaptest" ) +const ( + // Window duration over which to consider log messages "noisy." + noisyMessageWindowDuration = 5 * time.Second + // Count threshold within `noisyMessageWindowDuration` after which to + // consider log messages "noisy." + noisyMessageCountThreshold = 3 +) + type ( impl struct { name string @@ -26,6 +34,15 @@ type ( // avoid that. This function is a no-op for non-test loggers. See `NewTestAppender` // documentation for more details. testHelper func() + + // Whether or not to de-duplicate noisy logs. + dedupNoisyLogs bool + // Map of messages to counts of that message being `Write`ten within window. + recentMessageCounts map[string]int + // Map of messages to last `LogEntry` with that message within window. + recentMessageEntries map[string]LogEntry + // Start of current window. + recentMessageWindowStart time.Time } // LogEntry embeds a zapcore Entry and slice of Fields. @@ -84,6 +101,14 @@ func (imp *impl) Sublogger(subname string) Logger { imp.appenders, imp.registry, imp.testHelper, + // Inherit _whether_ we should deduplicate noisy logs from parent. However, + // subloggers should handle their own de-duplication with their own maps + // and windows. This design avoids races and allows logs with identical + // messages from different loggers to be considered unique. + imp.dedupNoisyLogs, + make(map[string]int), + make(map[string]LogEntry), + time.Now(), } // If there are multiple callers racing to create the same logger name (e.g: `viam.networking`), @@ -198,6 +223,42 @@ func (imp *impl) shouldLog(logLevel Level) bool { } func (imp *impl) Write(entry *LogEntry) { + if imp.dedupNoisyLogs { + // If we have have entered a new recentMessage window, output noisy logs from + // the last window. + if time.Since(imp.recentMessageWindowStart) > noisyMessageWindowDuration { + for message, count := range imp.recentMessageCounts { + if count > noisyMessageCountThreshold { + collapsedEntry := imp.recentMessageEntries[entry.Message] + collapsedEntry.Message = fmt.Sprintf("Message logged %d times in past %v: %s", + count, noisyMessageWindowDuration, message) + + imp.testHelper() + for _, appender := range imp.appenders { + err := appender.Write(collapsedEntry.Entry, collapsedEntry.Fields) + if err != nil { + fmt.Fprint(os.Stderr, err) + } + } + } + } + + // Clear maps and reset window. + clear(imp.recentMessageCounts) + clear(imp.recentMessageEntries) + imp.recentMessageWindowStart = time.Now() + } + + // Track entry in recentMessage maps. + imp.recentMessageCounts[entry.Message]++ + imp.recentMessageEntries[entry.Message] = *entry + + if imp.recentMessageCounts[entry.Message] > noisyMessageCountThreshold { + // If entry's message is reportedly "noisy," return early. + return + } + } + imp.testHelper() for _, appender := range imp.appenders { err := appender.Write(entry.Entry, entry.Fields) diff --git a/logging/logging.go b/logging/logging.go index 40060c14933..60fbcdd01bd 100644 --- a/logging/logging.go +++ b/logging/logging.go @@ -2,8 +2,10 @@ package logging import ( + "os" "sync" "testing" + "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -11,6 +13,8 @@ import ( "go.viam.com/utils" ) +const dedupNoisyLogsEnvVar = "VIAM_DEDUP_LOGS" + var ( globalMu sync.RWMutex globalLogger = NewDebugLogger("global") @@ -18,8 +22,19 @@ var ( // GlobalLogLevel should be used whenever a zap logger is created that wants to obey the debug // flag from the CLI or robot config. GlobalLogLevel = zap.NewAtomicLevelAt(zap.InfoLevel) + + // Whether to de-duplicate noisy logs; obtained from value of + // `dedupNoisyLogsEnvVar` and defaults to true. Export env var to "false" to + // turn off de-duplicating logic. + dedupNoisyLogs = false ) +func init() { + if dedupNoisyLogEnvVal := os.Getenv(dedupNoisyLogsEnvVar); dedupNoisyLogEnvVal == "false" { + dedupNoisyLogs = false + } +} + // ReplaceGlobal replaces the global loggers. func ReplaceGlobal(logger Logger) { globalMu.Lock() @@ -62,11 +77,15 @@ func NewZapLoggerConfig() zap.Config { // NewLogger returns a new logger that outputs Info+ logs to stdout in UTC. func NewLogger(name string) Logger { logger := &impl{ - name: name, - level: NewAtomicLevelAt(INFO), - appenders: []Appender{NewStdoutAppender()}, - registry: newRegistry(), - testHelper: func() {}, + name: name, + level: NewAtomicLevelAt(INFO), + appenders: []Appender{NewStdoutAppender()}, + registry: newRegistry(), + testHelper: func() {}, + dedupNoisyLogs: dedupNoisyLogs, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } logger.registry.registerLogger(name, logger) @@ -78,11 +97,15 @@ func NewLogger(name string) Logger { func NewLoggerWithRegistry(name string) (Logger, *Registry) { reg := newRegistry() logger := &impl{ - name: name, - level: NewAtomicLevelAt(INFO), - appenders: []Appender{NewStdoutAppender()}, - registry: reg, - testHelper: func() {}, + name: name, + level: NewAtomicLevelAt(INFO), + appenders: []Appender{NewStdoutAppender()}, + registry: reg, + testHelper: func() {}, + dedupNoisyLogs: dedupNoisyLogs, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } logger.registry.registerLogger(name, logger) @@ -92,11 +115,15 @@ func NewLoggerWithRegistry(name string) (Logger, *Registry) { // NewDebugLogger returns a new logger that outputs Debug+ logs to stdout in UTC. func NewDebugLogger(name string) Logger { logger := &impl{ - name: name, - level: NewAtomicLevelAt(DEBUG), - appenders: []Appender{NewStdoutAppender()}, - registry: newRegistry(), - testHelper: func() {}, + name: name, + level: NewAtomicLevelAt(DEBUG), + appenders: []Appender{NewStdoutAppender()}, + registry: newRegistry(), + testHelper: func() {}, + dedupNoisyLogs: dedupNoisyLogs, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } logger.registry.registerLogger(name, logger) @@ -107,11 +134,15 @@ func NewDebugLogger(name string) Logger { // pre-existing appenders/outputs. func NewBlankLogger(name string) Logger { logger := &impl{ - name: name, - level: NewAtomicLevelAt(DEBUG), - appenders: []Appender{}, - registry: newRegistry(), - testHelper: func() {}, + name: name, + level: NewAtomicLevelAt(DEBUG), + appenders: []Appender{}, + registry: newRegistry(), + testHelper: func() {}, + dedupNoisyLogs: dedupNoisyLogs, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } logger.registry.registerLogger(name, logger) @@ -136,6 +167,8 @@ func NewObservedTestLogger(tb testing.TB) (Logger, *observer.ObservedLogs) { }, registry: newRegistry(), testHelper: tb.Helper, + // Only prod loggers should de-duplicate noisy logs. + dedupNoisyLogs: false, } return logger, observedLogs @@ -155,6 +188,8 @@ func NewObservedTestLoggerWithRegistry(tb testing.TB, name string) (Logger, *obs }, registry: registry, testHelper: tb.Helper, + // Only prod loggers should de-duplicate noisy logs. + dedupNoisyLogs: false, } return logger, observedLogs, registry @@ -189,6 +224,8 @@ func NewInMemoryLogger(tb testing.TB) *MemLogger { }, registry: newRegistry(), testHelper: tb.Helper, + // Only prod loggers should de-duplicate noisy logs. + dedupNoisyLogs: false, } memLogger := &MemLogger{logger, tb, observedLogs} From 7603ab1b527d95ad6d44477bf1ddbe5fad9f4c4d Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Mon, 18 Nov 2024 16:32:31 -0500 Subject: [PATCH 02/12] typo --- logging/logging.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/logging/logging.go b/logging/logging.go index 60fbcdd01bd..309cf8dc7c7 100644 --- a/logging/logging.go +++ b/logging/logging.go @@ -13,6 +13,9 @@ import ( "go.viam.com/utils" ) +// Environment variable to control whether noisy logs are de-deduplicated. Set +// to "false" to turn off de-duplicating logic; de-duplication logic is enabled +// by default. const dedupNoisyLogsEnvVar = "VIAM_DEDUP_LOGS" var ( @@ -26,7 +29,7 @@ var ( // Whether to de-duplicate noisy logs; obtained from value of // `dedupNoisyLogsEnvVar` and defaults to true. Export env var to "false" to // turn off de-duplicating logic. - dedupNoisyLogs = false + dedupNoisyLogs = true ) func init() { From 192dad9f9c603db5af2dba77842faa39a5d9a8c0 Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Tue, 19 Nov 2024 15:38:14 -0500 Subject: [PATCH 03/12] up window --- logging/impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logging/impl.go b/logging/impl.go index 41ee35803ee..da6a8c24734 100644 --- a/logging/impl.go +++ b/logging/impl.go @@ -17,7 +17,7 @@ import ( const ( // Window duration over which to consider log messages "noisy." - noisyMessageWindowDuration = 5 * time.Second + noisyMessageWindowDuration = 10 * time.Second // Count threshold within `noisyMessageWindowDuration` after which to // consider log messages "noisy." noisyMessageCountThreshold = 3 From c487bb86b417b90605b73813f3f37bd4490d5246 Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Wed, 11 Dec 2024 18:14:03 -0500 Subject: [PATCH 04/12] add locking --- logging/impl.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/logging/impl.go b/logging/impl.go index da6a8c24734..da6da669ae6 100644 --- a/logging/impl.go +++ b/logging/impl.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "runtime" + "sync" "testing" "time" @@ -37,6 +38,9 @@ type ( // Whether or not to de-duplicate noisy logs. dedupNoisyLogs bool + + // recentMessageMu guards the recentMessage fields below. + recentMessageMu sync.Mutex // Map of messages to counts of that message being `Write`ten within window. recentMessageCounts map[string]int // Map of messages to last `LogEntry` with that message within window. @@ -103,9 +107,11 @@ func (imp *impl) Sublogger(subname string) Logger { imp.testHelper, // Inherit _whether_ we should deduplicate noisy logs from parent. However, // subloggers should handle their own de-duplication with their own maps - // and windows. This design avoids races and allows logs with identical - // messages from different loggers to be considered unique. + // and windows. This design avoids locking across all loggers and allows + // logs with identical messages from different loggers to be considered + // unique. imp.dedupNoisyLogs, + sync.Mutex{}, make(map[string]int), make(map[string]LogEntry), time.Now(), @@ -224,8 +230,9 @@ func (imp *impl) shouldLog(logLevel Level) bool { func (imp *impl) Write(entry *LogEntry) { if imp.dedupNoisyLogs { - // If we have have entered a new recentMessage window, output noisy logs from + // If we have entered a new recentMessage window, output noisy logs from // the last window. + imp.recentMessageMu.Lock() if time.Since(imp.recentMessageWindowStart) > noisyMessageWindowDuration { for message, count := range imp.recentMessageCounts { if count > noisyMessageCountThreshold { @@ -255,8 +262,10 @@ func (imp *impl) Write(entry *LogEntry) { if imp.recentMessageCounts[entry.Message] > noisyMessageCountThreshold { // If entry's message is reportedly "noisy," return early. + imp.recentMessageMu.Unlock() return } + imp.recentMessageMu.Unlock() } imp.testHelper() From 6d699447344a4ffe79b7d66f62ca7eb9ef41a473 Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Sun, 22 Dec 2024 17:08:29 -0500 Subject: [PATCH 05/12] undo motor changes --- components/motor/fake/motor.go | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/components/motor/fake/motor.go b/components/motor/fake/motor.go index d90d733195c..68b082ca215 100644 --- a/components/motor/fake/motor.go +++ b/components/motor/fake/motor.go @@ -18,7 +18,6 @@ import ( "go.viam.com/rdk/logging" "go.viam.com/rdk/operation" "go.viam.com/rdk/resource" - "go.viam.com/utils" ) var ( @@ -80,6 +79,7 @@ func init() { // direction. type Motor struct { resource.Named + resource.TriviallyCloseable mu sync.Mutex powerPct float64 @@ -93,29 +93,14 @@ type Motor struct { OpMgr *operation.SingleOperationManager Logger logging.Logger - - logWorkers *utils.StoppableWorkers } // NewMotor creates a new fake motor. func NewMotor(ctx context.Context, deps resource.Dependencies, conf resource.Config, logger logging.Logger) (motor.Motor, error) { - logWorker := func(ctx context.Context) { - for { - if ctx.Err() != nil { - logger.Info("stopping log worker") - return - } - - logger.Info("here is an annoying spammy message") - time.Sleep(10 * time.Millisecond) - } - } - m := &Motor{ - Named: conf.ResourceName().AsNamed(), - Logger: logger, - OpMgr: operation.NewSingleOperationManager(), - logWorkers: utils.NewBackgroundStoppableWorkers(logWorker), + Named: conf.ResourceName().AsNamed(), + Logger: logger, + OpMgr: operation.NewSingleOperationManager(), } if err := m.Reconfigure(ctx, deps, conf); err != nil { return nil, err @@ -446,8 +431,3 @@ func (m *Motor) IsMoving(ctx context.Context) (bool, error) { defer m.mu.Unlock() return math.Abs(m.powerPct) >= 0.005, nil } - -func (m *Motor) Close(ctx context.Context) error { - m.logWorkers.Stop() - return nil -} From 3dd6dfff1a0adf5e5c65dd12a8c0d1bdd09fc9e6 Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Mon, 23 Dec 2024 13:47:09 -0500 Subject: [PATCH 06/12] include fields in identicality calculations --- logging/impl.go | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/logging/impl.go b/logging/impl.go index da6da669ae6..cbe00ac1c9e 100644 --- a/logging/impl.go +++ b/logging/impl.go @@ -62,6 +62,31 @@ type ( } ) +// String stringifies a `LogEntry`. Should be used to emplace a log entry in +// `recentMessageEntries`, i.e. `LogEntry`s that `String`ify identically should +// be treated as identical with respect to noisiness and deduplication. Should +// not be used for actual writing of the entry to an appender. +func (le *LogEntry) String() string { + ret := le.Message + for _, field := range le.Fields { + ret += " " + field.Key + " " + + // Assume field's value is held in one of `Integer`, `Interface`, or + // `String`. Otherwise (field has no value or is equivalent to 0 or "") use + // the string "undefined". + if field.Integer != 0 { + ret += fmt.Sprintf("%d", field.Integer) + } else if field.Interface != nil { + ret += fmt.Sprintf("%v", field.Interface) + } else if field.String != "" { + ret += field.String + } else { + ret += "undefined" + } + } + return ret +} + func (imp *impl) NewLogEntry() *LogEntry { ret := &LogEntry{} ret.Time = time.Now() @@ -234,11 +259,11 @@ func (imp *impl) Write(entry *LogEntry) { // the last window. imp.recentMessageMu.Lock() if time.Since(imp.recentMessageWindowStart) > noisyMessageWindowDuration { - for message, count := range imp.recentMessageCounts { + for stringifiedEntry, count := range imp.recentMessageCounts { if count > noisyMessageCountThreshold { - collapsedEntry := imp.recentMessageEntries[entry.Message] + collapsedEntry := imp.recentMessageEntries[stringifiedEntry] collapsedEntry.Message = fmt.Sprintf("Message logged %d times in past %v: %s", - count, noisyMessageWindowDuration, message) + count, noisyMessageWindowDuration, collapsedEntry.Message) imp.testHelper() for _, appender := range imp.appenders { @@ -257,10 +282,11 @@ func (imp *impl) Write(entry *LogEntry) { } // Track entry in recentMessage maps. - imp.recentMessageCounts[entry.Message]++ - imp.recentMessageEntries[entry.Message] = *entry + stringifiedEntry := entry.String() + imp.recentMessageCounts[stringifiedEntry]++ + imp.recentMessageEntries[stringifiedEntry] = *entry - if imp.recentMessageCounts[entry.Message] > noisyMessageCountThreshold { + if imp.recentMessageCounts[stringifiedEntry] > noisyMessageCountThreshold { // If entry's message is reportedly "noisy," return early. imp.recentMessageMu.Unlock() return From afe9b2e282736f22de649de2f0d5f3d3ee5ba628 Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Mon, 23 Dec 2024 14:37:17 -0500 Subject: [PATCH 07/12] add basic testing --- logging/impl.go | 15 ++-- logging/impl_test.go | 169 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 7 deletions(-) diff --git a/logging/impl.go b/logging/impl.go index cbe00ac1c9e..0de084ea484 100644 --- a/logging/impl.go +++ b/logging/impl.go @@ -16,11 +16,11 @@ import ( "go.uber.org/zap/zaptest" ) -const ( - // Window duration over which to consider log messages "noisy." +var ( + // Window duration over which to consider log messages "noisy.". noisyMessageWindowDuration = 10 * time.Second // Count threshold within `noisyMessageWindowDuration` after which to - // consider log messages "noisy." + // consider log messages "noisy.". noisyMessageCountThreshold = 3 ) @@ -74,13 +74,14 @@ func (le *LogEntry) String() string { // Assume field's value is held in one of `Integer`, `Interface`, or // `String`. Otherwise (field has no value or is equivalent to 0 or "") use // the string "undefined". - if field.Integer != 0 { + switch { + case field.Integer != 0: ret += fmt.Sprintf("%d", field.Integer) - } else if field.Interface != nil { + case field.Interface != nil: ret += fmt.Sprintf("%v", field.Interface) - } else if field.String != "" { + case field.String != "": ret += field.String - } else { + default: ret += "undefined" } } diff --git a/logging/impl_test.go b/logging/impl_test.go index 45a190a9cb7..fa6b8c9a8a7 100644 --- a/logging/impl_test.go +++ b/logging/impl_test.go @@ -9,7 +9,9 @@ import ( "strconv" "strings" "testing" + "time" + "go.uber.org/zap/zapcore" "go.viam.com/test" ) @@ -420,3 +422,170 @@ func TestLoggingWithFields(t *testing.T) { assertLogMatches(t, notStdout, `2023-10-30T09:12:09.459Z DEBUG impl logging/impl_test.go:200 Debugw log {"traceKey":"foobar","k":"v","key":"value"}`) } + +func TestLogEntryStringify(t *testing.T) { + testCases := []struct { + name string + logEntry *LogEntry + expectedStringification string + }{ + { + "no fields", + &LogEntry{ + Entry: zapcore.Entry{ + Message: "these are not the droids you are looking for", + }, + }, + "these are not the droids you are looking for", + }, + { + "fields", + &LogEntry{ + Entry: zapcore.Entry{ + Message: "these are not the droids you are looking for", + }, + Fields: []zapcore.Field{ + { + Key: "obi", + String: "wan", + }, + { + Key: "r2d", + Integer: 2, + }, + { + Key: "c3", + Interface: "po", + }, + }, + }, + "these are not the droids you are looking for obi wan r2d 2 c3 po", + }, + { + "undefined field", + &LogEntry{ + Entry: zapcore.Entry{ + Message: "these are not the droids you are looking for", + }, + Fields: []zapcore.Field{ + { + Key: "obi", + }, + }, + }, + "these are not the droids you are looking for obi undefined", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualStringification := tc.logEntry.String() + test.That(t, actualStringification, test.ShouldEqual, tc.expectedStringification) + }) + } +} + +func TestLoggingDeduplication(t *testing.T) { + // Create a logger object that will write to the `notStdout` buffer. + notStdout := &bytes.Buffer{} + logger := &impl{ + name: "impl", + level: NewAtomicLevelAt(DEBUG), + appenders: []Appender{NewWriterAppender(notStdout)}, + registry: newRegistry(), + testHelper: func() {}, + dedupNoisyLogs: true, + // Initialize recent message fields as `NewLogger` would. + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), + } + + // Artificially lower noisy message window for testing. + originalNoisyMessageWindowDuration := noisyMessageWindowDuration + noisyMessageWindowDuration = 500 * time.Millisecond + defer func() { + noisyMessageWindowDuration = originalNoisyMessageWindowDuration + }() + + // Log 4 identical messages (same sublogger, messages, and fields) in quick + // succession. Sleep for noisy message window duration. Assert that a final, + // separate log is an aggregation log. + identicalMsg := "identical message" + loggerWith := logger.WithFields("key", "value") + for range 3 { + loggerWith.Info(identicalMsg) + assertLogMatches(t, notStdout, + `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value"}`) + } + loggerWith.Info(identicalMsg) // not output due to being noisy + time.Sleep(noisyMessageWindowDuration) + loggerWith.Info("foo") // log arbitrary message to force output of aggregated message + assertLogMatches(t, notStdout, + `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 4 times in past 500ms: identical message {"key":"value"}`) + assertLogMatches(t, notStdout, + `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 foo {"key":"value"}`) + + // Assert aggregation resets after sleep (same aggregation occurs again.) + for range 3 { + loggerWith.Info(identicalMsg) + assertLogMatches(t, notStdout, + `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value"}`) + } + loggerWith.Info(identicalMsg) // not output due to being noisy + time.Sleep(noisyMessageWindowDuration) + loggerWith.Info("foo") // log arbitrary message to force output of aggregated message + assertLogMatches(t, notStdout, + `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 4 times in past 500ms: identical message {"key":"value"}`) + assertLogMatches(t, notStdout, + `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 foo {"key":"value"}`) + + // TODO(benji): Fix and uncomment the following tests to test more deduplication logic. + // + // Assert that using a different sublogger uses separate aggregation. + // loggerWith2 := logger.WithFields("key", "value2") + // for range 3 { + //loggerWith.Info(identicalMsg) + //assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value"}`) + + // loggerWith2.Info(identicalMsg) + // assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) + //} + //loggerWith.Info(identicalMsg) // not output due to being noisy + //loggerWith2.Info(identicalMsg) // not output due to being noisy + //assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) + //time.Sleep(noisyMessageWindowDuration) + //assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 5 times in past 500ms: identical message`) + //assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 5 times in past 500ms: identical message`) + + //// Assert that using different fields uses separate aggregation. + // for range 5 { + // loggerWith2.Infow(identicalMsg, "key2", "value") + //assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2", "key2": "value"}`) + //} + //loggerWith2.Info(identicalMsg) // not output due to being noisy + //time.Sleep(noisyMessageWindowDuration) + //assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 5 times in past 500ms: identical message`) + + //// Assert that using different levels does _not_ use separate aggregation. + // for range 2 { + // loggerWith.Info(identicalMsg) + //assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) + //} + //for range 2 { + //loggerWith.Error(identicalMsg) + //assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z ERROR impl logging/impl_test.go:132 identical message {"key":"value2"}`) + //} + //loggerWith2.Debug(identicalMsg) + //time.Sleep(noisyMessageWindowDuration) + //assertLogMatches(t, notStdout, + //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 5 times in past 1s: identical message`) +} From 404203a4d9ba2114920abd3c30db7d599b222797 Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Mon, 23 Dec 2024 14:51:42 -0500 Subject: [PATCH 08/12] lint --- logging/impl_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/logging/impl_test.go b/logging/impl_test.go index fa6b8c9a8a7..76be6c1fd81 100644 --- a/logging/impl_test.go +++ b/logging/impl_test.go @@ -544,13 +544,13 @@ func TestLoggingDeduplication(t *testing.T) { // Assert that using a different sublogger uses separate aggregation. // loggerWith2 := logger.WithFields("key", "value2") // for range 3 { - //loggerWith.Info(identicalMsg) + // loggerWith.Info(identicalMsg) //assertLogMatches(t, notStdout, //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value"}`) // loggerWith2.Info(identicalMsg) // assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) + // `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) //} //loggerWith.Info(identicalMsg) // not output due to being noisy //loggerWith2.Info(identicalMsg) // not output due to being noisy @@ -565,7 +565,7 @@ func TestLoggingDeduplication(t *testing.T) { //// Assert that using different fields uses separate aggregation. // for range 5 { // loggerWith2.Infow(identicalMsg, "key2", "value") - //assertLogMatches(t, notStdout, + // assertLogMatches(t, notStdout, //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2", "key2": "value"}`) //} //loggerWith2.Info(identicalMsg) // not output due to being noisy @@ -576,7 +576,7 @@ func TestLoggingDeduplication(t *testing.T) { //// Assert that using different levels does _not_ use separate aggregation. // for range 2 { // loggerWith.Info(identicalMsg) - //assertLogMatches(t, notStdout, + // assertLogMatches(t, notStdout, //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) //} //for range 2 { From 25e5a21dda2557c9a2669e8c4b51acaffe643bab Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Mon, 23 Dec 2024 15:00:20 -0500 Subject: [PATCH 09/12] please forgive me lint --- logging/impl_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/logging/impl_test.go b/logging/impl_test.go index 76be6c1fd81..e1aa360e23e 100644 --- a/logging/impl_test.go +++ b/logging/impl_test.go @@ -545,14 +545,14 @@ func TestLoggingDeduplication(t *testing.T) { // loggerWith2 := logger.WithFields("key", "value2") // for range 3 { // loggerWith.Info(identicalMsg) - //assertLogMatches(t, notStdout, + // assertLogMatches(t, notStdout, //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value"}`) // loggerWith2.Info(identicalMsg) // assertLogMatches(t, notStdout, // `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) //} - //loggerWith.Info(identicalMsg) // not output due to being noisy + // loggerWith.Info(identicalMsg) // not output due to being noisy //loggerWith2.Info(identicalMsg) // not output due to being noisy //assertLogMatches(t, notStdout, //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) @@ -566,7 +566,7 @@ func TestLoggingDeduplication(t *testing.T) { // for range 5 { // loggerWith2.Infow(identicalMsg, "key2", "value") // assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2", "key2": "value"}`) + // `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2", "key2": "value"}`) //} //loggerWith2.Info(identicalMsg) // not output due to being noisy //time.Sleep(noisyMessageWindowDuration) @@ -577,7 +577,7 @@ func TestLoggingDeduplication(t *testing.T) { // for range 2 { // loggerWith.Info(identicalMsg) // assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) + // `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) //} //for range 2 { //loggerWith.Error(identicalMsg) From 3fa24e8fb9d416681d08460802ce432fa66faa61 Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Mon, 23 Dec 2024 15:23:09 -0500 Subject: [PATCH 10/12] please leave me alone nondeterministic lint --- logging/impl_test.go | 50 +++----------------------------------------- 1 file changed, 3 insertions(+), 47 deletions(-) diff --git a/logging/impl_test.go b/logging/impl_test.go index e1aa360e23e..f098321658c 100644 --- a/logging/impl_test.go +++ b/logging/impl_test.go @@ -539,53 +539,9 @@ func TestLoggingDeduplication(t *testing.T) { assertLogMatches(t, notStdout, `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 foo {"key":"value"}`) - // TODO(benji): Fix and uncomment the following tests to test more deduplication logic. + // TODO(benji): Add the following assertions to test more deduplication logic. // // Assert that using a different sublogger uses separate aggregation. - // loggerWith2 := logger.WithFields("key", "value2") - // for range 3 { - // loggerWith.Info(identicalMsg) - // assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value"}`) - - // loggerWith2.Info(identicalMsg) - // assertLogMatches(t, notStdout, - // `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) - //} - // loggerWith.Info(identicalMsg) // not output due to being noisy - //loggerWith2.Info(identicalMsg) // not output due to being noisy - //assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) - //time.Sleep(noisyMessageWindowDuration) - //assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 5 times in past 500ms: identical message`) - //assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 5 times in past 500ms: identical message`) - - //// Assert that using different fields uses separate aggregation. - // for range 5 { - // loggerWith2.Infow(identicalMsg, "key2", "value") - // assertLogMatches(t, notStdout, - // `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2", "key2": "value"}`) - //} - //loggerWith2.Info(identicalMsg) // not output due to being noisy - //time.Sleep(noisyMessageWindowDuration) - //assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 5 times in past 500ms: identical message`) - - //// Assert that using different levels does _not_ use separate aggregation. - // for range 2 { - // loggerWith.Info(identicalMsg) - // assertLogMatches(t, notStdout, - // `2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 identical message {"key":"value2"}`) - //} - //for range 2 { - //loggerWith.Error(identicalMsg) - //assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z ERROR impl logging/impl_test.go:132 identical message {"key":"value2"}`) - //} - //loggerWith2.Debug(identicalMsg) - //time.Sleep(noisyMessageWindowDuration) - //assertLogMatches(t, notStdout, - //`2023-10-30T13:19:45.806Z INFO impl logging/impl_test.go:132 Message logged 5 times in past 1s: identical message`) + // Assert that using different fields uses separate aggregation. + // Assert that using different levels does _not_ use separate aggregation. } From e57b5609935dc8cb44173117b5a179341d83bd5d Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Mon, 23 Dec 2024 15:51:58 -0500 Subject: [PATCH 11/12] use disable log deduplication value from JSON config --- config/config.go | 35 ++++++++++++---------- config/proto_conversions.go | 4 +++ go.mod | 2 +- go.sum | 4 +-- logging/impl.go | 11 +------ logging/impl_test.go | 58 +++++++++++++++++++++++-------------- logging/logging.go | 51 ++++++++++++-------------------- robot/impl/local_robot.go | 9 ++++++ 8 files changed, 92 insertions(+), 82 deletions(-) diff --git a/config/config.go b/config/config.go index 5ac34ad9d27..edb033277d3 100644 --- a/config/config.go +++ b/config/config.go @@ -71,6 +71,10 @@ type Config struct { // Revision contains the current revision of the config. Revision string + // DisableLogDeduplication controls whether deduplication of noisy logs + // should be turned off. Defaults to false. + DisableLogDeduplication bool + // toCache stores the JSON marshalled version of the config to be cached. It should be a copy of // the config pulled from cloud with minor changes. // This version is kept because the config is changed as it moves through the system. @@ -86,21 +90,22 @@ type MaintenanceConfig struct { // NOTE: This data must be maintained with what is in Config. type configData struct { - Cloud *Cloud `json:"cloud,omitempty"` - Modules []Module `json:"modules,omitempty"` - Remotes []Remote `json:"remotes,omitempty"` - Components []resource.Config `json:"components,omitempty"` - Processes []pexec.ProcessConfig `json:"processes,omitempty"` - Services []resource.Config `json:"services,omitempty"` - Packages []PackageConfig `json:"packages,omitempty"` - Network NetworkConfig `json:"network"` - Auth AuthConfig `json:"auth"` - Debug bool `json:"debug,omitempty"` - DisablePartialStart bool `json:"disable_partial_start"` - EnableWebProfile bool `json:"enable_web_profile"` - LogConfig []logging.LoggerPatternConfig `json:"log,omitempty"` - Revision string `json:"revision,omitempty"` - MaintenanceConfig *MaintenanceConfig `json:"maintenance,omitempty"` + Cloud *Cloud `json:"cloud,omitempty"` + Modules []Module `json:"modules,omitempty"` + Remotes []Remote `json:"remotes,omitempty"` + Components []resource.Config `json:"components,omitempty"` + Processes []pexec.ProcessConfig `json:"processes,omitempty"` + Services []resource.Config `json:"services,omitempty"` + Packages []PackageConfig `json:"packages,omitempty"` + Network NetworkConfig `json:"network"` + Auth AuthConfig `json:"auth"` + Debug bool `json:"debug,omitempty"` + DisablePartialStart bool `json:"disable_partial_start"` + EnableWebProfile bool `json:"enable_web_profile"` + LogConfig []logging.LoggerPatternConfig `json:"log,omitempty"` + Revision string `json:"revision,omitempty"` + MaintenanceConfig *MaintenanceConfig `json:"maintenance,omitempty"` + DisableLogDeduplication bool `json:"disable_log_deduplication"` } // AppValidationStatus refers to the. diff --git a/config/proto_conversions.go b/config/proto_conversions.go index e318523d6fc..d1ecb8e1aac 100644 --- a/config/proto_conversions.go +++ b/config/proto_conversions.go @@ -106,6 +106,10 @@ func FromProto(proto *pb.RobotConfig, logger logging.Logger) (*Config, error) { cfg.MaintenanceConfig = maintenanceConfig } + if proto.DisableLogDeduplication { + cfg.DisableLogDeduplication = true + } + return &cfg, nil } diff --git a/go.mod b/go.mod index ae0d6f3459a..5c837c27a8a 100644 --- a/go.mod +++ b/go.mod @@ -75,7 +75,7 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - go.viam.com/api v0.1.372 + go.viam.com/api v0.1.373 go.viam.com/test v1.2.4 go.viam.com/utils v0.1.118 goji.io v2.0.2+incompatible diff --git a/go.sum b/go.sum index ff9673c7a94..9b91e00af7c 100644 --- a/go.sum +++ b/go.sum @@ -1513,8 +1513,8 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.viam.com/api v0.1.372 h1:Al9P7yojBDdNVAF7nrr5BAbzCvb+vrSp8N7BitbV0mQ= -go.viam.com/api v0.1.372/go.mod h1:g5eipXHNm0rQmW7DWya6avKcmzoypLmxnMlAaIsE5Ls= +go.viam.com/api v0.1.373 h1:1vpxO9thQFeEOJhvGWcjHAHyBH5EKHWKTaJahAo5b/A= +go.viam.com/api v0.1.373/go.mod h1:g5eipXHNm0rQmW7DWya6avKcmzoypLmxnMlAaIsE5Ls= go.viam.com/test v1.2.4 h1:JYgZhsuGAQ8sL9jWkziAXN9VJJiKbjoi9BsO33TW3ug= go.viam.com/test v1.2.4/go.mod h1:zI2xzosHdqXAJ/kFqcN+OIF78kQuTV2nIhGZ8EzvaJI= go.viam.com/utils v0.1.118 h1:Kp6ebrCBiYReeSC1XnWPTjtBJoTUsQ6YWAomQkQF/mE= diff --git a/logging/impl.go b/logging/impl.go index 0de084ea484..14c0cc5f946 100644 --- a/logging/impl.go +++ b/logging/impl.go @@ -36,9 +36,6 @@ type ( // documentation for more details. testHelper func() - // Whether or not to de-duplicate noisy logs. - dedupNoisyLogs bool - // recentMessageMu guards the recentMessage fields below. recentMessageMu sync.Mutex // Map of messages to counts of that message being `Write`ten within window. @@ -131,12 +128,6 @@ func (imp *impl) Sublogger(subname string) Logger { imp.appenders, imp.registry, imp.testHelper, - // Inherit _whether_ we should deduplicate noisy logs from parent. However, - // subloggers should handle their own de-duplication with their own maps - // and windows. This design avoids locking across all loggers and allows - // logs with identical messages from different loggers to be considered - // unique. - imp.dedupNoisyLogs, sync.Mutex{}, make(map[string]int), make(map[string]LogEntry), @@ -255,7 +246,7 @@ func (imp *impl) shouldLog(logLevel Level) bool { } func (imp *impl) Write(entry *LogEntry) { - if imp.dedupNoisyLogs { + if !DisableLogDeduplication.Load() { // If we have entered a new recentMessage window, output noisy logs from // the last window. imp.recentMessageMu.Lock() diff --git a/logging/impl_test.go b/logging/impl_test.go index f098321658c..267b668cb34 100644 --- a/logging/impl_test.go +++ b/logging/impl_test.go @@ -148,11 +148,14 @@ func TestConsoleOutputFormat(t *testing.T) { // A logger object that will write to the `notStdout` buffer. notStdout := &bytes.Buffer{} impl := &impl{ - name: "impl", - level: NewAtomicLevelAt(DEBUG), - appenders: []Appender{NewWriterAppender(notStdout)}, - registry: newRegistry(), - testHelper: func() {}, + name: "impl", + level: NewAtomicLevelAt(DEBUG), + appenders: []Appender{NewWriterAppender(notStdout)}, + registry: newRegistry(), + testHelper: func() {}, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } impl.Info("impl Info log") @@ -213,11 +216,14 @@ func TestContextLogging(t *testing.T) { notStdout := &bytes.Buffer{} // The default log level is error. logger := &impl{ - name: "impl", - level: NewAtomicLevelAt(ERROR), - appenders: []Appender{NewWriterAppender(notStdout)}, - registry: newRegistry(), - testHelper: func() {}, + name: "impl", + level: NewAtomicLevelAt(ERROR), + appenders: []Appender{NewWriterAppender(notStdout)}, + registry: newRegistry(), + testHelper: func() {}, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } logger.CDebug(ctxNoDebug, "Debug log") @@ -288,11 +294,14 @@ func TestSublogger(t *testing.T) { // A logger object that will write to the `notStdout` buffer. notStdout := &bytes.Buffer{} logger := &impl{ - name: "impl", - level: NewAtomicLevelAt(DEBUG), - appenders: []Appender{NewWriterAppender(notStdout)}, - registry: newRegistry(), - testHelper: func() {}, + name: "impl", + level: NewAtomicLevelAt(DEBUG), + appenders: []Appender{NewWriterAppender(notStdout)}, + registry: newRegistry(), + testHelper: func() {}, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } logger.Info("info log") @@ -306,6 +315,13 @@ func TestSublogger(t *testing.T) { } func TestLoggingWithFields(t *testing.T) { + // Disable log deduplication for this test, as it logs "noisily" and makes + // assertions on those logs. + DisableLogDeduplication.Store(true) + defer func() { + DisableLogDeduplication.Store(false) + }() + // A logger object that will write to the `notStdout` buffer. notStdout := &bytes.Buffer{} var logger Logger @@ -488,13 +504,11 @@ func TestLoggingDeduplication(t *testing.T) { // Create a logger object that will write to the `notStdout` buffer. notStdout := &bytes.Buffer{} logger := &impl{ - name: "impl", - level: NewAtomicLevelAt(DEBUG), - appenders: []Appender{NewWriterAppender(notStdout)}, - registry: newRegistry(), - testHelper: func() {}, - dedupNoisyLogs: true, - // Initialize recent message fields as `NewLogger` would. + name: "impl", + level: NewAtomicLevelAt(DEBUG), + appenders: []Appender{NewWriterAppender(notStdout)}, + registry: newRegistry(), + testHelper: func() {}, recentMessageCounts: make(map[string]int), recentMessageEntries: make(map[string]LogEntry), recentMessageWindowStart: time.Now(), diff --git a/logging/logging.go b/logging/logging.go index 309cf8dc7c7..e6b551e815e 100644 --- a/logging/logging.go +++ b/logging/logging.go @@ -2,22 +2,17 @@ package logging import ( - "os" "sync" "testing" "time" + "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest/observer" "go.viam.com/utils" ) -// Environment variable to control whether noisy logs are de-deduplicated. Set -// to "false" to turn off de-duplicating logic; de-duplication logic is enabled -// by default. -const dedupNoisyLogsEnvVar = "VIAM_DEDUP_LOGS" - var ( globalMu sync.RWMutex globalLogger = NewDebugLogger("global") @@ -26,18 +21,11 @@ var ( // flag from the CLI or robot config. GlobalLogLevel = zap.NewAtomicLevelAt(zap.InfoLevel) - // Whether to de-duplicate noisy logs; obtained from value of - // `dedupNoisyLogsEnvVar` and defaults to true. Export env var to "false" to - // turn off de-duplicating logic. - dedupNoisyLogs = true + // Whether to disable de-duplicating noisy logs; defaults to false and can be + // specified in robot config. + DisableLogDeduplication = atomic.Bool{} ) -func init() { - if dedupNoisyLogEnvVal := os.Getenv(dedupNoisyLogsEnvVar); dedupNoisyLogEnvVal == "false" { - dedupNoisyLogs = false - } -} - // ReplaceGlobal replaces the global loggers. func ReplaceGlobal(logger Logger) { globalMu.Lock() @@ -85,7 +73,6 @@ func NewLogger(name string) Logger { appenders: []Appender{NewStdoutAppender()}, registry: newRegistry(), testHelper: func() {}, - dedupNoisyLogs: dedupNoisyLogs, recentMessageCounts: make(map[string]int), recentMessageEntries: make(map[string]LogEntry), recentMessageWindowStart: time.Now(), @@ -105,7 +92,6 @@ func NewLoggerWithRegistry(name string) (Logger, *Registry) { appenders: []Appender{NewStdoutAppender()}, registry: reg, testHelper: func() {}, - dedupNoisyLogs: dedupNoisyLogs, recentMessageCounts: make(map[string]int), recentMessageEntries: make(map[string]LogEntry), recentMessageWindowStart: time.Now(), @@ -123,7 +109,6 @@ func NewDebugLogger(name string) Logger { appenders: []Appender{NewStdoutAppender()}, registry: newRegistry(), testHelper: func() {}, - dedupNoisyLogs: dedupNoisyLogs, recentMessageCounts: make(map[string]int), recentMessageEntries: make(map[string]LogEntry), recentMessageWindowStart: time.Now(), @@ -142,7 +127,6 @@ func NewBlankLogger(name string) Logger { appenders: []Appender{}, registry: newRegistry(), testHelper: func() {}, - dedupNoisyLogs: dedupNoisyLogs, recentMessageCounts: make(map[string]int), recentMessageEntries: make(map[string]LogEntry), recentMessageWindowStart: time.Now(), @@ -168,10 +152,11 @@ func NewObservedTestLogger(tb testing.TB) (Logger, *observer.ObservedLogs) { NewTestAppender(tb), observerCore, }, - registry: newRegistry(), - testHelper: tb.Helper, - // Only prod loggers should de-duplicate noisy logs. - dedupNoisyLogs: false, + registry: newRegistry(), + testHelper: tb.Helper, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } return logger, observedLogs @@ -189,10 +174,11 @@ func NewObservedTestLoggerWithRegistry(tb testing.TB, name string) (Logger, *obs NewTestAppender(tb), observerCore, }, - registry: registry, - testHelper: tb.Helper, - // Only prod loggers should de-duplicate noisy logs. - dedupNoisyLogs: false, + registry: registry, + testHelper: tb.Helper, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } return logger, observedLogs, registry @@ -225,10 +211,11 @@ func NewInMemoryLogger(tb testing.TB) *MemLogger { appenders: []Appender{ observerCore, }, - registry: newRegistry(), - testHelper: tb.Helper, - // Only prod loggers should de-duplicate noisy logs. - dedupNoisyLogs: false, + registry: newRegistry(), + testHelper: tb.Helper, + recentMessageCounts: make(map[string]int), + recentMessageEntries: make(map[string]LogEntry), + recentMessageWindowStart: time.Now(), } memLogger := &MemLogger{logger, tb, observedLogs} diff --git a/robot/impl/local_robot.go b/robot/impl/local_robot.go index 0822cfcdef7..2b8aa7fc75c 100644 --- a/robot/impl/local_robot.go +++ b/robot/impl/local_robot.go @@ -1134,6 +1134,15 @@ func (r *localRobot) reconfigure(ctx context.Context, newConfig *config.Config, var allErrs error + // Check incoming disable log deduplication value for any diff. + if newConfig.DisableLogDeduplication != logging.DisableLogDeduplication.Load() { + state := "enabled" + if newConfig.DisableLogDeduplication { + state = "disabled" + } + r.Logger().CInfof(ctx, "noisy log deduplication now %s", state) + } + // Sync Packages before reconfiguring rest of robot and resolving references to any packages // in the config. // TODO(RSDK-1849): Make this non-blocking so other resources that do not require packages can run before package sync finishes. From f8ecdca0b8c57b91c1fa5ba9832d3d69850cce0a Mon Sep 17 00:00:00 2001 From: Benji Rewis Date: Mon, 23 Dec 2024 16:10:06 -0500 Subject: [PATCH 12/12] lint --- logging/logging.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logging/logging.go b/logging/logging.go index e6b551e815e..424dfdf89f5 100644 --- a/logging/logging.go +++ b/logging/logging.go @@ -21,8 +21,8 @@ var ( // flag from the CLI or robot config. GlobalLogLevel = zap.NewAtomicLevelAt(zap.InfoLevel) - // Whether to disable de-duplicating noisy logs; defaults to false and can be - // specified in robot config. + // DisableLogDeduplication controls whether to disable de-duplicating noisy + // logs; defaults to false and can be specified in robot config. DisableLogDeduplication = atomic.Bool{} )