lock on global rand causes latency spike

[cch123]

// Update samples a new value.
func (s *UniformSample) Update(v int64) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	s.count++
	if len(s.values) < s.reservoirSize {
		s.values = append(s.values, v)
	} else {
		r := rand.Int63n(s.count).  // -------------------> here
		if r < int64(len(s.values)) {
			s.values[int(r)] = v
		}
	}
}

When latency increase, the goroutine stack:

10760 @ 0x42f81f 0x4401d9 0x4401af 0x43ff4d 0x474df9 0x95709a 0x952d0d 0x17466fc 0x17462c4 0x174c8a8 0x174c88a 0x174e308 0x1755b78 0x1749432 0x17588e8 0xf90a54 0xc9670d 0x45d061
#	0x43ff4c	sync.runtime_SemacquireMutex+0x3c				/Users/xargin/sdk/go1.12.17/src/runtime/sema.go:71
#	0x474df8	sync.(*Mutex).Lock+0x108					/Users/xargin/sdk/go1.12.17/src/sync/mutex.go:134
#	0x957099	github.com/rcrowley/go-metrics.(*UniformSample).Update+0x39	/Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/sample.go:508
#	0x952d0c	github.com/rcrowley/go-metrics.(*StandardHistogram).Update+0x3c	/Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/histogram.go:199
#	0x17466fb	mosn.io/mosn/pkg/proxy.(*downStream).requestMetrics+0x2db	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:257
#	0x17462c3	mosn.io/mosn/pkg/proxy.(*downStream).cleanStream+0x153		/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:218
#	0x174c8a7	mosn.io/mosn/pkg/proxy.(*downStream).endStream+0x137		/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:179
#	0x174c889	mosn.io/mosn/pkg/proxy.(*downStream).appendData+0x119		/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1001
#	0x174e307	mosn.io/mosn/pkg/proxy.(*downStream).onUpstreamData+0x37	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1155
#	0x1755b77	mosn.io/mosn/pkg/proxy.(*upstreamRequest).receiveData+0x67	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/upstream.go:138
#	0x1749431	mosn.io/mosn/pkg/proxy.(*downStream).receive+0x22a1		/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:586
#	0x17588e7	mosn.io/mosn/pkg/proxy.(*downStream).OnReceive.func1+0xd7	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:366
#	0xf90a53	mosn.io/mosn/pkg/sync.(*workerPool).ScheduleAuto.func1+0x23	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/sync/workerpool.go:156
#	0xc9670c	mosn.io/pkg/utils.GoWithRecover.func1+0x4c			/Users/xargin/mosn/vendor/mosn.io/pkg/utils/goroutine.go:51

1 @ 0x42f81f 0x4401d9 0x4401af 0x43ff4d 0x474df9 0x504a3d 0x5040ba 0x5040ac 0x957187 0x957172 0x952d0d 0x17465d0 0x17462c4 0x174c8a8 0x174c88a 0x174e308 0x1755b78 0x1749432 0x17588e8 0xf90a54 0xc9670d 0x45d061
#	0x43ff4c	sync.runtime_SemacquireMutex+0x3c				/Users/xargin/sdk/go1.12.17/src/runtime/sema.go:71
#	0x474df8	sync.(*Mutex).Lock+0x108					/Users/xargin/sdk/go1.12.17/src/sync/mutex.go:134
#	0x504a3c	math/rand.(*lockedSource).Int63+0x2c				/Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:380
#	0x5040b9	math/rand.(*Rand).Int63+0x69					/Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:85
#	0x5040ab	math/rand.(*Rand).Int63n+0x5b					/Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:117
#	0x957186	math/rand.Int63n+0x126						/Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:319
#	0x957171	github.com/rcrowley/go-metrics.(*UniformSample).Update+0x111	/Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/sample.go:514
#	0x952d0c	github.com/rcrowley/go-metrics.(*StandardHistogram).Update+0x3c	/Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/histogram.go:199
#	0x17465cf	mosn.io/mosn/pkg/proxy.(*downStream).requestMetrics+0x1af	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:248
#	0x17462c3	mosn.io/mosn/pkg/proxy.(*downStream).cleanStream+0x153		/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:218
#	0x174c8a7	mosn.io/mosn/pkg/proxy.(*downStream).endStream+0x137		/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:179
#	0x174c889	mosn.io/mosn/pkg/proxy.(*downStream).appendData+0x119		/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1001
#	0x174e307	mosn.io/mosn/pkg/proxy.(*downStream).onUpstreamData+0x37	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1155
#	0x1755b77	mosn.io/mosn/pkg/proxy.(*upstreamRequest).receiveData+0x67	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/upstream.go:138
#	0x1749431	mosn.io/mosn/pkg/proxy.(*downStream).receive+0x22a1		/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:586
#	0x17588e7	mosn.io/mosn/pkg/proxy.(*downStream).OnReceive.func1+0xd7	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:366
#	0xf90a53	mosn.io/mosn/pkg/sync.(*workerPool).ScheduleAuto.func1+0x23	/Users/xargin/mosn/vendor/mosn.io/mosn/pkg/sync/workerpool.go:156
#	0xc9670c	mosn.io/pkg/utils.GoWithRecover.func1+0x4c			/Users/xargin/mosn/vendor/mosn.io/pkg/utils/goroutine.go:51