lock on global rand causes latency spike
cch123 opened this issue · comments
Xargin commented
// Update samples a new value.
func (s *UniformSample) Update(v int64) {
s.mutex.Lock()
defer s.mutex.Unlock()
s.count++
if len(s.values) < s.reservoirSize {
s.values = append(s.values, v)
} else {
r := rand.Int63n(s.count). // -------------------> here
if r < int64(len(s.values)) {
s.values[int(r)] = v
}
}
}
When latency increase, the goroutine stack:
10760 @ 0x42f81f 0x4401d9 0x4401af 0x43ff4d 0x474df9 0x95709a 0x952d0d 0x17466fc 0x17462c4 0x174c8a8 0x174c88a 0x174e308 0x1755b78 0x1749432 0x17588e8 0xf90a54 0xc9670d 0x45d061
# 0x43ff4c sync.runtime_SemacquireMutex+0x3c /Users/xargin/sdk/go1.12.17/src/runtime/sema.go:71
# 0x474df8 sync.(*Mutex).Lock+0x108 /Users/xargin/sdk/go1.12.17/src/sync/mutex.go:134
# 0x957099 github.com/rcrowley/go-metrics.(*UniformSample).Update+0x39 /Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/sample.go:508
# 0x952d0c github.com/rcrowley/go-metrics.(*StandardHistogram).Update+0x3c /Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/histogram.go:199
# 0x17466fb mosn.io/mosn/pkg/proxy.(*downStream).requestMetrics+0x2db /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:257
# 0x17462c3 mosn.io/mosn/pkg/proxy.(*downStream).cleanStream+0x153 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:218
# 0x174c8a7 mosn.io/mosn/pkg/proxy.(*downStream).endStream+0x137 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:179
# 0x174c889 mosn.io/mosn/pkg/proxy.(*downStream).appendData+0x119 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1001
# 0x174e307 mosn.io/mosn/pkg/proxy.(*downStream).onUpstreamData+0x37 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1155
# 0x1755b77 mosn.io/mosn/pkg/proxy.(*upstreamRequest).receiveData+0x67 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/upstream.go:138
# 0x1749431 mosn.io/mosn/pkg/proxy.(*downStream).receive+0x22a1 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:586
# 0x17588e7 mosn.io/mosn/pkg/proxy.(*downStream).OnReceive.func1+0xd7 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:366
# 0xf90a53 mosn.io/mosn/pkg/sync.(*workerPool).ScheduleAuto.func1+0x23 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/sync/workerpool.go:156
# 0xc9670c mosn.io/pkg/utils.GoWithRecover.func1+0x4c /Users/xargin/mosn/vendor/mosn.io/pkg/utils/goroutine.go:51
1 @ 0x42f81f 0x4401d9 0x4401af 0x43ff4d 0x474df9 0x504a3d 0x5040ba 0x5040ac 0x957187 0x957172 0x952d0d 0x17465d0 0x17462c4 0x174c8a8 0x174c88a 0x174e308 0x1755b78 0x1749432 0x17588e8 0xf90a54 0xc9670d 0x45d061
# 0x43ff4c sync.runtime_SemacquireMutex+0x3c /Users/xargin/sdk/go1.12.17/src/runtime/sema.go:71
# 0x474df8 sync.(*Mutex).Lock+0x108 /Users/xargin/sdk/go1.12.17/src/sync/mutex.go:134
# 0x504a3c math/rand.(*lockedSource).Int63+0x2c /Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:380
# 0x5040b9 math/rand.(*Rand).Int63+0x69 /Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:85
# 0x5040ab math/rand.(*Rand).Int63n+0x5b /Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:117
# 0x957186 math/rand.Int63n+0x126 /Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:319
# 0x957171 github.com/rcrowley/go-metrics.(*UniformSample).Update+0x111 /Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/sample.go:514
# 0x952d0c github.com/rcrowley/go-metrics.(*StandardHistogram).Update+0x3c /Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/histogram.go:199
# 0x17465cf mosn.io/mosn/pkg/proxy.(*downStream).requestMetrics+0x1af /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:248
# 0x17462c3 mosn.io/mosn/pkg/proxy.(*downStream).cleanStream+0x153 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:218
# 0x174c8a7 mosn.io/mosn/pkg/proxy.(*downStream).endStream+0x137 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:179
# 0x174c889 mosn.io/mosn/pkg/proxy.(*downStream).appendData+0x119 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1001
# 0x174e307 mosn.io/mosn/pkg/proxy.(*downStream).onUpstreamData+0x37 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1155
# 0x1755b77 mosn.io/mosn/pkg/proxy.(*upstreamRequest).receiveData+0x67 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/upstream.go:138
# 0x1749431 mosn.io/mosn/pkg/proxy.(*downStream).receive+0x22a1 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:586
# 0x17588e7 mosn.io/mosn/pkg/proxy.(*downStream).OnReceive.func1+0xd7 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:366
# 0xf90a53 mosn.io/mosn/pkg/sync.(*workerPool).ScheduleAuto.func1+0x23 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/sync/workerpool.go:156
# 0xc9670c mosn.io/pkg/utils.GoWithRecover.func1+0x4c /Users/xargin/mosn/vendor/mosn.io/pkg/utils/goroutine.go:51