Why is ristretto using so much memory?
caofengl opened this issue · comments
package main
import (
"fmt"
"math/rand"
"strconv"
"github.com/dgraph-io/ristretto"
)
const (
maxCnt = 1e7
maxLoop = 1e8
)
func main() {
cahchePool, err := ristretto.NewCache(&ristretto.Config{
NumCounters: 1 * 1e7,
MaxCost: 200 * (1 << 20), // allocate 200M, but running it need 2GB, why?(when i run this program, it kill by OOM)
BufferItems: 64,
Metrics: true,
})
if err != nil {
panic(err)
}
key := "def48b5abb6388d3cbbb18e550f47b4cYB6eRI3cK1VN2qCEHp8kvuMuH20dq10cYDcG2e"
for i := 0; i < maxLoop; i++ {
suffix := strconv.Itoa(rand.Intn(maxCnt))
cahchePool.Set(key+suffix, suffix, int64(len(key+suffix)))
if (i % maxCnt) == 0 {
fmt.Println(cahchePool.Metrics)
}
}
cnt := 0
for i := 0; i < maxCnt; i++ {
if _, found := cahchePool.Get(key + strconv.Itoa(i)); found {
cnt++
}
}
fmt.Println("cnt:", cnt, "\n", cahchePool.Metrics)
}I used ristretto in our company's project, and I really need help!
Can the author help give the correct code example(use 200M memory)? My English is very poor, thank you very much!
package main
import (
"fmt"
"math/rand"
"strconv"
"sync"
"time"
"github.com/dgraph-io/ristretto"
)
const (
maxLoop = 1e8
maxCnt = 1e7
)
func main() {
cost := int64(100 * 1024 * 1024)
cache, err := ristretto.NewCache(&ristretto.Config{
NumCounters: cost / 8,
MaxCost: cost / 88, // key+value size
BufferItems: 64,
})
if err != nil {
panic(err)
}
key := `def48b5abb6388d3cbbb18e550f47b4cYB6eRI3cK1VN2qCEHp8kvuMuH20dq10cYDcG2e
var wg sync.WaitGroup
for i := 0; i < maxLoop/maxCnt; i++ {
wg.Add(1)
go func() {
now := time.Now()
for i := 0; i < maxCnt; i++ {
val := strconv.Itoa(rand.Intn(maxCnt))
cache.Set(key+val, val, 1)
}
fmt.Println(time.Since(now).Seconds())
wg.Done()
}()
}
wg.Wait()
cnt := 0
now := time.Now()
for i := 0; i < maxCnt; i++ {
_, found := cache.Get(key + strconv.Itoa(i))
if found {
cnt++
}
}
fmt.Println(time.Since(now).Seconds())
fmt.Println(cnt)
time.Sleep(10 * 60 * time.Second)
}
write
196.145246674
196.594516091
198.114088657
198.93801043
201.966241441
202.236675922
204.914710305
204.955456302
205.115802574
205.308419901
read
24.981067788
count
1191525package main
import (
"fmt"
"math/rand"
"strconv"
"sync"
"time"
"github.com/coocood/freecache"
)
const (
maxLoop = 1e8
maxCnt = 1e7
)
func main() {
cacheSize := 100 * 1024 * 1024 // 1GB
cache := freecache.NewCache(cacheSize)
expire := 60 * 60 // expire in 60 seconds
key := `def48b5abb6388d3cbbb18e550f47b4cYB6eRI3cK1VN2qCEHp8kvuMuH20dq10cYDcG2e
var wg sync.WaitGroup
for i := 0; i < maxLoop/maxCnt; i++ {
wg.Add(1)
go func() {
now := time.Now()
for i := 0; i < maxCnt; i++ {
val := strconv.Itoa(rand.Intn(maxCnt))
cache.Set([]byte(key+val), []byte(val), expire)
}
fmt.Println(time.Since(now).Seconds())
wg.Done()
}()
}
wg.Wait()
cnt := 0
now := time.Now()
for i := 0; i < maxCnt; i++ {
_, err := cache.Get([]byte(key + strconv.Itoa(i)))
if err == nil {
cnt++
}
}
fmt.Println(time.Since(now).Seconds())
fmt.Println("entry count ", cnt, cache.EntryCount())
time.Sleep(10 * 60 * time.Second)
}
write
143.327503682
144.094319665
144.148188694
145.014152054
145.317289531
146.429781022
146.803659841
146.842320393
146.899385914
146.907347694
read
9.807717003
count
entry count 868063 868063memory contrash
freecache: 300M // keep 300M
ristretto: 1.7G // up to 1.7G, time.Sleep(10 * 60 * time.Second) -> 660M
Hi.
Sorry for not answering your question more promptly. We have been at work trying to release Dgraph 2.0.0.
I am not really sure why the differences in memory exist but I suspect you might have run into the bug described in #107.
The key in your examples are very big so if they escape to the heap they will use a lot more memory. For now, I suggest using smaller keys (uint64 for example).
Re-open the issue to test github-issue porting task.
