Increased memory consumption when serializing long slices.
RekGRpth opened this issue · comments
When serializing long slices, increased memory consumption is possible, mine exceeds 4 gigabytes! Steps to reproduce:
save this
package main
import (
"fmt"
"os"
"gopkg.in/yaml.v2"
)
type TOC struct {
StatisticsEntries []MetadataEntry
}
type MetadataEntry struct {
Name string
Value string
Other string
}
func panicOnErr(err error) {
if err != nil {
panic(err)
}
}
func main() {
toc := TOC{}
size := 1000000
for i := 0; i < size; i++ {
name := fmt.Sprintf("Name %v", i)
value := fmt.Sprintf("Value %v", i)
toc.StatisticsEntries = append(toc.StatisticsEntries, MetadataEntry{Name: name, Value: value})
}
fmt.Println("press enter 1")
var s string
fmt.Scanf("%s", &s)
file, _ := os.OpenFile("test.yaml", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
defer file.Close()
enc := yaml.NewEncoder(file)
defer enc.Close()
err := enc.Encode(toc)
panicOnErr(err)
fmt.Println("press enter 2")
fmt.Scanf("%s", &s)
}to file test.go and run it
go run test.go
press enter 1
press enter 2
bash-4.2$ GODEBUG=gctrace=1 go run my.go
gc 1 @0.003s 1%: 0.014+0.21+0.010 ms clock, 0.22+0.058/0.48/0.019+0.17 ms cpu, 3->3->0 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 2 @0.007s 2%: 0.10+0.41+0.010 ms clock, 1.6+0.17/0.81/0.13+0.17 ms cpu, 3->4->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 3 @0.010s 8%: 0.62+0.76+0.017 ms clock, 9.9+0.17/1.3/0.024+0.27 ms cpu, 3->4->2 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 4 @0.012s 8%: 0.056+0.37+0.025 ms clock, 0.90+0.11/0.70/0.55+0.41 ms cpu, 3->4->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 5 @0.012s 8%: 0.012+0.30+0.011 ms clock, 0.20+0.077/0.67/0.12+0.18 ms cpu, 3->3->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 6 @0.013s 8%: 0.018+0.26+0.011 ms clock, 0.29+0.043/0.58/0.22+0.17 ms cpu, 3->3->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 7 @0.016s 7%: 0.009+0.25+0.008 ms clock, 0.15+0.062/0.59/0.54+0.14 ms cpu, 3->3->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 8 @0.018s 7%: 0.032+0.24+0.012 ms clock, 0.52+0.27/0.56/0.021+0.20 ms cpu, 3->4->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 9 @0.019s 7%: 0.034+0.23+0.011 ms clock, 0.55+0/0.70/0.24+0.17 ms cpu, 3->3->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 10 @0.020s 7%: 0.011+0.23+0.010 ms clock, 0.18+0.062/0.68/0.19+0.16 ms cpu, 3->3->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 11 @0.021s 7%: 0.008+0.22+0.009 ms clock, 0.13+0.061/0.55/0.72+0.14 ms cpu, 3->3->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
# command-line-arguments
gc 1 @0.000s 4%: 0.008+0.19+0.004 ms clock, 0.12+0.14/0.43/0.012+0.075 ms cpu, 4->4->3 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 2 @0.001s 4%: 0.007+0.28+0.005 ms clock, 0.12+0.034/0.53/0.38+0.092 ms cpu, 7->7->6 MB, 7 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 3 @0.012s 1%: 0.014+0.92+0.005 ms clock, 0.23+0.60/0.43/0.057+0.083 ms cpu, 14->14->10 MB, 14 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 1 @0.003s 1%: 0.009+0.35+0.008 ms clock, 0.15+0.088/0.42/0+0.12 ms cpu, 3->3->1 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 2 @0.005s 2%: 0.008+1.0+0.008 ms clock, 0.13+0/1.2/0+0.12 ms cpu, 4->4->2 MB, 4 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 3 @0.008s 2%: 0.009+1.5+0.009 ms clock, 0.15+0/1.8/0.30+0.14 ms cpu, 6->6->3 MB, 6 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 4 @0.014s 2%: 0.008+1.1+0.005 ms clock, 0.12+0.31/1.0/0.013+0.082 ms cpu, 6->6->4 MB, 6 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 5 @0.018s 2%: 0.009+3.1+0.009 ms clock, 0.14+0.005/3.8/0.29+0.14 ms cpu, 12->12->6 MB, 12 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 6 @0.025s 2%: 0.007+0.72+0.009 ms clock, 0.12+0.086/2.0/0.57+0.14 ms cpu, 12->13->8 MB, 13 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 7 @0.032s 3%: 0.009+4.5+0.015 ms clock, 0.14+0.17/5.5/0.70+0.25 ms cpu, 24->24->12 MB, 24 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 8 @0.045s 3%: 0.013+3.6+0.009 ms clock, 0.21+0.039/4.9/0+0.14 ms cpu, 24->25->16 MB, 25 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 9 @0.061s 2%: 0.010+2.5+0.005 ms clock, 0.17+0.13/5.4/1.1+0.090 ms cpu, 32->32->21 MB, 33 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 10 @0.075s 3%: 0.013+10+0.006 ms clock, 0.21+0.90/14/0.053+0.099 ms cpu, 59->59->30 MB, 59 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 11 @0.106s 2%: 0.016+6.1+0.010 ms clock, 0.25+0.091/5.2/5.1+0.17 ms cpu, 59->60->39 MB, 61 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 12 @0.137s 2%: 0.016+2.7+0.009 ms clock, 0.26+0.10/9.2/6.4+0.15 ms cpu, 76->76->49 MB, 78 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 13 @0.176s 2%: 0.020+3.3+0.011 ms clock, 0.33+0.16/11/8.2+0.18 ms cpu, 97->97->63 MB, 99 MB goal, 0 MB stacks, 0 MB globals, 16 P
press enter 1
gc 14 @5.300s 0%: 0.016+11+0.008 ms clock, 0.26+9.9/12/0.004+0.13 ms cpu, 125->127->82 MB, 127 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 15 @5.354s 0%: 0.020+11+0.010 ms clock, 0.32+2.5/30/19+0.16 ms cpu, 180->180->137 MB, 180 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 16 @5.432s 0%: 0.019+15+0.005 ms clock, 0.31+2.5/30/38+0.083 ms cpu, 292->292->170 MB, 292 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 17 @5.560s 0%: 0.057+25+0.006 ms clock, 0.92+2.0/45/61+0.10 ms cpu, 422->422->258 MB, 422 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 18 @5.703s 0%: 0.019+48+0.006 ms clock, 0.30+3.5/72/87+0.098 ms cpu, 549->549->360 MB, 549 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 19 @5.936s 0%: 0.030+58+0.007 ms clock, 0.48+7.5/97/136+0.12 ms cpu, 814->814->518 MB, 814 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 20 @6.301s 0%: 0.027+90+0.009 ms clock, 0.44+14/142/172+0.15 ms cpu, 1228->1228->766 MB, 1228 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 21 @6.840s 0%: 0.019+138+0.007 ms clock, 0.31+23/218/286+0.11 ms cpu, 1874->1874->1153 MB, 1874 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 22 @7.698s 0%: 0.020+216+0.006 ms clock, 0.33+33/340/447+0.099 ms cpu, 2885->2885->1757 MB, 2885 MB goal, 0 MB stacks, 0 MB globals, 16 P
gc 23 @9.234s 1%: 0.021+833+0.008 ms clock, 0.34+71/1082/913+0.12 ms cpu, 4464->4464->2702 MB, 4464 MB goal, 0 MB stacks, 0 MB globals, 16 P
press enter 2