github.com/cockroachdb/pebble/internal/metamorphic: TestMeta failed
cockroach-teamcity opened this issue · comments
github.com/cockroachdb/pebble/internal/metamorphic.TestMeta failed with artifacts on refs/heads/master @ 3e083df5434f:
db1.Set("tnxymid@37", "hq") // <nil> #3595
db1.Get("mgzpk") // [""] pebble: not found #3596
db1.RangeKeyUnset("utuwjgy", "xbphspppcq", "@36") // <nil> #3597
db1.Set("ajpcc@36", "jnqyxbtypm") // <nil> #3598
db1.Get("rijkvssyas@8") // [""] pebble: not found #3599
db1.Set("ztjifkbh@29", "") // <nil> #3600
batch31 = db1.NewIndexedBatch() #3601
db1.Delete("zzsirw@2") // <nil> #3604
// INFO: [JOB 186] WAL deleted 000140
// INFO: [JOB 187] sstable deleted 000142
// INFO: [JOB 187] sstable deleted 000144
db1.Set("hsoomucv@20", "vtlxv") // <nil> #3605
batch31.Set("rcafolevxi@38", "atqzzgkgzeukbnqke") // <nil> #3602
batch31.RangeKeyDelete("fixu", "maayqhtuien") // <nil> #3603
batch31.LogData("brlrwwqstunullzcjkc") // <nil> #3608
snap32.Get("ajpcc@16") // ["cghlploj"] <nil> #3606
db1.Get("maayqhtuien") // [""] pebble: not found #3607
batch31.Get("cdztxmjddd") // [""] pebble: not found #3609
batch31.Get("rpnjihjiy@27") // ["k"] <nil> #3610
batch31.Set("zzsirw", "hxxgxctyyglflq") // <nil> #3613
batch32 = db1.NewBatch() #3611
batch32.SingleDelete("xbphspppcq@28", true /* maybeReplaceDelete */) // <nil> #3612
// INFO: [JOB 189] WAL created 000146
batch31.Commit() // <nil> #3614
batch31.Close() // <nil> #3615
// INFO: [JOB 190] ingesting: sstable created 000147
// INFO: [JOB 190] ingested L0:000147 (833B)
db1.Ingest(batch32) // <nil> #3616
db1.Set("rijkvssyas@15", "iwbvezamtunhb") // <nil> #3617
db1.Set("uqrh@38", "ecevbnjmlosfeakkjy") // <nil> #3618
db1.Get("gpkkakkz@26") // [""] pebble: not found #3619
snap32.Get("tplpde@10") // ["iznpkcqzitxnnd"] <nil> #3620
db1.Set("ztjifkbh@31", "unsocr") // <nil> #3621
db1.Set("uqrh@38", "tkuiol") // <nil> #3622
db1.RatchetFormatMajorVersion(016) // <nil> #3623
snap32.Get("wecglzntjdj@13") // [""] pebble: not found #3624
db1.Set("gpkkakkz@38", "ebhucsohmo") // <nil> #3625
db1.Set("utuwjgy@31", "wjvheuqtqydauk") // <nil> #3626
snap33 = db1.NewSnapshot("bznmsqqfp", "maayqhtuien", "pbfwihf", "tnxymid") #3627
snap33.Get("ioxje@24") // [""] <nil> #3628
db1.Set("tnxymid@15", "d") // <nil> #3629
snap32.Get("wecglzntjdj") // ["wj"] <nil> #3630
db1.Set("edagrxybvrz@25", "qsbgpyjykvtmf") // <nil> #3631
snap32.Get("ajpcc@15") // ["jjx"] <nil> #3632
db1.Get("ajpcc") // ["n"] <nil> #3633
// INFO: [JOB 191] flushing 2 memtables (604B) to L0
// INFO: [JOB 191] flushing: sstable created 000148
To reduce: go test ./internal/metamorphic -tags invariants -run 'TestMeta$' --run-dir /artifacts/meta/240209-051646.4012513623071/random-025 --try-to-reduce -v
--- FAIL: TestMeta/execution/random-025 (0.40s)
Help
To reproduce, try:
go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run 'TestMeta$' ./internal/metamorphic -seed 1707455806401473334 -ops "uniform:5000-10000"
=== NAME TestMeta/execution/random-025
meta.go:240:
===== SEED =====
1707455806401473334
===== ERR =====
exit status 2
===== OUT =====
panic: incremental L0 sublevel generation produced different output than regeneration: 0.1:
000147:[xbphspppcq@28#454,DEL-xbphspppcq@28#454,DEL]
0.0:
000148:[ajpcc@36#447,SET-zzsirw@2#449,DELSIZED]
!= 0.1:
000148:[ajpcc@36#447,SET-zzsirw@2#449,DELSIZED]
0.0:
000147:[xbphspppcq@28#454,DEL-xbphspppcq@28#454,DEL]
goroutine 503 [running]:
github.com/cockroachdb/pebble/internal/manifest.(*BulkVersionEdit).Apply(0xc000535100, 0xc00086e000, 0x0?, 0x0?, 0x0?, 0x3e80, 0xc0004c7bf0)
github.com/cockroachdb/pebble/internal/manifest/external/com_github_cockroachdb_pebble/internal/manifest/version_edit.go:1103 +0x19b9
github.com/cockroachdb/pebble/internal/manifest.AccumulateIncompleteAndApplySingleVE(0xc000023280, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0xc000535258, 0xc000535248)
github.com/cockroachdb/pebble/internal/manifest/external/com_github_cockroachdb_pebble/internal/manifest/version_edit.go:890 +0xe5
github.com/cockroachdb/pebble.(*versionSet).logAndApply.func1(0xc0001dac80, 0xc000023280, 0xc000535418, 0xc0005353f8, 0xc00007f380?, 0x0, 0x4f0a89?, 0xc00007f380?, 0xbf)
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/version_set.go:514 +0x1d9
github.com/cockroachdb/pebble.(*versionSet).logAndApply(0xc0001dac80, 0xc0004c7980?, 0xc000023280, 0xc1698b30131811bd?, 0x0, 0xc000535970)
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/version_set.go:566 +0x36e
github.com/cockroachdb/pebble.(*DB).flush1(0xc0000f0000)
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2000 +0x131f
github.com/cockroachdb/pebble.(*DB).flush.func1({0xe99830, 0xc0004b6ba0})
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1717 +0xee
runtime/pprof.Do({0xe99788?, 0x13f5a20?}, {{0xc00007c380?, 0xc0000f0000?, 0xc0002ecfd0?}}, 0xc0002ecfa8)
GOROOT/src/runtime/pprof/runtime.go:51 +0x9d
github.com/cockroachdb/pebble.(*DB).flush(0xc0002ecf98?)
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1710 +0x52
created by github.com/cockroachdb/pebble.(*DB).maybeScheduleFlush in goroutine 45
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1626 +0xdb
===== OPTIONS =====
[Version]
pebble_version=0.1
[Options]
bytes_per_sync=131072
cache_size=16
cleaner=archive
compaction_debt_concurrency=1073741824
comparer=pebble.internal.testkeys
disable_wal=false
flush_delay_delete_range=0s
flush_delay_range_key=240ms
flush_split_bytes=32768
format_major_version=17
l0_compaction_concurrency=3
l0_compaction_file_threshold=1
l0_compaction_threshold=83
l0_stop_writes_threshold=83
lbase_max_bytes=134217728
level_multiplier=320
max_concurrent_compactions=1
max_manifest_file_size=134217728
max_open_files=1000
mem_table_size=2048
mem_table_stop_writes_threshold=3
min_deletion_rate=67108864
merger=pebble.concatenate
multilevel_compaction_heuristic=wamp(0.00, false)
read_compaction_rate=16000
read_sampling_multiplier=16
strict_wal_tail=true
table_cache_shards=8
validate_on_ingest=false
wal_dir=
wal_bytes_per_sync=0
max_writer_concurrency=2
force_writer_parallelism=true
secondary_cache_size_bytes=0
create_on_shared=0
disable_delete_only_compactions=true
[Level "0"]
block_restart_interval=59
block_size=8192
block_size_threshold=73
compression=ZSTD
filter_policy=none
filter_type=table
index_block_size=8388608
target_file_size=4096
[TestOptions]
delete_sized=true
replace_single_delete=true
threads=7
enable_value_blocks=true
seed_efos=1275343413250672924
With verbosity:
panic: incremental L0 sublevel generation produced different output than regeneration: 0.1:
000156:[xbphspppcq@28#454,DEL-xbphspppcq@28#454,DEL] seqnums:[454-454] points:[xbphspppcq@28#454,DEL-xbphspppcq@28#454,DEL]
0.0:
000155:[ajpcc@36#447,SET-zzsirw@2#449,DELSIZED] seqnums:[443-454] points:[ajpcc@36#447,SET-zzsirw@2#449,DELSIZED] ranges:[fixu#452,RANGEKEYDEL-xbphspppcq#inf,RANGEKEYUNSET]
!= 0.1:
000155:[ajpcc@36#447,SET-zzsirw@2#449,DELSIZED] seqnums:[443-454] points:[ajpcc@36#447,SET-zzsirw@2#449,DELSIZED] ranges:[fixu#452,RANGEKEYDEL-xbphspppcq#inf,RANGEKEYUNSET]
0.0:
000156:[xbphspppcq@28#454,DEL-xbphspppcq@28#454,DEL] seqnums:[454-454] points:[xbphspppcq@28#454,DEL-xbphspppcq@28#454,DEL]
The 000156 [#454-#454] sstable was created here through an ingestion. The 000155 [#443-#454] sstable was the result of a memtable flush forced by the ingestion.
batch32 = db1.NewBatch() #3611
batch32.SingleDelete("xbphspppcq@28", true /* maybeReplaceDelete */) // <nil> #3612
// INFO: [JOB 207] flushing 2 memtables (557B) to L0
// INFO: [JOB 207] flushing: sstable created 000155
// INFO: [JOB 208] ingesting: sstable created 000156
// INFO: [JOB 208] ingested L0:000156 (833B)
db1.Ingest(batch32) // <nil> #3616
How is it possible for something in the memtable to have the exact same seqnum as an ingestion? That seems like the root of the issue here; the flush probably should have had a 443-453 seqnum range and the ingest 454-454.
Yes, there's definitely a seqnum inversion from equal largest seqnums.
This appears to be a consequence of the existence of the LogData operation. There's no failure if I comment out the contents of Batch.LogData to be a noop. I'm suspicious there's a lingering issue with the calculation of b.count and the numbers of sequence numbers allocated, allowing the seqnum counter to repeat itself.
github.com/cockroachdb/pebble/internal/metamorphic.TestMeta failed with artifacts on refs/heads/master @ 3e083df5434f:
// INFO: [JOB 379] WAL created 000266
batch25.Commit() // <nil> #3746
batch25.Close() // <nil> #3747
db1.SingleDelete("lqhqhg@10", false /* maybeReplaceDelete */) // <nil> #3749
db1.Get("iazfrjppeal") // [""] pebble: not found #3752
db1.Set("qaxmnjnucbq@10", "qq") // <nil> #3754
db1.Delete("xnqnrlcg@6") // <nil> #3755
snap25.Get("cxjhk@6") // [""] pebble: not found #3757
db1.Merge("zlsydpeqmxu@11", "occoo") // <nil> #3759
db1.Merge("jljadki@11", "ydzhocaecbcjkm") // <nil> #3762
db1.DeleteRange("ikkb@12", "ndpillsaf@8") // <nil> #3772
db1.SingleDelete("lmaptvzka@2", false /* maybeReplaceDelete */) // <nil> #3775
snap25.Get("cacnhn@13") // [""] pebble: not found #3776
db1.Delete("liyzzz@12") // <nil> #3777
db1.Merge("iyuo@10", "intgmhswhu") // <nil> #3778
db1.Delete("umhr@12") // <nil> #3780
db1.DeleteRange("kfpzpgm@1", "ygbe@1") // <nil> #3786
db1.SingleDelete("usiriixhrdn@2", false /* maybeReplaceDelete */) // <nil> #3791
// INFO: write stall beginning: memtable count limit reached
// INFO: [JOB 378] WAL deleted 000259
// INFO: [JOB 381] compacting(default) L0 [000265 000261] (2.3KB) Score=100.00 + L6 [000264] (5.8KB) Score=0.00; OverlappingRatio: Single 2.46, Multi 0.00
// INFO: [JOB 381] compacting: sstable created 000267
// INFO: [JOB 381] compacted(default) L0 [000265 000261] (2.3KB) Score=100.00 + L6 [000264] (5.8KB) Score=0.00 -> L6 [000267] (5.9KB), in 0.0s (0.0s total), output rate 9.0MB/s
// FATAL: checker failed with error: tombstone lkkcbbuj@13-usiriixhrdn@2:{(#1141,RANGEDEL)} in flushable-batch deletes key lqhqhg@10#1140,SINGLEDEL in memtable
// INFO: [JOB 382] flushing 2 memtables (120B) to L0
// INFO: [JOB 382] flushing: sstable created 000268
// FATAL: checker failed with error: tombstone lkkcbbuj@13-usiriixhrdn@2:{(#1141,RANGEDEL)} in L0.0: fileNum=000268 deletes key lqhqhg@10#1140,SINGLEDEL in memtable
// INFO: [JOB 382] flushed 2 memtables (120B) to L0 [000268] (1.1KB), in 0.0s (0.0s total), output rate 6.9MB/s
// INFO: [JOB 383] WAL created 000269
// INFO: write stall ending
db1.SingleDelete("kprzsd@13", false /* maybeReplaceDelete */) // <nil> #3792
// INFO: [JOB 381] sstable deleted 000261
// INFO: [JOB 381] sstable deleted 000264
// INFO: [JOB 381] sstable deleted 000265
// INFO: [JOB 382] WAL deleted 000262
// INFO: [JOB 385] flushing 1 memtable (684B) to L0
// INFO: [JOB 385] flushing: sstable created 000270
// INFO: [JOB 385] flushed 1 memtable (684B) to L0 [000270] (1.1KB), in 0.0s (0.0s total), output rate 7.1MB/s
// INFO: [JOB 386] compacting(default) L0 [000268 000270] (2.2KB) Score=100.00 + L6 [000267] (5.9KB) Score=0.00; OverlappingRatio: Single 2.67, Multi 0.00
// INFO: [JOB 386] compacting: sstable created 000271
// INFO: [JOB 386] compacted(default) L0 [000268 000270] (2.2KB) Score=100.00 + L6 [000267] (5.9KB) Score=0.00 -> L6 [000271] (6.0KB), in 0.0s (0.0s total), output rate 11MB/s
// INFO: [JOB 385] WAL deleted 000266
// INFO: [JOB 386] sstable deleted 000267
// INFO: [JOB 386] sstable deleted 000268
// INFO: [JOB 386] sstable deleted 000270
To reduce: go test ./internal/metamorphic -tags invariants -run 'TestMeta$' --run-dir /artifacts/meta/240210-065310.4301752779833/standard-008 --try-to-reduce -v
--- FAIL: TestMeta/execution/standard-008 (0.50s)
=== RUN TestMeta/execution
--- FAIL: TestMeta/execution (0.00s)
Help
To reproduce, try:
go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run 'TestMeta$' ./internal/metamorphic -seed 1707547990430597266 -ops "uniform:5000-10000"github.com/cockroachdb/pebble/internal/metamorphic.TestMeta failed with artifacts on refs/heads/master @ 3e083df5434f:
iter64.SeekLT("uibih@2", "")
iter64.SeekPrefixGE("oxniiafqpu@12")
iter64.SeekGE("ioiv@3", "")
db1.Get("anzdpuhpfg")
iter64.First()
db1.Merge("bgdid@3", "lrwfxlrymzshpakbte")
db1.Get("uujlfyosz@6")
iter64.Prev("")
iter64.First()
snap45.Close()
iter64.Last()
db1.Merge("gbmqvawop@10", "mtzcqiibbpsfsvmtz")
db1.Merge("rbssro@3", "vemnuznqjutwztwz")
iter64.Last()
iter64.SeekLT("uibih@6", "")
iter64.Prev("")
db1.Merge("mxaw", "uopmkfkunuolequvp")
iter64.First()
iter64.Prev("")
iter64.SeekLT("ecfeek@2", "")
db1.Get("ohcd@10")
iter64.SeekLT("vshdihlq@2", "")
iter64.SetBounds("anzdpuhpfg", "anzdpuhpfg")
iter64.SeekGE("anzdpuhpfg", "")
iter64.Prev("")
iter64.Prev("")
iter64.Next("")
iter64.Last()
iter64.SeekLT("bhkoimravv@6", "")
iter64.SeekGE("xjffk@6", "")
db1.Set("eufqrv@14", "szb")
db1.Set("kcwxfe", "a")
iter64.SetBounds("etskc", "kisssgn")
iter64.Prev("")
db1.DeleteRange("awbfxbv@2", "lpmxj")
db1.SingleDelete("auoa", false /* maybeReplaceDelete */)
iter64.Prev("")
iter64.Close()
db1.Close()
To reduce: go test ./internal/metamorphic -tags invariants -run 'TestMeta$' --compare "/artifacts/meta/240211-055033.3433683724136/{standard-000,standard-008}" --try-to-reduce -v
=== RUN TestMeta/execution/random-017
=== PAUSE TestMeta/execution/random-017
=== CONT TestMeta/execution/random-017
=== RUN TestMeta/execution/random-020
=== PAUSE TestMeta/execution/random-020
=== CONT TestMeta/execution/random-020
=== RUN TestMeta/execution/standard-010
=== PAUSE TestMeta/execution/standard-010
=== CONT TestMeta/execution/standard-010
Help
To reproduce, try:
go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run 'TestMeta$' ./internal/metamorphic -seed 1707630633343420014 -ops "uniform:5000-10000"github.com/cockroachdb/pebble/internal/metamorphic.TestMeta failed with artifacts on refs/heads/master @ 38e3430fe43b:
db1.SingleDelete("fssbrrsp@2", true /* maybeReplaceDelete */) // <nil> #613
db1.Merge("mdiksvzf@4", "xbcawcyraeyrrdxs") // <nil> #615
db1.Get("cnxtqygub@4") // [""] pebble: not found #616
iter1.SeekLT("qwzrhkt", "") // [false] <nil> #618
snap4 = db1.NewSnapshot("bllbsthxcge", "fndwh", "fssbrrsp", "kzwkcrwro", "mdiksvzf", "yyckbbcc") #625
db1.Set("fssbrrsp@2", "tbtnnzo") // <nil> #626
db1.Delete("nmtqhkcv@4") // <nil> #627
iter4.Close() // <nil> #628
iter5.Close() // <nil> #629
iter2.Close() // <nil> #630
iter1.Close() // <nil> #631
snap4.Close() // <nil> #632
db1.Restart() #633
db1.Set("odju@4", "siluyorr") // <nil> #634
db1.Set("krdzh@4", "wwdegqgaafnski") // <nil> #635
db1.Get("nkpfkrpny") // [""] pebble: not found #636
db1.SingleDelete("akrozhnckh@2", false /* maybeReplaceDelete */) // <nil> #637
db1.Set("vmskod", "sewdupgmcn") // <nil> #638
db1.Get("aigldhmjn") // [""] pebble: not found #639
db1.Set("odju", "tqw") // <nil> #640
db1.Merge("fndwh", "evolufhjqfkzp") // <nil> #641
db1.Get("krdzh@4") // ["wwdegqgaafnski"] <nil> #642
db1.Delete("ipgoggnt") // <nil> #643
db1.Merge("xjiqtzjhor@4", "foinrzktszdkolmbz") // <nil> #644
// INFO: write stall beginning: memtable count limit reached
// INFO: [JOB 61] WAL deleted 000043
// INFO: [JOB 64] sstable deleted 000025
// INFO: [JOB 64] sstable deleted 000028
// INFO: [JOB 65] sstable deleted 000027
// INFO: [JOB 65] sstable deleted 000030
// INFO: [JOB 65] sstable deleted 000031
// INFO: [JOB 65] sstable deleted 000034
// INFO: [JOB 65] sstable deleted 000036
// INFO: [JOB 66] sstable deleted 000023
// INFO: [JOB 67] flushing 2 memtables (904B) to L0
// INFO: [JOB 67] flushing: sstable created 000049
// FATAL: checker failed with error: tombstone ipgoggnt-nkpfkrpny@4:{(#208,RANGEDEL)} in L0.1: fileNum=000049 deletes key ipgoggnt#207,SINGLEDEL in memtable
// INFO: [JOB 67] flushed 2 memtables (904B) to L0 [000049] (1.2KB), in 0.0s (0.0s total), output rate 4.0MB/s
// INFO: [JOB 67] WAL deleted 000045
// INFO: [JOB 68] WAL created 000050
// INFO: write stall ending
db1.SingleDelete("tzcmpxwvenw@1", false /* maybeReplaceDelete */) // <nil> #645
// INFO: [JOB 70] compacting(default) L0 [000047 000049] (2.3KB) Score=100.00 + L6 [000046] (1.4KB) Score=0.00; OverlappingRatio: Single 0.62, Multi 0.00
// INFO: [JOB 70] compacting: sstable created 000051
// INFO: [JOB 70] compacted(default) L0 [000047 000049] (2.3KB) Score=100.00 + L6 [000046] (1.4KB) Score=0.00 -> L6 [000051] (1.1KB), in 0.0s (0.0s total), output rate 4.1MB/s
// INFO: [JOB 71] flushing 1 memtable (691B) to L0
// INFO: [JOB 71] flushing: sstable created 000052
To reduce: go test ./internal/metamorphic -tags invariants -run 'TestMeta$' --run-dir /artifacts/meta/240212-064552.2531691485892/standard-008 --try-to-reduce -v
--- FAIL: TestMeta/execution/standard-008 (0.22s)
Help
To reproduce, try:
go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run 'TestMeta$' ./internal/metamorphic -seed 1707720352253122519 -ops "uniform:5000-10000"