cockroachdb / pebble

RocksDB/LevelDB inspired key-value database in Go

Geek Repo:Geek Repo

Github PK Tool:Github PK Tool

github.com/cockroachdb/pebble/internal/metamorphic: TestMeta failed

cockroach-teamcity opened this issue · comments

github.com/cockroachdb/pebble/internal/metamorphic.TestMeta failed with artifacts on refs/heads/master @ 1cce3d01f6d9:

        // INFO: [JOB 9] flushing: sstable created 000294
        iter45.First() // [true,"jsgbqucnax@18",<no point>,["jsgbqucnax@18","ujvibniqb")=>{"@88"="kipqmhfc"}*] <nil> #4112
        // INFO: [JOB 9] flushing: sstable created 000295
        iter50.SeekGE("jsgbqucnax@10", "") // [true,"jsgbqucnax@10",<no point>,["jsgbqucnax@10","qwwwnz@55")=>{"@88"="kipqmhfc"}*] <nil> #4187
        // INFO: [JOB 9] flushing: sstable created 000296
        // INFO: [JOB 9] flushing: sstable created 000297
        iter50.Prev("") // [false] <nil> #4188
        // INFO: [JOB 9] flushing: sstable created 000298
        iter47.First() // [true,"aknpkox@48","mzchrqhhg",<no range>] <nil> #4176
        // INFO: [JOB 9] flushing: sstable created 000299
        // INFO: [JOB 9] flushing: sstable created 000300
        iter48.Prev("") // [true,"ujvibniqb",<no point>,["ujvibniqb","ujvibniqb@56")=>{"@88"="kipqmhfc","@72"="rlocaqqd"}*] <nil> #4137
        iter47.SeekPrefixGE("behruxrud@24") // [true,"behruxrud@24","gpegchdxidnz",<no range>] <nil> #4180
        iter45.Prev("jsgbqucnax@35") // [invalid] <nil> #4116
        iter45.SeekPrefixGE("gjtzgjulw@24") // [false] pebble: SeekPrefixGE supplied with key outside of lower bound #4117
        iter48.Prev("") // [true,"hjznd",<no point>,["hjznd","ujvibniqb")=>{"@88"="kipqmhfc"}*] <nil> #4139
        db1.Get("jsgbqucnax@46") // ["larmphffmyyvwuiz"] <nil> #4140
        // INFO: [JOB 9] flushing: sstable created 000301
        // INFO: [JOB 10] WAL created 000302
        iter45.First() // [true,"jsgbqucnax@18",<no point>,["jsgbqucnax@18","ujvibniqb")=>{"@88"="kipqmhfc"}*] <nil> #4119
        iter45.SeekLT("ujvibniqb@30", "") // [true,"ujvibniqb",<no point>,["ujvibniqb","ujvibniqb@6")=>{"@88"="kipqmhfc","@72"="rlocaqqd"}*] <nil> #4121
        iter47.First() // [true,"aknpkox@48","mzchrqhhg",<no range>] <nil> #4190
        iter45.SeekPrefixGE("hjznd@41") // [false] pebble: SeekPrefixGE supplied with key outside of lower bound #4125
        iter45.First() // [true,"jsgbqucnax@18",<no point>,["jsgbqucnax@18","ujvibniqb")=>{"@88"="kipqmhfc"}*] <nil> #4148
        iter45.SeekGE("elxwvjvb@90", "") // [true,"jsgbqucnax@18",<no point>,["jsgbqucnax@18","ujvibniqb")=>{"@88"="kipqmhfc"}] <nil> #4149
        iter45.Last() // [true,"ujvibniqb",<no point>,["ujvibniqb","ujvibniqb@6")=>{"@88"="kipqmhfc","@72"="rlocaqqd"}*] <nil> #4155
        iter45.SeekPrefixGE("qwwwnz@75") // [true,"qwwwnz@75",<no point>,["qwwwnz","qwwwnz\x00")=>{"@88"="kipqmhfc"}*] <nil> #4158
        iter45.SeekPrefixGE("ujvibniqb@30") // [true,"ujvibniqb@30",<no point>,["ujvibniqb","ujvibniqb@6")=>{"@88"="kipqmhfc","@72"="rlocaqqd"}*] <nil> #4159
        iter45.Last() // [true,"ujvibniqb",<no point>,["ujvibniqb","ujvibniqb@6")=>{"@88"="kipqmhfc","@72"="rlocaqqd"}] <nil> #4164
        iter45.SeekGE("behruxrud@6", "") // [true,"jsgbqucnax@18",<no point>,["jsgbqucnax@18","ujvibniqb")=>{"@88"="kipqmhfc"}*] <nil> #4165
        // INFO: [JOB 9] flushing: sstable created 000303
        // INFO: [JOB 9] flushing: sstable created 000304
        // INFO: [JOB 9] flushing: sstable created 000305
        // INFO: [JOB 9] flushed 1 memtable (1.4KB) to L0 [000285 000286 000287 000288 000289 000290 000291 000292 000293 000294 000295 000296 000297 000298 000299 000300 000301 000303 000304 000305] (19KB), in 0.0s (0.0s total), output rate 727KB/s
        // INFO: [JOB 9] WAL deleted 000282
        // INFO: [JOB 11] flushing 1 memtable (96B) to L0
        // INFO: [JOB 11] flushing: sstable created 000306
        // INFO: [JOB 11] flushing: sstable created 000307
        // INFO: [JOB 11] flushed 1 memtable (96B) to L0 [000306 000307] (1.5KB), in 0.0s (0.0s total), output rate 727KB/s
        // INFO: [JOB 11] WAL deleted 000284
        // INFO: [JOB 12] compacting(default) L0 [000031 000043 000059 000053 000062 000063 000078 000077 000102 000101 000091 000100 000093 000092 000117 000118 000107 000105 000104 000106 000103 000147 000146 000127 000145 000128 000129 000131 000130 000144 000158 000159 000151 000150 000149 000181 000184 000192 000191 000182 000185 000187 000183 000190 000188 000189 000186 000203 000198 000202 000201 000199 000200 000210 000223 000222 000224 000211 000217 000218 000213 000212 000220 000221 000219 000214 000215 000216 000209 000207 000226 000229 000227 000228 000236 000242 000239 000243 000241 000237 000248 000247 000250 000249 000251 000246 000245 000240 000244 000238 000234 000263 000256 000262 000255 000261 000260 000257 000254 000259 000258 000276 000277 000278 000275 000270 000266 000274 000269 000267 000265 000271 000268 000264 000273 000272 000286 000293 000295 000290 000294 000303 000288 000289 000287 000291 000301 000305 000292 000299 000296 000285 000300 000304 000298 000297 000307 000306] (133KB) Score=0.00 + L6 [000035 000168 000169 000170 000171 000172 000173 000174 000175 000176 000177 000178 000179 000180 000075] (24KB) Score=0.00; OverlappingRatio: Single 0.18, Multi 0.00
        // INFO: [JOB 12] compacting: sstable created 000308
        // INFO: [JOB 12] compacting: sstable created 000309
        // INFO: [JOB 12] compacting: sstable created 000310
        // INFO: [JOB 12] compacting: sstable created 000311
        // INFO: [JOB 12] compacting: sstable created 000312
        // INFO: [JOB 12] compacting: sstable created 000313
        // INFO: [JOB 12] compacting: sstable created 000314
        // INFO: [JOB 12] compacting: sstable created 000315
        --- FAIL: TestMeta/execution/random-027 (0.86s)
Help

To reproduce, try:

go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run TestMeta$ ./internal/metamorphic -seed 1703314987907323074 -ops "uniform:5000-10000"

This test on roachdash | Improve this report!

        ===== SEED =====
        1703314987907323074
        ===== ERR =====
        exit status 2
        ===== OUT =====
        panic: single del invariant violations on key "dturib@51" [recovered]
                panic: single del invariant violations on key "dturib@51"

        goroutine 938 [running]:
        github.com/cockroachdb/pebble.(*DB).runCompaction.func1()
                /Users/nickt/Development/pebble/compaction.go:2849 +0xff
        panic({0x100819080?, 0xc000718880?})
                /nix/store/hyzs4fhjw7zmwra5scss7hapgw22rcm5-go-1.21.5/share/go/src/runtime/panic.go:920 +0x270
        github.com/cockroachdb/pebble/metamorphic.RunOnce.defaultTestOptions.defaultOptions.func1({0xc000396150?, 0x78?, 0x10056b565?})
                /Users/nickt/Development/pebble/metamorphic/options.go:250 +0x5b
        github.com/cockroachdb/pebble.(*compactionIter).skipDueToSingleDeleteElision(0xc0009016c0)
                /Users/nickt/Development/pebble/compaction_iter.go:1122 +0x17d
        github.com/cockroachdb/pebble.(*compactionIter).Next(0xc0009016c0)
                /Users/nickt/Development/pebble/compaction_iter.go:465 +0x38a
        github.com/cockroachdb/pebble.(*DB).runCompaction(0xc00028e480, 0xa, 0xc000a56000)
                /Users/nickt/Development/pebble/compaction.go:3437 +0x287b
        github.com/cockroachdb/pebble.(*DB).compact1(0xc00028e480, 0xc000a56000, 0xc000a48480)
                /Users/nickt/Development/pebble/compaction.go:2675 +0x19e
        github.com/cockroachdb/pebble.(*DB).compact.func1({0x1009a9c60, 0xc0004fc930})
                /Users/nickt/Development/pebble/compaction.go:2639 +0xa7
        runtime/pprof.Do({0x1009a9bf0?, 0x100ecb760?}, {{0xc0000a3020?, 0x0?, 0xbd?}}, 0xc0008f9f88)
                /nix/store/hyzs4fhjw7zmwra5scss7hapgw22rcm5-go-1.21.5/share/go/src/runtime/pprof/runtime.go:51 +0x9d
        github.com/cockroachdb/pebble.(*DB).compact(0x0?, 0x0?, 0x0?)
                /Users/nickt/Development/pebble/compaction.go:2636 +0x65
        created by github.com/cockroachdb/pebble.(*DB).maybeScheduleCompactionPicker in goroutine 69
                /Users/nickt/Development/pebble/compaction.go:2329 +0x61d

The sequence of ops looks off - I don't think the test harness should be writing a SingleDel, given the key has been set twice.

$ grep 'dturib@51' internal/metamorphic/_meta/231226-074127.951453009372/ops
snap7.Get("dturib@51")
iter19.SetBounds("borban@43", "dturib@51")
iter19.SeekLT("txkl@29", "dturib@51")
db1.Get("dturib@51")
db1.Set("dturib@51", "puozmcrdcvlnunlbhxh")                     <-- 1st SET
db1.Set("dturib@51", "iwslepyzmjkzj")                           <-- 2nd SET
iter39.SeekPrefixGE("dturib@51")
iter39.SetBounds("dturib@51", "jsgbqucnax@35")
iter38.SetBounds("dturib@51", "jsgbqucnax@34")
db1.SingleDelete("dturib@51", false /* maybeReplaceDelete */)  <-- SINGLEDEL
iter47.SetBounds("dturib@51", "gjtzgjulw@41")
iter47.SeekGE("dturib@51", "")
iter47.SetBounds("dturib@51", "dturib@51")
iter47.SeekLT("dturib@51", "")
iter62.SeekGE("dturib@51", "")
iter67.SetBounds("dturib@51", "ujvibniqb@7")
iter67.SeekGE("dturib@51", "")
iter67.SetBounds("dturib@51", "dturib@51")
iter77.SetBounds("dturib@51", "gjtzgjulw@41")
iter77.SetBounds("aknpkox@95", "dturib@51")
iter91.Prev("dturib@51")

There are probably range deletes that make it permissible. I’d guess this is the same issue that motivated disabling the assertions in Cockroach: delete-only compactions can break the assertions.

Yeah, forgot about the range dels:

...
db1.Set("dturib@51", "puozmcrdcvlnunlbhxh")
db1.Set("dturib@51", "iwslepyzmjkzj")
...
batch27 = db1.NewBatch()
batch27.DeleteRange("borban@37", "ujvibniqb@31")
...
db1.Ingest(batch28, batch27)
...
db1.SingleDelete("dturib@51", false /* maybeReplaceDelete */)

Linking to cockroachdb/cockroach#116889 for posterity.

github.com/cockroachdb/pebble/internal/metamorphic.TestMeta failed with artifacts on refs/heads/master @ 1cce3d01f6d9:

        db1.DeleteRange("mbbjevizcq@14", "sujtjuh@10") // <nil> #7411
        db1.Set("nbrcwzbox@15", "dfybarb") // <nil> #7412
        iter69.Last() // [true,"bxdctdkpoe",<no point>,["bxdctdkpoe","ceiqjioaua@11")=>{"@13"="ksnyesjaefwuiwfmjon","@5"="xeriihsjfaerdtox","@3"="kdkmlnzuotikpdnxrij"}*] <nil> #7413
        db1.Merge("wdyazlgzfh", "wcsluvgkkfbgma") // <nil> #7414
        iter69.Next("") // [false] <nil> #7415
        iter69.Prev("") // [true,"bxdctdkpoe",<no point>,["bxdctdkpoe","ceiqjioaua@11")=>{"@13"="ksnyesjaefwuiwfmjon","@5"="xeriihsjfaerdtox","@3"="kdkmlnzuotikpdnxrij"}*] <nil> #7416
        iter69.Prev("iotyuvpv") // [invalid] <nil> #7417
        snap53 = db1.NewSnapshot("dastn", "gvcppyr", "halhgu", "mpvye", "rmcn", "spwo", "yqcbdmoqkkj", "zmftysf") #7418
        iter69.Last() // [true,"bxdctdkpoe",<no point>,["bxdctdkpoe","ceiqjioaua@11")=>{"@13"="ksnyesjaefwuiwfmjon","@5"="xeriihsjfaerdtox","@3"="kdkmlnzuotikpdnxrij"}*] <nil> #7419
        iter69.Prev("") // [true,"btnkmqym@10","ny",["bsahjyggbal@8","bxdctdkpoe")=>{"@5"="xeriihsjfaerdtox","@3"="kdkmlnzuotikpdnxrij"}*] <nil> #7420
        iter69.SeekLT("occglosrl@18", "") // [true,"bxdctdkpoe",<no point>,["bxdctdkpoe","ceiqjioaua@11")=>{"@13"="ksnyesjaefwuiwfmjon","@5"="xeriihsjfaerdtox","@3"="kdkmlnzuotikpdnxrij"}*] <nil> #7421
        db1.SingleDelete("xhelwl@8", false /* maybeReplaceDelete */) // <nil> #7422
        batch53 = db1.NewBatch() #7423
        iter69.SetBounds("qkniz@11", "zmftysf@4") // <nil> #7424
        iter69.SeekGE("qkniz@11", "") // [true,"qkniz@11",<no point>,["qkniz@11","rpia")=>{"@5"="xeriihsjfaerdtox"}*] <nil> #7425
        iter69.Prev("") // [false] <nil> #7426
        iter69.Close() // <nil> #7427
        snap53.Close() // <nil> #7428
        snap52.Close() // <nil> #7429
        batch53.Close() // <nil> #7430
        db1.Restart() #7431
        batch54 = db1.NewIndexedBatch() #7432
        batch54.RangeKeyDelete("petqfrhujoh", "qbgaww") // <nil> #7433
        batch54.Delete("dzdorxsboug@3") // <nil> #7434
        batch54.Delete("rcbhxqh@5") // <nil> #7435
        db1.Delete("vipt@12") // <nil> #7436
        db1.SingleDelete("htjsbkaxh@11", false /* maybeReplaceDelete */) // <nil> #7437
        // INFO: [JOB 159] ingesting: sstable created 000165
        // INFO: [JOB 160] flushing 1 memtable (5.2KB) to L0
        // INFO: [JOB 160] flushing: sstable created 000166
        // INFO: [JOB 160] flushing: sstable created 000167
        // INFO: [JOB 160] flushing: sstable created 000168
        // INFO: [JOB 160] flushing: sstable created 000169
        // INFO: [JOB 160] flushing: sstable created 000170
        // INFO: [JOB 160] MANIFEST created 000171
        // INFO: [JOB 160] flushed 1 memtable (5.2KB) to L0 [000166 000167 000168 000169 000170] (6.8KB), in 0.0s (0.0s total), output rate 4.4MB/s
        // INFO: [JOB 160] MANIFEST deleted 000153
        // INFO: [JOB 159] ingested L0:000165 (1.1KB)
        db1.Ingest(batch54) // <nil> #7438
        db1.Delete("jgdkf@11") // <nil> #7439
        db1.Merge("lbuui@16", "mijiqbid") // <nil> #7440
        db1.Merge("ellnfukxapk@3", "o") // <nil> #7441
        db1.SingleDelete("gbnixi@11", false /* maybeReplaceDelete */) // <nil> #7442
        // INFO: [JOB 162] compacting(default) L0 [000151 000150 000149 000157 000156 000155 000162 000160 000159 000161 000169 000170 000166 000168 000167 000165] (26KB) Score=0.00 + L6 [000146] (4.7KB) Score=0.00; OverlappingRatio: Single 0.18, Multi 0.00
        // INFO: [JOB 162] compacting: sstable created 000172
        db1.Compact("qkniz@11", "rpia@16", false /* parallelize */) // <nil> #7443
        db1.Set("tcdgvvk", "jzsriseuixg") // <nil> #7444
        db1.SingleDelete("mbbjevizcq@14", false /* maybeReplaceDelete */) // <nil> #7445
        db1.SingleDelete("tftwfd@10", false /* maybeReplaceDelete */) // <nil> #7446
        --- FAIL: TestMeta/execution/random-012 (0.97s)
Help

To reproduce, try:

go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run TestMeta$ ./internal/metamorphic -seed 1703836492125878456 -ops "uniform:5000-10000"

This test on roachdash | Improve this report!

github.com/cockroachdb/pebble/internal/metamorphic.TestMeta failed with artifacts on refs/heads/master @ 1cce3d01f6d9:

        iter13.SeekGE("agvjcqz@3", "") // [true,"trsojfbzlr@3",<no point>,["trsojfbzlr@3","unrfs")=>{""="etyidktnzl"}*] <nil> #1657
        iter13.First() // [true,"trsojfbzlr@3",<no point>,["trsojfbzlr@3","unrfs")=>{""="etyidktnzl"}] <nil> #1674
        iter13.Next("") // [true,"unrfs",<no point>,["unrfs","vrxantlkqbi")=>{""="etyidktnzl","@2"="ziidzvq"}*] <nil> #1708
        db1.Set("qpqrmzgra@3", "yupjltxymeuda") // <nil> #1580
        iter14 = db1.NewIter("", "", 2 /* key types */, 0, 0, false /* use L6 filters */, "" /* masking suffix */) // <nil> #1582
        iter13.SeekLT("etfeunjex@2", "") // [false] <nil> #1719
        iter13.Last() // [true,"vrxantlkqbi",<no point>,["vrxantlkqbi","vssps")=>{"@2"="ziidzvq"}*] <nil> #1729
        iter14.Prev("") // [true,"ztiapwugywr@3","rsfwvgzcjyimc",<no range>] <nil> #1596
        iter14.SeekPrefixGE("dptsjfv@2") // [true,"dptsjfv@2",<no point>,["dptsjfv","dptsjfv\x00")=>{""="etyidktnzl"}*] <nil> #1608
        snap6.Get("lgye") // [""] pebble: not found #1583
        iter14.SeekPrefixGE("okxfyw@3") // [true,"okxfyw@3",<no point>,["okxfyw","okxfyw\x00")=>{""="etyidktnzl","@3"="sjigfxly","@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1633
        iter6.SeekPrefixGE("wusu") // [false] pebble: SeekPrefixGE supplied with key outside of upper bound #1588
        iter6.SeekLT("keja@2", "") // [false] <nil> #1607
        iter6.Next("") // [true,"unrfs@3",<no point>,["unrfs@3","vssps@2")=>{"@2"=""}*] <nil> #1609
        iter14.SeekLT("jhms", "") // [true,"gpqtalhvpj@3","ssjfe",["gdnmgcosu","kqwcg")=>{""="etyidktnzl"}*] <nil> #1688
        iter14.Next("") // [true,"keja@2","bchjcizffogxbwfh",["gdnmgcosu","kqwcg")=>{""="etyidktnzl"}] <nil> #1701
        iter6.SetBounds("wusu", "zpjvfusphi@1") // <nil> #1630
        iter14.Next("") // [true,"kmvq","rvft",["gdnmgcosu","kqwcg")=>{""="etyidktnzl"}] <nil> #1714
        iter14.SetOptions("", "", 2 /* key types */, 4, 5, false /* use L6 filters */, "" /* masking suffix */) // <nil> #1722
        iter6.SeekGE("wusu", "") // [true,"wusu",<no point>,["wusu","wzvhznzgs")=>{"@2"=""}*] <nil> #1631
        iter6.Prev("") // [false] <nil> #1632
        iter14.SeekGE("olipqeiikdb@2", "onrixdjomov@1") // [valid,"olipqeiikdb@2",<no point>,["olipqeiikdb","vrxantlkqbi")=>{""="etyidktnzl","@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1723
        iter14.Last() // [true,"ymzvfqdvtcr@4","s",["xhdkgxmz","zpjvfusphi")=>{"@2"="ziidzvq"}*] <nil> #1736
        iter14.SeekGE("kqwcg@2", "") // [true,"kqwcg@2",<no point>,["kqwcg","miucldg")=>{""="etyidktnzl","@3"="sjigfxly"}*] <nil> #1741
        iter14.Prev("") // [true,"kqwcg",<no point>,["kqwcg","miucldg")=>{""="etyidktnzl","@3"="sjigfxly"}] <nil> #1748
        iter14.SeekGE("fjsemj@3", "") // [true,"fjsemj@3",<no point>,["fgbk","gdnmgcosu")=>{""="etyidktnzl","@2"="oxztufcdoxsudmc"}*] <nil> #1766
        iter14.SeekLT("kqwcg@3", "") // [true,"kqwcg",<no point>,["kqwcg","miucldg")=>{""="etyidktnzl","@3"="sjigfxly"}*] <nil> #1788
        iter14.SetBounds("miucldg@4", "vyvrg@4") // <nil> #1805
        iter14.SeekLT("vyvrg@4", "") // [true,"vssps","pdyxcjrlqifwrnk",["vrxantlkqbi","vyvrg@4")=>{"@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1806
        iter14.Next("") // [false] <nil> #1807
        iter14.First() // [true,"miucldg@4","sfsb",["miucldg@4","olipqeiikdb")=>{""="etyidktnzl","@3"="sjigfxly","@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1809
        iter14.Prev("") // [false] <nil> #1825
        iter14.SeekGE("fjsemj@3", "") // [true,"miucldg@4","sfsb",["miucldg@4","olipqeiikdb")=>{""="etyidktnzl","@3"="sjigfxly","@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1869
        iter14.Last() // [true,"vssps","pdyxcjrlqifwrnk",["vrxantlkqbi","vyvrg@4")=>{"@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1881
        iter14.SeekGE("dyoh@3", "xgsnex@1") // [valid,"miucldg@4","sfsb",["miucldg@4","olipqeiikdb")=>{""="etyidktnzl","@3"="sjigfxly","@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1907
        iter14.SeekPrefixGE("moseugmkzr@4") // [true,"moseugmkzr@4",<no point>,["moseugmkzr","moseugmkzr\x00")=>{""="etyidktnzl","@3"="sjigfxly","@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1909
        iter14.First() // [true,"miucldg@4","sfsb",["miucldg@4","olipqeiikdb")=>{""="etyidktnzl","@3"="sjigfxly","@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1963
        iter14.SeekPrefixGE("nxkqj") // [true,"nxkqj",<no point>,["nxkqj","nxkqj\x00")=>{""="etyidktnzl","@3"="sjigfxly","@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1965
        iter14.SeekLT("xgsnex@1", "") // [true,"vssps","pdyxcjrlqifwrnk",["vrxantlkqbi","vyvrg@4")=>{"@2"="kesetjhhagwwhwyjwcm"}*] <nil> #1997
        iter14.SeekGE("cttqtkhsu@1", "jhms") // [invalid] <nil> #1998
        // INFO: [JOB 135] flushed 1 memtable (715B) to L0 [000098] (1.3KB), in 0.0s (0.0s total), output rate 1.6MB/s
        iter14.Last() // [true,"vssps","pdyxcjrlqifwrnk",["vrxantlkqbi","vyvrg@4")=>{"@2"="kesetjhhagwwhwyjwcm"}*] <nil> #2005
        // INFO: [JOB 135] WAL deleted 000093
        iter14.Next("miucldg@3") // [invalid] <nil> #2044
        db1.Merge("fjsemj@3", "idlgeezefoynzhsmqa") // <nil> #1591
        snap10 = db1.NewSnapshot("dyoh", "fgbk", "fwcvbylfs", "jrigphw", "mbxfxatbcpo", "mdhppft", "moseugmkzr", "nxkqj", "qiqip", "unrfs") #1595
        db1.Set("miucldg@3", "shrsanqgfkfilaiipk") // <nil> #1598
        batch10 = db1.NewBatch() #1604
        batch10.Merge("kbxlql", "n") // <nil> #1615
        --- FAIL: TestMeta/execution/standard-008 (0.39s)
Help

To reproduce, try:

go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run TestMeta$ ./internal/metamorphic -seed 1703920756486732416 -ops "uniform:5000-10000"

This test on roachdash | Improve this report!