internal/metamorphic: TestMeta failed [wal data race]
cockroach-teamcity opened this issue · comments
github.com/cockroachdb/pebble/internal/metamorphic.TestMeta failed with artifacts on refs/heads/master @ f3e3d4aa49cb:
db1.Merge("fyorzp@1", "jggzzdkuq") // <nil> #444
db1.Set("tgmhkt@3", "zctskfym") // <nil> #445
db1.Merge("aemlpgwe@1", "atyzsieek") // <nil> #446
db1.Merge("jznlesedt@1", "c") // <nil> #447
db1.Delete("lqewi@3") // <nil> #448
db1.Delete("ritdl") // <nil> #449
// INFO: [JOB 9] WAL created 000008
db1.RangeKeySet("rpyibvpalk", "zyclxzcav", "", "gmmpkydnxedwmmev") // <nil> #450
// INFO: [JOB 10] flushing 1 memtable (2.7KB) to L0
db1.Delete("dgarwwrt@1") // <nil> #451
db1.Get("daziyx@2") // [""] pebble: not found #452
// INFO: [JOB 10] flushing: sstable created 000009
db1.Get("nsqodb@2") // [""] pebble: not found #453
db1.SingleDelete("cxuqm@2", true /* maybeReplaceDelete */) // <nil> #454
db1.Merge("tgmhkt@3", "jd") // <nil> #455
batch2 = db1.NewIndexedBatch() #456
db1.Get("ritdl") // [""] pebble: not found #457
db1.Ingest(batch2) // <nil> #458
db1.SingleDelete("toxvzr@2", false /* maybeReplaceDelete */) // <nil> #459
db1.Delete("nsqodb@2") // <nil> #460
snap2.Get("dmnjbbylph@2") // ["eriggvj"] <nil> #461
db1.Merge("sgdfwy", "bsdytugfefihzbdzrewn") // <nil> #462
db1.DeleteRange("dgarwwrt@1", "sgdfwy") // <nil> #463
iter5 = db1.NewIter("", "", 2 /* key types */, 3, 4, false /* use L6 filters */, "" /* masking suffix */) // <nil> #464
// INFO: [JOB 10] MANIFEST created 000010
iter5.SeekLT("dgarwwrt@1", "") // [true,"aemlpgwe",<no point>,["aemlpgwe","jamzfwrelqav")=>{"@2"="ug"}*] <nil> #466
db1.Merge("ritdl", "tninxzbjcvxyu") // <nil> #465
// INFO: [JOB 10] flushed 1 memtable (2.7KB) to L0 [000009] (2.2KB), in 0.0s (0.0s total), output rate 923KB/s
// INFO: [JOB 9] WAL created 000008
// INFO: [JOB 10] WAL deleted 000006
// INFO: [JOB 10] WAL deleted 000006
// INFO: [JOB 10] WAL deleted 000006
// INFO: [JOB 9] WAL created 000008
db1.DeleteRange("cvgic@4", "yjnxexrlu") // <nil> #470
iter6 = snap2.NewIter("mucngspupati@1", "nsqodb@2", 0 /* key types */, 0, 0, false /* use L6 filters */, "" /* masking suffix */) // <nil> #473
db1.Get("smhxocse") // [""] pebble: not found #481
db1.Merge("czczvvcji@2", "qkncyy") // <nil> #485
// INFO: [JOB 9] WAL created 000008
iter6.Prev("") // [false] <nil> #474
db1.Merge("khtkhd@6", "") // <nil> #487
iter5.SeekLT("ritdl@1", "mvjmkheecw@2") // [valid,"mvjmkheecw",<no point>,["mvjmkheecw","rpyibvpalk")=>{"@4"="iwlopskfaxomnckoufxu"}*] <nil> #467
db1.SingleDelete("sgdfwy", false /* maybeReplaceDelete */) // <nil> #489
db1.Get("jznlesedt@1") // [""] pebble: not found #491
iter6.Last() // [false] <nil> #476
iter6.SeekPrefixGE("mvjmkheecw@2") // [false] <nil> #477
iter6.Last() // [false] <nil> #482
iter5.SeekLT("dgarwwrt@1", "") // [true,"aemlpgwe",<no point>,["aemlpgwe","jamzfwrelqav")=>{"@2"="ug"}*] <nil> #468
To reduce: go test ./internal/metamorphic -tags invariants -run 'TestMeta$' --run-dir /artifacts/meta/240306-053547.660374439949/random-003 --try-to-reduce -v
--- FAIL: TestMeta/execution/random-003 (0.78s)
Help
To reproduce, try:
go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run 'TestMeta$' ./internal/metamorphic -seed 1709703347660761312 -ops "uniform:5000-10000"
[00:41:07] : [Step 1/1] meta.go:240:
[00:41:07] : [Step 1/1] ===== SEED =====
[00:41:07] : [Step 1/1] 1709703347660761312
[00:41:07] : [Step 1/1] ===== ERR =====
[00:41:07] : [Step 1/1] exit status 66
[00:41:07] : [Step 1/1] ===== OUT =====
[00:41:07] : [Step 1/1] ==================
[00:41:07] : [Step 1/1] WARNING: DATA RACE
[00:41:07] : [Step 1/1] Write at 0x00c000612258 by goroutine 56:
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal.(*recordQueue).push()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal/external/com_github_cockroachdb_pebble/wal/failover_writer.go:125 +0x4e9
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal.(*failoverWriter).WriteRecord()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal/external/com_github_cockroachdb_pebble/wal/failover_writer.go:449 +0x134
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble.(*DB).commitWrite()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:952 +0x6ce
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble.(*DB).commitWrite-fm()
[00:41:07] : [Step 1/1] <autogenerated>:1 +0x55
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble.(*commitPipeline).prepare()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:467 +0x3e1
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble.(*commitPipeline).Commit()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:316 +0x14a
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble.(*DB).applyInternal()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:843 +0x49a
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble.(*DB).Apply()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:772 +0x10a
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble.(*DB).Set()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:609 +0x10b
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/metamorphic.(*setOp).run()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/metamorphic/external/com_github_cockroachdb_pebble/metamorphic/ops.go:417 +0x155
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/metamorphic.Execute.func1()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/metamorphic/external/com_github_cockroachdb_pebble/metamorphic/meta.go:612 +0x28c
[00:41:07] : [Step 1/1] golang.org/x/sync/errgroup.(*Group).Go.func1()
[00:41:07] : [Step 1/1] golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:78 +0x91
[00:41:07] : [Step 1/1]
[00:41:07] : [Step 1/1] Previous read at 0x00c000612258 by goroutine 114:
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal.(*recordQueue).pop()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal/external/com_github_cockroachdb_pebble/wal/failover_writer.go:194 +0x24e
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal.(*failoverWriter).doneSyncCallback()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal/external/com_github_cockroachdb_pebble/wal/failover_writer.go:649 +0x5e
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal.(*failoverWriter).doneSyncCallback-fm()
[00:41:07] : [Step 1/1] <autogenerated>:1 +0x51
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record.(*pendingSyncsWithHighestSyncIndex).pop()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:313 +0xe4
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record.(*LogWriter).flushPending()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:766 +0x361
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:695 +0x6e4
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop-fm()
[00:41:07] : [Step 1/1] <autogenerated>:1 +0x47
[00:41:07] : [Step 1/1] runtime/pprof.Do()
[00:41:07] : [Step 1/1] GOROOT/src/runtime/pprof/runtime.go:51 +0x117
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record.NewLogWriter.func2()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:562 +0x91
[00:41:07] : [Step 1/1]
[00:41:07] : [Step 1/1] Goroutine 56 (running) created at:
[00:41:07] : [Step 1/1] golang.org/x/sync/errgroup.(*Group).Go()
[00:41:07] : [Step 1/1] golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x124
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/metamorphic.Execute()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/metamorphic/external/com_github_cockroachdb_pebble/metamorphic/meta.go:589 +0xa8
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/metamorphic.RunOnce()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/metamorphic/external/com_github_cockroachdb_pebble/metamorphic/meta.go:549 +0x21b3
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/internal/metamorphic.runTestMeta()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/internal/metamorphic/external/com_github_cockroachdb_pebble/internal/metamorphic/meta_test.go:87 +0x657
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/internal/metamorphic.TestMeta()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/internal/metamorphic/external/com_github_cockroachdb_pebble/internal/metamorphic/meta_test.go:54 +0x28
[00:41:07] : [Step 1/1] testing.tRunner()
[00:41:07] : [Step 1/1] GOROOT/src/testing/testing.go:1689 +0x21e
[00:41:07] : [Step 1/1] testing.(*T).Run.gowrap1()
[00:41:07] : [Step 1/1] GOROOT/src/testing/testing.go:1742 +0x44
[00:41:07] : [Step 1/1]
[00:41:07] : [Step 1/1] Goroutine 114 (running) created at:
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record.NewLogWriter()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:561 +0x8e4
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal.(*failoverWriter).switchToNewDir.func1()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal/external/com_github_cockroachdb_pebble/wal/failover_writer.go:566 +0x68c
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal.(*stopper).runAsync.func1()
[00:41:07] : [Step 1/1] github.com/cockroachdb/pebble/wal/external/com_github_cockroachdb_pebble/wal/failover_manager.go:787 +0x33
[00:41:07] : [Step 1/1] ==================