github.com/cockroachdb/pebble/internal/metamorphic: TestMetaTwoInstance failed
cockroach-teamcity opened this issue · comments
github.com/cockroachdb/pebble/internal/metamorphic.TestMetaTwoInstance failed with artifacts on refs/heads/master @ 3e083df5434f:
// INFO: [JOB 312] WAL deleted 000210
// INFO: [JOB 357] flushed 1 memtable (717B) to L0 [000247] (1.5KB), in 0.0s (0.0s total), output rate 4.0MB/s
// INFO: [JOB 358] WAL created 000248
// INFO: write stall ending
batch52.Commit() // <nil> #6974
batch52.Close() // <nil> #6975
db1.DeleteRange("goohzzgsy@11", "xrrvg@1") // <nil> #6979
batch53 = db1.NewIndexedBatch() #6981
db1.Ingest(batch53) // <nil> #6982
db1.DeleteRange("gcxugmdz@2", "vxnjyvioiky@2") // <nil> #6988
db1.Set("uygghlj@6", "ijunzk") // <nil> #6991
db1.RangeKeySet("bsxlxc", "dkufnrx", "@6", "wvztvexsxomdojcgq") // <nil> #6992
db1.Set("ieexebqmu@9", "csl") // <nil> #6995
db1.Get("qlsyvwqjbg") // [""] pebble: not found #6998
db1.Delete("oyld@6") // <nil> #7000
db1.Delete("pdzd@9") // <nil> #7002
db1.Delete("mqduvgfeauc@3") // <nil> #7031
db1.LogData("") // <nil> #7045
// INFO: [JOB 360] ingesting: sstable created 000250
// INFO: write stall beginning: memtable count limit reached
// INFO: [JOB 357] WAL deleted 000244
// INFO: [JOB 362] compacting(default) L0 [000245 000247] (2.7KB) Score=100.00 + L6 [000243] (1.4KB) Score=0.00; OverlappingRatio: Single 0.51, Multi 0.00
// INFO: [JOB 362] compacting: sstable created 000251
// INFO: [JOB 362] compacted(default) L0 [000245 000247] (2.7KB) Score=100.00 + L6 [000243] (1.4KB) Score=0.00 -> L6 [000251] (1.4KB), in 0.0s (0.0s total), output rate 5.0MB/s
// FATAL: checker failed with error: encountered tombstone wzfh-xrrvg@1:{(#799,RANGEDEL)} in memtable that has a lower seqnum than the same tombstone in memtable
// INFO: [JOB 363] flushing 2 memtables (359B) to L0
// INFO: [JOB 363] flushing: sstable created 000252
// INFO: [JOB 363] flushed 2 memtables (359B) to L0 [000252] (1.3KB), in 0.0s (0.0s total), output rate 6.9MB/s
// INFO: [JOB 364] WAL created 000253
// INFO: write stall ending
// INFO: [JOB 365] flushing 1 memtable (531B) to L0
// INFO: [JOB 365] flushing: sstable created 000254
// INFO: [JOB 365] flushed 1 memtable (531B) to L0 [000254] (1.2KB), in 0.0s (0.0s total), output rate 6.4MB/s
// INFO: [JOB 362] sstable deleted 000243
// INFO: [JOB 362] sstable deleted 000245
// INFO: [JOB 362] sstable deleted 000247
// INFO: [JOB 363] WAL deleted 000246
// INFO: [JOB 365] WAL deleted 000248
// INFO: [JOB 360] ingested L0:000250 (1.5KB)
db2.Replicate(db1, "dsyqb", "wzfh") // <nil> #7054
// INFO: [JOB 367] compacting(default) L0 [000252 000254] (2.6KB) Score=100.00 + L6 [000251] (1.4KB) Score=0.00; OverlappingRatio: Single 0.55, Multi 0.00
// INFO: [JOB 367] compacting: sstable created 000255
// INFO: [JOB 367] compacted(default) L0 [000252 000254] (2.6KB) Score=100.00 + L6 [000251] (1.4KB) Score=0.00 -> L6 [000255] (1.2KB), in 0.0s (0.0s total), output rate 4.3MB/s
// INFO: [JOB 367] sstable deleted 000251
// INFO: [JOB 367] sstable deleted 000252
// INFO: [JOB 367] sstable deleted 000254
db1.RangeKeyUnset("avjqzdfnr", "ppjgeiaoeqj", "") // <nil> #7058
To reduce: go test ./internal/metamorphic -tags invariants -run 'TestMetaTwoInstance$' --run-dir /artifacts/meta/240209-063400.3933891069011/standard-008 --try-to-reduce -v
--- FAIL: TestMetaTwoInstance/execution/standard-008 (0.75s)
Help
To reproduce, try:
go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run 'TestMetaTwoInstance$' ./internal/metamorphic -seed 1707460440392994789 -ops "uniform:5000-10000"
=== CONT TestMetaTwoInstance/execution/standard-008
meta.go:240:
===== SEED =====
1707460440392994789
===== ERR =====
exit status 1
===== OUT =====
Seed: 0
checker failed with error: encountered tombstone wzfh-xrrvg@1:{(#799,RANGEDEL)} in memtable that has a lower seqnum than the same tombstone in memtable
===== OPTIONS =====
[Version]
pebble_version=0.1
[Options]
bytes_per_sync=524288
cache_size=8388608
cleaner=archive
compaction_debt_concurrency=1073741824
comparer=pebble.internal.testkeys
disable_wal=false
flush_delay_delete_range=0s
flush_delay_range_key=0s
flush_split_bytes=4194304
format_major_version=13
l0_compaction_concurrency=10
l0_compaction_file_threshold=500
l0_compaction_threshold=4
l0_stop_writes_threshold=12
lbase_max_bytes=67108864
max_concurrent_compactions=1
max_manifest_file_size=134217728
max_open_files=1000
mem_table_size=2000
mem_table_stop_writes_threshold=2
min_deletion_rate=0
merger=pebble.concatenate
multilevel_compaction_heuristic=wamp(0.00, false)
read_compaction_rate=16000
read_sampling_multiplier=16
strict_wal_tail=true
table_cache_shards=8
validate_on_ingest=false
wal_dir=
wal_bytes_per_sync=0
max_writer_concurrency=0
force_writer_parallelism=false
secondary_cache_size_bytes=0
create_on_shared=0
[Level "0"]
block_restart_interval=16
block_size=4096
block_size_threshold=90
compression=Snappy
filter_policy=rocksdb.BuiltinBloomFilter
filter_type=table
index_block_size=4096
target_file_size=2097152
[TestOptions]
threads=16
===== OPS =====
github.com/cockroachdb/pebble/internal/metamorphic.TestMetaTwoInstance failed with artifacts on refs/heads/master @ 3e083df5434f:
iter55.Prev("bvtvahoez@6")
iter60.SetBounds("tnnehkpiro", "zanfkwiifx")
iter60.SeekGE("tnnehkpiro", "")
iter60.Prev("")
iter60.Prev("")
iter60.Next("")
iter53.Last()
iter53.SetBounds("ktpswfsec@2", "ssozcbit@5")
iter53.SeekLT("ssozcbit@5", "")
iter53.Next("")
iter53.First()
iter58.SeekGE("cstgtyddcq@2", "")
db1.Set("htprnoloxbq", "feutffekgijqgexgh")
db1.Delete("mwuhndci")
iter58.SeekGE("fryereku@4", "")
iter60.SeekLT("splv@1", "")
iter53.Next("")
iter60.Prev("")
iter58.Next("")
iter55.SetBounds("bgch@11", "bgch@11")
iter55.SeekGE("bgch@11", "")
iter55.Prev("")
iter55.Next("")
db1.Delete("demngs@5")
iter60.SeekGE("kavglhtrpua@3", "")
iter58.NextPrefix()
iter60.Last()
iter53.SetBounds("bjfefrqwo@3", "khxak")
iter53.SeekLT("khxak", "")
iter53.Prev("")
iter53.First()
db1.Set("ovcyh@15", "hcior")
iter60.SeekLT("xrkcngcec@7", "")
iter58.SeekGE("houxyk@9", "")
snap35.Get("cstgtyddcq@2")
db2.Get("fqcxueyje")
db2.RatchetFormatMajorVersion(017)
iter58.Prev("")
iter53.Close()
iter60.Close()
iter55.Close()
iter58.Close()
snap35.Close()
db1.Close()
db2.Close()
To reduce: go test ./internal/metamorphic -tags invariants -run 'TestMetaTwoInstance$' --compare "/artifacts/meta/240211-070019.7323657095505/{standard-000,standard-008}" --try-to-reduce -v
=== RUN TestMetaTwoInstance/execution/standard-015
=== PAUSE TestMetaTwoInstance/execution/standard-015
=== CONT TestMetaTwoInstance/execution/standard-015
Help
To reproduce, try:
go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run 'TestMetaTwoInstance$' ./internal/metamorphic -seed 1707634819732350543 -ops "uniform:5000-10000"github.com/cockroachdb/pebble/internal/metamorphic.TestMetaTwoInstance failed with artifacts on refs/heads/master @ 38e3430fe43b:
=== RUN TestMetaTwoInstance/execution/standard-022
=== PAUSE TestMetaTwoInstance/execution/standard-022
=== CONT TestMetaTwoInstance/execution/standard-022
=== RUN TestMetaTwoInstance/execution/random-007
=== PAUSE TestMetaTwoInstance/execution/random-007
=== CONT TestMetaTwoInstance/execution/random-007
=== RUN TestMetaTwoInstance/execution/random-009
=== PAUSE TestMetaTwoInstance/execution/random-009
=== CONT TestMetaTwoInstance/execution/random-009
=== RUN TestMetaTwoInstance/execution/random-022
=== PAUSE TestMetaTwoInstance/execution/random-022
=== CONT TestMetaTwoInstance/execution/random-022
=== RUN TestMetaTwoInstance/execution/standard-001
=== PAUSE TestMetaTwoInstance/execution/standard-001
=== CONT TestMetaTwoInstance/execution/standard-001
=== RUN TestMetaTwoInstance/execution/standard-019
=== PAUSE TestMetaTwoInstance/execution/standard-019
=== CONT TestMetaTwoInstance/execution/standard-019
=== RUN TestMetaTwoInstance/execution/standard-023
=== PAUSE TestMetaTwoInstance/execution/standard-023
=== CONT TestMetaTwoInstance/execution/standard-023
=== RUN TestMetaTwoInstance/compare/standard-002
=== RUN TestMetaTwoInstance/execution/random-006
=== PAUSE TestMetaTwoInstance/execution/random-006
=== CONT TestMetaTwoInstance/execution/random-006
=== RUN TestMetaTwoInstance/execution/random-011
=== PAUSE TestMetaTwoInstance/execution/random-011
=== CONT TestMetaTwoInstance/execution/random-011
=== RUN TestMetaTwoInstance/execution/standard-009
=== PAUSE TestMetaTwoInstance/execution/standard-009
=== CONT TestMetaTwoInstance/execution/standard-009
=== RUN TestMetaTwoInstance/execution/standard-012
=== PAUSE TestMetaTwoInstance/execution/standard-012
=== CONT TestMetaTwoInstance/execution/standard-012
=== RUN TestMetaTwoInstance/execution
=== RUN TestMetaTwoInstance/execution/random-004
=== PAUSE TestMetaTwoInstance/execution/random-004
=== CONT TestMetaTwoInstance/execution/random-004
=== RUN TestMetaTwoInstance/execution/standard-025
=== PAUSE TestMetaTwoInstance/execution/standard-025
=== CONT TestMetaTwoInstance/execution/standard-025
=== RUN TestMetaTwoInstance/compare
=== RUN TestMetaTwoInstance/execution/standard-017
=== PAUSE TestMetaTwoInstance/execution/standard-017
=== CONT TestMetaTwoInstance/execution/standard-017
=== RUN TestMetaTwoInstance/execution/standard-011
=== PAUSE TestMetaTwoInstance/execution/standard-011
=== CONT TestMetaTwoInstance/execution/standard-011
=== RUN TestMetaTwoInstance/compare/standard-003
=== RUN TestMetaTwoInstance/compare/standard-007
Help
To reproduce, try:
go test -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run 'TestMetaTwoInstance$' ./internal/metamorphic -seed 1707715236525809201 -ops "uniform:5000-10000"