apple / foundationdb

Code location and commit hash:

foundationdb/fdbserver/worker.actor.cpp

Line 1508 in d47a2ab

std::string type = role.roleName + "Failed";

Ensemble ID: 20230121-044637-nightly_correctness_main_x86_64-460a17ab47cd772b

Information to Reproduce: BuggifyEnabled="1" DeterminismCheck="0" FaultInjectionEnabled="1" TestFile="tests/fast/GetMappedRange.toml"

Trace snippet:

  <StorageServerFailed Severity="40" ErrorKind="Unset" Time="163.692544" DateTime="2023-01-21T05:55:41Z" Type="StorageServerFailed" Machine="[abcd::2:1:1:0]:1" ID="52df2300dbdbe2ae" Error="tenants_disabled" ErrorDescription="Tenants have been disabled in the cluster" ErrorCode="2136" Reason="Error" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2f3e9a1 0x2f51f36 0x2f51b0e 0x2f50142 0x2f505a6 0x1a2fa09 0x2f643ad 0x2f63f8b 0x2008b78 0x2008ff6 0x1a2fa09 0x2f50823 0x1a42858 0x2447d76 0x24478cc 0x24467e4 0x2447428 0x2447293 0x2008ea8 0x2008a84 0x1a42858 0x2461273 0x246115b 0x245bc94 0x2450f08 0x4209248 0x4208f44 0x2a7a578 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default" Roles="LR,SS,TL"/>
  <StorageServerFailed Severity="40" ErrorKind="Unset" Time="164.306793" DateTime="2023-01-21T05:55:41Z" Type="StorageServerFailed" Machine="[abcd::2:0:1:1]:1" ID="778ac1ba5ea1ed3c" Error="tenants_disabled" ErrorDescription="Tenants have been disabled in the cluster" ErrorCode="2136" Reason="Error" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2f3e9a1 0x2f51f36 0x2f51b0e 0x2f50142 0x2f505a6 0x1a2fa09 0x2f643ad 0x2f63f8b 0x2008b78 0x2008ff6 0x1a2fa09 0x2f50823 0x1a42858 0x2447d76 0x24478cc 0x24467e4 0x2447428 0x2447293 0x2008ea8 0x2008a84 0x1a42858 0x2461273 0x246115b 0x245bc94 0x2450f08 0x4209248 0x4208f44 0x2a7a578 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default" Roles="CP,CS,DD,GP,LR,MS,RK,SS,TL"/>
  <TestFailure Severity="40" ErrorKind="Unset" Time="15152.586655" DateTime="2023-01-21T06:01:35Z" Type="TestFailure" Machine="0.0.0.0:0" ID="0000000000000000" Error="timed_out" ErrorDescription="Operation timed out" ErrorCode="1004" Reason="Test timed out" Timeout="15000" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2f05397 0x2f08803 0x2ef7649 0x2f05a04 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default"/>
  <TesterFailed Severity="40" ErrorKind="Unset" Time="15153.587316" DateTime="2023-01-21T06:01:35Z" Type="TesterFailed" Machine="[abcd::3:4:3:8]:1" ID="52da8e21af62ef31" Error="timed_out" ErrorDescription="Operation timed out" ErrorCode="1004" Reason="Error" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2f3e9a1 0x2ee8cf1 0x2eeba37 0x1a2fa09 0x2127f24 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default" Roles="TS"/>
  <TesterFailed Severity="40" ErrorKind="Unset" Time="15153.587766" DateTime="2023-01-21T06:01:35Z" Type="TesterFailed" Machine="[abcd::3:4:3:4]:1" ID="361c7add963ca949" Error="timed_out" ErrorDescription="Operation timed out" ErrorCode="1004" Reason="Error" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2f3e9a1 0x2ee8cf1 0x2eeba37 0x1a2fa09 0x2127f24 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default" Roles="TS"/>
  <TesterFailed Severity="40" ErrorKind="Unset" Time="15153.587780" DateTime="2023-01-21T06:01:35Z" Type="TesterFailed" Machine="[abcd::3:4:3:1]:1" ID="6870c8483a2085bb" Error="timed_out" ErrorDescription="Operation timed out" ErrorCode="1004" Reason="Error" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2f3e9a1 0x2ee8cf1 0x2eeba37 0x1a2fa09 0x2127f24 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default" Roles="TS"/>
  <TesterFailed Severity="40" ErrorKind="Unset" Time="15153.587799" DateTime="2023-01-21T06:01:35Z" Type="TesterFailed" Machine="[abcd::3:4:3:7]:1" ID="82b3d27f64fd9e63" Error="timed_out" ErrorDescription="Operation timed out" ErrorCode="1004" Reason="Error" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2f3e9a1 0x2ee8cf1 0x2eeba37 0x1a2fa09 0x2127f24 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default" Roles="TS"/>
  <TesterFailed Severity="40" ErrorKind="Unset" Time="15153.587926" DateTime="2023-01-21T06:01:35Z" Type="TesterFailed" Machine="[abcd::3:4:3:5]:1" ID="64fc9d35067f35af" Error="timed_out" ErrorDescription="Operation timed out" ErrorCode="1004" Reason="Error" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2f3e9a1 0x2ee8cf1 0x2eeba37 0x1a2fa09 0x2127f24 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default" Roles="TS"/>
  <TesterFailed Severity="40" ErrorKind="Unset" Time="15153.588115" DateTime="2023-01-21T06:01:35Z" Type="TesterFailed" Machine="[abcd::3:4:3:2]:1" ID="0138304f40306c81" Error="timed_out" ErrorDescription="Operation timed out" ErrorCode="1004" Reason="Error" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2f3e9a1 0x2ee8cf1 0x2eeba37 0x1a2fa09 0x2127f24 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default" Roles="TS"/>
  <TestFailure Severity="40" ErrorKind="Unset" Time="15153.588599" DateTime="2023-01-21T06:01:35Z" Type="TestFailure" Machine="[abcd::3:4:3:6]:1" ID="8981946ddb6df391" Error="operation_cancelled" ErrorDescription="Asynchronous operation cancelled" ErrorCode="1101" Reason="Error starting workload" Workload="GetMappedRange" ThreadID="13263738797810762438" Backtrace="addr2line -e fdbserver.debug -p -C -f -i 0x430a4d1 0x2ee641c 0x2ee838f 0x2ee256a 0x1af4106 0x1af3852 0x2127f18 0x1a42858 0x41ed615 0x2d48477 0x7fd7fb9cc555" LogGroup="default" Roles="TS"/>
  <WarningLimitExceeded Severity="30" WarningCount="22561"/>

I think this is a duplicate of #9198

fast/GetMappedRange.toml runs into StorageServerFailed by tenants_disabled