dividePoly crashes on the attempt to add 8th point
elsid opened this issue · comments
This happens here because poly2Vert has value 7. There is a stack allocated buffer used here that does not provide enough capacity for this. Looking at adcd4f4 it seems it should not. So there is a problem in the logic leading to extra data being generated. I had to modify the code adding bounds check to get a snapshot right before the crash.
Stack trace:
Thread 20 (Thread 0x7fffaf7fe6c0 (LWP 104677) "openmw"):
#0 0x00007ffff0aac83c in () at /usr/lib/libc.so.6
#1 0x00007ffff0a5c668 in raise () at /usr/lib/libc.so.6
#2 0x00007ffff0a444b8 in abort () at /usr/lib/libc.so.6
#3 0x000055555957c2b6 in (anonymous namespace)::Span<float>::operator[](int) const (this=0x7fffaf7fcf10, index=21) at /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:247
#4 0x000055555957ae6f in dividePoly(float const*, int, float*, int*, float*, int*, float, rcAxis, float const*) (inVertsPtr=0x7fffaf7fd0c4, inVertsCount=7, outVerts1Ptr=0x7fffaf7fd118, outVerts1Count=0x7fffaf7fcfe0, outVerts2Ptr=0x7fffaf7fd16c, outVerts2Count=0x7fffaf7fcfe4, axisOffset=7228336.5, axis=RC_AXIS_X, outEnd=0x7fffaf7fd1c0) at /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:326
sameSide = true
inVertA = 6
inVertB = 5
__PRETTY_FUNCTION__ = "void dividePoly(const float*, int, float*, int*, float*, int*, float, rcAxis, const float*)"
inVerts = {values = 0x7fffaf7fd0c4, count = 21}
outVerts1 = {values = 0x7fffaf7fd118, count = 42}
outVerts2 = {values = 0x7fffaf7fd16c, count = 21}
inVertAxisDelta = {0, -1, -0.5, 0, 0, 0, 0, 4.59163468e-41, -2.32663888e-10, 4.59163468e-41, 7228368, 1.57225688e-42}
poly1Vert = 6
poly2Vert = 7
#5 0x000055555957b4c4 in rasterizeTri(float const*, float const*, float const*, unsigned char, rcHeightfield&, float const*, float const*, float, float, float, int) (v0=0x7fffaf7fd310, v1=0x7fffaf7fd31c, v2=0x7fffaf7fd328, areaID=1 '\001', heightfield=..., heightfieldBBMin=0x7fffaf7fd578, heightfieldBBMax=0x7fffaf7fd584, cellSize=0.200000003, inverseCellSize=5, inverseCellHeight=5, flagMergeThreshold=5) at /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:447
spanMinCellIndex = 282
spanMaxCellIndex = 283
cx = 7228336.5
spanMin = 56.4897079
spanMax = 56.4897079
x = 3
minX = 7228336.5
maxX = 7228337.5
x0 = 2
nv = 5
cellZ = 7228337.5
x1 = 7
nv2 = 7
z = 7
triBBMin = {7228336.5, -3.7455883, 7228336.5}
triBBMax = {7228368, -3.7455883, 7228368}
w = 160
h = 160
by = 60.2352943
z0 = 2
z1 = 159
bufSize = 84
buf = {7228337.5, -3.7455883, 7228337.5, 7228336.5, -3.7455883, 7228337.5, 7228336.5, -3.7455883, 7228368, 7228368, -3.7455883, 7228368, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337.5, 7228337.5, -3.7455883, 7228337.5, 7228337, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337.5, 7228336.5, -3.7455883, 7228337.5, 7228336.5, -3.7455883, 7228337.5, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337.5, -2.32665665e-10, 4.59163468e-41, 0, 7228336.5, -3.7455883, 7228337.5, 7228336.5, -3.7455883, 7228337.5, 7228337.5, -3.7455883, 7228337.5, 7228337, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337, 7228336.5, -3.7455883, 7228337}
in = 0x7fffaf7fd070
inRow = 0x7fffaf7fd0c4
bufEnd = 0x7fffaf7fd1c0
p1 = 0x7fffaf7fd118
p2 = 0x7fffaf7fd16c
nvRow = 6
nvIn = 4
#6 0x000055555957bb83 in rcRasterizeTriangles(rcContext*, float const*, int, int const*, unsigned char const*, int, rcHeightfield&, int) (context=0x7fffaf7fd540, verts=0x7fffaf7fd310, tris=0x7fffaf7fd2f0, triAreaIDs=0x7fffaf7fd2ee "\001\001", numTris=2, heightfield=..., flagMergeThreshold=5) at /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:541
v0 = 0x7fffaf7fd310
v1 = 0x7fffaf7fd31c
v2 = 0x7fffaf7fd328
triIndex = 0
__PRETTY_FUNCTION__ = "bool rcRasterizeTriangles(rcContext*, const float*, int, const int*, const unsigned char*, int, rcHeightfield&, int)"
timer = {m_ctx = 0x7fffaf7fd540, m_label = RC_TIMER_RASTERIZE_TRIANGLES}
inverseCellSize = 5
inverseCellHeight = 5
#7 0x0000555559469589 in DetourNavigator::(anonymous namespace)::rasterizeTriangles(DetourNavigator::RecastContext&, DetourNavigator::(anonymous namespace)::Rectangle const&, DetourNavigator::AreaType, DetourNavigator::(anonymous namespace)::RecastParams const&, rcHeightfield&) (context=..., rectangle=..., areaType=DetourNavigator::AreaType_water, params=..., solid=...) at /home/elsid/dev/openmw/components/detournavigator/makenavmesh.cpp:227
vertices = {_M_elems = {7228336.5, -3.7455883, 7228336.5, 7228336.5, -3.7455883, 7228368, 7228368, -3.7455883, 7228368, 7228368, -3.7455883, 7228336.5}}
indices = {_M_elems = {0, 1, 2, 0, 2, 3}}
areas = {_M_elems = "\001\001"}
#8 0x0000555559469727 in DetourNavigator::(anonymous namespace)::rasterizeTriangles(DetourNavigator::RecastContext&, float, std::vector<DetourNavigator::CellWater, std::allocator<DetourNavigator::CellWater> > const&, DetourNavigator::RecastSettings const&, DetourNavigator::(anonymous namespace)::RecastParams const&, DetourNavigator::TileBounds const&, rcHeightfield&) (context=..., agentHalfExtentsZ=66.5, water=..., settings=..., params=..., realTileBounds=..., solid=...) at /home/elsid/dev/openmw/components/detournavigator/makenavmesh.cpp:244
rectangle = {mBounds = {mMin = {_v = {7228336.5, 7228336.5}}, mMax = {_v = {7228368, 7228368}}}, mHeight = -3.7455883}
intersection = {<std::_Optional_base<DetourNavigator::TileBounds, true, true>> = {<std::_Optional_base_impl<DetourNavigator::TileBounds, std::_Optional_base<DetourNavigator::TileBounds, true, true> >> = {<No data fields>}, _M_payload = {<std::_Optional_payload_base<DetourNavigator::TileBounds>> = {_M_payload = {_M_empty = {<No data fields>}, _M_value = {mMin = {_v = {245763440, 245763440}}, mMax = {_v = {245764512, 245764512}}}}, _M_engaged = true}, <No data fields>}}, <std::_Enable_copy_move<true, true, true, true, std::optional<DetourNavigator::TileBounds> >> = {<No data fields>}, <No data fields>}
cellTileBounds = {mMin = {_v = {245760000, 245760000}}, mMax = {_v = {245768192, 245768192}}}
cellWater = @0x7fffa0000bb0: {mCellPosition = {_v = {30000, 30000}}, mWater = {mCellSize = 8192, mLevel = -1}}
__for_range = @0x7fffa0000cd8: {<std::_Vector_base<DetourNavigator::CellWater, std::allocator<DetourNavigator::CellWater> >> = {_M_impl = {<std::allocator<DetourNavigator::CellWater>> = {<std::__new_allocator<DetourNavigator::CellWater>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<DetourNavigator::CellWater, std::allocator<DetourNavigator::CellWater> >::_Vector_impl_data> = {_M_start = 0x7fffa0000bb0, _M_finish = 0x7fffa0000bc0, _M_end_of_storage = 0x7fffa0000bc0}, <No data fields>}}, <No data fields>}
__for_begin = {_M_current = 0x7fffa0000bb0}
__for_end = {_M_current = 0x7fffa0000bc0}
#9 0x0000555559469b42 in DetourNavigator::(anonymous namespace)::rasterizeTriangles(DetourNavigator::RecastContext&, DetourNavigator::TilePosition const&, float, DetourNavigator::RecastMesh const&, DetourNavigator::RecastSettings const&, DetourNavigator::(anonymous namespace)::RecastParams const&, rcHeightfield&) (context=..., tilePosition=..., agentHalfExtentsZ=66.5, recastMesh=..., settings=..., params=..., solid=...) at /home/elsid/dev/openmw/components/detournavigator/makenavmesh.cpp:287
realTileBounds = {mMin = {_v = {245763440, 245763440}}, mMax = {_v = {245764512, 245764512}}}
#10 0x000055555946aedd in DetourNavigator::prepareNavMeshTileData(DetourNavigator::RecastMesh const&, std::basic_string_view<char, std::char_traits<char> >, osg::Vec2i const&, DetourNavigator::AgentBounds const&, DetourNavigator::RecastSettings const&) (recastMesh=..., worldspace=..., tilePosition=..., agentBounds=..., settings=...) at /home/elsid/dev/openmw/components/detournavigator/makenavmesh.cpp:528
context = {<rcContext> = {_vptr.rcContext = 0x5555596d9290 <vtable for DetourNavigator::RecastContext+16>, m_logEnabled = true, m_timerEnabled = true}, mPrefix = {_M_dataplus = {<std::allocator<char>> = {<std::__new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fffa0000fa0 "Worldspace: sys::default; tile position: 282357, 282357; agent bounds: AgentBounds {AgentShapeType::Cylinder, {29.28, 28.48, 66.5}}; "}, _M_string_length = 133, {_M_local_buf = "\205\000\000\000\000\000\000\000(\260\204cUU\000", _M_allocated_capacity = 133}}}
minZ = @0x7fffaf7fd518: -2048
maxZ = @0x7fffaf7fd51c: 0
solid = {width = 160, height = 160, bmin = {7228336, -60.2352943, 7228336}, bmax = {7228368, 0, 7228368}, cs = 0.200000003, ch = 0.200000003, spans = 0x7fffa0001038, pools = 0x7fffa0033048, freelist = 0x7fffa0033100}
params = {mSampleDist = 1.20000005, mSampleMaxError = 0.200000003, mMaxEdgeLen = 60, mWalkableClimb = 5, mWalkableHeight = 20, mWalkableRadius = 5}
result = {_M_t = {<std::__uniq_ptr_impl<DetourNavigator::PreparedNavMeshData, std::default_delete<DetourNavigator::PreparedNavMeshData> >> = {_M_t = {<std::_Tuple_impl<0, DetourNavigator::PreparedNavMeshData*, std::default_delete<DetourNavigator::PreparedNavMeshData> >> = {<std::_Tuple_impl<1, std::default_delete<DetourNavigator::PreparedNavMeshData> >> = {<std::_Head_base<1, std::default_delete<DetourNavigator::PreparedNavMeshData>, true>> = {_M_head_impl = {<No data fields>}}, <No data fields>}, <std::_Head_base<0, DetourNavigator::PreparedNavMeshData*, false>> = {_M_head_impl = 0x7fffaf7fd520}, <No data fields>}, <No data fields>}}, <No data fields>}}
#11 0x0000555559450883 in DetourNavigator::AsyncNavMeshUpdater::processInitialJob(DetourNavigator::Job&, Misc::ScopeGuarded<DetourNavigator::NavMeshCacheItem>&) (this=0x55555a8dfbc0, job=..., navMeshCacheItem=...) at /home/elsid/dev/openmw/components/detournavigator/asyncnavmeshupdater.cpp:437
recastMesh = {<std::__shared_ptr<DetourNavigator::RecastMesh, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<DetourNavigator::RecastMesh, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fffa0000c80, _M_refcount = {_M_pi = 0x7fffa0000c70}}, <No data fields>}
cachedNavMeshData = {mOwner = 0x0, mIterator = {_M_node = 0x0}}
preparedNavMeshData = {_M_t = {<std::__uniq_ptr_impl<DetourNavigator::PreparedNavMeshData, std::default_delete<DetourNavigator::PreparedNavMeshData> >> = {_M_t = {<std::_Tuple_impl<0, DetourNavigator::PreparedNavMeshData*, std::default_delete<DetourNavigator::PreparedNavMeshData> >> = {<std::_Tuple_impl<1, std::default_delete<DetourNavigator::PreparedNavMeshData> >> = {<std::_Head_base<1, std::default_delete<DetourNavigator::PreparedNavMeshData>, true>> = {_M_head_impl = {<No data fields>}}, <No data fields>}, <std::_Head_base<0, DetourNavigator::PreparedNavMeshData*, false>> = {_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}, <No data fields>}}
preparedNavMeshDataPtr = 0x0
offMeshConnections = {<std::_Vector_base<DetourNavigator::OffMeshConnection, std::allocator<DetourNavigator::OffMeshConnection> >> = {_M_impl = {<std::allocator<DetourNavigator::OffMeshConnection>> = {<std::__new_allocator<DetourNavigator::OffMeshConnection>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<DetourNavigator::OffMeshConnection, std::allocator<DetourNavigator::OffMeshConnection> >::_Vector_impl_data> = {_M_start = 0x7fffaf7fd690, _M_finish = 0x55555944f6dd <DetourNavigator::getDistance(osg::Vec2i const&, osg::Vec2i const&)+66>, _M_end_of_storage = 0x7fffaf7fd730}, <No data fields>}}, <No data fields>}
__PRETTY_FUNCTION__ = "DetourNavigator::JobStatus DetourNavigator::AsyncNavMeshUpdater::processInitialJob(DetourNavigator::Job&, DetourNavigator::GuardedNavMeshCacheItem&)"
status = 21845
#12 0x00005555594489c6 in DetourNavigator::AsyncNavMeshUpdater::processJob(DetourNavigator::Job&) (this=0x55555a8dfbc0, job=...) at /home/elsid/dev/openmw/components/detournavigator/asyncnavmeshupdater.cpp:392
navMeshCacheItem = {<std::__shared_ptr<Misc::ScopeGuarded<DetourNavigator::NavMeshCacheItem>, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<Misc::ScopeGuarded<DetourNavigator::NavMeshCacheItem>, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x55556003e650, _M_refcount = {_M_pi = 0x55556003e640}}, <No data fields>}
playerTile = {_v = {282357, 282357}}
maxTiles = 512
#13 0x00005555594484a2 in DetourNavigator::AsyncNavMeshUpdater::process() (this=0x55555a8dfbc0) at /home/elsid/dev/openmw/components/detournavigator/asyncnavmeshupdater.cpp:337
status = (unknown: 0x5a8dfc98)
job = {_M_node = 0x55556384aff0}
#14 0x0000555559447397 in operator()() const (__closure=0x55555a7846d8) at /home/elsid/dev/openmw/components/detournavigator/asyncnavmeshupdater.cpp:161
this = 0x55555a8dfbc0
i = 140736137779152
#15 0x000055555944edc5 in std::__invoke_impl<void, DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> >(std::__invoke_other, struct {...} &&) (__f=...) at /usr/include/c++/13.2.1/bits/invoke.h:61
#16 0x000055555944ed35 in std::__invoke<DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> >(struct {...} &&) (__fn=...) at /usr/include/c++/13.2.1/bits/invoke.h:96
#17 0x000055555944ecb6 in std::thread::_Invoker<std::tuple<DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55555a7846d8) at /usr/include/c++/13.2.1/bits/std_thread.h:292
#18 0x000055555944ec6e in std::thread::_Invoker<std::tuple<DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> > >::operator()(void) (this=0x55555a7846d8) at /usr/include/c++/13.2.1/bits/std_thread.h:299
#19 0x000055555944ec32 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> > > >::_M_run(void) (this=0x55555a7846d0) at /usr/include/c++/13.2.1/bits/std_thread.h:244
#20 0x00007ffff0ce1943 in std::execute_native_thread_routine(void*) (__p=0x55555a7846d0) at /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:104
#21 0x00007ffff0aaa9eb in () at /usr/lib/libc.so.6
#22 0x00007ffff0b2e7cc in () at /usr/lib/libc.so.6
Original AddressSanitizer report:
==103277==ERROR: AddressSanitizer: stack-buffer-overflow on address 0x7efdeabbc280 at pc 0x55afc201ec3f bp 0x7efded0cb480 sp 0x7efded0cb470
WRITE of size 4 at 0x7efdeabbc280 thread T18
#0 0x55afc201ec3e in rcVcopy(float*, float const*) /home/elsid/dev/recastnavigation/build/gcc/asan/install/include/recastnavigation/Recast.h:774
#1 0x55afc2296ce0 in dividePoly /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:281
#2 0x55afc2297c46 in rasterizeTri /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:400
#3 0x55afc22989bf in rcRasterizeTriangles(rcContext*, float const*, int, int const*, unsigned char const*, int, rcHeightfield&, int) /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:494
#4 0x55afc2018e60 in rasterizeTriangles /home/elsid/dev/openmw/components/detournavigator/makenavmesh.cpp:227
#5 0x55afc2019346 in rasterizeTriangles /home/elsid/dev/openmw/components/detournavigator/makenavmesh.cpp:244
#6 0x55afc2019f3b in rasterizeTriangles /home/elsid/dev/openmw/components/detournavigator/makenavmesh.cpp:287
#7 0x55afc201c8f7 in DetourNavigator::prepareNavMeshTileData(DetourNavigator::RecastMesh const&, std::basic_string_view<char, std::char_traits<char> >, osg::Vec2i const&, DetourNavigator::AgentBounds const&, DetourNavigator::RecastSettings const&) /home/elsid/dev/openmw/components/detournavigator/makenavmesh.cpp:528
#8 0x55afc1fde356 in DetourNavigator::AsyncNavMeshUpdater::processInitialJob(DetourNavigator::Job&, Misc::ScopeGuarded<DetourNavigator::NavMeshCacheItem>&) /home/elsid/dev/openmw/components/detournavigator/asyncnavmeshupdater.cpp:437
#9 0x55afc1fc81c1 in DetourNavigator::AsyncNavMeshUpdater::processJob(DetourNavigator::Job&) /home/elsid/dev/openmw/components/detournavigator/asyncnavmeshupdater.cpp:392
#10 0x55afc1fc7463 in DetourNavigator::AsyncNavMeshUpdater::process() /home/elsid/dev/openmw/components/detournavigator/asyncnavmeshupdater.cpp:337
#11 0x55afc1fc4e2d in operator() /home/elsid/dev/openmw/components/detournavigator/asyncnavmeshupdater.cpp:161
#12 0x55afc1fda8ae in __invoke_impl<void, DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> > /usr/include/c++/13.2.1/bits/invoke.h:61
#13 0x55afc1fda81e in __invoke<DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> > /usr/include/c++/13.2.1/bits/invoke.h:96
#14 0x55afc1fda79f in _M_invoke<0> /usr/include/c++/13.2.1/bits/std_thread.h:292
#15 0x55afc1fda757 in operator() /usr/include/c++/13.2.1/bits/std_thread.h:299
#16 0x55afc1fda71b in _M_run /usr/include/c++/13.2.1/bits/std_thread.h:244
#17 0x7efe23ae1942 in execute_native_thread_routine /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:104
#18 0x7efe237bd9ea (/usr/lib/libc.so.6+0x8c9ea) (BuildId: 8bfe03f6bf9b6a6e2591babd0bbc266837d8f658)
#19 0x7efe238417cb (/usr/lib/libc.so.6+0x1107cb) (BuildId: 8bfe03f6bf9b6a6e2591babd0bbc266837d8f658)
Address 0x7efdeabbc280 is located in stack of thread T18 at offset 640 in frame
#0 0x55afc2296e18 in rasterizeTri /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:311
This frame has 11 object(s):
[48, 52) 'nvRow' (line 351)
[64, 68) 'nvIn' (line 352)
[80, 84) 'nv' (line 393)
[96, 100) 'nv2' (line 394)
[112, 120) 'in' (line 343)
[144, 152) 'inRow' (line 344)
[176, 184) 'p1' (line 345)
[208, 216) 'p2' (line 346)
[240, 252) 'triBBMin' (line 313)
[272, 284) 'triBBMax' (line 318)
[304, 640) 'buf' (line 342) <== Memory access at offset 640 overflows this variable
HINT: this may be a false positive if your program uses some custom stack unwind mechanism, swapcontext or vfork
(longjmp and C++ exceptions *are* supported)
Thread T18 created by T0 here:
#0 0x7efe2b04a497 in __interceptor_pthread_create /usr/src/debug/gcc/gcc/libsanitizer/asan/asan_interceptors.cpp:208
#1 0x7efe23ae1a29 in __gthread_create /usr/src/debug/gcc/gcc-build/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:663
#2 0x7efe23ae1a29 in std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:172
#3 0x55afc1fd0292 in construct_at<std::thread, DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> > /usr/include/c++/13.2.1/bits/stl_construct.h:97
#4 0x55afc1fcefac in construct<std::thread, DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> > /usr/include/c++/13.2.1/bits/alloc_traits.h:539
#5 0x55afc1fcefac in _M_realloc_insert<DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> > /usr/include/c++/13.2.1/bits/vector.tcc:468
#6 0x55afc1fcd693 in emplace_back<DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(const DetourNavigator::Settings&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb>&&)::<lambda()> > /usr/include/c++/13.2.1/bits/vector.tcc:123
#7 0x55afc1fc51c1 in DetourNavigator::AsyncNavMeshUpdater::AsyncNavMeshUpdater(DetourNavigator::Settings const&, DetourNavigator::TileCachedRecastMeshManager&, DetourNavigator::OffMeshConnectionsManager&, std::unique_ptr<DetourNavigator::NavMeshDb, std::default_delete<DetourNavigator::NavMeshDb> >&&) /home/elsid/dev/openmw/components/detournavigator/asyncnavmeshupdater.cpp:161
#8 0x55afc20685f5 in DetourNavigator::NavMeshManager::NavMeshManager(DetourNavigator::Settings const&, std::unique_ptr<DetourNavigator::NavMeshDb, std::default_delete<DetourNavigator::NavMeshDb> >&&) /home/elsid/dev/openmw/components/detournavigator/navmeshmanager.cpp:59
#9 0x55afc203026d in DetourNavigator::NavigatorImpl::NavigatorImpl(DetourNavigator::Settings const&, std::unique_ptr<DetourNavigator::NavMeshDb, std::default_delete<DetourNavigator::NavMeshDb> >&&) /home/elsid/dev/openmw/components/detournavigator/navigatorimpl.cpp:14
#10 0x55afc2025578 in std::__detail::_MakeUniq<DetourNavigator::NavigatorImpl>::__single_object std::make_unique<DetourNavigator::NavigatorImpl, DetourNavigator::Settings const&, std::unique_ptr<DetourNavigator::NavMeshDb, std::default_delete<DetourNavigator::NavMeshDb> > >(DetourNavigator::Settings const&, std::unique_ptr<DetourNavigator::NavMeshDb, std::default_delete<DetourNavigator::NavMeshDb> >&&) /usr/include/c++/13.2.1/bits/unique_ptr.h:1070
#11 0x55afc20238d6 in DetourNavigator::makeNavigator(DetourNavigator::Settings const&, std::filesystem::__cxx11::path const&) /home/elsid/dev/openmw/components/detournavigator/navigator.cpp:30
#12 0x55afbfea574a in MWWorld::World::init(osgViewer::Viewer*, osg::ref_ptr<osg::Group>, SceneUtil::WorkQueue*, SceneUtil::UnrefQueue&) /home/elsid/dev/openmw/apps/openmw/mwworld/worldimp.cpp:302
#13 0x55afc0b05a25 in OMW::Engine::prepareEngine() /home/elsid/dev/openmw/apps/openmw/engine.cpp:826
#14 0x55afc0b08147 in OMW::Engine::go() /home/elsid/dev/openmw/apps/openmw/engine.cpp:929
#15 0x55afc0af071e in runApplication(int, char**) /home/elsid/dev/openmw/apps/openmw/main.cpp:231
#16 0x55afc1ce6d62 in wrapApplication(int (*)(int, char**), int, char**, std::basic_string_view<char, std::char_traits<char> >) /home/elsid/dev/openmw/components/debug/debugging.cpp:361
#17 0x55afc0af094d in main /home/elsid/dev/openmw/apps/openmw/main.cpp:243
#18 0x7efe23758ccf (/usr/lib/libc.so.6+0x27ccf) (BuildId: 8bfe03f6bf9b6a6e2591babd0bbc266837d8f658)
SUMMARY: AddressSanitizer: stack-buffer-overflow /home/elsid/dev/recastnavigation/build/gcc/asan/install/include/recastnavigation/Recast.h:774 in rcVcopy(float*, float const*)
Shadow bytes around the buggy address:
0x7efdeabbc000: f1 f1 f1 f1 f1 f1 04 f2 04 f2 04 f2 04 f2 00 f2
0x7efdeabbc080: f2 f2 00 f2 f2 f2 00 f2 f2 f2 00 f2 f2 f2 00 04
0x7efdeabbc100: f2 f2 00 04 f2 f2 00 00 00 00 00 00 00 00 00 00
0x7efdeabbc180: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x7efdeabbc200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
=>0x7efdeabbc280:[f3]f3 f3 f3 f3 f3 f3 f3 00 00 00 00 00 00 00 00
0x7efdeabbc300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x7efdeabbc380: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x7efdeabbc400: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x7efdeabbc480: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x7efdeabbc500: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
Shadow byte legend (one shadow byte represents 8 application bytes):
Addressable: 00
Partially addressable: 01 02 03 04 05 06 07
Heap left redzone: fa
Freed heap region: fd
Stack left redzone: f1
Stack mid redzone: f2
Stack right redzone: f3
Stack after return: f5
Stack use after scope: f8
Global redzone: f9
Global init order: f6
Poisoned by user: f7
Container overflow: fc
Array cookie: ac
Intra object redzone: bb
ASan internal: fe
Left alloca redzone: ca
Right alloca redzone: cb
==103277==ABORTING
Here is unit test reproducing the issue: elsid@3e115d7
AddressSanitizer report:
=================================================================
==18517==ERROR: AddressSanitizer: stack-buffer-overflow on address 0x75c481831280 at pc 0x57523532ca07 bp 0x7ffee556b7a0 sp 0x7ffee556b790
WRITE of size 4 at 0x75c481831280 thread T0
#0 0x57523532ca06 in rcVcopy(float*, float const*) /home/elsid/dev/recastnavigation/Tests/../Recast/Include/Recast.h:774
#1 0x5752355509ea in dividePoly /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:281
#2 0x575235551950 in rasterizeTri /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:400
#3 0x5752355526c9 in rcRasterizeTriangles(rcContext*, float const*, int, int const*, unsigned char const*, int, rcHeightfield&, int) /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:494
#4 0x575235325137 in CATCH2_INTERNAL_TEST_107 /home/elsid/dev/recastnavigation/Tests/Recast/Tests_Recast.cpp:967
#5 0x57523539cb50 in invoke /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_amalgamated.cpp:6571
#6 0x57523540cb60 in Catch::TestCaseHandle::invoke() const /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_all.hpp:7210
#7 0x575235391215 in Catch::RunContext::invokeActiveTestCase() /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_amalgamated.cpp:5613
#8 0x575235390a94 in Catch::RunContext::runCurrentTest(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_amalgamated.cpp:5576
#9 0x57523538c5d4 in Catch::RunContext::runTest(Catch::TestCaseHandle const&) /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_amalgamated.cpp:5300
#10 0x575235353fb5 in execute /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_amalgamated.cpp:1102
#11 0x575235356c7b in Catch::Session::runInternal() /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_amalgamated.cpp:1324
#12 0x57523535620c in Catch::Session::run() /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_amalgamated.cpp:1255
#13 0x5752354340fb in int Catch::Session::run<char>(int, char const* const*) /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_all.hpp:5166
#14 0x575235385106 in main /home/elsid/dev/recastnavigation/Tests/Contrib/catch2/catch_amalgamated.cpp:4450
#15 0x75c484245ccf (/usr/lib/libc.so.6+0x27ccf) (BuildId: 8bfe03f6bf9b6a6e2591babd0bbc266837d8f658)
#16 0x75c484245d89 in __libc_start_main (/usr/lib/libc.so.6+0x27d89) (BuildId: 8bfe03f6bf9b6a6e2591babd0bbc266837d8f658)
#17 0x5752352a8874 in _start (/home/elsid/dev/recastnavigation/build/gcc/asan/Tests/Tests+0x228874) (BuildId: e00578540157788430dcf2f4edccb9160f408bd1)
Address 0x75c481831280 is located in stack of thread T0 at offset 640 in frame
#0 0x575235550b22 in rasterizeTri /home/elsid/dev/recastnavigation/Recast/Source/RecastRasterization.cpp:311
This frame has 11 object(s):
[48, 52) 'nvRow' (line 351)
[64, 68) 'nvIn' (line 352)
[80, 84) 'nv' (line 393)
[96, 100) 'nv2' (line 394)
[112, 120) 'in' (line 343)
[144, 152) 'inRow' (line 344)
[176, 184) 'p1' (line 345)
[208, 216) 'p2' (line 346)
[240, 252) 'triBBMin' (line 313)
[272, 284) 'triBBMax' (line 318)
[304, 640) 'buf' (line 342) <== Memory access at offset 640 overflows this variable
HINT: this may be a false positive if your program uses some custom stack unwind mechanism, swapcontext or vfork
(longjmp and C++ exceptions *are* supported)
SUMMARY: AddressSanitizer: stack-buffer-overflow /home/elsid/dev/recastnavigation/Tests/../Recast/Include/Recast.h:774 in rcVcopy(float*, float const*)
Shadow bytes around the buggy address:
0x75c481831000: f1 f1 f1 f1 f1 f1 04 f2 04 f2 04 f2 04 f2 00 f2
0x75c481831080: f2 f2 00 f2 f2 f2 00 f2 f2 f2 00 f2 f2 f2 00 04
0x75c481831100: f2 f2 00 04 f2 f2 00 00 00 00 00 00 00 00 00 00
0x75c481831180: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x75c481831200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
=>0x75c481831280:[f3]f3 f3 f3 f3 f3 f3 f3 00 00 00 00 00 00 00 00
0x75c481831300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x75c481831380: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x75c481831400: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x75c481831480: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
0x75c481831500: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
Shadow byte legend (one shadow byte represents 8 application bytes):
Addressable: 00
Partially addressable: 01 02 03 04 05 06 07
Heap left redzone: fa
Freed heap region: fd
Stack left redzone: f1
Stack mid redzone: f2
Stack right redzone: f3
Stack after return: f5
Stack use after scope: f8
Global redzone: f9
Global init order: f6
Poisoned by user: f7
Container overflow: fc
Array cookie: ac
Intra object redzone: bb
ASan internal: fe
Left alloca redzone: ca
Right alloca redzone: cb
==18517==ABORTING
It seems like this is related to the floating point limitations. 7228336 used for input is >= (1 << 22) in other words 23 significant bits are required for the fraction part and it's a borderline for the 32-bit float. If I add tests to use values starting with 1 << 21 and with 1 << 22, first one passes and second one leads to crash. In 1 << 22 case and the original one this expression gives subsequently the same result for some iterations and all different for 1 << 21:
SECTION("22 significant bits")
{
const int shift = 21;
rcContext ctx;
const float verts[] = {
static_cast<float>(1 << shift), -3.7455883f, static_cast<float>(1 << shift),
static_cast<float>(1 << shift), -3.7455883f, static_cast<float>(1 << shift) + 32,
static_cast<float>(1 << shift) + 32, -3.7455883f, static_cast<float>(1 << shift) + 32,
static_cast<float>(1 << shift) + 32, -3.7455883f, static_cast<float>(1 << shift)
};
const int numVerts = 4;
const int tris[] = {
0, 1, 2,
0, 2, 3
};
const unsigned char triAreaIDs[] = {1, 2};
const int numTris = 2;
const float bmin[3] = {static_cast<float>(1 << shift), -60.2352943f,static_cast<float>(1 << shift)};
const float bmax[3] = {static_cast<float>(1 << shift) + 32, 0.0f, static_cast<float>(1 << shift) + 32};
const float cellSize = 0.200000003f;
const float cellHeight = 0.200000003f;
const int width = 160;
const int height = 160;
rcHeightfield solid;
REQUIRE(rcCreateHeightfield(&ctx, solid, width, height, bmin, bmax, cellSize, cellHeight));
const int flagMergeThreshold = 1;
REQUIRE(rcRasterizeTriangles(&ctx, verts, numVerts, tris, triAreaIDs, numTris, solid, flagMergeThreshold));
}