dividePoly crashes on the attempt to add 8th point #687
Comments
|
Here is unit test reproducing the issue: elsid@3e115d7 AddressSanitizer report: |
|
It seems like this is related to the floating point limitations. SECTION("22 significant bits")
{
const int shift = 21;
rcContext ctx;
const float verts[] = {
static_cast<float>(1 << shift), -3.7455883f, static_cast<float>(1 << shift),
static_cast<float>(1 << shift), -3.7455883f, static_cast<float>(1 << shift) + 32,
static_cast<float>(1 << shift) + 32, -3.7455883f, static_cast<float>(1 << shift) + 32,
static_cast<float>(1 << shift) + 32, -3.7455883f, static_cast<float>(1 << shift)
};
const int numVerts = 4;
const int tris[] = {
0, 1, 2,
0, 2, 3
};
const unsigned char triAreaIDs[] = {1, 2};
const int numTris = 2;
const float bmin[3] = {static_cast<float>(1 << shift), -60.2352943f,static_cast<float>(1 << shift)};
const float bmax[3] = {static_cast<float>(1 << shift) + 32, 0.0f, static_cast<float>(1 << shift) + 32};
const float cellSize = 0.200000003f;
const float cellHeight = 0.200000003f;
const int width = 160;
const int height = 160;
rcHeightfield solid;
REQUIRE(rcCreateHeightfield(&ctx, solid, width, height, bmin, bmax, cellSize, cellHeight));
const int flagMergeThreshold = 1;
REQUIRE(rcRasterizeTriangles(&ctx, verts, numVerts, tris, triAreaIDs, numTris, solid, flagMergeThreshold));
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This happens here because
poly2Verthas value 7. There is a stack allocated buffer used here that does not provide enough capacity for this. Looking at adcd4f4 it seems it should not. So there is a problem in the logic leading to extra data being generated. I had to modify the code adding bounds check to get a snapshot right before the crash.Stack trace:
Original AddressSanitizer report:
The text was updated successfully, but these errors were encountered: