Fix isPointInTriangle in 3d (#6882) (#7106)

* Fix isPointInTriangle in 3d. With tests (#6882)
Solution have been find by tylerhasman & tommyettinger but never added

* Spotless formatting and spelling/grammar edits.

---------

Co-authored-by: Tommy Ettinger <tommy.ettinger@gmail.com>

gdx/src/com/badlogic/gdx/math/Intersector.java

@@ -41,7 +41,10 @@ private Intersector () {
 	private final static FloatArray floatArray2 = new FloatArray();
 
 	/** Returns whether the given point is inside the triangle. This assumes that the point is on the plane of the triangle. No
-	 * check is performed that this is the case.
+	 * check is performed that this is the case. <br>
+	 * If the Vector3 parameters contain both small and large values, such as one that contains 0.0001 and one that contains
+	 * 10000000.0, this can fail due to floating-point imprecision.
+	 * 
 	 * @param t1 the first vertex of the triangle
 	 * @param t2 the second vertex of the triangle
 	 * @param t3 the third vertex of the triangle
@@ -51,15 +54,12 @@ public static boolean isPointInTriangle (Vector3 point, Vector3 t1, Vector3 t2,
 		v1.set(t2).sub(point);
 		v2.set(t3).sub(point);
 
-		float ab = v0.dot(v1);
-		float ac = v0.dot(v2);
-		float bc = v1.dot(v2);
-		float cc = v2.dot(v2);
+		v1.crs(v2);
+		v2.crs(v0);
 
-		if (bc * ac - cc * ab < 0) return false;
-		float bb = v1.dot(v1);
-		if (ab * bc - ac * bb < 0) return false;
-		return true;
+		if (v1.dot(v2) < 0f) return false;
+		v0.crs(v2.set(t2).sub(point));
+		return (v1.dot(v0) >= 0f);
 	}
 
 	/** Returns true if the given point is inside the triangle. */

gdx/test/com/badlogic/gdx/math/IntersectorTest.java

@@ -183,4 +183,44 @@ public void testIntersectPlanes () {
 		assertEquals(-57.7337f, intersection.y, 0.1f);
 		assertEquals(100, intersection.z, 0.1f);
 	}
+
+	@Test
+	public void testIsPointInTriangle2D () {
+		assertFalse(Intersector.isPointInTriangle(new Vector2(0.1f, 0), new Vector2(0, 0), new Vector2(1, 1), new Vector2(-1, -1)));
+
+		assertTrue(Intersector.isPointInTriangle(new Vector2(0, 0.1f), new Vector2(-1, 1), new Vector2(1, 1), new Vector2(-1, -2)));
+	}
+
+	@Test
+	public void testIsPointInTriangle3D () {
+		// 2D ---
+		assertFalse(Intersector.isPointInTriangle(new Vector3(0.1f, 0, 0), new Vector3(0, 0, 0), new Vector3(1, 1, 0),
+			new Vector3(-1, -1, 0)));
+
+		assertTrue(Intersector.isPointInTriangle(new Vector3(0, 0.1f, 0), new Vector3(-1, 1, 0), new Vector3(1, 1, 0),
+			new Vector3(-1, -2, 0)));
+
+		// 3D ---
+		assertTrue(Intersector.isPointInTriangle(new Vector3(0.2f, 0, 1.25f), new Vector3(-1, 1, 0), new Vector3(1.4f, 0.99f, 2.5f),
+			new Vector3(-1, -2, 0)));
+		// 1.2f away.
+		assertFalse(Intersector.isPointInTriangle(new Vector3(2.6f, 0, 3.75f), new Vector3(-1, 1, 0),
+			new Vector3(1.4f, 0.99f, 2.5f), new Vector3(-1, -2, 0)));
+		// In an edge.
+		assertTrue(Intersector.isPointInTriangle(new Vector3(0, -0.5f, 0.5f), new Vector3(-1, 1, 0), new Vector3(1, 1, 1),
+			new Vector3(-1, -2, 0)));
+		// Really close to the edge.
+		float epsilon = 0.0000001f; // One more 0 will fail.
+		float almost1 = 1 - epsilon;
+		assertFalse(Intersector.isPointInTriangle(new Vector3(0, -0.5f, 0.5f), new Vector3(-1, 1, 0), new Vector3(almost1, 1, 1),
+			new Vector3(-1, -2, 0)));
+
+		// A really long distance away.
+		assertFalse(Intersector.isPointInTriangle(new Vector3(199f, 1f, 500f), new Vector3(-1, 1, 0), new Vector3(1, 1, 5f),
+			new Vector3(-1, -2, 0)));
+
+		assertFalse(Intersector.isPointInTriangle(new Vector3(-5120.8345f, 8946.126f, -3270.5813f),
+			new Vector3(50.008057f, 22.20586f, 124.62208f), new Vector3(62.282288f, 22.205864f, 109.665924f),
+			new Vector3(70.92052f, 7.205861f, 115.437805f)));
+	}
 }