Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

NDArraysRegressionFixture: regression on arrays with arbitrary shape. #72

Merged
merged 4 commits into from Sep 15, 2021
Merged

NDArraysRegressionFixture: regression on arrays with arbitrary shape. #72

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
62e86ed
NDArraysRegressionFixture: regression on arrays with arbitrary shape.
tovrstra Sep 3, 2021
fade52f
Improve NDArraysRegression code
tovrstra Sep 7, 2021
af561d7
Improve error messages and add tests for corner cases
tovrstra Sep 8, 2021
f03cdce
Fix broken tests
tovrstra Sep 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 2 additions & 3 deletions src/pytest_regressions/dataframe_regression.py
View file
Open in desktop
Expand Up @@ -123,8 +123,7 @@ def _check_fn(self, obtained_filename, expected_filename):
self._check_data_types(k, obtained_column, expected_column)
self._check_data_shapes(obtained_column, expected_column)

data_type = obtained_column.values.dtype
if data_type in [float, np.float16, np.float32, np.float64]:
if np.issubdtype(obtained_column.values.dtype, np.inexact):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please keep in mind that this will change behavior a bit, because complex numbers are also np.inexact:

>>> a = np.array([], dtype=np.complex128)
>>> np.issubdtype(a.dtype, np.inexact)
True

Perhaps add a test for this case (ignore me if you already did, by the time I'm writing this, I didn't finished the review yet), just to make sure it doesn't crash or anything...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's correct. I wanted to support complex numbers and there is a unit test for it.

not_close_mask = ~np.isclose(
obtained_column.values,
expected_column.values,
Expand All @@ -138,7 +137,7 @@ def _check_fn(self, obtained_filename, expected_filename):
diff_ids = np.where(not_close_mask)[0]
diff_obtained_data = obtained_column[diff_ids]
diff_expected_data = expected_column[diff_ids]
if data_type == bool:
if obtained_column.values.dtype == bool:
diffs = np.logical_xor(obtained_column, expected_column)[diff_ids]
else:
diffs = np.abs(obtained_column - expected_column)[diff_ids]
Expand Down
160 changes: 103 additions & 57 deletions src/pytest_regressions/ndarrays_regression.py
View file
Open in desktop
Expand Up @@ -7,7 +7,7 @@ class NDArraysRegressionFixture:
"""

THRESHOLD = 100
ROWFORMAT = "{:>10s} {:>20s} {:>20s} {:>20s}\n"
ROWFORMAT = "{:>15s} {:>20s} {:>20s} {:>20s}\n"

def __init__(self, datadir, original_datadir, request):
"""
Expand Down Expand Up @@ -36,21 +36,24 @@ def _check_data_types(self, key, obtained_array, expected_array):

__tracebackhide__ = True

obtained_data_type = obtained_array.dtype
expected_data_type = expected_array.dtype
if obtained_data_type != expected_data_type:
if obtained_array.dtype != expected_array.dtype:
# Check if both data types are comparable as numbers (float, int, short, bytes, etc...)
if np.issubdtype(obtained_data_type, np.number) and np.issubdtype(
expected_data_type, np.number
if np.issubdtype(obtained_array.dtype, np.number) and np.issubdtype(
expected_array.dtype, np.number
):
return
# Check if both data types are comparable as strings
if np.issubdtype(obtained_array.dtype, str) and np.issubdtype(
expected_array.dtype, str
):
return

# In case they are not, assume they are not comparable
error_msg = (
"Data types are not the same.\n"
"key: %s\n"
"Obtained: %s\n"
"Expected: %s\n" % (key, obtained_data_type, expected_data_type)
f"key: {key}\n"
f"Obtained: {obtained_array.dtype}\n"
f"Expected: {expected_array.dtype}\n"
)
raise AssertionError(error_msg)

Expand All @@ -61,14 +64,12 @@ def _check_data_shapes(self, key, obtained_array, expected_array):
"""
__tracebackhide__ = True

obtained_data_shape = obtained_array.shape
expected_data_shape = expected_array.shape
if obtained_data_shape != expected_data_shape:
if obtained_array.shape != expected_array.shape:
error_msg = (
"Shapes are not the same.\n"
"Key: %s\n"
"Obtained: %s\n"
"Expected: %s\n" % (key, obtained_data_shape, expected_data_shape)
f"Key: {key}\n"
f"Obtained: {obtained_array.shape}\n"
f"Expected: {expected_array.shape}\n"
)
raise AssertionError(error_msg)

Expand All @@ -90,31 +91,33 @@ def _check_fn(self, obtained_filename, expected_filename):
obtained_data = dict(np.load(str(obtained_filename)))
expected_data = dict(np.load(str(expected_filename)))

# Check mismatches in the keys.
if set(obtained_data) != set(expected_data):
error_msg = (
"They keys in the obtained results differ from the expected results.\n"
)
error_msg += " Matching keys: "
error_msg += str(list(set(obtained_data) & set(expected_data)))
error_msg += "\n"
error_msg += " New in obtained: "
error_msg += str(list(set(obtained_data) - set(expected_data)))
error_msg += "\n"
error_msg += " Missing from obtained: "
error_msg += str(list(set(expected_data) - set(obtained_data)))
error_msg += "\n"
error_msg += "To update values, use --force-regen option.\n\n"
raise AssertionError(error_msg)

# Compare the contents of the arrays.
comparison_tables_dict = {}
for k in obtained_data.keys():
obtained_array = obtained_data[k]
for k, obtained_array in obtained_data.items():
expected_array = expected_data.get(k)

if expected_array is None:
error_msg = f"Could not find key '{k}' in the expected results.\n"
error_msg += "Keys in the obtained data table: ["
for k in obtained_data.keys():
error_msg += f"'{k}', "
error_msg += "]\n"
error_msg += "Keys in the expected data table: ["
for k in expected_data.keys():
error_msg += f"'{k}', "
error_msg += "]\n"
error_msg += "To update values, use --force-regen option.\n\n"
raise AssertionError(error_msg)

tolerance_args = self._tolerances_dict.get(k, self._default_tolerance)

self._check_data_types(k, obtained_array, expected_array)
self._check_data_shapes(k, obtained_array, expected_array)

data_type = obtained_array.dtype
if data_type in [float, np.float16, np.float32, np.float64]:
if np.issubdtype(obtained_array.dtype, np.inexact):
not_close_mask = ~np.isclose(
obtained_array,
expected_array,
Expand All @@ -125,27 +128,77 @@ def _check_fn(self, obtained_filename, expected_filename):
not_close_mask = obtained_array != expected_array

if np.any(not_close_mask):
diff_ids = np.nonzero(not_close_mask)
if not_close_mask.ndim == 0:
diff_ids = [()]
else:
diff_ids = np.array(np.nonzero(not_close_mask)).T
comparison_tables_dict[k] = (
np.array(diff_ids).T,
obtained_array[diff_ids],
expected_array[diff_ids],
expected_array.size,
expected_array.shape,
diff_ids,
obtained_array[not_close_mask],
expected_array[not_close_mask],
)

if len(comparison_tables_dict) > 0:
error_msg = "Values are not sufficiently close.\n"
error_msg += "To update values, use --force-regen option.\n\n"
for k, (
size,
shape,
diff_ids,
obtained_array,
expected_array,
) in comparison_tables_dict.items():
# Summary
error_msg += f"{k}:\n Shape: {shape}\n"
pct = 100 * len(diff_ids) / size
error_msg += (
f" Number of differences: {len(diff_ids)} / {size} ({pct:.1f}%)\n"
)
if np.issubdtype(obtained_array.dtype, np.number) and len(diff_ids) > 1:
error_msg += (
" Statistics are computed for differing elements only.\n"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice messages for the statistics, complete and informative! 👍

)

abs_errors = abs(obtained_array - expected_array)
error_msg += " Stats for abs(obtained - expected):\n"
error_msg += f" Max: {abs_errors.max()}\n"
error_msg += f" Mean: {abs_errors.mean()}\n"
error_msg += f" Median: {np.median(abs_errors)}\n"

error_msg += (
f" Stats for abs(obtained - expected) / abs(expected):\n"
)
expected_nonzero = np.array(np.nonzero(expected_array)).T
rel_errors = abs(
(
obtained_array[expected_nonzero]
- expected_array[expected_nonzero]
)
/ expected_array[expected_nonzero]
)
if len(rel_errors) != len(abs_errors):
pct = 100 * len(rel_errors) / len(abs_errors)
error_msg += f" Number of (differing) non-zero expected results: {len(rel_errors)} / {len(abs_errors)} ({pct:.1f}%)\n"
error_msg += f" Relative errors are computed for the non-zero expected results.\n"
else:
rel_errors = abs(
(obtained_array - expected_array) / expected_array
)
error_msg += f" Max: {rel_errors.max()}\n"
error_msg += f" Mean: {rel_errors.mean()}\n"
error_msg += f" Median: {np.median(rel_errors)}\n"

# Details results
error_msg += " Individual errors:\n"
if len(diff_ids) > self.THRESHOLD:
error_msg += f"Only showing first {self.THRESHOLD} mismatches.\n"
error_msg += (
f" Only showing first {self.THRESHOLD} mismatches.\n"
)
diff_ids = diff_ids[: self.THRESHOLD]
obtained_array = obtained_array[: self.THRESHOLD]
expected_array = expected_array[: self.THRESHOLD]
error_msg += f"{k}:\n"
error_msg += self.ROWFORMAT.format(
"Index",
"Obtained",
Expand All @@ -155,15 +208,18 @@ def _check_fn(self, obtained_filename, expected_filename):
for diff_id, obtained, expected in zip(
diff_ids, obtained_array, expected_array
):
diff_id_str = ", ".join(str(i) for i in diff_id)
if len(diff_id) != 1:
diff_id_str = f"({diff_id_str})"
error_msg += self.ROWFORMAT.format(
",".join(str(i) for i in diff_id),
diff_id_str,
str(obtained),
str(expected),
str(obtained - expected)
if isinstance(obtained, np.number)
else "",
)
error_msg += "\n\n"
error_msg += "\n"
raise AssertionError(error_msg)

def _dump_fn(self, data_object, filename):
Expand Down Expand Up @@ -249,22 +305,12 @@ def test_some_data(ndarrays_regression):
data_dict[key] = np.asarray(array)

for key, array in data_dict.items():
# Skip assertion if an array of strings
if (array.dtype == "O") and (type(array[0]) is str):
continue
# Rejected: timedelta, datetime, objects, zero-terminated bytes, unicode strings and raw data
assert array.dtype not in [
"m",
"M",
"O",
"S",
"a",
"U",
"V",
], "Only numeric data is supported on ndarrays_regression fixture.\n" "Array '%s' with type '%s' was given.\n" % (
key,
str(array.dtype),
)
# Rejected: timedelta, datetime, objects, zero-terminated bytes and raw data
if array.dtype in ["m", "M", "O", "S", "a", "V"]:
raise TypeError(
"Only numeric data is supported on ndarrays_regression fixture.\n"
f"Array '{key}' with type '{array.dtype}' was given.\n"
)

if tolerances is None:
tolerances = {}
Expand Down
5 changes: 5 additions & 0 deletions tests/test_dataframe_regression.py
View file
Open in desktop
Expand Up @@ -238,6 +238,11 @@ def test_string_array(dataframe_regression):
data1 = {"potato": ["delicious", "nutritive", "yummy"]}
dataframe_regression.check(pd.DataFrame.from_dict(data1))

# TODO: The following fails with a confusing error message.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

# Try wrong data
# data1 = {"potato": ["delicious", "nutritive", "yikes"]}
# dataframe_regression.check(pd.DataFrame.from_dict(data1))


def test_non_pandas_dataframe(dataframe_regression):
data = np.ones(shape=(10, 10))
Expand Down