Add small tests and improvement for mpl-use-full-test-name

README.rst

@@ -222,6 +222,37 @@ decorator:
 This will make the test insensitive to changes in e.g. the freetype
 library.
 
+
+Using full test paths for output names
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Use full test name starting at root directory for outputs
+
+.. code:: python
+
+    # pytest.ini|tox.ini|setup.cfg
+    [tool:pytest]
+    mpl-use-full-test-name = True
+
+Example:
+
+.. code:: bash
+
+    # test configuration
+    tests/
+        tests_1/
+            test_pytest_mpl.py
+        tests_2/
+            test_pytest_mpl.py
+
+    # output
+    baseline/
+        tests_1.test_pytest_mpl.xxx
+        tests_2.test_pytest_mpl.xxx
+
+See also `pytest -h`
+
+
 Test failure example
 --------------------
 

pytest_mpl/plugin.py

@@ -43,6 +43,8 @@
 
 import pytest
 
+from pytest_mpl.utils import wrap_message
+
 SHAPE_MISMATCH_ERROR = """Error: Image dimensions did not match.
   Expected shape: {expected_shape}
     {expected_path}
@@ -459,7 +461,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir):
         savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})
 
         hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None)
-        hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute()
+        if self.config.getini("mpl-use-full-test-name"):
+            hash_library_filename = hash_library_filename.absolute()
+        else:
+            hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute()
 
         if not Path(hash_library_filename).exists():
             pytest.fail(f"Can't find hash library at path {hash_library_filename}")
@@ -470,15 +475,21 @@ def compare_image_to_hash_library(self, item, fig, result_dir):
         test_hash = self.generate_image_hash(item, fig)
 
         if hash_name not in hash_library:
-            return (f"Hash for test '{hash_name}' not found in {hash_library_filename}. "
-                    f"Generated hash is {test_hash}.")
+            error_message = (f"Hash for test '{hash_name}'\n"
+                             f"not found in {hash_library_filename}.\n"
+                             f"Generated hash is {test_hash}.")
+            error_message = wrap_message(error_message)
+            return error_message
 
         if test_hash == hash_library[hash_name]:
             return
 
-        error_message = (f"Hash {test_hash} doesn't match hash "
-                         f"{hash_library[hash_name]} in library "
-                         f"{hash_library_filename} for test {hash_name}.")
+        error_message = (f"Hash {test_hash}\n"
+                         f"doesn't match\n"
+                         f"hash {hash_library[hash_name]}\n"
+                         f"in library {hash_library_filename}\n"
+                         f"for test {hash_name}.")
+        error_message = wrap_message(error_message)
 
         # If the compare has only been specified with hash and not baseline
         # dir, don't attempt to find a baseline image at the default path.
@@ -497,7 +508,7 @@ def compare_image_to_hash_library(self, item, fig, result_dir):
 
         if baseline_image is None:
             error_message += f"\nUnable to find baseline image {baseline_image_path}."
-            return error_message
+            return wrap_message(error_message)
 
         # Override the tolerance (if not explicitly set) to 0 as the hashes are not forgiving
         tolerance = compare.kwargs.get('tolerance', None)

pytest_mpl/utils.py

@@ -0,0 +1,8 @@
+import textwrap
+
+CHAR_WIDTH = 120
+
+
+def wrap_message(msg):
+    wrapped = "\n".join(textwrap.wrap(msg, CHAR_WIDTH, break_long_words=False))
+    return wrapped

tests/test_pytest_mpl.py

@@ -49,10 +49,24 @@ def assert_pytest_fails_with(args, output_substring):
         subprocess.check_output([sys.executable, '-m', 'pytest', '-s'] + args)
     except subprocess.CalledProcessError as exc:
         output = exc.output.decode()
-        assert output_substring in output, output
+        assert_is_in_output(output, output_substring)
         return output
 
 
+def assert_is_in_output(output, output_substring):
+    """Remove all formatting characters before running assertion"""
+    A = output_substring.replace("\n", " ").replace(" ", "")
+    B = output.replace("\n", " ").replace(" ", "")
+    assert A in B, output
+
+
+def assert_is_not_in_output(output, output_substring):
+    """Remove all formatting characters before running assertion"""
+    A = output_substring.replace("\n", " ").replace(" ", "")
+    B = output.replace("\n", " ").replace(" ", "")
+    assert A not in B, output
+
+
 @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local,
                                tolerance=DEFAULT_TOLERANCE)
 def test_succeeds():
@@ -289,14 +303,14 @@ def test_hash_fails(tmpdir):
     # If we use --mpl, it should detect that the figure is wrong
     output = assert_pytest_fails_with(['--mpl', test_file], "doesn't match hash FAIL in library")
     # We didn't specify a baseline dir so we shouldn't attempt to find one
-    assert "Unable to find baseline image" not in output, output
+    assert_is_not_in_output(output, "Unable to find baseline image")
 
     # Check that the summary path is printed and that it exists.
     output = assert_pytest_fails_with(['--mpl', test_file, '--mpl-generate-summary=html'],
                                       "doesn't match hash FAIL in library")
     # We didn't specify a baseline dir so we shouldn't attempt to find one
     print_message = "A summary of the failed tests can be found at:"
-    assert print_message in output, output
+    assert_is_in_output(output, print_message)
     printed_path = Path(output.split(print_message)[1].strip())
     assert printed_path.exists()
 
@@ -329,19 +343,19 @@ def test_hash_fail_hybrid(tmpdir):
     output = assert_pytest_fails_with(['--mpl', test_file,
                                        rf'--mpl-baseline-path={hash_baseline_dir_abs / "fail"}'],
                                       "doesn't match hash FAIL in library")
-    assert "Error: Image files did not match." in output, output
+    assert_is_in_output(output, "Error: Image files did not match.")
 
     # Assert reports missing baseline image
     output = assert_pytest_fails_with(['--mpl', test_file,
                                        '--mpl-baseline-path=/not/a/path'],
                                       "doesn't match hash FAIL in library")
-    assert "Unable to find baseline image" in output, output
+    assert_is_in_output(output, "Unable to find baseline image")
 
     # Assert reports image comparison succeeds
     output = assert_pytest_fails_with(['--mpl', test_file,
                                        rf'--mpl-baseline-path={hash_baseline_dir_abs / "succeed"}'],
                                       "doesn't match hash FAIL in library")
-    assert "However, the comparison to the baseline image succeeded." in output, output
+    assert_is_in_output(output, "However, the comparison to the baseline image succeeded.")
 
     # If we don't use --mpl option, the test should succeed
     code = call_pytest([test_file])

tests/test_use_full_name.py

@@ -0,0 +1,143 @@
+import os
+import subprocess
+import sys
+import configparser
+import json
+
+TEST_FILE = """
+import pytest
+import matplotlib.pyplot as plt
+@pytest.mark.mpl_image_compare
+def test_plot():
+    fig = plt.figure()
+    ax = fig.add_subplot(1,1,1)
+    ax.plot([1,2,2])
+    return fig
+"""
+
+
+def assert_is_in_output(output, output_substring):
+    """Remove all formatting characters before running assertion"""
+    A = output_substring.replace("\n", " ").replace(" ", "")
+    B = output.replace("\n", " ").replace(" ", "")
+    assert A in B, output
+
+
+def call_pytest(args, cwd):
+    """Run python -m pytest -s [args] from cwd and return its output"""
+    try:
+        output = subprocess.check_output(
+            [sys.executable, "-m", "pytest", "-s"] + args, cwd=cwd, shell=False
+        )
+        return output.decode()
+    except subprocess.CalledProcessError as exc:
+        output = exc.output.decode()
+        return output
+
+
+def make_ini_file(mpl_use_full_test_name, path):
+    config = configparser.ConfigParser()
+    config["tool:pytest"] = {"mpl-use-full-test-name": mpl_use_full_test_name}
+    with open(os.path.join(path, "setup.cfg"), "w") as configfile:
+        config.write(configfile)
+
+
+def make_test_code_file(path, fname):
+    with open(os.path.join(path, fname), "w") as f:
+        f.write(TEST_FILE)
+
+
+def make_test_subdir(pardir, subdir_name):
+    sub_dir = pardir.mkdir(subdir_name)
+    with open(os.path.join(sub_dir, "__init__.py"), "w"):
+        pass
+    return sub_dir
+
+
+def test_success(tmpdir):
+    basedir = tmpdir
+    tests_basedir = make_test_subdir(basedir, "tests")
+    subtest_dir1 = make_test_subdir(tests_basedir, "test_1")
+    subtest_dir2 = make_test_subdir(tests_basedir, "test_2")
+    # Create test files with identical names but in different test directories
+    make_test_code_file(tests_basedir, "test_foo.py")
+    make_test_code_file(subtest_dir1, "test_foo.py")
+    make_test_code_file(subtest_dir2, "test_foo.py")
+    make_ini_file(True, basedir)
+
+    has_lib_file = os.path.join("mpl_generate_dir", "baseline_hashes.json")
+
+    output1 = call_pytest(
+        args=["--mpl-generate-hash-library", has_lib_file, "tests"],
+        cwd=basedir.strpath,
+    )
+    output2 = call_pytest(
+        args=["--mpl", "--mpl-hash-library", has_lib_file, "tests"],
+        cwd=basedir.strpath,
+    )
+    assert_is_in_output(output1, "3 passed")
+    assert_is_in_output(output2, "3 passed")
+
+
+def test_missing_hash(tmpdir):
+    basedir = tmpdir
+    tests_basedir = make_test_subdir(basedir, "tests")
+    subtest_dir1 = make_test_subdir(tests_basedir, "test_1")
+    subtest_dir2 = make_test_subdir(tests_basedir, "test_2")
+    make_test_code_file(tests_basedir, "test_foo.py")
+    make_test_code_file(subtest_dir1, "test_foo.py")
+    make_test_code_file(subtest_dir2, "test_foo.py")
+    make_ini_file(True, basedir)
+
+    has_lib_file = os.path.join("mpl_generate_dir", "baseline_hashes.json")
+
+    output1 = call_pytest(
+        args=["--mpl-generate-hash-library", has_lib_file, "tests"],
+        cwd=basedir.strpath,
+    )
+    # Manually delete one entry from the hash lib
+    with open(os.path.join(basedir, has_lib_file), "r") as handle:
+        hash_lib = json.loads(handle.read())
+    hash_lib.pop(list(hash_lib.keys())[0])
+    with open(os.path.join(basedir, has_lib_file), "w") as handle:
+        json.dump(hash_lib, handle)
+
+    output2 = call_pytest(
+        args=["--mpl", "--mpl-hash-library", has_lib_file, "tests"],
+        cwd=basedir.strpath,
+    )
+    assert_is_in_output(output1, "3 passed")
+    assert_is_in_output(output2, "1 failed, 2 passed")
+    assert_is_in_output(output2, "Hash for test 'tests.test_foo.test_plot' not found")
+
+
+def test_incorrect_hash(tmpdir):
+    basedir = tmpdir
+    tests_basedir = make_test_subdir(basedir, "tests")
+    subtest_dir1 = make_test_subdir(tests_basedir, "test_1")
+    subtest_dir2 = make_test_subdir(tests_basedir, "test_2")
+    make_test_code_file(tests_basedir, "test_foo.py")
+    make_test_code_file(subtest_dir1, "test_foo.py")
+    make_test_code_file(subtest_dir2, "test_foo.py")
+    make_ini_file(True, basedir)
+
+    has_lib_file = os.path.join("mpl_generate_dir", "baseline_hashes.json")
+
+    output1 = call_pytest(
+        args=["--mpl-generate-hash-library", has_lib_file, "tests"],
+        cwd=basedir.strpath,
+    )
+    # Manually delete one entry from the hash lib
+    with open(os.path.join(basedir, has_lib_file), "r") as handle:
+        hash_lib = json.loads(handle.read())
+    hash_lib[list(hash_lib.keys())[0]] = 12345
+    with open(os.path.join(basedir, has_lib_file), "w") as handle:
+        json.dump(hash_lib, handle)
+
+    output2 = call_pytest(
+        args=["--mpl", "--mpl-hash-library", has_lib_file, "tests"],
+        cwd=basedir.strpath,
+    )
+    assert_is_in_output(output1, "3 passed")
+    assert_is_in_output(output2, "1 failed, 2 passed")
+    assert_is_in_output(output2, "doesn't match")