Fix false negative results, add more tests (#9)

* Increase verbosity of CI tests

* Add pickle and integration tests

* Workaround missing readline issue on Windows

* Add Jupyter requirement for integration tests

* Fix the tests not failing (ipython not returning the correct exit status)

* Install cairo on Mac

* Attempt to fix file permission error on Windows

See https://stackoverflow.com/a/51126234/6646912

* Add pycairo pip install for Mac

* Use raw string for pickle path to protect from issues on Windows

https://stackoverflow.com/a/46011113/6646912

* Narrow down coverage scope

* Remove outdated TODO note

* Try to figure out the problem with Windows

* Try with older version of ffmpeg on Windows

* Try with conda

* Add conda to path?

* Manually add to path

* Try with ffmpeg provided by imagemagick.app

* Trying in blind

* Give up, allow failures on Windows

.travis.yml

@@ -18,10 +18,12 @@ jobs:
       language: shell
       before_install:
         - brew install ffmpeg
+        - brew install cairo
+        - pip3 install pycairo --user
     - name: "Python 3.7 on Windows"
       os: windows
       language: shell
-      env: PATH=/c/Python37:/c/Python37/Scripts:$PATH
+      env: PATH=/c/Python37:/c/Python37/Scripts:/c/tools/miniconda3/:/c/tools/miniconda3/Scripts:$PATH
       before_install:
          - choco install python --version 3.7.4
          - choco install ffmpeg
@@ -30,6 +32,10 @@ jobs:
          # https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycairo
          # https://stackoverflow.com/a/48132109/6646912
          - pip3 install https://download.lfd.uci.edu/pythonlibs/t7epjj8p/pycairo-1.18.2-cp37-cp37m-win_amd64.whl --user
+         # temporary workaround for https://github.com/3b1b/manim/pull/672
+         - pip3 install pyreadline==2.1 --user
+  allow_failures:
+    - os: windows
 
 install:
   - pip3 install --upgrade pip || python -m pip install --upgrade pip --user
@@ -39,7 +45,7 @@ install:
   - python3 setup.py install || python setup.py install
 
 script:
-  - ipython -m pytest -- --cov=.
+  - python3 ipytest.py --cov=jupyter_manim -v || python ipytest.py --cov=jupyter_manim -v
 
 after_success:
   - codecov

ipytest.py

@@ -0,0 +1,18 @@
+import sys
+from IPython.core.interactiveshell import InteractiveShell
+from IPython import start_ipython
+import runpy
+
+
+def run_module(self, mod_name, where):
+    where.update(
+        runpy.run_module(
+            str(mod_name), run_name="__main__",
+            alter_sys=True
+        )
+    )
+
+
+InteractiveShell.safe_run_module = run_module
+
+result = start_ipython(['-m', 'pytest', '--'] + sys.argv[1:])

jupyter_manim/__init__.py

@@ -49,7 +49,7 @@ def write(self, s):
 import pickle
 from warnings import warn
 try:
-    with open('{pickle_path}', 'rb') as f:
+    with open(r'{pickle_path}', 'rb') as f:
         objects_from_notebook = pickle.load(f)
 except pickle.PickleError as e:
     warn('Could not unpickle the global objects from the notebook', e)
@@ -76,7 +76,7 @@ def is_pickable(obj):
 
 
 def find_ipython_frame(frames):
-    for frame in inspect.stack():
+    for frame in frames:
         if frame.filename.startswith('<ipython-input-'):
             return frame
     return None
@@ -128,6 +128,8 @@ def export_globals(self):
             warn('Pickling failed: ' + str(e))
             yield None
         finally:
+            if not f.closed:
+                f.close()
             os.remove(f.name)
 
     def parse_arguments(self, line) -> Tuple[Dict, List]:
@@ -203,7 +205,6 @@ def catch_path_and_forward(lines):
                 enter = stack.enter_context
 
                 if settings['export_variables']:
-                    # TODO test this with pytest
                     pickle_path = enter(self.export_globals())
 
                     if pickle_path:

tests/requirements.txt

@@ -1,3 +1,5 @@
 pytest==5.3.0
 pytest-cov==2.5.1
 codecov
+papermill
+jupyter

tests/test_manim.py

@@ -1,11 +1,99 @@
+import inspect
+import pickle
 import re
+from typing import NamedTuple
 
-from jupyter_manim import ManimMagics
+import pytest
+from papermill import execute_notebook
 
+from jupyter_manim import ManimMagics, find_ipython_frame
+import jupyter_manim
 
-def test_pickle():
-    # TODO
-    pass
+
+SHAPES_EXAMPLE = """
+from manimlib.scene.scene import Scene
+from manimlib.mobject.geometry import Circle
+from manimlib.animation.creation import ShowCreation
+
+class Shapes(Scene):
+
+    def construct(self):
+        circle = Circle()
+        self.play(ShowCreation(circle))
+"""
+
+
+SHAPES_OUTPUT_REGEXP = r"""
+    <video
+      width="854"
+      height="480"
+      autoplay="autoplay"
+      controls
+    >
+        <source src="videos/(.*?)/1440p60/Shapes\.mp4" type="video/mp4">
+    </video>
+"""
+
+
+@pytest.mark.manim_dependent
+def test_integration():
+    output_notebook = execute_notebook(
+        'Example.ipynb',
+        'Example_result.ipynb',
+    )
+    for cell in output_notebook['cells']:
+        assert cell['metadata']['papermill']['exception'] is False
+
+    last_cell_outputs = output_notebook['cells'][-1]['outputs']
+    if len(last_cell_outputs) != 1:
+        print(last_cell_outputs)
+    assert len(last_cell_outputs) == 1
+    output_data = last_cell_outputs[0]['data']
+    assert re.match(
+        # the example notebook uses low quality option for faster rendering
+        SHAPES_OUTPUT_REGEXP.replace('/1440p60/', '/480p15/'),
+        output_data['text/html']
+    )
+
+
+class MockFrameInfo(NamedTuple):
+    filename: str
+
+
+def test_find_ipython_frame():
+    ipython_frame = MockFrameInfo(filename='<ipython-input-918-9c9a2cba73bf>')
+    parent_frame = MockFrameInfo(filename='/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py')
+
+    frame = find_ipython_frame([ipython_frame, parent_frame])
+    assert frame is ipython_frame
+
+    frame = find_ipython_frame([parent_frame])
+    assert frame is None
+
+
+PICKLE_A = 'this should be pickled '
+_PICKLE_B = 'this should not'
+
+
+def test_pickle(monkeypatch):
+    magics_manager = ManimMagics()
+
+    def mock_frame(stack):
+        """Return the frame of test_pickle"""
+        return inspect.stack()[0]
+
+    monkeypatch.setattr(jupyter_manim, 'find_ipython_frame', mock_frame)
+
+    with magics_manager.export_globals() as pickled_file:
+        assert type(pickled_file) is str
+        assert pickled_file.endswith('.pickle')
+
+        with open(pickled_file, 'rb') as f:
+            unpickled = pickle.load(f)
+
+        assert 'PICKLE_A' in unpickled
+        assert unpickled['PICKLE_A'] == PICKLE_A
+        assert '_PICKLE_B' not in unpickled
 
 
 def test_arguments_resolution():
@@ -48,37 +136,17 @@ def test_help(capsys):
     assert not captured.err.strip()
 
 
-SHAPES_EXAMPLE = """
-from manimlib.scene.scene import Scene
-from manimlib.mobject.geometry import Circle
-from manimlib.animation.creation import ShowCreation
-
-class Shapes(Scene):
-
-    def construct(self):
-        circle = Circle()
-        self.play(ShowCreation(circle))
-"""
-
-
+@pytest.mark.manim_dependent
 def test_cell_magic():
     magics_manager = ManimMagics()
     result = magics_manager.manim('Shapes', SHAPES_EXAMPLE)
-    assert re.match(r"""
-    <video
-      width="854"
-      height="480"
-      autoplay="autoplay"
-      controls
-    >
-        <source src="videos/(.*?)/1440p60/Shapes\.mp4" type="video/mp4">
-    </video>
-    """, result.data)
+    assert re.match(SHAPES_OUTPUT_REGEXP, result.data)
 
 
+@pytest.mark.manim_dependent
 def test_cell_base64():
     magics_manager = ManimMagics()
-    result = magics_manager.manim('Shapes --base64', SHAPES_EXAMPLE)
+    result = magics_manager.manim('Shapes --base64 --low_quality', SHAPES_EXAMPLE)
     assert re.match(r"""
     <video
       width="854"