Merge pull request #26 from escaped/feat/django-storages-support

feat: django-storages support, fixes #13

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+* support for django-storages, @lifenautjoe & @bashu
+
 ### Changed
 
 * remove deprecation warnings

CONTRIBUTORS.md

@@ -3,5 +3,6 @@
 * [@bashu](https://github.com/bashu)
 * [@escaped](https://github.com/escaped/)
 * [@goranpavlovic](https://github.com/goranpavlovic)
+* [@lifenautjoe](https://github.com/lifenautjoe)
 * [@mabuelhagag](https://github.com/mabuelhagag)
 

test_proj/conftest.py

@@ -1,12 +1,22 @@
+import enum
 import os
+import shutil
+from pathlib import Path
+from typing import IO, Any, Generator
 
 import pytest
 from django.core.files import File
+from django.core.files.storage import FileSystemStorage
 
 from test_proj.media_library.models import Video
 from video_encoding.backends.ffmpeg import FFmpegBackend
 
 
+class StorageType(enum.Enum):
+    LOCAL = enum.auto()
+    REMOTE = enum.auto()
+
+
 @pytest.fixture
 def video_path():
     path = os.path.dirname(os.path.abspath(__file__))
@@ -19,9 +29,95 @@ def ffmpeg():
 
 
 @pytest.fixture
-def video(video_path):
-    video = Video()
+def local_video(video_path) -> Generator[Video, None, None]:
+    """
+    Return a video object which is stored locally.
+    """
+    video = Video.objects.create()
     video.file.save('test.MTS', File(open(video_path, 'rb')), save=True)
-    video.save()
-    video.refresh_from_db()
-    return video
+    try:
+        yield video
+    finally:
+        try:
+            video.file.delete()
+        except ValueError:
+            # file has already been deleted
+            pass
+
+        for format in video.format_set.all():
+            format.file.delete()
+
+        video.delete()
+
+
+@pytest.fixture
+def remote_video(local_video) -> Generator[Video, None, None]:
+    """
+    Return a video which is stored "remotely".
+    """
+    storage_path = Path(local_video.file.path).parent
+
+    remote_video = local_video
+    remote_video.file.storage = FakeRemoteStorage(storage_path)
+    yield remote_video
+
+
+@pytest.fixture(params=StorageType)
+def video(
+    request, local_video: Video, remote_video: Video
+) -> Generator[Video, None, None]:
+    """
+    Return a locally and a remotely stored video.
+    """
+    storage_type = request.param
+
+    if storage_type == StorageType.LOCAL:
+        yield local_video
+    elif storage_type == StorageType.REMOTE:
+        yield remote_video
+    else:
+        raise ValueError(f"Invalid storage type {storage_type}")
+
+
+class FakeRemoteStorage(FileSystemStorage):
+    """
+    Fake remote storage which does not support accessing a file by path.
+    """
+
+    def __init__(self, root_path: Path) -> None:
+        super().__init__()
+        self.root_path = root_path
+
+    def delete(self, name: str) -> None:
+        file_path = self.__path(name)
+        file_path.unlink()
+
+    def exists(self, name: str) -> bool:
+        return self.__path(name).exists()
+
+    def open(self, name: str, mode: str) -> IO[Any]:
+        return open(self.__path(name), mode)
+
+    def path(self, *args, **kwargs):
+        raise NotImplementedError("Remote storage does not implement path()")
+
+    def _save(self, name: str, content: File) -> str:
+        file_path = self.__path(name)
+        folder_path = file_path.parent
+
+        if not folder_path.is_dir():
+            file_path.parent.mkdir(parents=True)
+
+        if hasattr(content, 'temporary_file_path'):
+            shutil.move(content.temporary_file_path(), file_path)
+        else:
+            with open(file_path, 'wb') as fp:
+                fp.write(content.read())
+
+        return str(file_path)
+
+    def __path(self, name: str) -> Path:
+        """
+        Return path to local file.
+        """
+        return self.root_path / name

test_proj/media_library/tests/test_fields.py

@@ -8,8 +8,8 @@
 
 
 @pytest.mark.django_db
-def test_info_forward(ffmpeg, video):
-    media_info = ffmpeg.get_media_info(video.file.path)
+def test_info_forward(ffmpeg, video, video_path):
+    media_info = ffmpeg.get_media_info(video_path)
 
     assert video.duration == media_info['duration']
     assert video.width == media_info['width']

test_proj/media_library/tests/test_files.py

@@ -1,3 +1,5 @@
+import pytest
+
 from video_encoding.files import VideoFile
 
 
@@ -8,3 +10,13 @@ def test_videofile(ffmpeg, video_path):
     assert video_file.duration == media_info['duration']
     assert video_file.width == media_info['width']
     assert video_file.height == media_info['height']
+
+
+@pytest.mark.django_db
+def test_videofile__with_storages(ffmpeg, video, video_path):
+    media_info = ffmpeg.get_media_info(video_path)
+
+    video_file = video.file
+    assert video_file.duration == media_info['duration']
+    assert video_file.width == media_info['width']
+    assert video_file.height == media_info['height']

test_proj/media_library/tests/test_managers.py

@@ -5,20 +5,20 @@
 
 
 @pytest.fixture
-def video_format(video):
+def video_format(local_video):
     return Format.objects.create(
-        object_id=video.pk,
-        content_type=ContentType.objects.get_for_model(video),
+        object_id=local_video.pk,
+        content_type=ContentType.objects.get_for_model(local_video),
         field_name='file',
         format='mp4_hd',
         progress=100,
     )
 
 
 @pytest.mark.django_db
-def test_related_manager(video):
-    assert hasattr(video.format_set, 'complete')
-    assert hasattr(video.format_set, 'in_progress')
+def test_related_manager(local_video):
+    assert hasattr(local_video.format_set, 'complete')
+    assert hasattr(local_video.format_set, 'in_progress')
 
 
 @pytest.mark.django_db

video_encoding/files.py

@@ -1,8 +1,7 @@
-import os
-
 from django.core.files import File
 
 from .backends import get_backend
+from .utils import get_local_path
 
 
 class VideoFile(File):
@@ -41,9 +40,10 @@ def _get_video_info(self):
         """
         if not hasattr(self, '_info_cache'):
             encoding_backend = get_backend()
-            try:
-                path = os.path.abspath(self.path)
-            except AttributeError:
-                path = os.path.abspath(self.name)
-            self._info_cache = encoding_backend.get_media_info(path)
+
+            with get_local_path(self) as local_path:
+                info_cache = encoding_backend.get_media_info(local_path)
+
+            self._info_cache = info_cache
+
         return self._info_cache

video_encoding/tasks.py

@@ -6,10 +6,12 @@
 from django.core.files import File
 
 from .backends import get_backend
+from .backends.base import BaseEncodingBackend
 from .config import settings
 from .exceptions import VideoEncodingError
 from .fields import VideoField
 from .models import Format
+from .utils import get_local_path
 
 
 def convert_all_videos(app_label, model_name, object_pk):
@@ -40,55 +42,64 @@ def convert_video(fieldfile, force=False):
     instance = fieldfile.instance
     field = fieldfile.field
 
-    filename = os.path.basename(fieldfile.path)
-    source_path = fieldfile.path
+    with get_local_path(fieldfile) as source_path:
 
-    encoding_backend = get_backend()
+        encoding_backend = get_backend()
 
-    for options in settings.VIDEO_ENCODING_FORMATS[encoding_backend.name]:
-        video_format, created = Format.objects.get_or_create(
-            object_id=instance.pk,
-            content_type=ContentType.objects.get_for_model(instance),
-            field_name=field.name,
-            format=options['name'],
-        )
+        for options in settings.VIDEO_ENCODING_FORMATS[encoding_backend.name]:
+            video_format, created = Format.objects.get_or_create(
+                object_id=instance.pk,
+                content_type=ContentType.objects.get_for_model(instance),
+                field_name=field.name,
+                format=options['name'],
+            )
 
-        # do not reencode if not requested
-        if video_format.file and not force:
-            continue
-        else:
-            # set progress to 0
-            video_format.reset_progress()
+            # do not reencode if not requested
+            if video_format.file and not force:
+                continue
 
-        # TODO do not upscale videos
+            try:
+                _encode(source_path, video_format, encoding_backend, options)
+            except VideoEncodingError:
+                # TODO handle with more care
+                video_format.delete()
+                continue
 
-        _, target_path = tempfile.mkstemp(
-            suffix='_{name}.{extension}'.format(**options)
-        )
 
-        try:
-            encoding = encoding_backend.encode(
-                source_path, target_path, options['params']
-            )
-            while encoding:
-                try:
-                    progress = next(encoding)
-                except StopIteration:
-                    break
-                video_format.update_progress(progress)
-        except VideoEncodingError:
-            # TODO handle with more care
-            video_format.delete()
-            os.remove(target_path)
-            continue
+def _encode(
+    source_path: str,
+    video_format: Format,
+    encoding_backend: BaseEncodingBackend,
+    options: dict,
+) -> None:
+    """
+    Encode video and continously report encoding progress.
+    """
+    # TODO do not upscale videos
+    # TODO move logic to Format model
+
+    with tempfile.NamedTemporaryFile(
+        suffix='_{name}.{extension}'.format(**options)
+    ) as file_handler:
+        target_path = file_handler.name
+
+        # set progress to 0
+        video_format.reset_progress()
+
+        encoding = encoding_backend.encode(source_path, target_path, options['params'])
+        while encoding:
+            try:
+                progress = next(encoding)
+            except StopIteration:
+                break
+            video_format.update_progress(progress)
 
         # save encoded file
+        filename = os.path.basename(source_path)
+        # TODO remove existing file?
         video_format.file.save(
             '{filename}_{name}.{extension}'.format(filename=filename, **options),
             File(open(target_path, mode='rb')),
         )
 
         video_format.update_progress(100)  # now we are ready
-
-        # remove temporary file
-        os.remove(target_path)

video_encoding/utils.py

@@ -0,0 +1,33 @@
+import contextlib
+import os
+import tempfile
+from typing import Generator
+
+from django.core.files import File
+
+
+@contextlib.contextmanager
+def get_local_path(fieldfile: File) -> Generator[str, None, None]:
+    """
+    Get a local file to work with from a file retrieved from a FileField.
+    """
+    if not hasattr(fieldfile, 'storage'):
+        # Its a local file with no storage abstraction
+        try:
+            yield os.path.abspath(fieldfile.path)
+        except AttributeError:
+            yield os.path.abspath(fieldfile.name)
+    else:
+        storage = fieldfile.storage
+        try:
+            # Try to access with path
+            yield storage.path(fieldfile.path)
+        except (NotImplementedError, AttributeError):
+            # Storage doesnt support absolute paths,
+            # download file to a temp local dir
+            with tempfile.NamedTemporaryFile(mode="wb", delete=False) as temp_file:
+                storage_file = storage.open(fieldfile.name, 'rb')
+
+                temp_file.write(storage_file.read())
+                temp_file.flush()
+                yield temp_file.name