MRG: Merge pull request #121 from octue/release/0.1.10

Release: 0.1.10

.github/workflows/python-ci.yml

@@ -32,14 +32,17 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
-      - name: Install Tox and any other packages
-        run: pip install tox
-      - name: Run Tox
+      - name: Install package
+        run: pip install -r requirements-dev.txt
+      - name: Run tests
         env:
           GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCP_SERVICE_ACCOUNT }}
           TEST_PROJECT_NAME: ${{ secrets.TEST_PROJECT_NAME }}
           TEST_BUCKET_NAME: ${{ secrets.TEST_BUCKET_NAME }}
-        run: tox -e py
+        run: |
+          coverage run --source octue -m unittest discover
+          coverage report --show-missing
+          coverage xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:

README.md

@@ -61,7 +61,6 @@ pyenv virtualenv 3.8 myenv              # Makes a virtual environment for you to
 pyend activate myenv                    # Activates the virtual environment so you don't screw up other installations
 pip install -r requirements-dev.txt     # Installs the testing and code formatting utilities
 pre-commit install                      # Installs the pre-commit code formatting hooks in the git repo
-tox                                     # Runs the tests with coverage. NB you can also just set up pycharm or vscode to run these.
 ```
 
 - Adopt a Test Driven Development approach to implementing new features or fixing bugs.

octue/resources/datafile.py

@@ -113,7 +113,7 @@ def from_cloud(cls, project_name, bucket_name, datafile_path, timestamp=None):
         :return Datafile:
         """
         metadata = GoogleCloudStorageClient(project_name).get_metadata(bucket_name, datafile_path)
-        custom_metadata = metadata["metadata"]
+        custom_metadata = metadata.get("metadata") or {}
 
         datafile = cls(
             timestamp=custom_metadata.get("timestamp", timestamp),

octue/utils/cloud/emulators.py

@@ -0,0 +1,91 @@
+import os
+import socket
+from contextlib import closing
+from gcp_storage_emulator.server import create_server
+
+
+class GoogleCloudStorageEmulator:
+    """A local emulator for Google Cloud Storage
+
+    :param str host:
+    :param int port:
+    :param str default_bucket:
+    :return None:
+    """
+
+    def __init__(self, host="localhost", port=9090, in_memory=True, default_bucket=os.environ["TEST_BUCKET_NAME"]):
+        self._server = create_server(host, port, in_memory=in_memory, default_bucket=default_bucket)
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.stop()
+
+    def start(self):
+        """Start the emulator. Do nothing if it's already started on the given host and port.
+
+        :return None:
+        """
+        try:
+            self._server.start()
+        except RuntimeError:
+            pass
+
+    def stop(self):
+        """Stop the emulator.
+
+        :return None:
+        """
+        self._server.stop()
+
+
+def get_free_tcp_port():
+    """Get a free TCP port.
+
+    :return int:
+    """
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        s.bind(("", 0))
+        _, port = s.getsockname()
+        return port
+
+
+class GoogleCloudStorageEmulatorTestResultModifier:
+    """A class providing `startTestRun` and `endTestRun` methods for use by a `unittest.TestResult` that start up a
+    Google Cloud Storage emulator on a free port before the tests run and stop it after they've all run.
+
+    :param str host:
+    :param bool in_memory:
+    :param str default_bucket_name:
+    :return None:
+    """
+
+    STORAGE_EMULATOR_HOST_ENVIRONMENT_VARIABLE_NAME = "STORAGE_EMULATOR_HOST"
+
+    def __init__(self, host="localhost", in_memory=True, default_bucket_name=os.environ["TEST_BUCKET_NAME"]):
+        port = get_free_tcp_port()
+        self.storage_emulator_host = f"http://{host}:{port}"
+
+        self.storage_emulator = GoogleCloudStorageEmulator(
+            host=host, port=port, in_memory=in_memory, default_bucket=default_bucket_name
+        )
+
+    def startTestRun(self):
+        """Start the Google Cloud Storage emulator before starting the test run.
+
+        :param unittest.TestResult test_result:
+        :return None:
+        """
+        os.environ[self.STORAGE_EMULATOR_HOST_ENVIRONMENT_VARIABLE_NAME] = self.storage_emulator_host
+        self.storage_emulator.start()
+
+    def stopTestRun(self):
+        """Stop the Google Cloud Storage emulator before starting the test run.
+
+        :param unittest.TestResult test_result:
+        :return None:
+        """
+        self.storage_emulator.stop()
+        del os.environ[self.STORAGE_EMULATOR_HOST_ENVIRONMENT_VARIABLE_NAME]

octue/utils/cloud/storage/client.py

@@ -103,7 +103,7 @@ def get_metadata(self, bucket_name, path_in_bucket, timeout=_DEFAULT_TIMEOUT):
         bucket = self.client.get_bucket(bucket_or_name=bucket_name)
         metadata = bucket.get_blob(blob_name=self._strip_leading_slash(path_in_bucket), timeout=timeout)._properties
 
-        if metadata["metadata"] is not None:
+        if metadata.get("metadata") is not None:
             metadata["metadata"] = {key: json.loads(value) for key, value in metadata["metadata"].items()}
 
         return metadata

requirements-dev.txt

@@ -1,7 +1,6 @@
 
 # Testing
 # ------------------------------------------------------------------------------
-tox
 pluggy
 gcp-storage-emulator>=2021.2.17
 

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name="octue",
-    version="0.1.9",
+    version="0.1.10",
     py_modules=["cli"],
     install_requires=[
         "blake3>=0.1.8",
@@ -27,7 +27,7 @@
         "google-cloud-pubsub>=2.2.0",
         "google-cloud-storage>=1.35.1",
         "twined==0.0.16",
-    ],  # Dev note: you also need to bump twined in tox.ini
+    ],
     url="https://www.github.com/octue/octue-sdk-python",
     license="MIT",
     author="Thomas Clark (github: thclark)",

tests/__init__.py

@@ -1,30 +1,12 @@
 import os
 import unittest
 
-from tests.emulators import GoogleCloudStorageEmulator
+from octue.utils.cloud.emulators import GoogleCloudStorageEmulatorTestResultModifier
 
 
 TESTS_DIR = os.path.dirname(__file__)
-storage_emulator = GoogleCloudStorageEmulator()
 
 
-def startTestRun(instance):
-    """Start the test run, running any code in this function first.
-
-    :param unittest.TestResult instance:
-    :return None:
-    """
-    storage_emulator.start()
-
-
-def stopTestRun(instance):
-    """Finish the test run, running any code in this function first.
-
-    :param unittest.TestResult instance:
-    :return None:
-    """
-    storage_emulator.stop()
-
-
-setattr(unittest.TestResult, "startTestRun", startTestRun)
-setattr(unittest.TestResult, "stopTestRun", stopTestRun)
+test_result_modifier = GoogleCloudStorageEmulatorTestResultModifier()
+setattr(unittest.TestResult, "startTestRun", test_result_modifier.startTestRun)
+setattr(unittest.TestResult, "stopTestRun", test_result_modifier.stopTestRun)

tests/utils/cloud/storage/test_client.py

@@ -10,10 +10,14 @@
 
 
 class TestUploadFileToGoogleCloud(BaseTestCase):
-    PROJECT_NAME = os.environ["TEST_PROJECT_NAME"]
-    TEST_BUCKET_NAME = os.environ["TEST_BUCKET_NAME"]
-    FILENAME = "my_file.txt"
-    storage_client = GoogleCloudStorageClient(project_name=PROJECT_NAME, credentials=OCTUE_MANAGED_CREDENTIALS)
+    @classmethod
+    def setUpClass(cls):
+        cls.PROJECT_NAME = os.environ["TEST_PROJECT_NAME"]
+        cls.TEST_BUCKET_NAME = os.environ["TEST_BUCKET_NAME"]
+        cls.FILENAME = "my_file.txt"
+        cls.storage_client = GoogleCloudStorageClient(
+            project_name=cls.PROJECT_NAME, credentials=OCTUE_MANAGED_CREDENTIALS
+        )
 
     def test_upload_and_download_file(self):
         """Test that a file can be uploaded to Google Cloud storage and downloaded again."""