Merge pull request #522 from radical-cybertools/fix/integration_tests

Fix/integration tests

.github/workflows/python-app.yml

@@ -10,10 +10,16 @@ on:
     branches: [ devel ]
 
 jobs:
-  build:
+  test:
 
     runs-on: ubuntu-latest
-
+
+    services:
+      rabbitmq:
+        image: rabbitmq
+        ports:
+        # will assign a random free host port
+        - 5672/tcp
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python 3.6
@@ -27,35 +33,76 @@ jobs:
         pip install psutil
         pip install hypothesis
         pip install coverage
-        pip install flake8
-        pip install flake8-per-file-ignores
-        pip install pylint
         pip install codecov
         pip install pytest
         pip install pytest-xdist
         pip install pytest-timeout
         pip install timeout_decorator
     - name: Test with pytest
+      env:
+        RMQ_HOSTNAME: localhost
+        RMQ_PORT: ${{ job.services.rabbitmq.ports[5672] }} # get randomly assigned published port
+        RMQ_USERNAME: guest
+        RMQ_PASSWORD: guest
       run: |
         LOC=/opt/hostedtoolcache/Python/3.6.12/x64/lib/python3.6/site-packages
-        coverage run --include=$LOC/radical/entk/* -m pytest -ra --timeout=600 -vvv --showlocals tests/test_component tests/test_utils/
+        coverage run --include=$LOC/radical/entk/* -m pytest -ra --timeout=600 -vvv --showlocals tests/test_component tests/test_utils/ tests/test_integration
     - name: Codecov
       uses: codecov/codecov-action@v1.0.15
+
+  flake8:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install . --upgrade
+        pip install flake8
+        pip install flake8-per-file-ignores
     - name: Lint with flake8
       run: |
-        # stop the build if there are Python syntax errors or undefined names
         LOG=`git log -n 1 | grep Merge`
-        OLD=`echo $LOG | cut -d ' ' -f2`
-        NEW=`echo $LOG | cut -d ' ' -f4`
-        echo $LOC
-        git log -n 1
-        flake8 $PWD/src/radical/
+        NEW=`echo $LOG | cut -d ' ' -f2`
+        OLD=`echo $LOG | cut -d ' ' -f3`
+        DIFF=`git diff  --name-only --diff-filter=db $NEW..$OLD`
+        DIFF=$(echo $DIFF | grep -o -e '\b[^ ]*.py\b' | sed "s|^|$PWD/|")
+        test -z "$DIFF" && echo 'nothing to flake8' || flake8 $DIFF
+
+  pylint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install . --upgrade
+        pip install psutil
+        pip install hypothesis
+        pip install coverage
+        pip install codecov
+        pip install pytest
+        pip install pytest-xdist
+        pip install pytest-timeout
+        pip install timeout_decorator
+        pip install pylint
     - name: Lint with pylint
       run: |
-        pylint $PWD/src/radical/
-        # # stop the build if there are Python syntax errors or undefined names
-        # LOG=`git log -n 1 | grep Merge`
-        # OLD=`echo $LOG | cut -d ' ' -f2`
-        # NEW=`echo $LOG | cut -d ' ' -f4`
-        # DIFF=`git diff --name-only --diff-filter=b $OLD...$NEW`
-        # test -z \"$DIFF\" && echo 'nothing to pylint' || pylint $DIFF
+        LOG=`git log -n 1 | grep Merge`
+        NEW=`echo $LOG | cut -d ' ' -f2`
+        OLD=`echo $LOG | cut -d ' ' -f3`
+        DIFF=`git diff  --name-only --diff-filter=db $NEW..$OLD`
+        DIFF=$(echo $DIFF | grep -o -e '\b[^ ]*.py\b' | sed "s|^|$PWD/|")
+        test -z "$DIFF" && echo 'nothing to pylint' || pylint $DIFF

src/radical/entk/exceptions.py

@@ -1,6 +1,7 @@
 __author__ = "Vivek Balasubramanian <vivek.balasubramanian@rutgers.edu>"
 __copyright__ = "Copyright 2017, http://radical.rutgers.edu"
 __license__ = "MIT"
+# pylint: disable=useless-super-delegation
 
 
 class EnTKError(Exception):
@@ -73,5 +74,4 @@ def __init__(self, obj, missing_attribute):
         msg = 'Attribute %s in %s undefined' % (str(missing_attribute), str(obj))
         super(MissingError, self).__init__(msg)
 
-
 # pylint: enable=useless-super-delegation

src/radical/entk/pipeline.py

@@ -150,9 +150,9 @@ def state_history(self):
 
         return self._state_history
 
-    # ------------------------------------------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     # Setter functions
-    # ------------------------------------------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
 
     @name.setter
     def name(self, value):
@@ -161,7 +161,10 @@ def name(self, value):
                 raise ValueError(obj=self._uid,
                                 attribute='name',
                                 actual_value=value,
-                                expected_value="Using ',' in an object's name will corrupt the profiling and internal mapping tables")
+                                expected_value="Using ',' in an object's " +
+                                               "name will corrupt the " +
+                                               "profiling and internal " +
+                                               "mapping tables")
             else:
                 self._name = value
 

tests/test_component/test_pipeline.py

@@ -3,6 +3,7 @@
 
 from unittest import TestCase
 from random import shuffle
+import string
 
 from   hypothesis import given, settings
 import hypothesis.strategies as st
@@ -50,12 +51,12 @@ def test_pipeline_initialization(self, mocked_generate_id, mocked_Lock,
         self.assertFalse(p.completed)
 
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch('radical.utils.generate_id', return_value='pipeline.0000')
     @mock.patch('threading.Lock', return_value='test_lock')
     @mock.patch('threading.Event', return_value='test_event')
-    @given(t=st.text(),
+    @given(t=st.text(alphabet=string.ascii_letters + ',', min_size=10).filter(lambda x: ',' in x),
         l=st.lists(st.text()),
         i=st.integers().filter(lambda x: type(x) == int),
         b=st.booleans(),
@@ -67,21 +68,24 @@ def test_pipeline_assignment_exceptions(self, mocked_generate_id,
         p = Pipeline()
 
         data_type = [t, l, i, b, se]
+        print(data_type)
 
         for data in data_type:
-
             if not isinstance(data, str):
                 with self.assertRaises(TypeError):
                     p.name = data
 
+            if isinstance(data,str):
+                with self.assertRaises(ValueError):
+                    p.name = data
+
             with self.assertRaises(TypeError):
                 p.stages = data
 
             with self.assertRaises(TypeError):
                 p.add_stages(data)
 
-
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch('radical.utils.generate_id', return_value='pipeline.0000')
     @mock.patch('threading.Lock', return_value='test_lock')
@@ -99,7 +103,7 @@ def test_pipeline_stage_assignment(self, mocked_generate_id, mocked_Lock,
         self.assertEqual(p.stages[0], s)
 
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch('radical.utils.generate_id', return_value='pipeline.0000')
     @mock.patch('threading.Lock', return_value='test_lock')
@@ -136,7 +140,7 @@ def test_pipeline_state_assignment(self, mocked_generate_id, mocked_Lock,
             self.assertEqual(p._state, val)
             self.assertEqual(p._state_history, state_history)
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch('radical.utils.generate_id', return_value='pipeline.0000')
     @mock.patch('threading.Lock', return_value='test_lock')
@@ -156,7 +160,7 @@ def test_pipeline_stage_addition(self, mocked_generate_id, mocked_Lock,
         self.assertEqual(p.stages[1], s2)
 
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch.object(Pipeline, '__init__', return_value=None)
     def test_pipeline_to_dict(self, mocked_init):
@@ -180,7 +184,7 @@ def test_pipeline_to_dict(self, mocked_init):
                              'completed': False})
 
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch.object(Pipeline, '__init__', return_value=None)
     def test_pipeline_from_dict(self, mocked_init):
@@ -203,7 +207,7 @@ def test_pipeline_from_dict(self, mocked_init):
         self.assertEqual(p.completed, d['completed'])
 
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch.object(Pipeline, '__init__', return_value=None)
     def test_pipeline_increment_stage(self, mocked_init):
@@ -230,7 +234,7 @@ def test_pipeline_increment_stage(self, mocked_init):
         self.assertTrue(p._completed_flag.is_set())
 
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch.object(Pipeline, '__init__', return_value=None)
     def test_pipeline_decrement_stage(self, mocked_init):
@@ -252,7 +256,7 @@ def test_pipeline_decrement_stage(self, mocked_init):
         self.assertFalse(p._completed_flag.is_set(), False)
 
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch.object(Pipeline, '__init__', return_value=None)
     @given(t=st.text(),
@@ -278,7 +282,7 @@ def test_pipeline_validate_entities(self, mocked_init,  t, l, i, b, se):
         self.assertEqual([s1,s2], p._validate_entities([s1,s2]))
 
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch.object(Pipeline, '__init__', return_value=None)
     def test_pipeline_validate(self, mocked_init):
@@ -304,7 +308,7 @@ def test_pipeline_validate(self, mocked_init):
         p._state = states.INITIAL
         p._validate()
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch.object(Pipeline, '__init__', return_value=None)
     def test_pipeline_properties(self, mocked_init):
@@ -345,7 +349,7 @@ def test_pipeline_properties(self, mocked_init):
         p._state_history = ['state1','state2']
         self.assertEqual(p.state_history, ['state1','state2'])
 
-    # ------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     #
     @mock.patch.object(Pipeline, '__init__', return_value=None)
     def test_pipeline_suspend(self, mocked_init):

tests/test_component/test_task.py

@@ -6,7 +6,6 @@
 
 from unittest import TestCase
 
-import pytest
 
 from radical.entk.task import Task
 from radical.entk import states
@@ -267,7 +266,7 @@ def test_dict_to_task(self):
 
         # make sure the type checks kick in
         d = 'test'
-        with pytest.raises(ree.TypeError):
+        with self.assertRaises(ree.TypeError):
             t = Task(from_dict=d)
 
     # --------------------------------------------------------------------------