Update test harness (Py3.11) (#2744)

* Test Python 3.11 in test harness

* Update requirements for Python 3.11

* Fix failing operation test
(writing log to disk)

* Increase stacklevel of warnings to appease flake8
B028

* Use our own AsyncMock
The one from unittest doesn't exist in Python 3.7.
Also consolidate a second async mock wrapper for code reuse.

* Update SonarCloud GitHub action version

* Refactor duplicate string to appease sonarcloud

* Update GitHub Actions versions

* Fix warning stacklevel

.github/workflows/publish_docker_image.yml

@@ -21,18 +21,18 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 

.github/workflows/quality.yml

@@ -29,14 +29,16 @@ jobs:
             toxenv: py39,style,coverage-ci
           - python-version: 3.10.9
             toxenv: py310,style,coverage-ci
+          - python-version: 3.11
+            toxenv: py311,style,coverage-ci
 
     steps:
-      - uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
         with:
           submodules: recursive
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
       - name: Setup python
-        uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -50,7 +52,7 @@ jobs:
       - name: Override Coverage Source Path for Sonar
         run: sed -i "s/<source>\/home\/runner\/work\/caldera\/caldera/<source>\/github\/workspace/g" /home/runner/work/caldera/caldera/coverage.xml
       - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@156db6fef3e168e4972abb76de0b32bbce8ec77a
+        uses: SonarSource/sonarcloud-github-action@5875562561d22a34be0c657405578705a169af6c
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/security.yml

@@ -22,11 +22,11 @@ jobs:
             toxenv: safety
 
     steps:
-      - uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
         with:
           submodules: recursive
       - name: Setup python
-        uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies

.github/workflows/stale.yml

@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
 
     steps:
-    - uses: actions/stale@9c1b1c6e115ca2af09755448e0dbba24e5061cc8
+    - uses: actions/stale@a20b814fb01b71def3bd6f56e7494d667ddf28da
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-label: 'no-issue-activity'

app/service/data_svc.py

@@ -41,6 +41,8 @@
 PAYLOADS_CONFIG_SPECIAL_KEY = 'special_payloads'
 PAYLOADS_CONFIG_EXTENSIONS_KEY = 'extensions'
 
+DEPRECATION_WARNING_LOAD = "Function deprecated and will be removed in a future update. Use load_yaml_file"
+
 
 class DataService(DataServiceInterface, BaseService):
 
@@ -240,15 +242,15 @@ async def load_requirements_from_list(self, requirements: list):
         return [RequirementSchema().load(entry) for entry in requirements]
 
     async def load_adversary_file(self, filename, access):
-        warnings.warn("Function deprecated and will be removed in a future update. Use load_yaml_file", DeprecationWarning)
+        warnings.warn(DEPRECATION_WARNING_LOAD, DeprecationWarning, stacklevel=2)
         await self.load_yaml_file(Adversary, filename, access)
 
     async def load_source_file(self, filename, access):
-        warnings.warn("Function deprecated and will be removed in a future update. Use load_yaml_file", DeprecationWarning)
+        warnings.warn(DEPRECATION_WARNING_LOAD, DeprecationWarning, stacklevel=2)
         await self.load_yaml_file(Source, filename, access)
 
     async def load_objective_file(self, filename, access):
-        warnings.warn("Function deprecated and will be removed in a future update. Use load_yaml_file", DeprecationWarning)
+        warnings.warn(DEPRECATION_WARNING_LOAD, DeprecationWarning, stacklevel=2)
         await self.load_yaml_file(Objective, filename, access)
 
     async def load_yaml_file(self, object_class, filename, access):

requirements.txt

@@ -1,5 +1,5 @@
 aiohttp-jinja2==1.5.0
-aiohttp==3.8.1
+aiohttp==3.8.4
 aiohttp_session==2.9.0
 aiohttp-security==0.4.0
 aiohttp-apispec==2.2.3
@@ -19,7 +19,7 @@ donut-shellcode==0.9.2
 marshmallow-enum==1.5.1
 ldap3==2.8.1
 lxml~=4.9.1  # debrief
-reportlab==3.5.67  # debrief
+reportlab==3.6.12  # debrief
 svglib==1.0.1  # debrief
 Markdown==3.3.3  # training
 dnspython==2.1.0

tests/conftest.py

@@ -2,7 +2,6 @@
 import os.path
 
 import pytest
-from unittest.mock import AsyncMock
 import random
 import string
 import uuid
@@ -63,6 +62,7 @@
 from app.api.rest_api import RestApi
 
 from app import version
+from tests import AsyncMock
 
 DIR = os.path.dirname(os.path.abspath(__file__))
 CONFIG_DIR = os.path.join(DIR, '..', 'conf')

tests/objects/test_operation.py

@@ -17,6 +17,7 @@
 from app.objects.secondclass.c_result import Result
 from app.objects.secondclass.c_fact import Fact
 from app.utility.base_object import BaseObject
+from app.utility.base_world import BaseWorld
 
 LINK1_DECIDE_TIME = MOCK_LINK_FINISH_TIME = '2021-01-01T08:00:00Z'
 LINK1_COLLECT_TIME = '2021-01-01T08:01:00Z'
@@ -74,6 +75,12 @@ def _encode_command(command_str):
     return _encode_command
 
 
+@pytest.fixture
+def setup_op_config():
+    BaseWorld.apply_config(name='main', config={'exfil_dir': '/tmp/caldera',
+                                                'reports_dir': '/tmp'})
+
+
 @pytest.fixture
 def op_for_event_logs(operation_agent, operation_adversary, executor, ability, operation_link, encoded_command,
                       parse_datestring):
@@ -263,6 +270,9 @@ def test_event_logs(self, event_loop, op_for_event_logs, operation_agent, file_s
         event_logs = event_loop.run_until_complete(op_for_event_logs.event_logs(file_svc, data_svc))
         assert event_logs == want
 
+    @pytest.mark.usefixtures(
+        "setup_op_config"
+    )
     def test_writing_event_logs_to_disk(self, event_loop, op_for_event_logs, operation_agent, file_svc, data_svc,
                                         event_log_op_start_time, op_agent_creation_time, fire_event_mock):
         event_loop.run_until_complete(data_svc.remove('agents', match=dict(unique=operation_agent.unique)))

tests/services/test_planning_svc.py

@@ -11,6 +11,7 @@
 from app.objects.secondclass.c_fact import Fact
 from app.objects.secondclass.c_requirement import Requirement
 from app.utility.base_world import BaseWorld
+from tests import AsyncMock
 
 
 stop_bucket_exhaustion_params = [
@@ -69,13 +70,6 @@ def __init__(self, **kwargs):
     return PlannerStub(**kwargs)
 
 
-def async_wrapper(return_value):
-    """Creates an async method that returns a constant value for mocking purposes."""
-    async def wrap(*args, **kwargs):
-        return return_value
-    return wrap
-
-
 @pytest.fixture
 async def setup_planning_test(executor, ability, agent, operation, data_svc, event_svc, init_base_world):
     texecutor = executor(name='sh', platform='darwin', command='mkdir test', cleanup='rm -rf test')
@@ -355,9 +349,9 @@ async def test_trim_links(self, setup_planning_test, planning_svc):
             Fact(trait='a.b.e', value='3'),
         ]
 
-        operation.all_facts = async_wrapper(facts)
+        operation.all_facts = AsyncMock(return_value=facts)
         operation.planner = MagicMock()
-        planning_svc.load_module = async_wrapper(RequirementFake())
+        planning_svc.load_module = AsyncMock(return_value=RequirementFake())
         link.ability.requirements = [Requirement('fake_requirement', [{'fake': 'relationship'}])]
 
         trimmed_links = await planning_svc.trim_links(operation, [link], agent)

tox.ini

@@ -6,7 +6,7 @@
 [tox]
 skipsdist = True
 envlist =
-    py{37,38,39,310}
+    py{37,38,39,310,311}
     style
     coverage
     safety
@@ -25,7 +25,7 @@ deps =
     coverage
 commands =
     py37: coverage run -p -m pytest --tb=short -Werror --asyncio-mode=auto tests
-    py{38,39,310}: coverage run -p -m pytest --tb=short --asyncio-mode=auto tests
+    py{38,39,310,311}: coverage run -p -m pytest --tb=short --asyncio-mode=auto tests
 
 [testenv:style]
 deps = pre-commit