Merge branch 'main' into dependabot/pip/uvicorn-0.29.0

.github/workflows/linters.yml

@@ -8,7 +8,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Install dependencies
@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Install dependencies

.github/workflows/tests.yml

@@ -20,7 +20,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Install dependencies

db_revisions/versions/0040045eae14_add_validation_expired_state.py

@@ -5,6 +5,7 @@
 Create Date: 2024-04-11 13:08:20.850470
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op, context

db_revisions/versions/19fccbf914bc_add_submission_time_to_submission.py

@@ -5,6 +5,7 @@
 Create Date: 2024-02-06 15:50:28.836237
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op

db_revisions/versions/4ca961a003e1_create_filing_task_table.py

@@ -5,6 +5,7 @@
 Create Date: 2024-01-30 12:59:15.720135
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op

db_revisions/versions/4e8ae26c1a22_create_filing_task_state_table.py

@@ -5,6 +5,7 @@
 Create Date: 2024-01-30 13:02:52.041229
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op, context

db_revisions/versions/5492f53d1fa5_alter_validation_json_to_validation_results.py

@@ -5,6 +5,7 @@
 Create Date: 2024-05-01 13:40:54.288361
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op

db_revisions/versions/7356a7d7036d_add_total_records_to_submission.py

@@ -5,6 +5,7 @@
 Create Date: 2024-05-07 11:02:46.846411
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op

db_revisions/versions/7a1b7eab0167_accept_updates.py

@@ -5,6 +5,7 @@
 Create Date: 2024-03-13 14:38:34.324557
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op, context

db_revisions/versions/8e8faceb46bd_add_filename_to_submission_table.py

@@ -5,6 +5,7 @@
 Create Date: 2024-03-06 10:43:59.261556
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op

db_revisions/versions/bbc51b08d22f_240304_change_filing_task_state_to_.py

@@ -5,6 +5,7 @@
 Create Date: 2024-03-04 12:02:27.253888
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op

db_revisions/versions/c89920be2e66_add_validation_error_state.py

@@ -5,6 +5,7 @@
 Create Date: 2024-04-18 13:06:48.162639
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op, context

db_revisions/versions/fb46d55283d6_add_signature_table.py

@@ -5,6 +5,7 @@
 Create Date: 2024-03-13 11:41:47.815220
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op

pyproject.toml

@@ -10,33 +10,32 @@ packages = [{ include = "sbl_filing_api", from = "src" }]
 python = ">=3.12,<4"
 fastapi = "^0.109.1"
 uvicorn = "^0.29.0"
-python-keycloak = "^3.7.0"
 sqlalchemy = "^2.0.23"
 psycopg2-binary = "^2.9.9"
 asyncpg = "^0.29.0"
 regtech-api-commons = {git = "https://github.com/cfpb/regtech-api-commons.git"}
 regtech-data-validator = {git = "https://github.com/cfpb/regtech-data-validator.git"}
 regtech-regex = {git = "https://github.com/cfpb/regtech-regex.git"}
-python-multipart = "^0.0.7"
-boto3 = "^1.33.12"
+python-multipart = "^0.0.9"
+boto3 = "^1.34.105"
 alembic = "^1.12.0"
 async-lru = "^2.0.4"
-httpx = "^0.26.0"
+httpx = "^0.27.0"
+ujson = "^5.10.0"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.4.3"
-pytest-cov = "^4.1.0"
+pytest = "^8.2.0"
+pytest-cov = "^5.0.0"
 pytest-mock = "^3.12.0"
 pytest-env = "^1.1.3"
-pytest-alembic = "^0.10.7"
+pytest-alembic = "^0.11.0"
 pytest-asyncio = "^0.23.2"
-moto = "^4.2.13"
-aiosqlite = "^0.19.0"
+aiosqlite = "^0.20.0"
 
 
 [tool.poetry.group.linters.dependencies]
-ruff = "0.1.6"
-black = "23.11.0"
+ruff = "0.4.4"
+black = "24.4.2"
 
 [build-system]
 requires = ["poetry-core"]

src/sbl_filing_api/entities/repos/submission_repo.py

@@ -110,6 +110,13 @@ async def expire_submission(submission_id: int):
         await upsert_helper(session, submission, SubmissionDAO)
 
 
+async def error_out_submission(submission_id: int):
+    async with SessionLocal() as session:
+        submission = await get_submission(session, submission_id)
+        submission.state = SubmissionState.VALIDATION_ERROR
+        await upsert_helper(session, submission, SubmissionDAO)
+
+
 async def upsert_filing_period(session: AsyncSession, filing_period: FilingPeriodDTO) -> FilingPeriodDAO:
     return await upsert_helper(session, filing_period, FilingPeriodDAO)
 

src/sbl_filing_api/routers/filing.py

@@ -220,6 +220,13 @@ async def get_submissions(request: Request, lei: str, period_code: str):
 @router.get("/institutions/{lei}/filings/{period_code}/submissions/latest", response_model=SubmissionDTO)
 @requires("authenticated")
 async def get_submission_latest(request: Request, lei: str, period_code: str):
+    filing = await repo.get_filing(request.state.db_session, lei, period_code)
+    if not filing:
+        raise RegTechHttpException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            name="Filing Not Found",
+            detail=f"There is no Filing for LEI {lei} in period {period_code}, unable to get latest submission for it.",
+        )
     result = await repo.get_latest_submission(request.state.db_session, lei, period_code)
     if result:
         return result
@@ -317,6 +324,13 @@ async def put_contact_info(request: Request, lei: str, period_code: str, contact
 )
 @requires("authenticated")
 async def get_latest_submission_report(request: Request, lei: str, period_code: str):
+    filing = await repo.get_filing(request.state.db_session, lei, period_code)
+    if not filing:
+        raise RegTechHttpException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            name="Filing Not Found",
+            detail=f"There is no Filing for LEI {lei} in period {period_code}, unable to get latest submission for it.",
+        )
     latest_sub = await repo.get_latest_submission(request.state.db_session, lei, period_code)
     if latest_sub:
         file_data = submission_processor.get_from_storage(

src/sbl_filing_api/services/multithread_handler.py

@@ -21,7 +21,12 @@ def handle_submission(period_code: str, lei: str, submission: SubmissionDAO, con
 
 async def check_future(future, submission_id, exec_check):
     await asyncio.sleep(settings.expired_submission_check_secs)
-    if not future.done():
+    if future.done() and future.exception():
+        future.cancel()
+        exec_check["continue"] = False
+        await repo.error_out_submission(submission_id)
+        logger.error(f"Validation for submission {submission_id} did not complete due to an unexpected error.")
+    elif not future.done():
         future.cancel()
         exec_check["continue"] = False
         await repo.expire_submission(submission_id)

src/sbl_filing_api/services/submission_processor.py

@@ -1,4 +1,4 @@
-import json
+import ujson
 from typing import Generator
 import pandas as pd
 import importlib.metadata as imeta
@@ -114,7 +114,7 @@ async def validate_and_update_submission(
 
 
 def build_validation_results(result):
-    val_json = json.loads(df_to_json(result[1]))
+    val_json = ujson.loads(df_to_json(result[1]))
 
     if result[2] == ValidationPhase.SYNTACTICAL.value:
         val_res = {"syntax_errors": {"count": len(val_json), "details": val_json}}

tests/api/routers/test_filing_api.py

@@ -174,7 +174,9 @@ def test_unauthed_get_latest_submissions(
         res = client.get("/v1/filing/institutions/123456790/filings/2024/submissions/latest")
         assert res.status_code == 403
 
-    async def test_get_latest_submission(self, mocker: MockerFixture, app_fixture: FastAPI, authed_user_mock: Mock):
+    async def test_get_latest_submission(
+        self, mocker: MockerFixture, app_fixture: FastAPI, get_filing_mock: Mock, authed_user_mock: Mock
+    ):
         user_action_submit = UserActionDAO(
             id=2,
             user_id="123456-7890-ABCDEF-GHIJ",
@@ -183,6 +185,7 @@ async def test_get_latest_submission(self, mocker: MockerFixture, app_fixture: F
             action_type=UserActionType.SUBMIT,
             timestamp=datetime.datetime.now(),
         )
+
         mock = mocker.patch("sbl_filing_api.entities.repos.submission_repo.get_latest_submission")
         mock.return_value = SubmissionDAO(
             filing=1,
@@ -208,6 +211,12 @@ async def test_get_latest_submission(self, mocker: MockerFixture, app_fixture: F
         mock.assert_called_with(ANY, "1234567890ZXWVUTSR00", "2024")
         assert res.status_code == 204
 
+        # verify Filing Not Found RegTechHttpException returned when filing does not exist
+        get_filing_mock.return_value = None
+        client = TestClient(app_fixture)
+        res = client.get("/v1/filing/institutions/1234567890ZXWVUTSR00/filings/2024/submissions/latest")
+        assert res.status_code == 404
+
     def test_unauthed_get_submission_by_id(self, mocker: MockerFixture, app_fixture: FastAPI):
         client = TestClient(app_fixture)
         res = client.get("/v1/filing/institutions/123456790/filings/2024/submissions/1")
@@ -956,7 +965,9 @@ async def test_errors_sign_filing(
             == "There is no Filing for LEI 1234567890ABCDEFGH00 in period 2024, unable to sign a non-existent Filing."
         )
 
-    async def test_get_latest_sub_report(self, mocker: MockerFixture, app_fixture: FastAPI, authed_user_mock: Mock):
+    async def test_get_latest_sub_report(
+        self, mocker: MockerFixture, app_fixture: FastAPI, get_filing_mock: Mock, authed_user_mock: Mock
+    ):
         sub_mock = mocker.patch("sbl_filing_api.entities.repos.submission_repo.get_latest_submission")
         sub_mock.return_value = SubmissionDAO(
             id=1,
@@ -994,6 +1005,12 @@ async def test_get_latest_sub_report(self, mocker: MockerFixture, app_fixture: F
         sub_mock.assert_called_with(ANY, "1234567890ZXWVUTSR00", "2024")
         assert res.status_code == 204
 
+        # verify Filing Not Found RegTechHttpException returned when filing does not exist
+        get_filing_mock.return_value = None
+        client = TestClient(app_fixture)
+        res = client.get("/v1/filing/institutions/1234567890ZXWVUTSR00/filings/2024/submissions/latest/report")
+        assert res.status_code == 404
+
     async def test_get_sub_report(self, mocker: MockerFixture, app_fixture: FastAPI, authed_user_mock: Mock):
         sub_mock = mocker.patch("sbl_filing_api.entities.repos.submission_repo.get_submission")
         sub_mock.return_value = SubmissionDAO(

tests/entities/repos/test_submission_repo.py

@@ -30,7 +30,7 @@ class TestSubmissionRepo:
     async def setup(
         self, transaction_session: AsyncSession, mocker: MockerFixture, session_generator: async_scoped_session
     ):
-        mocker.patch.object(repo, "SessionLocal", return_value=session_generator)
+        mocker.patch.object(repo, "SessionLocal", return_value=session_generator())
 
         user_action1 = UserActionDAO(
             id=1,
@@ -416,6 +416,12 @@ async def test_add_submission(self, transaction_session: AsyncSession):
         assert res.submitter.user_email == user_action_submit.user_email
         assert res.submitter.action_type == UserActionType.SUBMIT
 
+    async def test_error_out_submission(self, transaction_session: AsyncSession):
+        await repo.error_out_submission(4)
+        expired_sub = await repo.get_submission(transaction_session, 4)
+        assert expired_sub.id == 4
+        assert expired_sub.state == SubmissionState.VALIDATION_ERROR
+
     async def test_update_submission(self, session_generator: async_scoped_session):
         user_action_submit = UserActionDAO(
             id=2,

tests/services/test_multithreader.py

@@ -12,35 +12,55 @@ async def mock_future(self, sleeptime):
         await asyncio.sleep(sleeptime)
         return
 
+    async def mock_future_exception(self):
+        await asyncio.sleep(2)
+        raise asyncio.InvalidStateError("Pool died.")
+
     async def test_future_checker(self, mocker: MockerFixture):
         exec_check = Manager().dict()
         exec_check["continue"] = True
 
         mocker.patch("sbl_filing_api.services.multithread_handler.settings.expired_submission_check_secs", 4)
-        repo_mock = mocker.patch("sbl_filing_api.entities.repos.submission_repo.expire_submission")
+        expire_mock = mocker.patch("sbl_filing_api.entities.repos.submission_repo.expire_submission")
+        error_mock = mocker.patch("sbl_filing_api.entities.repos.submission_repo.error_out_submission")
         log_mock = mocker.patch("sbl_filing_api.services.multithread_handler.logger")
 
-        future = asyncio.get_event_loop().create_task(self.mock_future(5))
+        future = asyncio.get_event_loop().create_task(self.mock_future_exception())
         cancel_mock = mocker.patch.object(future, "cancel")
         await check_future(future, 1, exec_check)
 
         assert not exec_check["continue"]
         cancel_mock.assert_called_once()
-        repo_mock.assert_called_with(1)
+        error_mock.assert_called_with(1)
+        log_mock.error.assert_called_with("Validation for submission 1 did not complete due to an unexpected error.")
+
+        log_mock.reset_mock()
+
+        future = asyncio.get_event_loop().create_task(self.mock_future(6))
+        cancel_mock = mocker.patch.object(future, "cancel")
+        exec_check["continue"] = True
+        await check_future(future, 1, exec_check)
+
+        assert not exec_check["continue"]
+        cancel_mock.assert_called_once()
+        expire_mock.assert_called_with(1)
         log_mock.warning.assert_called_with(
             "Validation for submission 1 did not complete within the expected timeframe, will be set to VALIDATION_EXPIRED."
         )
 
-        repo_mock.reset_mock()
+        expire_mock.reset_mock()
+        error_mock.reset_mock()
         log_mock.reset_mock()
         cancel_mock.reset_mock()
+
         future = asyncio.get_event_loop().create_task(self.mock_future(1))
         exec_check["continue"] = True
         await check_future(future, 2, exec_check)
 
         assert exec_check["continue"]
         assert not cancel_mock.called
-        assert not repo_mock.called
+        assert not expire_mock.called
+        assert not error_mock.called
         assert not log_mock.called
 
     async def test_handler(self, mocker: MockerFixture):