MRG: Merge pull request #608 from octue/support-automatic-question-re…

…trying Add ability to automatically retry failed multiple questions
octue · Nov 13, 2023 · 384e5a1 · 384e5a1
2 parents c481d28 + 5632abf
commit 384e5a1
Show file tree

Hide file tree

Showing 15 changed files with 668 additions and 346 deletions.
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
@@ -81,11 +81,15 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v3
 
+      - name: Install Poetry
+        uses: snok/install-poetry@v1.3.4
+
+      - name: Build a binary wheel and a source tarball
+        run: poetry build
+
       - name: Test package is publishable with PyPI test server
-        uses: JRubics/poetry-publish@v1.10
+        uses: pypa/gh-action-pypi-publish@v1.8.10
         with:
-          python_version: "3.9"
-          pypi_token: ${{ secrets.TEST_PYPI_TOKEN }}
-          repository_name: "testpypi"
-          repository_url: "https://test.pypi.org/legacy/"
-          ignore_dev_requirements: "yes"
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
+          verbose: true
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -83,18 +83,24 @@ jobs:
           prerelease: false
 
   publish:
-    runs-on: ubuntu-latest
     needs: release
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v3
 
-      - name: Build and publish latest package to PyPI
-        uses: JRubics/poetry-publish@v1.10
-        with:
-          python_version: "3.9"
-          pypi_token: ${{ secrets.PYPI_TOKEN }}
-          ignore_dev_requirements: "yes"
+      - name: Install Poetry
+        uses: snok/install-poetry@v1.3.4
+
+      - name: Build a binary wheel and a source tarball
+        run: poetry build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@v1.8.10
 
   docker:
     runs-on: ubuntu-latest

diff --git a/docs/source/asking_questions.rst b/docs/source/asking_questions.rst
@@ -64,8 +64,7 @@ sent back to the parent. If you're not sure how long a particular analysis might
 Asking multiple questions in parallel
 =====================================
 You can also ask multiple questions to a service in parallel. By default, if any of the questions fail, an error is
-raised and no answers are returned. If ``raise_errors=False`` is provided, answers are returned for all successful
-questions while errors are returned unraised for unsuccessful ones.
+raised and no answers are returned.
 
 .. code-block:: python
 
@@ -80,7 +79,16 @@ questions while errors are returned unraised for unsuccessful ones.
             {"output_values": {"different": "result"}, "output_manifest": None},
         ]
 
-This method uses threads, allowing all the questions to be asked at once instead of one after another.
+This method uses multithreading, allowing all the questions to be asked at once instead of one after another.
+
+Options:
+
+- If ``raise_errors=False`` is provided, answers are returned for all successful questions while unraised errors are
+  returned for unsuccessful ones
+- If ``raise_errors=False`` is provided with ``max_retries > 0``, failed questions are retried up to this number of
+  times
+- If ``raise_errors=False`` is provided with ``max_retries > 0`` and ``prevent_retries_when`` is set to a list of
+  exception types, failed questions are retried except for those whose exception types are in the list
 
 
 Asking a question within a service

diff --git a/docs/source/inter_service_compatibility.rst b/docs/source/inter_service_compatibility.rst
diff --git a/octue/cloud/deployment/google/cloud_run/Dockerfile-python311 b/octue/cloud/deployment/google/cloud_run/Dockerfile-python311
@@ -0,0 +1,49 @@
+FROM windpioneers/gdal-python:little-gecko-gdal-2.4.1-python-3.11-slim
+
+# Ensure print statements and log messages appear promptly in Cloud Logging.
+ENV PYTHONUNBUFFERED True
+
+ENV PROJECT_ROOT=/workspace
+WORKDIR $PROJECT_ROOT
+
+RUN apt-get update -y && apt-get install -y --fix-missing build-essential && rm -rf /var/lib/apt/lists/*
+
+# Install poetry.
+ENV POETRY_HOME=/root/.poetry
+ENV PATH "$POETRY_HOME/bin:$PATH"
+RUN curl -sSL https://install.python-poetry.org | python3 - && poetry config virtualenvs.create false;
+
+# Copy in the dependencies file(s) for caching. One or more of `requirements.txt`, `setup.py`, and `pyproject.toml and
+# `poetry.lock` must be present.
+COPY pyproject.tom[l] poetry.loc[k] setup.p[y] requirements.tx[t] ./
+
+# If `pyproject.toml` is present, install the dependencies only to utilise layer caching for quick rebuilds.
+RUN if [ -f "pyproject.toml" ]; then poetry install  \
+    --no-ansi  \
+    --no-interaction  \
+    --no-cache  \
+    --no-root  \
+    --only main;  \
+    fi
+
+# Copy local code to the application root directory.
+COPY . .
+
+# Install local packages if using poetry. Otherwise, install everything if using `setup.py` or `requirements.txt`.
+RUN if [ -f "pyproject.toml" ]; then poetry install --only main;  \
+    elif [ -f "setup.py" ]; then pip install --upgrade pip && pip install -e .;  \
+    elif [ -f "requirements.txt" ]; then pip install --upgrade pip && pip install -r requirements.txt; fi
+
+EXPOSE $PORT
+
+ENV USE_OCTUE_LOG_HANDLER=1
+ENV COMPUTE_PROVIDER=GOOGLE_CLOUD_RUN
+
+ARG GUNICORN_WORKERS=1
+ENV GUNICORN_WORKERS=$GUNICORN_WORKERS
+
+ARG GUNICORN_THREADS=8
+ENV GUNICORN_THREADS=$GUNICORN_THREADS
+
+# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling.
+CMD exec gunicorn --bind :$PORT --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --timeout 0 octue.cloud.deployment.google.cloud_run.flask_app:app
diff --git a/octue/metadata/recorded_questions.jsonl b/octue/metadata/recorded_questions.jsonl
@@ -80,3 +80,4 @@
 {"parent_sdk_version": "0.49.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"34216d6d-e32e-4c40-8b05-ee9bd71966c9\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppi1e5t2n\"}}, \"children\": null, \"message_number\": 0}", "attributes": {"question_uuid": "3b1224da-8e7f-4b1b-8ef3-c5cf1a5e698c", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.49.0"}}}
 {"parent_sdk_version": "0.49.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"34216d6d-e32e-4c40-8b05-ee9bd71966c9\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppi1e5t2n\"}}, \"children\": null, \"message_number\": 0}", "attributes": {"question_uuid": "3b1224da-8e7f-4b1b-8ef3-c5cf1a5e698c", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.49.1"}}}
 {"parent_sdk_version": "0.49.2", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"bdadcd12-a51b-4571-9db3-9d240bd15faa\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpw3232ii0\"}}, \"children\": null, \"message_number\": 0}", "attributes": {"question_uuid": "0b57da2c-38e2-4f8b-9368-fe35125da8fb", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.49.2"}}}
+{"parent_sdk_version": "0.50.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"2d618d16-2a2b-437b-8db7-d5139e1645f2\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpr7jpj4sp\"}}, \"children\": null, \"message_number\": 0}", "attributes": {"question_uuid": "7bc572dc-ab8b-4aed-a132-b0b509caceb3", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.50.0"}}}