MRG: Merge pull request #548 from octue/feature/support-poetry-based-…

…apps-on-cloud-run

Add periodic monitor messages and support poetry-based apps on Cloud Run

.pre-commit-config.yaml

@@ -23,8 +23,8 @@ repos:
       - id: black
         args: ["--line-length", "120"]
 
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
     hooks:
       - id: flake8
         language_version: python3

octue/cli.py

@@ -363,9 +363,17 @@ def deploy():
 )
 @click.option("--no-cache", is_flag=True, help="If provided, don't use the Docker cache.")
 @click.option("--update", is_flag=True, help="If provided, allow updates to an existing service.")
-def cloud_run(service_config, update, no_cache):
+@click.option(
+    "--revision-tag",
+    type=str,
+    default=None,
+    help="A tag to use for this revision of the service (e.g. 1.3.7). This overrides the `OCTUE_SERVICE_REVISION_TAG` "
+    "environment variable if it's present. If this option isn't given and the environment variable isn't present, a "
+    "random 'cool name' tag is generated e.g 'curious-capybara'.",
+)
+def cloud_run(service_config, update, no_cache, revision_tag):
     """Deploy a python app to Google Cloud Run as an Octue service or digital twin."""
-    CloudRunDeployer(service_config).deploy(update=update, no_cache=no_cache)
+    CloudRunDeployer(service_config, revision_tag=revision_tag).deploy(update=update, no_cache=no_cache)
 
 
 @deploy.command()
@@ -385,7 +393,15 @@ def cloud_run(service_config, update, no_cache):
     help="If provided, skip creating and running the build trigger and just deploy a pre-built image to Dataflow",
 )
 @click.option("--image-uri", type=str, default=None, help="The actual image URI to use when creating the Dataflow job.")
-def dataflow(service_config, no_cache, update, dataflow_job_only, image_uri):
+@click.option(
+    "--revision-tag",
+    type=str,
+    default=None,
+    help="A tag to use for this revision of the service (e.g. 1.3.7). This overrides the `OCTUE_SERVICE_REVISION_TAG` "
+    "environment variable if it's present. If this option isn't given and the environment variable isn't present, a "
+    "random 'cool name' tag is generated e.g 'curious-capybara'.",
+)
+def dataflow(service_config, no_cache, update, dataflow_job_only, image_uri, revision_tag):
     """Deploy a python app to Google Dataflow as an Octue service or digital twin."""
     if bool(importlib.util.find_spec("apache_beam")):
         # Import the Dataflow deployer only if the `apache-beam` package is available (due to installing `octue` with
@@ -397,7 +413,7 @@ def dataflow(service_config, no_cache, update, dataflow_job_only, image_uri):
             "`pip install octue[dataflow]`"
         )
 
-    deployer = DataflowDeployer(service_config)
+    deployer = DataflowDeployer(service_config, revision_tag=revision_tag)
 
     if dataflow_job_only:
         deployer.create_streaming_dataflow_job(image_uri=image_uri, update=update)

octue/cloud/deployment/google/base_deployer.py

@@ -4,6 +4,7 @@
 import time
 from abc import abstractmethod
 
+import coolname
 import yaml
 
 from octue.cloud.service_id import convert_service_id_to_pub_sub_form, create_service_sruid, get_service_sruid_parts
@@ -35,13 +36,20 @@ class BaseDeployer:
     pubsub-emulator 0.6.0
     ```
 
+    The service revision tag resolution order is:
+    1. The `revision_tag` variable if present
+    2. The `OCTUE_SERVICE_REVISION_TAG` environment variable if present (returned by `get_service_sruid_parts`)
+    3. A generated 'cool name'
+
     :param str octue_configuration_path: the path to the `octue.yaml` file if it's not in the current working directory
+    :param str|None image_uri_template: the image URI template to use to name and store images with Cloud Build
+    :param str|None revision_tag: a tag to use for this revision of the service (e.g. 1.3.7). This overrides the `OCTUE_SERVICE_REVISION_TAG` environment variable if it's present. If this option isn't given and the environment variable isn't present, a random 'cool name' tag is generated e.g 'curious-capybara'.
     :return None:
     """
 
     TOTAL_NUMBER_OF_STAGES = None
 
-    def __init__(self, octue_configuration_path, image_uri_template=None):
+    def __init__(self, octue_configuration_path, image_uri_template=None, revision_tag=None):
         self.service_configuration = ServiceConfiguration.from_file(octue_configuration_path)
 
         # Generated attributes.
@@ -52,6 +60,12 @@ def __init__(self, octue_configuration_path, image_uri_template=None):
             self.service_configuration
         )
 
+        if revision_tag:
+            self.service_revision_tag = revision_tag
+
+        if not self.service_revision_tag:
+            self.service_revision_tag = coolname.generate_slug(2)
+
         self.service_sruid = create_service_sruid(
             namespace=self.service_namespace,
             name=self.service_name,
@@ -76,6 +90,8 @@ def __init__(self, octue_configuration_path, image_uri_template=None):
             f"[SUCCESS] Service deployed - it can be questioned via Pub/Sub at {self.service_sruid!r}."
         )
 
+        print(f"Ready to deploy {self.service_sruid!r}.")
+
     @abstractmethod
     def deploy(self, no_cache=False, update=False):
         """Deploy the octue app.

octue/cloud/deployment/google/cloud_run/Dockerfile

@@ -6,14 +6,21 @@ ENV PYTHONUNBUFFERED True
 ENV PROJECT_ROOT=/app
 WORKDIR $PROJECT_ROOT
 
-RUN apt-get update -y && apt-get install -y --fix-missing build-essential && rm -rf /var/lib/apt/lists/*
+RUN apt-get update -y && apt-get install -y --fix-missing build-essential curl && rm -rf /var/lib/apt/lists/*
 
 COPY . .
 
-# Install requirements (supports requirements.txt and setup.py; both will be run if both are present.)
-RUN if [ ! -f "requirements.txt" ] && [ ! -f "setup.py" ]; then exit 1; fi
+# Install poetry if necessary.
+ENV POETRY_HOME=/etc/poetry
+ENV PATH "$POETRY_HOME/bin:$PATH"
+RUN if [ -f "pyproject.toml" ]; then curl -sSL https://install.python-poetry.org | python3 - && poetry config virtualenvs.create false; fi
+
+# Install dependencies - supports `setup.py`, `requirements.txt`, and `pyproject.toml` via poetry. All dependency files
+# are installed from if present.
+RUN if [ ! -f "requirements.txt" ] && [ ! -f "setup.py" ] && [ ! -f "pyproject.toml" ]; then exit 1; fi
 RUN if [ -f "requirements.txt" ]; then pip install --upgrade pip && pip install -r requirements.txt; fi
 RUN if [ -f "setup.py" ]; then pip install --upgrade pip && pip install -e .; fi
+RUN if [ -f "pyproject.toml" ]; then poetry install --only main; fi
 
 EXPOSE $PORT
 

octue/cloud/deployment/google/cloud_run/deployer.py

@@ -26,13 +26,15 @@ class CloudRunDeployer(BaseDeployer):
     ```
 
     :param str octue_configuration_path: the path to the `octue.yaml` file if it's not in the current working directory
+    :param str|None image_uri_template: the image URI template to use to name and store images with Cloud Build
+    :param str revision_tag: a tag to use for this revision of the service (e.g. 1.3.7). This overrides the `OCTUE_SERVICE_REVISION_TAG` environment variable if it's present. If this option isn't given and the environment variable isn't present, a random 'cool name' tag is generated e.g 'curious-capybara'.
     :return None:
     """
 
     TOTAL_NUMBER_OF_STAGES = 5
 
-    def __init__(self, octue_configuration_path, image_uri_template=None):
-        super().__init__(octue_configuration_path, image_uri_template)
+    def __init__(self, octue_configuration_path, image_uri_template=None, revision_tag=None):
+        super().__init__(octue_configuration_path, image_uri_template, revision_tag=revision_tag)
         self.build_trigger_description = f"Build the {self.service_sruid!r} service and deploy it to Cloud Run."
 
     def deploy(self, no_cache=False, update=False):
@@ -137,6 +139,7 @@ def _generate_cloud_build_configuration(self, no_cache=False):
                             f"--region={self.service_configuration.region}",
                             f"--memory={self.service_configuration.memory}",
                             f"--cpu={self.service_configuration.cpus}",
+                            f"--execution-environment={self.service_configuration.execution_environment}",
                             f"--set-env-vars={environment_variables}",
                             "--timeout=3600",
                             f"--concurrency={self.service_configuration.concurrency}",

octue/cloud/deployment/google/dataflow/deployer.py

@@ -32,13 +32,15 @@ class DataflowDeployer(BaseDeployer):
     ```
 
     :param str octue_configuration_path: the path to the `octue.yaml` file if it's not in the current working directory
+    :param str|None image_uri_template: the image URI template to use to name and store images with Cloud Build
+    :param str revision_tag: a tag to use for this revision of the service (e.g. 1.3.7). This overrides the `OCTUE_SERVICE_REVISION_TAG` environment variable if it's present. If this option isn't given and the environment variable isn't present, a random 'cool name' tag is generated e.g 'curious-capybara'.
     :return None:
     """
 
     TOTAL_NUMBER_OF_STAGES = 3
 
-    def __init__(self, octue_configuration_path, image_uri_template=None):
-        super().__init__(octue_configuration_path, image_uri_template)
+    def __init__(self, octue_configuration_path, image_uri_template=None, revision_tag=None):
+        super().__init__(octue_configuration_path, image_uri_template, revision_tag=revision_tag)
         self.build_trigger_description = f"Build the {self.service_sruid!r} service and deploy it to Dataflow."
 
         if not self.service_configuration.temporary_files_location:

octue/cloud/emulators/child.py

@@ -19,12 +19,12 @@ class ChildEmulator:
 
     :param str|None id: the ID of the child; a UUID is generated if none is provided
     :param dict|None backend: a dictionary including the key "name" with a value of the name of the type of backend (e.g. "GCPPubSubBackend") and key-value pairs for any other parameters the chosen backend expects; a mock backend is used if none is provided
-    :param str|None internal_service_name: the name to give to the internal service used to ask questions to the child; defaults to "<id>-parent"
+    :param str internal_service_name: the name to give to the internal service used to ask questions to the child
     :param list(dict)|None messages: the list of messages to send to the parent
     :return None:
     """
 
-    def __init__(self, id=None, backend=None, internal_service_name=None, messages=None):
+    def __init__(self, id=None, backend=None, internal_service_name="local/local:local", messages=None):
         self.messages = messages or []
 
         backend = copy.deepcopy(backend or {"name": "GCPPubSubBackend", "project_name": "emulated-project"})
@@ -36,7 +36,7 @@ def __init__(self, id=None, backend=None, internal_service_name=None, messages=N
 
         self._parent = MockService(
             backend=backend,
-            service_id=internal_service_name or f"{self.id}-parent",
+            service_id=internal_service_name,
             children={self._child.id: self._child},
         )
 

octue/configuration.py

@@ -42,6 +42,7 @@ def __init__(
         memory="128Mi",
         cpus=1,
         minimum_instances=0,
+        execution_environment="gen2",
         temporary_files_location=None,
         setup_file_path=None,
         service_account_email=None,
@@ -72,6 +73,7 @@ def __init__(
         self.memory = memory
         self.cpus = cpus
         self.minimum_instances = minimum_instances
+        self.execution_environment = execution_environment
 
         # Dataflow services only.
         self.temporary_files_location = temporary_files_location

octue/metadata/recorded_questions.jsonl

@@ -57,3 +57,4 @@
 {"parent_sdk_version": "0.40.2", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmprj5sl5al\\\"\\n    },\\n    \\\"id\\\": \\\"6ce744fb-fce6-4346-949a-109675ca7a5a\\\",\\n    \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "42d9a880-8672-4be6-a3ae-b8518a9e58db", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.2"}}}
 {"parent_sdk_version": "0.41.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpnc9s8hwm\\\"\\n    },\\n    \\\"id\\\": \\\"b21ca765-9dac-4291-a301-daa4d282f1fd\\\",\\n    \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "76ae0850-50b7-4c7d-a22b-c8bcd0ab2bb3", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.41.0"}}}
 {"parent_sdk_version": "0.41.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"6e57125c-3e32-407c-b0f8-6403d2db6d8a\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpsri5p05d\"}}, \"children\": null}", "attributes": {"question_uuid": "07bfa681-a93c-44e4-9ee7-a33e9518a77c", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.41.1"}}}
+{"parent_sdk_version": "0.42.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": {\"id\": \"0a0f80fc-0e81-4a3e-b345-84e7b3e049c6\", \"name\": null, \"datasets\": {\"my_dataset\": \"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmplu287_ud\"}}, \"children\": null}", "attributes": {"question_uuid": "fcd103db-6b9d-497b-a4d6-1c8f8c4403b6", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.42.0"}}}