MRG: Merge pull request #536 from octue/fix/use-octue-JSON-encoder-wh…

…en-uploading-manifests Fix manifest upload and analysis error logging
octue · Oct 20, 2022 · b026de9 · b026de9
2 parents 24b0ba0 + 993bbb1
commit b026de9
Show file tree

Hide file tree

Showing 10 changed files with 303 additions and 180 deletions.
diff --git a/docs/source/inter_service_compatibility.rst b/docs/source/inter_service_compatibility.rst
diff --git a/octue/cli.py b/octue/cli.py
@@ -158,7 +158,7 @@ def run(service_config, input_dir, output_file, output_manifest_file, monitor_me
         handle_monitor_message=monitor_message_handler,
     )
 
-    click.echo(json.dumps(analysis.output_values))
+    click.echo(json.dumps(analysis.output_values, cls=OctueJSONEncoder))
 
     if analysis.output_values and output_file:
         if not os.path.exists(os.path.dirname(output_file)):

diff --git a/octue/cloud/emulators/_pub_sub.py b/octue/cloud/emulators/_pub_sub.py
@@ -7,6 +7,7 @@
 from octue.cloud.pub_sub import Subscription, Topic
 from octue.cloud.pub_sub.service import Service
 from octue.resources import Manifest
+from octue.utils.encoders import OctueJSONEncoder
 
 
 logger = logging.getLogger(__name__)
@@ -306,7 +307,8 @@ def ask(
             self.children[service_id].answer(
                 MockMessage(
                     data=json.dumps(
-                        {"input_values": input_values, "input_manifest": input_manifest, "children": children}
+                        {"input_values": input_values, "input_manifest": input_manifest, "children": children},
+                        cls=OctueJSONEncoder,
                     ).encode(),
                     question_uuid=question_uuid,
                     forward_logs=subscribe_to_logs,

diff --git a/octue/cloud/pub_sub/service.py b/octue/cloud/pub_sub/service.py
@@ -459,7 +459,7 @@ def _send_monitor_message(self, data, topic, timeout=30):
         self._send_message(
             {
                 "type": "monitor_message",
-                "data": json.dumps(data),
+                "data": json.dumps(data, cls=OctueJSONEncoder),
                 "message_number": topic.messages_published,
             },
             topic=topic,

diff --git a/octue/metadata/recorded_questions.jsonl b/octue/metadata/recorded_questions.jsonl
@@ -54,3 +54,4 @@
 {"parent_sdk_version": "0.39.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp664lr951\\\"\\n    },\\n    \\\"id\\\": \\\"57a78545-f40f-4971-bff9-b138bf1d6292\\\",\\n    \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "7a8eed0b-bd2d-4eb7-93b3-5eb8244c812d", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.39.0"}}}
 {"parent_sdk_version": "0.40.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppnbc6rcc\\\"\\n    },\\n    \\\"id\\\": \\\"31440887-beef-41a3-935f-08adbe8dbbdc\\\",\\n    \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "f66edd6e-517c-4c26-9d60-16cb508e5fe1", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.0"}}}
 {"parent_sdk_version": "0.40.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmppnbc6rcc\\\"\\n    },\\n    \\\"id\\\": \\\"31440887-beef-41a3-935f-08adbe8dbbdc\\\",\\n    \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "f66edd6e-517c-4c26-9d60-16cb508e5fe1", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.1"}}}
+{"parent_sdk_version": "0.40.2", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmprj5sl5al\\\"\\n    },\\n    \\\"id\\\": \\\"6ce744fb-fce6-4346-949a-109675ca7a5a\\\",\\n    \\\"name\\\": null\\n}\", \"children\": null}", "attributes": {"question_uuid": "42d9a880-8672-4be6-a3ae-b8518a9e58db", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.40.2"}}}