profile browser fixes: better resource usage + load retry (#1599)

- Backend: Use separate resource constraints for profiles: default
profile browser resources to either 'profile_browser_cpu' /
'profile_browser_memory' or single browser 'crawler_memory_base' /
'crawler_cpu_base', instead of scaled to the number of browser workers

- Frontend: check that profile html page is loading, keep retrying if
still getting nginx error instead of loading an iframe with the error.

- Fixes #1598
(Prepared for 1.9.4 release)

backend/btrixcloud/operator.py

@@ -304,6 +304,7 @@ def __init__(
 
         self._has_pod_metrics = False
         self.compute_crawler_resources()
+        self.compute_profile_resources()
 
         # to avoid background tasks being garbage collected
         # see: https://stackoverflow.com/a/74059981
@@ -336,6 +337,23 @@ def compute_crawler_resources(self):
         else:
             print(f"memory = {p['crawler_memory']}")
 
+    def compute_profile_resources(self):
+        """compute memory /cpu resources for a single profile browser"""
+        p = self.shared_params
+        # if no profile specific options provided, default to crawler base for one browser
+        profile_cpu = parse_quantity(
+            p.get("profile_browser_cpu") or p["crawler_cpu_base"]
+        )
+        profile_memory = parse_quantity(
+            p.get("profile_browser_memory") or p["crawler_memory_base"]
+        )
+        p["profile_cpu"] = profile_cpu
+        p["profile_memory"] = profile_memory
+
+        print("profile browser resources")
+        print(f"cpu = {profile_cpu}")
+        print(f"memory = {profile_memory}")
+
     async def async_init(self):
         """perform any async init here"""
         self._has_pod_metrics = await self.is_pod_metrics_available()

chart/app-templates/profilebrowser.yaml

@@ -78,8 +78,8 @@ spec:
 
       resources:
         limits:
-          memory: "{{ crawler_memory }}"
+          memory: "{{ profile_memory }}"
 
         requests:
-          cpu: "{{ crawler_cpu }}"
-          memory: "{{ crawler_memory }}"
+          cpu: "{{ profile_cpu }}"
+          memory: "{{ profile_memory }}"

chart/values.yaml

@@ -220,6 +220,11 @@ crawler_extra_memory_per_browser: 768Mi
 # crawler_memory = crawler_memory_base + crawler_memory_per_extra_browser * (crawler_browser_instances - 1)
 # crawler_memory:
 
+# optional: defaults to crawler_memory_base and crawler_cpu_base if not set
+# profile_browser_memory:
+#
+# profile_browser_cpu:
+
 # Other Crawler Settings
 # ----------------------
 

frontend/src/features/browser-profiles/profile-browser.ts

@@ -301,6 +301,18 @@ export class ProfileBrowser extends LiteElement {
 
       return;
     } else if (result.url) {
+      // check that the browser is actually available
+      // if not, continue waiting
+      // (will not work with local frontend due to CORS)
+      try {
+        const resp = await fetch(result.url, { method: "HEAD" });
+        if (!resp.ok) {
+          return;
+        }
+      } catch (e) {
+        // ignore
+      }
+
       if (this.initialNavigateUrl) {
         await this.navigateBrowser({ url: this.initialNavigateUrl });
       }