Merge V1.9.3 changes into main

backend/btrixcloud/crawls.py

@@ -441,6 +441,13 @@ async def inc_crawl_exec_time(self, crawl_id, exec_time, last_updated_time):
             },
         )
 
+    async def get_crawl_exec_last_update_time(self, crawl_id):
+        """get crawl last updated time"""
+        res = await self.crawls.find_one(
+            {"_id": crawl_id, "type": "crawl"}, projection=["_lut"]
+        )
+        return res and res.get("_lut")
+
     async def get_crawl_state(self, crawl_id):
         """return current crawl state of a crawl"""
         res = await self.crawls.find_one(

backend/btrixcloud/operator/crawls.py

@@ -232,7 +232,9 @@ async def sync_crawls(self, data: MCSyncData):
 
         else:
             status.scale = crawl.scale
-            status.lastUpdatedTime = to_k8s_date(dt_now())
+            now = dt_now()
+            await self.crawl_ops.inc_crawl_exec_time(crawl_id, 0, now)
+            status.lastUpdatedTime = to_k8s_date(now)
 
         children = self._load_redis(params, status, data.children)
 
@@ -828,12 +830,15 @@ async def increment_pod_exec_time(
         """inc exec time tracking"""
         now = dt_now()
 
-        if not status.lastUpdatedTime:
+        update_start_time = await self.crawl_ops.get_crawl_exec_last_update_time(
+            crawl_id
+        )
+
+        if not update_start_time:
+            await self.crawl_ops.inc_crawl_exec_time(crawl_id, 0, now)
             status.lastUpdatedTime = to_k8s_date(now)
             return
 
-        update_start_time = from_k8s_date(status.lastUpdatedTime)
-
         reason = None
         update_duration = (now - update_start_time).total_seconds()
 
@@ -907,16 +912,6 @@ async def increment_pod_exec_time(
                 max_duration = max(duration, max_duration)
 
         if exec_time:
-            if not await self.crawl_ops.inc_crawl_exec_time(
-                crawl_id, exec_time, status.lastUpdatedTime
-            ):
-                # if lastUpdatedTime is same as previous, something is wrong, don't update!
-                print(
-                    "Already updated for lastUpdatedTime, skipping execTime update!",
-                    flush=True,
-                )
-                return
-
             await self.org_ops.inc_org_time_stats(oid, exec_time, True)
             status.crawlExecTime += exec_time
             status.elapsedCrawlTime += max_duration
@@ -926,6 +921,7 @@ async def increment_pod_exec_time(
             flush=True,
         )
 
+        await self.crawl_ops.inc_crawl_exec_time(crawl_id, exec_time, now)
         status.lastUpdatedTime = to_k8s_date(now)
 
     def should_mark_waiting(self, state, started):

frontend/src/components/ui/copy-button.ts

@@ -57,6 +57,8 @@ export class CopyButton extends LitElement {
             ? this.content
             : msg("Copy")}
         ?hoist=${this.hoist}
+        @sl-hide=${this.stopProp}
+        @sl-after-hide=${this.stopProp}
       >
         <sl-icon-button
           name=${this.isCopied ? "check-lg" : this.name ? this.name : "files"}
@@ -82,4 +84,13 @@ export class CopyButton extends LitElement {
       button?.blur(); // Remove focus from the button to set it back to its default state
     }, 3000);
   }
+
+  /**
+   * Stop propgation of sl-tooltip events.
+   * Prevents bug where sl-dialog closes when tooltip closes
+   * https://github.com/shoelace-style/shoelace/issues/170
+   */
+  private stopProp(e: Event) {
+    e.stopPropagation();
+  }
 }

frontend/src/components/ui/tab-list.ts

@@ -4,6 +4,7 @@ import { property, queryAsync, customElement } from "lit/decorators.js";
 import { ifDefined } from "lit/directives/if-defined.js";
 
 const DEFAULT_PANEL_ID = "default-panel";
+// postcss-lit-disable-next-line
 export const TWO_COL_SCREEN_MIN_CSS = css`64.5rem`;
 
 /**

frontend/src/features/archived-items/crawl-pending-exclusions.ts

@@ -45,10 +45,11 @@ export class CrawlPendingExclusions extends LiteElement {
   render() {
     return html`
       <btrix-section-heading style="--margin: var(--sl-spacing-small)">
-        <div class="flex items-center justify-between">
+        <div class="flex w-full items-center justify-between">
           <div>${msg("Pending Exclusions")} ${this.renderBadge()}</div>
           ${this.total && this.total > this.pageSize
             ? html`<btrix-pagination
+                page=${this.page}
                 size=${this.pageSize}
                 totalCount=${this.total}
                 compact

frontend/src/features/crawl-workflows/queue-exclusion-table.ts

@@ -6,7 +6,7 @@ import RegexColorize from "regex-colorize";
 
 import type { SeedConfig } from "@/pages/org/types";
 import LiteElement, { html } from "@/utils/LiteElement";
-import { regexEscape } from "@/utils/string";
+import { regexEscape, regexUnescape } from "@/utils/string";
 import type { Exclusion } from "./queue-exclusion-form";
 import { type PageChangeEvent } from "@/components/ui/pagination";
 import { type TemplateResult, type PropertyValues } from "lit";
@@ -115,11 +115,12 @@ export class QueueExclusionTable extends LiteElement {
     this.results = this.exclusions
       .slice((this.page - 1) * this.pageSize, this.page * this.pageSize)
       .map((str: string) => {
+        // if escaped version of string, with '\' removed matches string, then consider it
+        // to be matching text, otherwise, regex
+        const isText = regexEscape(str.replace(/\\/g, "")) === str;
         return {
-          // if escaped version of string, with '\' removed matches string, then consider it
-          // to be matching text, otherwise, regex
-          type: regexEscape(str.replace(/\\/g, "")) === str ? "text" : "regex",
-          value: str,
+          type: isText ? "text" : "regex",
+          value: isText ? regexUnescape(str) : str,
         };
       });
   }

frontend/src/pages/org/workflow-editor.ts

@@ -27,7 +27,7 @@
 import Fuse from "fuse.js";
 
 import LiteElement, { html } from "@/utils/LiteElement";
-import { regexEscape } from "@/utils/string";
+import { regexEscape, regexUnescape } from "@/utils/string";
 import type { AuthState } from "@/utils/AuthService";
 import {
   getUTCSchedule,
@@ -495,7 +495,7 @@
       if (primarySeedConfig.include?.length) {
         formState.customIncludeUrlList = primarySeedConfig.include
           // Unescape regex
-          .map((url) => url.replace(/(\\|\/\.\*)/g, ""))
+          .map(regexUnescape)
           .join("\n");
         // if we have additional include URLs, set to "custom" scope here
         // to indicate 'Custom Page Prefix' option
@@ -596,7 +596,7 @@
       includeLinkedPages: Boolean(
         primarySeedConfig.extraHops || seedsConfig.extraHops,
       ),
       useSitemap: seedsConfig.useSitemap ?? defaultFormState.useSitemap,
       failOnFailedSeed:
         seedsConfig.failOnFailedSeed ?? defaultFormState.failOnFailedSeed,
       pageLimit:

frontend/src/utils/string.ts

@@ -5,3 +5,7 @@
 export function regexEscape(s: unknown) {
   return String(s).replace(/[\\^$*+?.()|[\]{}]/g, "\\$&");
 }
+
+export function regexUnescape(s: unknown) {
+  return String(s).replace(/(\\|\/\.\*)/g, "");
+}