Support attachment columns in Automations

packages/backend-core/src/objectStore/objectStore.ts

@@ -19,7 +19,7 @@ const streamPipeline = promisify(stream.pipeline)
 const STATE = {
   bucketCreationPromises: {},
 }
-const signedFilePrefix = "/files/signed"
+export const signedFilePrefix = "/files/signed"
 
 type ListParams = {
   ContinuationToken?: string
@@ -41,7 +41,7 @@ type UploadParams = BaseUploadParams & {
 }
 
 type StreamUploadParams = BaseUploadParams & {
-  stream: ReadStream
+  stream: ReadStream | NodeJS.ReadableStream | ReadableStream<Uint8Array> | null
 }
 
 const CONTENT_TYPE_MAP: any = {
@@ -174,11 +174,9 @@ export async function upload({
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
-  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+  if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    if (objectStore.putBucketLifecycleConfiguration) {
-      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-    }
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
   }
 
   let contentType = type
@@ -222,11 +220,9 @@ export async function streamUpload({
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
-  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+  if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    if (objectStore.putBucketLifecycleConfiguration) {
-      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-    }
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
   }
 
   // Set content type for certain known extensions
@@ -333,7 +329,11 @@ export function getPresignedUrl(
     const signedUrl = new URL(url)
     const path = signedUrl.pathname
     const query = signedUrl.search
-    return `${signedFilePrefix}${path}${query}`
+    if (path.startsWith(signedFilePrefix)) {
+      return `${path}${query}`
+    } else {
+      return `${signedFilePrefix}${path}${query}`
+    }
   }
 }
 

packages/backend-core/src/objectStore/utils.ts

@@ -1,9 +1,14 @@
-import { join } from "path"
+import path, { join } from "path"
 import { tmpdir } from "os"
 import fs from "fs"
 import env from "../environment"
 import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
-
+import * as objectStore from "./objectStore"
+import {
+  AutomationAttachment,
+  AutomationAttachmentContent,
+  BucketedContent,
+} from "@budibase/types"
 /****************************************************
  *      NOTE: When adding a new bucket - name       *
  *     sure that S3 usages (like budibase-infra)    *
@@ -55,3 +60,50 @@ export const bucketTTLConfig = (
 
   return params
 }
+
+async function processUrlAttachment(
+  attachment: AutomationAttachment
+): Promise<AutomationAttachmentContent> {
+  const response = await fetch(attachment.url)
+  if (!response.ok) {
+    throw new Error(`Unexpected response ${response.statusText}`)
+  }
+  const fallbackFilename = path.basename(new URL(attachment.url).pathname)
+  return {
+    filename: attachment.filename || fallbackFilename,
+    content: response.body,
+  }
+}
+
+export async function processObjectStoreAttachment(
+  attachment: AutomationAttachment
+): Promise<BucketedContent> {
+  const result = objectStore.extractBucketAndPath(attachment.url)
+
+  if (result === null) {
+    throw new Error("Invalid signed URL")
+  }
+
+  const { bucket, path } = result
+  const readStream = await objectStore.getReadStream(bucket, path)
+  const fallbackFilename = path.split("/").pop() || ""
+  return {
+    bucket,
+    path,
+    filename: attachment.filename || fallbackFilename,
+    content: readStream,
+  }
+}
+
+export async function processAutomationAttachment(
+  attachment: AutomationAttachment
+): Promise<AutomationAttachmentContent | BucketedContent> {
+  const isFullyFormedUrl =
+    attachment.url.startsWith("http://") ||
+    attachment.url.startsWith("https://")
+  if (isFullyFormedUrl) {
+    return await processUrlAttachment(attachment)
+  } else {
+    return await processObjectStoreAttachment(attachment)
+  }
+}

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -358,7 +358,8 @@
       value.customType !== "cron" &&
       value.customType !== "triggerSchema" &&
       value.customType !== "automationFields" &&
-      value.type !== "attachment"
+      value.type !== "attachment" &&
+      value.type !== "attachment_single"
     )
   }
 

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -2,6 +2,8 @@
   import { tables } from "stores/builder"
   import { Select, Checkbox, Label } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
+  import { FieldType } from "@budibase/types"
+
   import RowSelectorTypes from "./RowSelectorTypes.svelte"
   import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
@@ -14,7 +16,6 @@
   export let bindings
   export let isTestModal
   export let isUpdateRow
-
   $: parsedBindings = bindings.map(binding => {
     let clone = Object.assign({}, binding)
     clone.icon = "ShareAndroid"
@@ -26,15 +27,19 @@
 
   $: {
     table = $tables.list.find(table => table._id === value?.tableId)
-    schemaFields = Object.entries(table?.schema ?? {})
-    // surface the schema so the user can see it in the json
-    schemaFields.map(([, schema]) => {
+
+    // Just sorting attachment types to the bottom here for a cleaner UX
+    schemaFields = Object.entries(table?.schema ?? {}).sort(
+      ([, schemaA], [, schemaB]) =>
+        (schemaA.type === "attachment") - (schemaB.type === "attachment")
+    )
+
+    schemaFields.forEach(([, schema]) => {
       if (!schema.autocolumn && !value[schema.name]) {
         value[schema.name] = ""
       }
     })
   }
-
   const onChangeTable = e => {
     value["tableId"] = e.detail
     dispatch("change", value)
@@ -114,10 +119,16 @@
 </div>
 {#if schemaFields.length}
   {#each schemaFields as [field, schema]}
-    {#if !schema.autocolumn && schema.type !== "attachment"}
-      <div class="schema-fields">
+    {#if !schema.autocolumn}
+      <div
+        class:schema-fields={schema.type !== FieldType.ATTACHMENTS &&
+          schema.type !== FieldType.ATTACHMENT_SINGLE}
+      >
         <Label>{field}</Label>
-        <div class="field-width">
+        <div
+          class:field-width={schema.type !== FieldType.ATTACHMENTS &&
+            schema.type !== FieldType.ATTACHMENT_SINGLE}
+        >
           {#if isTestModal}
             <RowSelectorTypes
               {isTestModal}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,10 +1,12 @@
 <script>
   import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui"
+  import { FieldType } from "@budibase/types"
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import Editor from "components/integration/QueryEditor.svelte"
+  import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
 
   export let onChange
   export let field
@@ -22,6 +24,16 @@
   function schemaHasOptions(schema) {
     return !!schema.constraints?.inclusion?.length
   }
+
+  const handleAttachmentParams = keyValuObj => {
+    let params = {}
+    if (keyValuObj?.length) {
+      for (let param of keyValuObj) {
+        params[param.url] = param.filename
+      }
+    }
+    return params
+  }
 </script>
 
 {#if schemaHasOptions(schema) && schema.type !== "array"}
@@ -77,7 +89,31 @@
     on:change={e => onChange(e, field)}
     useLabel={false}
   />
-{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
+{:else if schema.type === FieldType.ATTACHMENTS || schema.type === FieldType.ATTACHMENT_SINGLE}
+  <div class="attachment-field-spacinng">
+    <KeyValueBuilder
+      on:change={e =>
+        onChange(
+          {
+            detail: e.detail.map(({ name, value }) => ({
+              url: name,
+              filename: value,
+            })),
+          },
+          field
+        )}
+      object={handleAttachmentParams(value[field])}
+      allowJS
+      {bindings}
+      keyBindings
+      customButtonText={"Add attachment"}
+      keyPlaceholder={"URL"}
+      valuePlaceholder={"Filename"}
+      noAddButton={schema.type === FieldType.ATTACHMENT_SINGLE &&
+        value[field].length >= 1}
+    />
+  </div>
+{:else if ["string", "number", "bigint", "barcodeqr", "array", "attachment"].includes(schema.type)}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}
     panel={AutomationBindingPanel}
@@ -90,3 +126,10 @@
     title={schema.name}
   />
 {/if}
+
+<style>
+  .attachment-field-spacinng {
+    margin-top: var(--spacing-s);
+    margin-bottom: var(--spacing-l);
+  }
+</style>

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -4,6 +4,7 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "dataBinding"
+  import { FieldType } from "@budibase/types"
 
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher, setContext } from "svelte"
@@ -102,6 +103,8 @@
     longform: value => !isJSBinding(value),
     json: value => !isJSBinding(value),
     boolean: isValidBoolean,
+    attachment: false,
+    attachment_single: false,
   }
 
   const isValid = value => {
@@ -116,7 +119,16 @@
     if (type === "json" && !isJSBinding(value)) {
       return "json-slot-icon"
     }
-    if (!["string", "number", "bigint", "barcodeqr"].includes(type)) {
+    if (
+      ![
+        "string",
+        "number",
+        "bigint",
+        "barcodeqr",
+        "attachment",
+        "attachment_single",
+      ].includes(type)
+    ) {
       return "slot-icon"
     }
     return ""
@@ -157,7 +169,7 @@
       {updateOnChange}
     />
   {/if}
-  {#if !disabled && type !== "formula"}
+  {#if !disabled && type !== "formula" && !disabled && type !== FieldType.ATTACHMENTS && !disabled && type !== FieldType.ATTACHMENT_SINGLE}
     <div
       class={`icon ${getIconClass(value, type)}`}
       on:click={() => {