Support attachment columns in Automations

packages/backend-core/src/objectStore/objectStore.ts

@@ -41,7 +41,7 @@ type UploadParams = BaseUploadParams & {
 }
 
 type StreamUploadParams = BaseUploadParams & {
-  stream: ReadStream
+  stream: ReadStream | NodeJS.ReadableStream | ReadableStream<Uint8Array> | null
 }
 
 const CONTENT_TYPE_MAP: any = {
@@ -222,11 +222,9 @@ export async function streamUpload({
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
-  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+  if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    if (objectStore.putBucketLifecycleConfiguration) {
-      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-    }
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
   }
 
   // Set content type for certain known extensions

packages/backend-core/src/objectStore/utils.ts

@@ -1,9 +1,10 @@
-import { join } from "path"
+import path, { join } from "path"
 import { tmpdir } from "os"
-import fs from "fs"
+import fs, { ReadStream } from "fs"
 import env from "../environment"
 import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
-
+import * as objectStore from "./objectStore"
+import { AutomationAttachment } from "@budibase/types"
 /****************************************************
  *      NOTE: When adding a new bucket - name       *
  *     sure that S3 usages (like budibase-infra)    *
@@ -55,3 +56,43 @@ export const bucketTTLConfig = (
 
   return params
 }
+
+export const processAutomationAttachment = async (
+  attachment: AutomationAttachment
+): Promise<{
+  filename: string
+  content:
+    | ReadStream
+    | NodeJS.ReadableStream
+    | ReadableStream<Uint8Array>
+    | null
+}> => {
+  const isFullyFormedUrl =
+    attachment.url.startsWith("http://") ||
+    attachment.url.startsWith("https://")
+
+  if (isFullyFormedUrl) {
+    const response = await fetch(attachment.url)
+    if (!response.ok) {
+      throw new Error(`unexpected response ${response.statusText}`)
+    }
+    const fallbackFilename = path.basename(new URL(attachment.url).pathname)
+    return {
+      filename: attachment.filename || fallbackFilename,
+      content: response?.body,
+    }
+  } else {
+    const url = attachment.url
+    const result = objectStore.extractBucketAndPath(url)
+    if (result === null) {
+      throw new Error("Invalid signed URL")
+    }
+    const { bucket, path } = result
+    const readStream = await objectStore.getReadStream(bucket, path)
+    const fallbackFilename = path.split("/").pop() || ""
+    return {
+      filename: attachment.filename || fallbackFilename,
+      content: readStream,
+    }
+  }
+}

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -358,7 +358,8 @@
       value.customType !== "cron" &&
       value.customType !== "triggerSchema" &&
       value.customType !== "automationFields" &&
-      value.type !== "attachment"
+      value.type !== "attachment" &&
+      value.type !== "attachment_single"
     )
   }
 

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -2,6 +2,8 @@
   import { tables } from "stores/builder"
   import { Select, Checkbox, Label } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
+  import { FieldType } from "@budibase/types"
+
   import RowSelectorTypes from "./RowSelectorTypes.svelte"
   import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
@@ -14,7 +16,6 @@
   export let bindings
   export let isTestModal
   export let isUpdateRow
-
   $: parsedBindings = bindings.map(binding => {
     let clone = Object.assign({}, binding)
     clone.icon = "ShareAndroid"
@@ -26,15 +27,19 @@
 
   $: {
     table = $tables.list.find(table => table._id === value?.tableId)
-    schemaFields = Object.entries(table?.schema ?? {})
-    // surface the schema so the user can see it in the json
-    schemaFields.map(([, schema]) => {
+
+    // Just sorting attachment types to the bottom here for a cleaner UX
+    schemaFields = Object.entries(table?.schema ?? {}).sort(
+      ([, schemaA], [, schemaB]) =>
+        (schemaA.type === "attachment") - (schemaB.type === "attachment")
+    )
+
+    schemaFields.forEach(([, schema]) => {
       if (!schema.autocolumn && !value[schema.name]) {
         value[schema.name] = ""
       }
     })
   }
-
   const onChangeTable = e => {
     value["tableId"] = e.detail
     dispatch("change", value)
@@ -114,10 +119,16 @@
 </div>
 {#if schemaFields.length}
   {#each schemaFields as [field, schema]}
-    {#if !schema.autocolumn && schema.type !== "attachment"}
-      <div class="schema-fields">
+    {#if !schema.autocolumn}
+      <div
+        class:schema-fields={schema.type !== FieldType.ATTACHMENTS &&
+          schema.type !== FieldType.ATTACHMENT_SINGLE}
+      >
         <Label>{field}</Label>
-        <div class="field-width">
+        <div
+          class:field-width={schema.type !== FieldType.ATTACHMENTS &&
+            schema.type !== FieldType.ATTACHMENT_SINGLE}
+        >
           {#if isTestModal}
             <RowSelectorTypes
               {isTestModal}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,10 +1,12 @@
 <script>
   import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui"
+  import { FieldType } from "@budibase/types"
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import Editor from "components/integration/QueryEditor.svelte"
+  import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
 
   export let onChange
   export let field
@@ -22,6 +24,16 @@
   function schemaHasOptions(schema) {
     return !!schema.constraints?.inclusion?.length
   }
+
+  const handleAttachmentParams = keyValuObj => {
+    let params = {}
+    if (keyValuObj?.length) {
+      for (let param of keyValuObj) {
+        params[param.url] = param.filename
+      }
+    }
+    return params
+  }
 </script>
 
 {#if schemaHasOptions(schema) && schema.type !== "array"}
@@ -77,7 +89,31 @@
     on:change={e => onChange(e, field)}
     useLabel={false}
   />
-{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
+{:else if schema.type === FieldType.ATTACHMENTS || schema.type === FieldType.ATTACHMENT_SINGLE}
+  <div class="attachment-field-spacinng">
+    <KeyValueBuilder
+      on:change={e =>
+        onChange(
+          {
+            detail: e.detail.map(({ name, value }) => ({
+              url: name,
+              filename: value,
+            })),
+          },
+          field
+        )}
+      object={handleAttachmentParams(value[field])}
+      allowJS
+      {bindings}
+      keyBindings
+      customButtonText={"Add attachment"}
+      keyPlaceholder={"URL"}
+      valuePlaceholder={"Filename"}
+      noAddButton={schema.type === FieldType.ATTACHMENT_SINGLE &&
+        value[field].length >= 1}
+    />
+  </div>
+{:else if ["string", "number", "bigint", "barcodeqr", "array", "attachment"].includes(schema.type)}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}
     panel={AutomationBindingPanel}
@@ -90,3 +126,10 @@
     title={schema.name}
   />
 {/if}
+
+<style>
+  .attachment-field-spacinng {
+    margin-top: var(--spacing-s);
+    margin-bottom: var(--spacing-l);
+  }
+</style>

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -102,6 +102,8 @@
     longform: value => !isJSBinding(value),
     json: value => !isJSBinding(value),
     boolean: isValidBoolean,
+    attachment: false,
+    attachment_single: false,
   }
 
   const isValid = value => {
@@ -116,7 +118,16 @@
     if (type === "json" && !isJSBinding(value)) {
       return "json-slot-icon"
     }
-    if (!["string", "number", "bigint", "barcodeqr"].includes(type)) {
+    if (
+      ![
+        "string",
+        "number",
+        "bigint",
+        "barcodeqr",
+        "attachment",
+        "attachment_single",
+      ].includes(type)
+    ) {
       return "slot-icon"
     }
     return ""
@@ -157,7 +168,7 @@
       {updateOnChange}
     />
   {/if}
-  {#if !disabled && type !== "formula"}
+  {#if !disabled && type !== "formula" && !disabled && type !== "attachment" && !disabled && type !== "attachment_single"}
     <div
       class={`icon ${getIconClass(value, type)}`}
       on:click={() => {

packages/server/src/automations/automationUtils.ts

@@ -4,8 +4,10 @@ import {
   encodeJSBinding,
 } from "@budibase/string-templates"
 import sdk from "../sdk"
-import { Row } from "@budibase/types"
+import { AutomationAttachment, Row } from "@budibase/types"
 import { LoopInput, LoopStepType } from "../definitions/automations"
+import { objectStore, context } from "@budibase/backend-core"
+import * as uuid from "uuid"
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -96,6 +98,52 @@ export function getError(err: any) {
   return typeof err !== "string" ? err.toString() : err
 }
 
+export async function sendAutomationAttachmentsToStorage(
+  tableId: string,
+  row: Row
+) {
+  let table = await sdk.tables.getTable(tableId)
+  const attachmentRows: { [key: string]: any } = {}
+
+  Object.entries(row).forEach(([prop, value]) => {
+    const schema = table.schema[prop]
+    if (Object.hasOwn(table.schema, prop) && schema?.type === "attachment") {
+      attachmentRows[prop] = value
+    }
+  })
+
+  for (const prop in attachmentRows) {
+    const attachments = attachmentRows[prop]
+    const updatedAttachments = await Promise.all(
+      attachments.map(async (attachment: AutomationAttachment) => {
+        let { content } = await objectStore.processAutomationAttachment(
+          attachment
+        )
+        const extension = attachment.filename.split(".").pop() || ""
+        const processedFileName = `${uuid.v4()}.${extension}`
+        const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+        if (content) {
+          await objectStore.streamUpload({
+            bucket: objectStore.ObjectStoreBuckets.APPS,
+            stream: content,
+            filename: s3Key,
+          })
+
+          return {
+            size: 10,
+            name: attachment.filename,
+            extension,
+            key: s3Key,
+          }
+        }
+      })
+    )
+    row[prop] = updatedAttachments
+  }
+
+  return row
+}
 export function substituteLoopStep(hbsString: string, substitute: string) {
   let checkForJS = isJSBinding(hbsString)
   let substitutedHbsString = ""

packages/server/src/automations/steps/createRow.ts

@@ -1,5 +1,9 @@
 import { save } from "../../api/controllers/row"
-import { cleanUpRow, getError } from "../automationUtils"
+import {
+  cleanUpRow,
+  getError,
+  sendAutomationAttachmentsToStorage,
+} from "../automationUtils"
 import { buildCtx } from "./utils"
 import {
   AutomationActionStepId,
@@ -89,6 +93,11 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
 
   try {
     inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row)
+
+    inputs.row = await sendAutomationAttachmentsToStorage(
+      inputs.row.tableId,
+      inputs.row
+    )
     await save(ctx)
     return {
       row: inputs.row,

packages/server/src/automations/steps/updateRow.ts

@@ -108,7 +108,15 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
 
   try {
     if (tableId) {
-      inputs.row = await automationUtils.cleanUpRow(tableId, inputs.row)
+      inputs.row = await automationUtils.cleanUpRow(
+        inputs.row.tableId,
+        inputs.row
+      )
+
+      inputs.row = await automationUtils.sendAutomationAttachmentsToStorage(
+        inputs.row.tableId,
+        inputs.row
+      )
     }
     await rowController.patch(ctx)
     return {

packages/types/src/documents/app/automation.ts

@@ -235,3 +235,8 @@ export interface AutomationMetadata extends Document {
   errorCount?: number
   automationChainCount?: number
 }
+
+export type AutomationAttachment = {
+  url: string
+  filename: string
+}