Support attachment columns in Automations

packages/backend-core/src/objectStore/objectStore.ts

@@ -13,13 +13,14 @@ import { bucketTTLConfig, budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 import fsp from "fs/promises"
+import { HeadObjectOutput } from "aws-sdk/clients/s3"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
 const STATE = {
   bucketCreationPromises: {},
 }
-export const signedFilePrefix = "/files/signed"
+export const SIGNED_FILE_PREFIX = "/files/signed"
 
 type ListParams = {
   ContinuationToken?: string
@@ -40,8 +41,13 @@ type UploadParams = BaseUploadParams & {
   path?: string | PathLike
 }
 
-type StreamUploadParams = BaseUploadParams & {
-  stream: ReadStream | NodeJS.ReadableStream | ReadableStream<Uint8Array> | null
+export type StreamTypes =
+  | ReadStream
+  | NodeJS.ReadableStream
+  | ReadableStream<Uint8Array>
+
+export type StreamUploadParams = BaseUploadParams & {
+  stream?: StreamTypes
 }
 
 const CONTENT_TYPE_MAP: any = {
@@ -329,11 +335,7 @@ export function getPresignedUrl(
     const signedUrl = new URL(url)
     const path = signedUrl.pathname
     const query = signedUrl.search
-    if (path.startsWith(signedFilePrefix)) {
-      return `${path}${query}`
-    } else {
-      return `${signedFilePrefix}${path}${query}`
-    }
+    return `${SIGNED_FILE_PREFIX}${path}${query}`
   }
 }
 
@@ -521,6 +523,24 @@ export async function getReadStream(
   return client.getObject(params).createReadStream()
 }
 
+export async function getObjectMetadata(
+  bucket: string,
+  path: string
+): Promise<HeadObjectOutput> {
+  bucket = sanitizeBucket(bucket)
+  path = sanitizeKey(path)
+
+  const client = ObjectStore(bucket)
+  const params = {
+    Bucket: bucket,
+    Key: path,
+  }
+
+  const metadata = await client.headObject(params).promise()
+
+  return metadata
+}
+
 /*
 Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
 the bucket and the path from it
@@ -530,7 +550,9 @@ export function extractBucketAndPath(
 ): { bucket: string; path: string } | null {
   const baseUrl = url.split("?")[0]
 
-  const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
+  const regex = new RegExp(
+    `^${SIGNED_FILE_PREFIX}/(?<bucket>[^/]+)/(?<path>.+)$`
+  )
   const match = baseUrl.match(regex)
 
   if (match && match.groups) {

packages/backend-core/src/objectStore/utils.ts

@@ -71,7 +71,7 @@ async function processUrlAttachment(
   const fallbackFilename = path.basename(new URL(attachment.url).pathname)
   return {
     filename: attachment.filename || fallbackFilename,
-    content: response.body,
+    content: response.body!,
   }
 }
 
@@ -84,12 +84,12 @@ export async function processObjectStoreAttachment(
     throw new Error("Invalid signed URL")
   }
 
-  const { bucket, path } = result
-  const readStream = await objectStore.getReadStream(bucket, path)
-  const fallbackFilename = path.split("/").pop() || ""
+  const { bucket, path: objectPath } = result
+  const readStream = await objectStore.getReadStream(bucket, objectPath)
+  const fallbackFilename = path.basename(objectPath)
   return {
     bucket,
-    path,
+    path: objectPath,
     filename: attachment.filename || fallbackFilename,
     content: readStream,
   }
@@ -99,8 +99,8 @@ export async function processAutomationAttachment(
   attachment: AutomationAttachment
 ): Promise<AutomationAttachmentContent | BucketedContent> {
   const isFullyFormedUrl =
-    attachment.url.startsWith("http://") ||
-    attachment.url.startsWith("https://")
+    attachment.url?.startsWith("http://") ||
+    attachment.url?.startsWith("https://")
   if (isFullyFormedUrl) {
     return await processUrlAttachment(attachment)
   } else {

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -27,7 +27,18 @@
 
   const handleAttachmentParams = keyValuObj => {
     let params = {}
-    if (keyValuObj?.length) {
+
+    if (
+      schema.type === FieldType.ATTACHMENT_SINGLE &&
+      Object.keys(keyValuObj).length === 0
+    ) {
+      return []
+    }
+    if (!Array.isArray(keyValuObj)) {
+      keyValuObj = [keyValuObj]
+    }
+
+    if (keyValuObj.length) {
       for (let param of keyValuObj) {
         params[param.url] = param.filename
       }
@@ -95,10 +106,15 @@
       on:change={e =>
         onChange(
           {
-            detail: e.detail.map(({ name, value }) => ({
-              url: name,
-              filename: value,
-            })),
+            detail:
+              schema.type === FieldType.ATTACHMENT_SINGLE
+                ? e.detail.length > 0
+                  ? { url: e.detail[0].name, filename: e.detail[0].value }
+                  : {}
+                : e.detail.map(({ name, value }) => ({
+                    url: name,
+                    filename: value,
+                  })),
           },
           field
         )}
@@ -109,11 +125,11 @@
       customButtonText={"Add attachment"}
       keyPlaceholder={"URL"}
       valuePlaceholder={"Filename"}
-      noAddButton={schema.type === FieldType.ATTACHMENT_SINGLE &&
-        value[field].length >= 1}
+      actionButtonDisabled={schema.type === FieldType.ATTACHMENT_SINGLE &&
+        Object.keys(value[field]).length >= 1}
     />
   </div>
-{:else if ["string", "number", "bigint", "barcodeqr", "array", "attachment"].includes(schema.type)}
+{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}
     panel={AutomationBindingPanel}

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -37,6 +37,7 @@
   export let customButtonText = null
   export let keyBindings = false
   export let allowJS = false
+  export let actionButtonDisabled = false
   export let compare = (option, value) => option === value
 
   let fields = Object.entries(object || {}).map(([name, value]) => ({
@@ -189,7 +190,14 @@
 {/if}
 {#if !readOnly && !noAddButton}
   <div>
-    <ActionButton icon="Add" secondary thin outline on:click={addEntry}>
+    <ActionButton
+      disabled={actionButtonDisabled}
+      icon="Add"
+      secondary
+      thin
+      outline
+      on:click={addEntry}
+    >
       {#if customButtonText}
         {customButtonText}
       {:else}

packages/server/src/automations/automationUtils.ts

@@ -8,6 +8,7 @@ import { AutomationAttachment, FieldType, Row } from "@budibase/types"
 import { LoopInput, LoopStepType } from "../definitions/automations"
 import { objectStore, context } from "@budibase/backend-core"
 import * as uuid from "uuid"
+import path from "path"
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -101,9 +102,12 @@ export function getError(err: any) {
 export async function sendAutomationAttachmentsToStorage(
   tableId: string,
   row: Row
-) {
-  let table = await sdk.tables.getTable(tableId)
-  const attachmentRows: Record<string, AutomationAttachment[]> = {}
+): Promise<Row> {
+  const table = await sdk.tables.getTable(tableId)
+  const attachmentRows: Record<
+    string,
+    AutomationAttachment[] | AutomationAttachment
+  > = {}
 
   for (const [prop, value] of Object.entries(row)) {
     const schema = table.schema[prop]
@@ -114,12 +118,15 @@ export async function sendAutomationAttachmentsToStorage(
       attachmentRows[prop] = value
     }
   }
-
   for (const [prop, attachments] of Object.entries(attachmentRows)) {
-    if (attachments.length) {
-      row[prop] = await Promise.all(
-        attachments.map(attachment => generateAttachmentRow(attachment))
-      )
+    if (Array.isArray(attachments)) {
+      if (attachments.length) {
+        row[prop] = await Promise.all(
+          attachments.map(attachment => generateAttachmentRow(attachment))
+        )
+      }
+    } else if (Object.keys(row[prop]).length > 0) {
+      row[prop] = await generateAttachmentRow(attachments)
     }
   }
 
@@ -128,55 +135,56 @@ export async function sendAutomationAttachmentsToStorage(
 
 async function generateAttachmentRow(attachment: AutomationAttachment) {
   const prodAppId = context.getProdAppId()
-  try {
-    let size: number | null | undefined
-    let s3Key: string
-    const client = objectStore.ObjectStore(objectStore.ObjectStoreBuckets.APPS)
 
+  async function uploadToS3(
+    extension: string,
+    content: objectStore.StreamTypes
+  ) {
+    const fileName = `${uuid.v4()}.${extension}`
+    const s3Key = `${prodAppId}/attachments/${fileName}`
+
+    await objectStore.streamUpload({
+      bucket: objectStore.ObjectStoreBuckets.APPS,
+      stream: content,
+      filename: s3Key,
+    })
+
+    return s3Key
+  }
+
+  async function getSize(s3Key: string) {
+    return (
+      (
+        await objectStore.getObjectMetadata(
+          objectStore.ObjectStoreBuckets.APPS,
+          s3Key
+        )
+      ).ContentLength || 0
+    )
+  }
+
+  try {
+    const { filename } = attachment
+    const extension = path.extname(filename)
     const attachmentResult = await objectStore.processAutomationAttachment(
       attachment
     )
-    const extension = attachment.filename.split(".").pop() || ""
 
-    if ("bucket" in attachmentResult && "path" in attachmentResult) {
-      const { path, content, bucket } = attachmentResult
-      let client = objectStore.ObjectStore(bucket)
-
-      if (path.includes(`${prodAppId}/attachments/`)) {
-        s3Key = path
-      } else {
-        const processedFileName = `${uuid.v4()}.${extension}`
-        s3Key = `${prodAppId}/attachments/${processedFileName}`
-        await objectStore.streamUpload({
-          bucket: objectStore.ObjectStoreBuckets.APPS,
-          stream: content,
-          filename: s3Key,
-        })
-      }
-      const metadata = await client
-        .headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key })
-        .promise()
-      size = metadata.ContentLength
+    let s3Key = ""
+    if (
+      "path" in attachmentResult &&
+      attachmentResult.path.startsWith(`${prodAppId}/attachments/`)
+    ) {
+      s3Key = attachmentResult.path
     } else {
-      const { content } = attachmentResult
-      const processedFileName = `${uuid.v4()}.${extension}`
-      s3Key = `${prodAppId}/attachments/${processedFileName}`
-
-      await objectStore.streamUpload({
-        bucket: objectStore.ObjectStoreBuckets.APPS,
-        stream: content,
-        filename: s3Key,
-      })
-
-      const metadata = await client
-        .headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key })
-        .promise()
-      size = metadata.ContentLength
+      s3Key = await uploadToS3(extension, attachmentResult.content)
     }
 
+    const size = await getSize(s3Key)
+
     return {
       size,
-      name: attachment.filename,
+      name: filename,
       extension,
       key: s3Key,
     }

packages/server/src/automations/tests/createRow.spec.ts

@@ -1,5 +1,4 @@
 import * as setup from "./utilities"
-import tk from "timekeeper"
 import { basicTableWithAttachmentField } from "../../tests/utilities/structures"
 import { objectStore } from "@budibase/backend-core"
 
@@ -18,14 +17,6 @@ describe("test the create row action", () => {
     }
   })
 
-  beforeAll(async () => {
-    tk.reset()
-  })
-
-  afterAll(async () => {
-    await setup.afterAll()
-  })
-
   afterAll(setup.afterAll)
 
   it("should be able to run the action", async () => {

packages/server/src/utilities/rowProcessor/index.ts

@@ -272,7 +272,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       }
     } else if (column.type === FieldType.ATTACHMENT_SINGLE) {
       for (let row of enriched) {
-        if (!row[property]) {
+        if (!row[property] || Object.keys(row[property]).length === 0) {
           continue
         }
 

packages/types/src/documents/app/automation.ts

@@ -244,11 +244,7 @@ export type AutomationAttachment = {
 
 export type AutomationAttachmentContent = {
   filename: string
-  content:
-    | ReadStream
-    | NodeJS.ReadableStream
-    | ReadableStream<Uint8Array>
-    | null
+  content: ReadStream | NodeJS.ReadableStream | ReadableStream<Uint8Array>
 }
 
 export type BucketedContent = AutomationAttachmentContent & {