From 1645d79987fe7e30b21452aaecb0ea952ca4e3d3 Mon Sep 17 00:00:00 2001 From: Caden Buckhalt Date: Tue, 19 Mar 2024 14:51:01 -0700 Subject: [PATCH 1/6] fix: assetId not unique --- utils/protocolImport.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/protocolImport.tsx b/utils/protocolImport.tsx index 84a4817c..cbc9ae5f 100644 --- a/utils/protocolImport.tsx +++ b/utils/protocolImport.tsx @@ -1,5 +1,6 @@ import type { Protocol } from '@codaco/shared-consts'; import type Zip from 'jszip'; +import { createId } from '@paralleldrive/cuid2'; // Fetch protocol.json as a parsed object from the protocol zip. export const getProtocolJson = async (protocolZip: Zip) => { @@ -63,7 +64,7 @@ export const getProtocolAssets = async ( } files.push({ - assetId: key, + assetId: `${key}_${createId()}`, // We cannot assume key is unique across protocols. Add a unique suffix. name: asset.source, type: asset.type, file: new File([file], asset.source), // Convert Blob to File with filename From bb4cfcec8871d0f53327fa86ebd18fd7ddba52dc Mon Sep 17 00:00:00 2001 From: Caden Buckhalt Date: Mon, 25 Mar 2024 12:25:50 -0700 Subject: [PATCH 2/6] wip feat: protocols can have identical assets implements uploading assets, adding protocols to db, deleting protocols --- hooks/useProtocolImport.tsx | 47 +++++++++++++++++++++++++++++++------ prisma/schema.prisma | 5 ++-- server/router.ts | 2 ++ server/routers/asset.ts | 17 ++++++++++++++ server/routers/protocol.ts | 43 ++++++++++++++++++++++++++++----- utils/protocolImport.tsx | 3 +-- 6 files changed, 99 insertions(+), 18 deletions(-) create mode 100644 server/routers/asset.ts diff --git a/hooks/useProtocolImport.tsx b/hooks/useProtocolImport.tsx index 18c85da2..b8db31be 100644 --- a/hooks/useProtocolImport.tsx +++ b/hooks/useProtocolImport.tsx @@ -17,6 +17,7 @@ import Link from '~/components/Link'; import { ErrorDetails } from '~/components/ErrorDetails'; import { XCircle } from 'lucide-react'; import type { assetInsertSchema } from '~/server/routers/protocol'; +import type { Asset } from '@prisma/client'; import type { z } from 'zod'; import { hash } from 'ohash'; import { AlertDialogDescription } from '~/components/ui/AlertDialog'; @@ -160,9 +161,40 @@ export const useProtocolImport = () => { } const assets = await getProtocolAssets(protocolJson, zip); - let assetsWithCombinedMetadata: z.infer = []; + + const newAssets: typeof assets = []; + + const assetsWithCombinedMetadata: z.infer = []; + + let newAssetsWithCombinedMetadata: z.infer = []; + + // If there are assets, first CHECK if they are already in the database + // and if they are, push them to the assetsWithCombinedMetadata. + // if they are not, push them to newAssets to be uploaded. if (assets.length > 0) { + assets.forEach((asset) => { + const { data: existingAsset }: { data: Asset | undefined } = + api.asset.get.useQuery(asset.assetId); + + if (existingAsset) { + assetsWithCombinedMetadata.push({ + key: existingAsset.key, + assetId: asset.assetId, + name: asset.name, + type: asset.type, + url: existingAsset.url, + size: existingAsset.size, + }); + } else { + newAssets.push(asset); + } + }); + } + + // we're going to upload the new assets + + if (newAssets.length > 0) { dispatch({ type: 'UPDATE_STATUS', payload: { @@ -177,18 +209,18 @@ export const useProtocolImport = () => { * track the current bytes uploaded per file (uploads are done in * parallel). */ - const totalBytesToUpload = assets.reduce((acc, asset) => { + const totalBytesToUpload = newAssets.reduce((acc, asset) => { return acc + asset.file.size; }, 0); const currentBytesUploaded: Record = {}; - const files = assets.map((asset) => asset.file); + const files = newAssets.map((asset) => asset.file); const uploadedFiles = await uploadFiles('assetRouter', { files, onUploadProgress({ progress, file }) { - const thisFileSize = assets.find((asset) => asset.name === file)! + const thisFileSize = newAssets.find((asset) => asset.name === file)! .file.size; // eg. 1000 const thisCompletedBytes = thisFileSize * (progress / 100); @@ -222,11 +254,11 @@ export const useProtocolImport = () => { /** * We now need to merge the metadata from the uploaded files with the * asset metadata from the protocol json, so that we can insert the - * assets into the database. + * newassets into the database. * * The 'name' prop matches across both - we can use that to merge them. */ - assetsWithCombinedMetadata = assets.map((asset) => { + newAssetsWithCombinedMetadata = newAssets.map((asset) => { const uploadedAsset = uploadedFiles.find( (uploadedFile) => uploadedFile.name === asset.name, ); @@ -259,7 +291,8 @@ export const useProtocolImport = () => { const result = await insertProtocol({ protocol: protocolJson, protocolName: fileName, - assets: assetsWithCombinedMetadata, + newAssets: newAssetsWithCombinedMetadata, + existingAssets: assetsWithCombinedMetadata, }); if (result.error) { diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 39b1d2cd..f3986392 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -33,10 +33,9 @@ model Asset { type String url String size Int - protocol Protocol @relation(fields: [protocolId], references: [id], onDelete: Cascade) - protocolId String // from db + protocols Protocol[] - @@index(fields: [protocolId, assetId, key]) + @@index(fields: [assetId, key]) } model Interview { diff --git a/server/router.ts b/server/router.ts index 71d3ba34..66cd0141 100644 --- a/server/router.ts +++ b/server/router.ts @@ -5,9 +5,11 @@ import { protocolRouter } from '~/server/routers/protocol'; import { participantRouter } from './routers/participant'; import { router } from './trpc'; import { dashboardRouter } from './routers/dashboard'; +import { assetRouter } from './routers/asset'; export const appRouter = router({ appSettings: appSettingsRouter, + asset: assetRouter, dashboard: dashboardRouter, session: sessionRouter, interview: interviewRouter, diff --git a/server/routers/asset.ts b/server/routers/asset.ts new file mode 100644 index 00000000..f2dbe09c --- /dev/null +++ b/server/routers/asset.ts @@ -0,0 +1,17 @@ +/* eslint-disable local-rules/require-data-mapper */ +import { prisma } from '~/utils/db'; +import { protectedProcedure, router } from '~/server/trpc'; +import { z } from 'zod'; + +export const assetRouter = router({ + get: protectedProcedure + .input(z.string()) + .query(async ({ input: assetId }) => { + const asset = await prisma.asset.findFirst({ + where: { + assetId, + }, + }); + return asset; + }), +}); diff --git a/server/routers/protocol.ts b/server/routers/protocol.ts index c8a5bd8e..6bba88dd 100644 --- a/server/routers/protocol.ts +++ b/server/routers/protocol.ts @@ -27,8 +27,32 @@ export const deleteProtocols = async (hashes: string[]) => { select: { id: true, name: true }, }); + // get all assets associated with the protocols to be deleted + // that are not associated with any other protocols const assets = await prisma.asset.findMany({ - where: { protocolId: { in: protocolsToBeDeleted.map((p) => p.id) } }, + where: { + AND: [ + { + protocols: { + some: { + id: { + in: protocolsToBeDeleted.map((p) => p.id), + }, + }, + }, + }, + // check if the asset is only associated with the protocols to be deleted + { + protocols: { + every: { + id: { + in: protocolsToBeDeleted.map((p) => p.id), + }, + }, + }, + }, + ], + }, select: { key: true }, }); // We put asset deletion in a separate try/catch because if it fails, we still @@ -145,13 +169,19 @@ export const protocolRouter = router({ .object({ protocol: z.unknown(), // TODO: replace this with zod schema version of Protocol type protocolName: z.string(), - assets: assetInsertSchema, + newAssets: assetInsertSchema, + existingAssets: assetInsertSchema, }) .passthrough() .parse(value); }) .mutation(async ({ input }) => { - const { protocol: inputProtocol, protocolName, assets } = input; + const { + protocol: inputProtocol, + protocolName, + newAssets, + existingAssets, + } = input; const protocol = inputProtocol as Protocol; @@ -169,7 +199,8 @@ export const protocolRouter = router({ codebook: protocol.codebook, description: protocol.description, assets: { - create: assets, + create: newAssets, + connect: existingAssets.map((a) => ({ key: a.key })), }, }, }); @@ -189,8 +220,8 @@ export const protocolRouter = router({ return { error: null, success: true }; } catch (e) { // Attempt to delete any assets we uploaded to storage - if (assets.length > 0) { - await deleteFilesFromUploadThing(assets.map((a) => a.key)); + if (newAssets.length > 0) { + await deleteFilesFromUploadThing(newAssets.map((a) => a.key)); } // Check for protocol already existing if (e instanceof Prisma.PrismaClientKnownRequestError) { diff --git a/utils/protocolImport.tsx b/utils/protocolImport.tsx index cbc9ae5f..84a4817c 100644 --- a/utils/protocolImport.tsx +++ b/utils/protocolImport.tsx @@ -1,6 +1,5 @@ import type { Protocol } from '@codaco/shared-consts'; import type Zip from 'jszip'; -import { createId } from '@paralleldrive/cuid2'; // Fetch protocol.json as a parsed object from the protocol zip. export const getProtocolJson = async (protocolZip: Zip) => { @@ -64,7 +63,7 @@ export const getProtocolAssets = async ( } files.push({ - assetId: `${key}_${createId()}`, // We cannot assume key is unique across protocols. Add a unique suffix. + assetId: key, name: asset.source, type: asset.type, file: new File([file], asset.source), // Convert Blob to File with filename From 9bccf3f184e3ba31f9077ffce9814c19334275b0 Mon Sep 17 00:00:00 2001 From: Caden Buckhalt Date: Mon, 25 Mar 2024 14:05:25 -0700 Subject: [PATCH 3/6] fix: wait until all assets are checked before uploading new --- hooks/useProtocolImport.tsx | 38 ++++++++++++++++++++++--------------- server/routers/asset.ts | 2 +- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/hooks/useProtocolImport.tsx b/hooks/useProtocolImport.tsx index b8db31be..64eaa28f 100644 --- a/hooks/useProtocolImport.tsx +++ b/hooks/useProtocolImport.tsx @@ -17,7 +17,6 @@ import Link from '~/components/Link'; import { ErrorDetails } from '~/components/ErrorDetails'; import { XCircle } from 'lucide-react'; import type { assetInsertSchema } from '~/server/routers/protocol'; -import type { Asset } from '@prisma/client'; import type { z } from 'zod'; import { hash } from 'ohash'; import { AlertDialogDescription } from '~/components/ui/AlertDialog'; @@ -38,6 +37,8 @@ export const useProtocolImport = () => { const { mutateAsync: getProtocolExists } = api.protocol.get.byHash.useMutation(); + const { mutateAsync: getAsset } = api.asset.get.useMutation(); + /** * This is the main job processing function. Takes a file, and handles all * the steps required to import it into the database, updating the job @@ -173,21 +174,28 @@ export const useProtocolImport = () => { // if they are not, push them to newAssets to be uploaded. if (assets.length > 0) { - assets.forEach((asset) => { - const { data: existingAsset }: { data: Asset | undefined } = - api.asset.get.useQuery(asset.assetId); - - if (existingAsset) { - assetsWithCombinedMetadata.push({ - key: existingAsset.key, - assetId: asset.assetId, - name: asset.name, - type: asset.type, - url: existingAsset.url, - size: existingAsset.size, - }); + const assetPromises = assets.map(async (asset) => { + const existingAsset = await getAsset(asset.assetId); + return existingAsset + ? { + key: existingAsset.key, + assetId: asset.assetId, + name: asset.name, + type: asset.type, + url: existingAsset.url, + size: existingAsset.size, + } + : asset; + }); + + const resolvedAssets = await Promise.all(assetPromises); + + // If the asset has a key, it's an existing asset + resolvedAssets.forEach((resolvedAsset) => { + if ('key' in resolvedAsset) { + assetsWithCombinedMetadata.push(resolvedAsset); } else { - newAssets.push(asset); + newAssets.push(resolvedAsset); } }); } diff --git a/server/routers/asset.ts b/server/routers/asset.ts index f2dbe09c..1ace039c 100644 --- a/server/routers/asset.ts +++ b/server/routers/asset.ts @@ -6,7 +6,7 @@ import { z } from 'zod'; export const assetRouter = router({ get: protectedProcedure .input(z.string()) - .query(async ({ input: assetId }) => { + .mutation(async ({ input: assetId }) => { const asset = await prisma.asset.findFirst({ where: { assetId, From d877e856a702412bc8af21997202e0720a55cb82 Mon Sep 17 00:00:00 2001 From: Caden Buckhalt Date: Mon, 25 Mar 2024 14:18:58 -0700 Subject: [PATCH 4/6] rm unneeded check in deletion, clean up comments --- hooks/useProtocolImport.tsx | 6 +++--- server/routers/protocol.ts | 27 ++++++--------------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/hooks/useProtocolImport.tsx b/hooks/useProtocolImport.tsx index 64eaa28f..f83c0a97 100644 --- a/hooks/useProtocolImport.tsx +++ b/hooks/useProtocolImport.tsx @@ -169,9 +169,9 @@ export const useProtocolImport = () => { let newAssetsWithCombinedMetadata: z.infer = []; - // If there are assets, first CHECK if they are already in the database - // and if they are, push them to the assetsWithCombinedMetadata. - // if they are not, push them to newAssets to be uploaded. + // Check if the assets are already in the database. + // If yes, add them to assetsWithCombinedMetadata to be connected to the protocol. + // If not, add them to newAssets to be uploaded. if (assets.length > 0) { const assetPromises = assets.map(async (asset) => { diff --git a/server/routers/protocol.ts b/server/routers/protocol.ts index 6bba88dd..8b422221 100644 --- a/server/routers/protocol.ts +++ b/server/routers/protocol.ts @@ -27,31 +27,16 @@ export const deleteProtocols = async (hashes: string[]) => { select: { id: true, name: true }, }); - // get all assets associated with the protocols to be deleted - // that are not associated with any other protocols + // Select assets that are ONLY associated with the protocols to be deleted const assets = await prisma.asset.findMany({ where: { - AND: [ - { - protocols: { - some: { - id: { - in: protocolsToBeDeleted.map((p) => p.id), - }, - }, - }, - }, - // check if the asset is only associated with the protocols to be deleted - { - protocols: { - every: { - id: { - in: protocolsToBeDeleted.map((p) => p.id), - }, - }, + protocols: { + every: { + id: { + in: protocolsToBeDeleted.map((p) => p.id), }, }, - ], + }, }, select: { key: true }, }); From a3be68e49cdca722f050a9a157639cfdd0f4cf87 Mon Sep 17 00:00:00 2001 From: Caden Buckhalt Date: Tue, 26 Mar 2024 08:06:28 -0700 Subject: [PATCH 5/6] refactor: router accepts array of ids and returns ids that do not exist makes it so that there is only one db call. existingIds are passed directly in insertProtocol and connected. removes needing to return entire existing object asset. --- hooks/useProtocolImport.tsx | 41 ++++++++++++++----------------------- server/routers/asset.ts | 14 ++++++++----- server/routers/protocol.ts | 6 +++--- 3 files changed, 27 insertions(+), 34 deletions(-) diff --git a/hooks/useProtocolImport.tsx b/hooks/useProtocolImport.tsx index f83c0a97..ffe9201d 100644 --- a/hooks/useProtocolImport.tsx +++ b/hooks/useProtocolImport.tsx @@ -37,7 +37,7 @@ export const useProtocolImport = () => { const { mutateAsync: getProtocolExists } = api.protocol.get.byHash.useMutation(); - const { mutateAsync: getAsset } = api.asset.get.useMutation(); + const { mutateAsync: getNewAssetIds } = api.asset.get.useMutation(); /** * This is the main job processing function. Takes a file, and handles all @@ -165,42 +165,31 @@ export const useProtocolImport = () => { const newAssets: typeof assets = []; - const assetsWithCombinedMetadata: z.infer = []; + const existingAssetIds: string[] = []; let newAssetsWithCombinedMetadata: z.infer = []; // Check if the assets are already in the database. - // If yes, add them to assetsWithCombinedMetadata to be connected to the protocol. + // If yes, add them to existingAssetIds to be connected to the protocol. // If not, add them to newAssets to be uploaded. - if (assets.length > 0) { - const assetPromises = assets.map(async (asset) => { - const existingAsset = await getAsset(asset.assetId); - return existingAsset - ? { - key: existingAsset.key, - assetId: asset.assetId, - name: asset.name, - type: asset.type, - url: existingAsset.url, - size: existingAsset.size, - } - : asset; - }); - - const resolvedAssets = await Promise.all(assetPromises); + try { + const newAssetIds = await getNewAssetIds( + assets.map((asset) => asset.assetId), + ); - // If the asset has a key, it's an existing asset - resolvedAssets.forEach((resolvedAsset) => { - if ('key' in resolvedAsset) { - assetsWithCombinedMetadata.push(resolvedAsset); + assets.forEach((asset) => { + if (newAssetIds.includes(asset.assetId)) { + newAssets.push(asset); } else { - newAssets.push(resolvedAsset); + existingAssetIds.push(asset.assetId); } }); + } catch (e) { + throw new Error('Error checking for existing assets'); } - // we're going to upload the new assets + // Upload the new assets if (newAssets.length > 0) { dispatch({ @@ -300,7 +289,7 @@ export const useProtocolImport = () => { protocol: protocolJson, protocolName: fileName, newAssets: newAssetsWithCombinedMetadata, - existingAssets: assetsWithCombinedMetadata, + existingAssetIds: existingAssetIds, }); if (result.error) { diff --git a/server/routers/asset.ts b/server/routers/asset.ts index 1ace039c..178fac90 100644 --- a/server/routers/asset.ts +++ b/server/routers/asset.ts @@ -5,13 +5,17 @@ import { z } from 'zod'; export const assetRouter = router({ get: protectedProcedure - .input(z.string()) - .mutation(async ({ input: assetId }) => { - const asset = await prisma.asset.findFirst({ + .input(z.array(z.string())) + .mutation(async ({ input: assetIds }) => { + const assets = await prisma.asset.findMany({ where: { - assetId, + assetId: { + in: assetIds, + }, }, }); - return asset; + const existingAssets = assets.map((asset) => asset.assetId); + // Return the assetIds that are not in the database + return assetIds.filter((assetId) => !existingAssets.includes(assetId)); }), }); diff --git a/server/routers/protocol.ts b/server/routers/protocol.ts index 8b422221..a42180a5 100644 --- a/server/routers/protocol.ts +++ b/server/routers/protocol.ts @@ -155,7 +155,7 @@ export const protocolRouter = router({ protocol: z.unknown(), // TODO: replace this with zod schema version of Protocol type protocolName: z.string(), newAssets: assetInsertSchema, - existingAssets: assetInsertSchema, + existingAssetIds: z.array(z.string()), }) .passthrough() .parse(value); @@ -165,7 +165,7 @@ export const protocolRouter = router({ protocol: inputProtocol, protocolName, newAssets, - existingAssets, + existingAssetIds, } = input; const protocol = inputProtocol as Protocol; @@ -185,7 +185,7 @@ export const protocolRouter = router({ description: protocol.description, assets: { create: newAssets, - connect: existingAssets.map((a) => ({ key: a.key })), + connect: existingAssetIds.map((assetId) => ({ assetId })), }, }, }); From 2ab8efd1ad3787f0d1ad50c51317ce4565c04680 Mon Sep 17 00:00:00 2001 From: Caden Buckhalt Date: Tue, 26 Mar 2024 08:24:01 -0700 Subject: [PATCH 6/6] improve asset route name --- hooks/useProtocolImport.tsx | 2 +- server/routers/asset.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/useProtocolImport.tsx b/hooks/useProtocolImport.tsx index ffe9201d..352fb401 100644 --- a/hooks/useProtocolImport.tsx +++ b/hooks/useProtocolImport.tsx @@ -37,7 +37,7 @@ export const useProtocolImport = () => { const { mutateAsync: getProtocolExists } = api.protocol.get.byHash.useMutation(); - const { mutateAsync: getNewAssetIds } = api.asset.get.useMutation(); + const { mutateAsync: getNewAssetIds } = api.asset.checkExisting.useMutation(); /** * This is the main job processing function. Takes a file, and handles all diff --git a/server/routers/asset.ts b/server/routers/asset.ts index 178fac90..46e49321 100644 --- a/server/routers/asset.ts +++ b/server/routers/asset.ts @@ -4,7 +4,7 @@ import { protectedProcedure, router } from '~/server/trpc'; import { z } from 'zod'; export const assetRouter = router({ - get: protectedProcedure + checkExisting: protectedProcedure .input(z.array(z.string())) .mutation(async ({ input: assetIds }) => { const assets = await prisma.asset.findMany({