WIP feat: use did storage as the storage engine

.env

@@ -1,3 +1,5 @@
 GENERATE_SOURCEMAP=false
 APP_PORT=3030
 REACT_APP_TITLE=image-bin
+
+DID_STORAGE_URL=http://cedcaa27-znkga5fuxuwntawzo6ugwlps92i9alparpfh.did.abtnet.io

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.7.1 (十月 27, 2022)
+
+- feat: use did storage as the storage engine 
+
 ## 0.7.0 (October 14, 2022)
 
 - fix: security for non-admin users

api/index.js

@@ -12,6 +12,7 @@ const fallback = require('express-history-api-fallback');
 const { name, version } = require('../package.json');
 const logger = require('./libs/logger');
 const env = require('./libs/env');
+const { apiRouter } = require('./routes');
 
 if (fs.existsSync(env.uploadDir) === false) {
   fs.mkdirSync(env.uploadDir, { recursive: true });
@@ -27,7 +28,7 @@ app.use(express.urlencoded({ extended: true, limit: env.maxUploadSize }));
 app.use('/uploads', express.static(env.uploadDir, { maxAge: '356d', immutable: true, index: false }));
 
 const router = express.Router();
-router.use('/api', require('./routes/upload'));
+router.use('/api', apiRouter);
 
 const isDevelopment = process.env.NODE_ENV === 'development';
 const isProduction = process.env.NODE_ENV === 'production' || process.env.ABT_NODE_SERVICE_ENV === 'production';

api/libs/api.js

@@ -0,0 +1,7 @@
+const { default: axios } = require('axios');
+
+const api = axios.create();
+
+module.exports = {
+  api,
+};

api/libs/storage-endpoint.js

@@ -0,0 +1,14 @@
+const urlJoin = require('url-join');
+
+/**
+ *
+ * @param {string} endpoint
+ * @param {string} objectKey
+ */
+function getPublicUrl(endpoint, objectKey) {
+  return urlJoin(endpoint.replace('object/', ''), 'public', objectKey);
+}
+
+module.exports = {
+  getPublicUrl,
+};

api/routes/.upload.js

@@ -0,0 +1,137 @@
+const fs = require('fs');
+const path = require('path');
+const multer = require('multer');
+const express = require('express');
+const joinUrl = require('url-join');
+const pick = require('lodash/pick');
+const middleware = require('@blocklet/sdk/lib/middlewares');
+const mime = require('mime-types');
+const { customRandom, urlAlphabet, random } = require('nanoid');
+
+const env = require('../libs/env');
+const Upload = require('../states/upload');
+const Folder = require('../states/folder');
+
+const uploadRouter = express.Router();
+const nanoid = customRandom(urlAlphabet, 24, random);
+const auth = middleware.auth({ roles: env.uploaderRoles });
+const user = middleware.user();
+const ensureAdmin = middleware.auth({ roles: ['admin', 'owner'] });
+const upload = multer({
+  storage: multer.diskStorage({
+    destination: env.uploadDir,
+    filename: (req, file, cb) => {
+      cb(null, `${Date.now()}-${nanoid()}.${mime.extension(file.mimetype)}`);
+    },
+  }),
+});
+
+uploadRouter.post('/', user, auth, upload.single('image'), async (req, res) => {
+  const obj = new URL(env.appUrl);
+  obj.protocol = req.get('x-forwarded-proto') || req.protocol;
+  obj.pathname = joinUrl(req.headers['x-path-prefix'] || '/', '/uploads', req.file.filename);
+
+  const doc = await Upload.insert({
+    ...pick(req.file, ['size', 'filename', 'mimetype', 'originalname']),
+    remark: req.body.remark || '',
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    createdBy: req.user.did,
+    updatedBy: req.user.did,
+  });
+
+  res.json({ url: obj.href, ...doc });
+});
+
+const DEFAULT_PAGE_SIZE = 20;
+const MAX_PAGE_SIZE = 100;
+uploadRouter.get('/', user, auth, async (req, res) => {
+  let page = Number(req.query.page || 1);
+  let pageSize = Number(req.query.pageSize || DEFAULT_PAGE_SIZE);
+
+  page = Number.isNaN(page) ? 1 : page;
+  pageSize = Number.isNaN(pageSize) ? DEFAULT_PAGE_SIZE : pageSize;
+  pageSize = pageSize > MAX_PAGE_SIZE ? MAX_PAGE_SIZE : pageSize;
+
+  const condition = {};
+  if (req.query.folderId) {
+    condition.folderId = req.query.folderId;
+  }
+
+  if (['guest', 'member'].includes(req.user.role)) {
+    condition.createdBy = req.user.did;
+  }
+
+  const uploads = await Upload.paginate({ condition, sort: { createdAt: -1 }, page, size: pageSize });
+  const total = await Upload.count(condition);
+
+  const folders = await Folder.cursor({}).sort({ createdAt: -1 }).exec();
+
+  res.jsonp({ uploads, folders, total, page, pageSize, pageCount: Math.ceil(total / pageSize) });
+});
+
+// preview image
+uploadRouter.get('/:filename', user, auth, async (req, res) => {
+  const doc = await Upload.findOne({ filename: req.params.filename });
+  res.jsonp(doc);
+});
+
+// remove upload
+uploadRouter.delete('/:id', user, ensureAdmin, async (req, res) => {
+  const doc = await Upload.findOne({ _id: req.params.id });
+  if (!doc) {
+    res.jsonp({ error: 'No such upload' });
+    return;
+  }
+
+  const result = await Upload.remove({ _id: req.params.id });
+  if (result) {
+    fs.unlinkSync(path.join(env.uploadDir, doc.filename));
+  }
+
+  res.jsonp(doc);
+});
+
+// create folder
+uploadRouter.post('/folders', user, ensureAdmin, async (req, res) => {
+  const name = req.body.name.trim();
+  if (!name) {
+    res.jsonp({ error: 'folder name required' });
+    return;
+  }
+
+  const exist = await Folder.findOne({ name });
+  if (exist) {
+    res.json(exist);
+    return;
+  }
+
+  const doc = await Folder.insert({
+    name,
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    createdBy: req.user.did,
+    updatedBy: req.user.did,
+  });
+
+  res.json(doc);
+});
+
+// move to folder
+uploadRouter.put('/:id', user, ensureAdmin, async (req, res) => {
+  const doc = await Upload.findOne({ _id: req.params.id });
+  if (!doc) {
+    res.jsonp({ error: 'No such upload' });
+    return;
+  }
+
+  const [, updatedDoc] = await Upload.update(
+    { _id: req.params.id },
+    { $set: pick(req.body, ['folderId']) },
+    { returnUpdatedDocs: true }
+  );
+
+  res.jsonp(updatedDoc);
+});
+
+module.exports = { uploadRouter };

api/routes/index.js

@@ -0,0 +1,12 @@
+const express = require('express');
+const { storageEndpointRouter } = require('./storage-endpoint');
+const { uploadRouter } = require('./upload');
+
+const apiRouter = express.Router();
+
+apiRouter.use('/uploads', uploadRouter);
+apiRouter.use('/storage-endpoint', storageEndpointRouter);
+
+module.exports = {
+  apiRouter,
+};

api/routes/storage-endpoint.js

@@ -0,0 +1,47 @@
+const express = require('express');
+const { isEmpty } = require('lodash');
+const { storageEndpointRepository } = require('../states/storage-endpoint');
+
+const storageEndpointRouter = express.Router();
+
+storageEndpointRouter.put(
+  '/',
+  /**
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async (req, res) => {
+    const { endpoint } = req.body;
+
+    if (isEmpty(endpoint)) {
+      return res.status(400).send('endpoint cannot be empty');
+    }
+
+    await storageEndpointRepository.write(endpoint);
+
+    return res.send('ok');
+  }
+);
+
+storageEndpointRouter.get(
+  '/',
+  /**
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async (req, res) => {
+    if (!(await storageEndpointRepository.exists())) {
+      return '';
+    }
+
+    const endpoint = await storageEndpointRepository.read();
+
+    return res.send(endpoint);
+  }
+);
+
+module.exports = {
+  storageEndpointRouter,
+};

api/routes/upload.js

@@ -1,18 +1,20 @@
-const fs = require('fs');
-const path = require('path');
+const fs = require('fs-extra');
 const multer = require('multer');
 const express = require('express');
 const joinUrl = require('url-join');
 const pick = require('lodash/pick');
 const middleware = require('@blocklet/sdk/lib/middlewares');
 const mime = require('mime-types');
 const { customRandom, urlAlphabet, random } = require('nanoid');
-
+const FormData = require('form-data');
 const env = require('../libs/env');
 const Upload = require('../states/upload');
 const Folder = require('../states/folder');
+const { api } = require('../libs/api');
+const { storageEndpointRepository } = require('../states/storage-endpoint');
+const { getPublicUrl } = require('../libs/storage-endpoint');
 
-const router = express.Router();
+const uploadRouter = express.Router();
 const nanoid = customRandom(urlAlphabet, 24, random);
 const auth = middleware.auth({ roles: env.uploaderRoles });
 const user = middleware.user();
@@ -26,26 +28,54 @@ const upload = multer({
   }),
 });
 
-router.post('/uploads', user, auth, upload.single('image'), async (req, res) => {
+uploadRouter.post('/', user, auth, upload.single('image'), async (req, res) => {
   const obj = new URL(env.appUrl);
   obj.protocol = req.get('x-forwarded-proto') || req.protocol;
   obj.pathname = joinUrl(req.headers['x-path-prefix'] || '/', '/uploads', req.file.filename);
 
+  const endpoint = await storageEndpointRepository.read();
+
+  const stream = fs.createReadStream(req.file.path);
+  const filename = req.file.originalname;
+  const putUrl = joinUrl(endpoint, filename);
+
+  const formData = new FormData();
+  formData.append('data', stream);
+
+  // @see: https://github.com/axios/axios#-automatic-serialization-to-formdata
+  await api({
+    url: putUrl,
+    method: 'PUT',
+    data: formData,
+    headers: {
+      'x-app-did': env.appId,
+      'x-skip-signature': true,
+      ...formData.getHeaders(),
+    },
+  }).catch((error) => console.error(error.message));
+
+  const publicUrl = getPublicUrl(endpoint, filename);
+
   const doc = await Upload.insert({
     ...pick(req.file, ['size', 'filename', 'mimetype', 'originalname']),
     remark: req.body.remark || '',
     createdAt: new Date().toISOString(),
     updatedAt: new Date().toISOString(),
     createdBy: req.user.did,
     updatedBy: req.user.did,
+    objectUrl: putUrl,
+    publicUrl,
   });
 
-  res.json({ url: obj.href, ...doc });
+  // eslint-disable-next-line no-console
+  console.log({ publicUrl });
+
+  return res.json({ url: doc.publicUrl, ...doc });
 });
 
 const DEFAULT_PAGE_SIZE = 20;
 const MAX_PAGE_SIZE = 100;
-router.get('/uploads', user, auth, async (req, res) => {
+uploadRouter.get('/', user, auth, async (req, res) => {
   let page = Number(req.query.page || 1);
   let pageSize = Number(req.query.pageSize || DEFAULT_PAGE_SIZE);
 
@@ -71,29 +101,37 @@ router.get('/uploads', user, auth, async (req, res) => {
 });
 
 // preview image
-router.get('/uploads/:filename', user, auth, async (req, res) => {
+uploadRouter.get('/:filename', user, auth, async (req, res) => {
   const doc = await Upload.findOne({ filename: req.params.filename });
   res.jsonp(doc);
 });
 
 // remove upload
-router.delete('/uploads/:id', user, ensureAdmin, async (req, res) => {
+uploadRouter.delete('/:id', user, ensureAdmin, async (req, res) => {
+  /**
+   * @type {ImageBin.Upload}
+   */
   const doc = await Upload.findOne({ _id: req.params.id });
   if (!doc) {
-    res.jsonp({ error: 'No such upload' });
-    return;
+    return res.jsonp({ error: 'No such upload' });
   }
 
-  const result = await Upload.remove({ _id: req.params.id });
-  if (result) {
-    fs.unlinkSync(path.join(env.uploadDir, doc.filename));
+  if (doc.objectUrl) {
+    await api.delete(doc.objectUrl, {
+      headers: {
+        'x-app-did': env.appId,
+        'x-skip-signature': true,
+      },
+    });
   }
 
-  res.jsonp(doc);
+  await Upload.remove({ _id: req.params.id });
+
+  return res.jsonp(doc);
 });
 
 // create folder
-router.post('/folders', user, ensureAdmin, async (req, res) => {
+uploadRouter.post('/folders', user, ensureAdmin, async (req, res) => {
   const name = req.body.name.trim();
   if (!name) {
     res.jsonp({ error: 'folder name required' });
@@ -118,7 +156,7 @@ router.post('/folders', user, ensureAdmin, async (req, res) => {
 });
 
 // move to folder
-router.put('/uploads/:id', user, ensureAdmin, async (req, res) => {
+uploadRouter.put('/:id', user, ensureAdmin, async (req, res) => {
   const doc = await Upload.findOne({ _id: req.params.id });
   if (!doc) {
     res.jsonp({ error: 'No such upload' });
@@ -134,4 +172,4 @@ router.put('/uploads/:id', user, ensureAdmin, async (req, res) => {
   res.jsonp(updatedDoc);
 });
 
-module.exports = router;
+module.exports = { uploadRouter };