Update deps; Fix: tar.gz download consumes much memory (fixes #221)

psi-4ward · May 9, 2022 · 6984d81 · 6984d81
1 parent 3a95b87
commit 6984d81
Show file tree

Hide file tree

Showing 4 changed files with 11,256 additions and 3,093 deletions.
diff --git a/config.dev.js b/config.dev.js
@@ -16,8 +16,8 @@ module.exports = {
   "uploadPass": false,
   "baseUrl": '/',
   "uploadAppPath": '/',
-  "maxFileSize": Math.pow(2, 20) * 15,
-  "maxBucketSize": Math.pow(2, 20) * 20,
+  // "maxFileSize": Math.pow(2, 20) * 15,
+  // "maxBucketSize": Math.pow(2, 20) * 20,
   "mailFrom": "PsiTransfer <psitransfer@psi.cx>"
   // "sslKeyFile": './tmp/cert.key',
   // "sslCertFile": './tmp/cert.pem',

diff --git a/lib/endpoints.js b/lib/endpoints.js
@@ -9,7 +9,7 @@ const assert = require('assert');
 const AES = require("crypto-js/aes");
 const MD5 = require("crypto-js/md5");
 const archiver = require('archiver');
-const zlib = require('zlib');
+const tar = require('tar-stream');
 const config = require('../config');
 const eventBus = require('./eventBus');
 const tusboy = require('./tusboy');
@@ -28,6 +28,7 @@ const downloadPage = pug.compileFile(path.join(__dirname, '../public/pug/downloa
 
 const store = new Store(config.uploadDir);
 const Db = require('./db');
+const { createGzip } = require("zlib");
 const db = new Db(config.uploadDir, store);
 db.init();
 const app = express();
@@ -208,29 +209,43 @@ app.get(`${ config.baseUrl }files/:fid`, async (req, res, next) => {
     }
     debug(`Download Bucket ${ sid }`);
 
-    if (format === 'zip') res.header('ContentType', 'application/zip');
-    if (format === 'tar.gz') res.header('ContentType', 'application/x-gtar');
     const filename = `${ sid }.${ format }`;
     res.header('Content-Disposition', `attachment; filename="${ filename }"`);
 
-    const archive = archiver(format === 'zip' ? 'zip' : 'tar');
-    archive.on('error', function(err) {
-      console.error(err);
-    });
+    if(format === 'zip') {
+      res.header('ContentType', 'application/zip');
+      const archive = archiver('zip');
+      archive.on('error', function(err) {
+        console.error(err);
+      });
+      archive.pipe(res);
 
-    bucket.forEach(info => {
-      archive.append(
-        fs.createReadStream(store.getFilename(info.metadata.sid + '++' + info.key)),
-        { name: info.metadata.name }
-      );
-    });
+      for (const info of bucket) {
+        await new Promise((resolve, reject) => {
+          const stream = fs.createReadStream(store.getFilename(info.metadata.sid + '++' + info.key));
+          stream.on('end', resolve);
+          archive.append(stream, { name: info.metadata.name });
+        });
+      }
 
-    if (format === 'tar.gz') {
-      archive.pipe(zlib.createGzip()).pipe(res);
+      await archive.finalize();
     } else {
-      archive.pipe(res);
+      res.header('ContentType', 'application/x-gtar');
+      const pack = tar.pack();
+      pack.pipe(createGzip()).pipe(res);
+
+      for (const info of bucket) {
+        await new Promise((resolve, reject) => {
+          const readStream = fs.createReadStream(store.getFilename(info.metadata.sid + '++' + info.key));
+          const entry = pack.entry({ name: info.metadata.name, size: info.size });
+          readStream.on('error', reject);
+          entry.on('error', reject);
+          entry.on('finish',resolve);
+          readStream.pipe(entry);
+        });
+      }
+      pack.finalize();
     }
-    archive.finalize();
 
     try {
       res.on('finish', async () => {