The maximum size of zip can bigger?

[heguixin]

I copy the logic of saving-multiple-files.html.
I did successfully implement the packaging function，but I am worried about the size limit of 4GB, how can I solve this problem
Could you give me some direction to increase the size limit of zip, hahaQAQ。

[jimmywarting]

you would need something that can generate zip64

[heguixin]

how can i do this, can u give me an example?

[jimmywarting]

my zip library don't have support for the extended zip64 specification... so maybe you would have to use something else. don't know what... think i remember someone tried to make a PR to me about adding support for zip64 extension

but i right now it only supports 4GiB

[heguixin]

It's so sad.Although i am so desperate to fix this,I am afraid to change to the source code.

[jimmywarting]

conflux is another zip create/read library... don't know if they have added zip64 support to it. was a long time since i helped out with that. they eventually turned it into something that i hate so i stop using it. All toe i was the original author who started writing it for them

searching for zip64 on npm also gives some alternative results https://www.npmjs.com/search?q=zip64

but there is also an possible alternative solution too. you could use file system access and write files directly to the disc instead of packing & unpacking every files that only waste processing time. But i find it somewhat restrictive to what you can/can't do

you can ask for a director to write to, it can't be the download folder, it can't be any folder that may contain executable system files, it dose not have a progress bar or any UI and you must implement that yourself. and you can only write file atomicly and not use "in-place" (but that only matters if you need to modify or append data to files) give it a try at least. might be worth it.

Also, if the files are coming from a network request from a backend, then i really recommend that you 1. zip the files on the backend, 2. respond back to the users with a content-dispoition header to save the files. then you don't need any client side solution to download everything and it works in the background too, so that you do not need to keep the browser tab open all the time

[heguixin]

I am very honored to get your advice, and then I tried the implementation of Conflux, which successfully helped me to achieve about 10GB of package download.

But I found that there was still a bottleneck, and the browser still crashed when I got around 16GB. May I ask whether this size can be extended? Or do I ask the user to make multiple download requests and eventually achieve manual splicing.

Will you continue to explore the problem of the size? If there is any update, could you please remind me？

[heguixin]

Conflux's 16GB limit may be the reason for my operation. Maybe the page crashed because I turned on debugger mode (F12). I closed the debug window while downloading and found that it worked. I continued to test the maximum size supported by myself.

[christianwengert]

I made some changes to the zip stuff (I am sorry I can't find the source anymore, and I had to adapt it a bit). But the following works in my application:

class Crc32 {
	constructor() {
		this.crc = -1
	}

	append(data) {
		let crc = this.crc | 0;
		const table = this.table;
		for (let offset = 0, len = data.length | 0; offset < len; offset++) {
			crc = (crc >>> 8) ^ table[(crc ^ data[offset]) & 0xFF]
		}
		this.crc = crc
	}

	get() {
		return ~this.crc
	}
}

Crc32.prototype.table = (() => {
	var i;
	var j;
	var t;
	var table = []
	for (i = 0; i < 256; i++) {
		t = i
		for (j = 0; j < 8; j++) {
			t = (t & 1)
				? (t >>> 1) ^ 0xEDB88320
				: t >>> 1
		}
		table[i] = t
	}
	return table
})()

const getDataHelper = byteLength => {
	var uint8 = new Uint8Array(byteLength)
	return {
		array: uint8,
		view: new DataView(uint8.buffer)
	}
}

const pump = zipObj => zipObj.reader.read().then(chunk => {
	if (chunk.done) return zipObj.writeFooter()
	const outputData = chunk.value
	zipObj.crc.append(outputData)
	zipObj.uncompressedLength += outputData.length
	zipObj.compressedLength += outputData.length
	zipObj.ctrl.enqueue(outputData)
})

/**
 * [createWriter description]
 * @param  {Object} underlyingSource [description]
 * @return {Boolean}                  [description]
 */
export function ZIP(underlyingSource) {
	const files = Object.create(null)
	const filenames = []
	const encoder = new TextEncoder()
	let offset = 0
	let activeZipIndex = 0
	let ctrl
	let activeZipObject, closed

	let overallSize = 0
	let zip64 = false

	function next() {
		activeZipIndex++
		activeZipObject = files[filenames[activeZipIndex]]
		if (activeZipObject) processNextChunk()
		else if (closed) closeZip()
	}

	const zipWriter = {
		enqueue(fileLike) {
			if (closed) throw new TypeError('Cannot enqueue a chunk into a readable stream that is closed or has been requested to be closed')

			let name = fileLike.name.trim()
			const date = new Date(typeof fileLike.lastModified === 'undefined' ? Date.now() : fileLike.lastModified)

			if (fileLike.directory && !name.endsWith('/')) name += '/'
			if (files[name]) throw new Error('File already exists.')
			overallSize += fileLike.size
			zip64 = true; //(overallSize >= 0xffffffff) ;

			const nameBuf = encoder.encode(name)
			filenames.push(name)


			const zipObject = files[name] = {
				level: 0,
				ctrl,
				directory: !!fileLike.directory,
				nameBuf,
				comment: encoder.encode(fileLike.comment || ''),
				compressedLength: 0,
				uncompressedLength: 0,
				extraArray: null,
				writeHeader() {
					var header = getDataHelper(26)
					var data = getDataHelper(30 + nameBuf.length)

					zipObject.offset = offset
					zipObject.header = header
					if (zipObject.level !== 0 && !zipObject.directory) {
						header.view.setUint16(4, 0x0800)
					}
					header.view.setUint32(0, 0x14000808)

					if (zip64) //Zip64 Min ver.
						header.view.setUint16(0, 45, true)

					header.view.setUint16(6, (((date.getHours() << 6) | date.getMinutes()) << 5) | date.getSeconds() / 2, true)
					header.view.setUint16(8, ((((date.getFullYear() - 1980) << 4) | (date.getMonth() + 1)) << 5) | date.getDate(), true)
					header.view.setUint16(22, nameBuf.length, true)
					data.view.setUint32(0, 0x504b0304)
					data.array.set(header.array, 4)
					data.array.set(nameBuf, 30)
					offset += data.array.length
					ctrl.enqueue(data.array)
				},
				writeFooter() {
					// if (this.compressedLength && this.compressedLength >= 0xffffffff) {
					//   zipObject.header.view.setUint16(0, 45)
					//   zip64 = true
					// }
					var footer = getDataHelper(zip64 ? 24 : 16)
					footer.view.setUint32(0, 0x504b0708)

					if (zipObject.crc) {
						zipObject.header.view.setUint32(10, zipObject.crc.get(), true)
						footer.view.setUint32(4, zipObject.crc.get(), true)
					}

					if (zip64) {
						let zip64Extra = getDataHelper(28)
						zipObject.header.view.setUint32(14, 0xffffffff, true)
						zipObject.header.view.setUint32(18, 0xffffffff, true)
						footer.view.setBigUint64(8, BigInt(zipObject.compressedLength), true)
						footer.view.setBigInt64(16, BigInt(zipObject.uncompressedLength), true)
						zip64Extra.view.setUint16(0, 0x0001, true)
						zip64Extra.view.setUint16(2, 24, true)
						zip64Extra.view.setBigUint64(4, BigInt(zipObject.uncompressedLength), true)
						zip64Extra.view.setBigUint64(12, BigInt(zipObject.compressedLength), true)
						zip64Extra.view.setBigUint64(20, BigInt(files[name].offset), true)
						files[name].extraArray = zip64Extra.array
					} else {
						zipObject.header.view.setUint32(14, zipObject.compressedLength, true)
						zipObject.header.view.setUint32(18, zipObject.uncompressedLength, true)
						footer.view.setUint32(8, zipObject.compressedLength, true)
						footer.view.setUint32(12, zipObject.uncompressedLength, true)

					}

					ctrl.enqueue(footer.array)
					offset += zipObject.compressedLength + footer.array.length
					next()
				},
				fileLike
			}

			if (!activeZipObject) {
				activeZipObject = zipObject
				processNextChunk()
			}
		},
		close() {
			if (closed) throw new TypeError('Cannot close a readable stream that has already been requested to be closed')
			if (!activeZipObject) closeZip()
			closed = true
		}
	};

	function closeZip() {
		var length = 0
		var index = 0
		var indexFilename, file
		// var zip64 = false
		let cdOffset = offset
		let totalEntries = filenames.length
		for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) {
			file = files[filenames[indexFilename]]
			length += 46 + file.nameBuf.length + file.comment.length
			if (file.extraArray) {
				length += file.extraArray.length
				// zip64 = true
			}
		}

		// if (cdOffset + length >= 0xffffffff || filenames.length >= 0xffff)
		//   zip64 = true

		const data = getDataHelper(length + 22 + (zip64 ? 56 + 20 : 0))
		for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) {
			file = files[filenames[indexFilename]]
			data.view.setUint32(index, 0x504b0102)
			data.view.setUint16(index + 4, 0x1400)
			data.array.set(file.header.array, index + 6)
			if (file.extraArray) {
				data.view.setUint16(index + 30, file.extraArray.length, true)
			}
			data.view.setUint16(index + 32, file.comment.length, true)
			if (file.directory) {
				data.view.setUint8(index + 38, 0x10)
			}
			if (file.offset >= 0xffffffff)
				data.view.setUint32(index + 42, 0xffffffff, true)
			else
				data.view.setUint32(index + 42, file.offset, true)

			data.array.set(file.nameBuf, index + 46)
			let extraLength = 0
			if (file.extraArray) {
				extraLength = file.extraArray.length
				data.array.set(file.extraArray, index + 46 + file.nameBuf.length)
			}
			data.array.set(file.comment, index + 46 + file.nameBuf.length + extraLength)
			index += 46 + file.nameBuf.length + file.comment.length + extraLength
		}
		if (zip64) {
			// Zip64 End of Central Directory record
			// 0: Signature
			data.view.setUint32(index, 0x504b0606);
			// 4: Size of zip64 EOCD
			data.view.setBigUint64(index + 4, BigInt(44), true);
			// 12: Version made By
			data.view.setUint16(index + 12, 45, true);
			// 14: version needed to extract
			data.view.setUint16(index + 14, 45, true);
			// 16: number of this disk
			// 20: number of the disk with the start of CD
			// 24: total number of entries in the central directory on this disk
			data.view.setBigUint64(index + 24, BigInt(totalEntries), true);
			// 32: total number of entries in the central directory
			data.view.setBigUint64(index + 32, BigInt(totalEntries), true);
			// 40: size of the central directory
			data.view.setBigUint64(index + 40, BigInt(length), true);
			// 48: Offset of start of central directory
			data.view.setBigUint64(index + 48, BigInt(cdOffset), true);
			index += 56

			// Zip64 End of Central Directory locator
			// 0: Signature
			data.view.setUint32(index, 0x504b0607);
			// 4: number of the disk with the zip64 EOCD
			// 8: Offset of the zip64 EOCD
			data.view.setBigUint64(index + 8, BigInt(cdOffset + length), true);
			// 16: total number of disks
			data.view.setUint32(index + 16, 1, true);
			index += 20

			// EOCD must set these values to 0xffff and 0xffffffff when using ZIP64 format
			totalEntries = 0xffff;
			cdOffset = 0xffffffff;
		}
		data.view.setUint32(index, 0x504b0506)
		data.view.setUint16(index + 8, totalEntries, true)
		data.view.setUint16(index + 10, totalEntries, true)
		data.view.setUint32(index + 12, length, true)
		data.view.setUint32(index + 16, cdOffset, true)
		ctrl.enqueue(data.array)
		ctrl.close()
	}

	function processNextChunk() {
		if (!activeZipObject) return
		if (activeZipObject.directory) return activeZipObject.writeFooter(activeZipObject.writeHeader())
		if (activeZipObject.reader) return pump(activeZipObject)
		if (activeZipObject.fileLike.stream) {
			activeZipObject.crc = new Crc32()
			activeZipObject.reader = activeZipObject.fileLike.stream().getReader()
			activeZipObject.writeHeader()
		} else next()
	}

	return new ReadableStream({
		start: c => {
			ctrl = c
			underlyingSource.start && Promise.resolve(underlyingSource.start(zipWriter))
		},
		pull() {
			return processNextChunk() || (
				underlyingSource.pull &&
				Promise.resolve(underlyingSource.pull(zipWriter))
			)
		}
	})
}

[heguixin]

What is the maximum ZIP size you will eventually download?

[christianwengert]

With zip64 basically unlimited

[gildas-lormeau]

I can't find the source anymore,

FYI, it's a fork of the old version (callback based API) of zip.js.

[lazerlipa]

Thank you very much for the code it helped me a lot but is there a way to know how big the file would be before i start given that i know the size of each file so i can show progress?

[heguixin]

I think I found the real reason my page crashed. The crash occurred when I found that the size of the downloaded file compared to the size of the service worker's download file exceeded 2GB. How do I deal with this problem?

[heguixin]

I reused its logic saving-multiple-files.html and then changed only the link and number of files to download in the pull function, and this crash happened.
for example:

<!DOCTYPE html>
<html lang="en" dir="ltr">
  <head>
    <meta charset="utf-8">
    <title>Saving multiple files as zip</title>
  </head>
  <body>
    <button id="$start">Start</button>

    <script src="https://cdn.jsdelivr.net/npm/web-streams-polyfill@2.0.2/dist/ponyfill.min.js"></script>

    <!--
    includes blob.stream() polyfill
    while Also making File constructor work in some browser that don't support it
    -->
    <script src="https://cdn.jsdelivr.net/gh/eligrey/Blob.js/Blob.js"></script>
    <script src="../StreamSaver.js"></script>
    <script src="zip-stream.js"></script>
    <script>
      $start.onclick = () => {
        const fileStream = streamSaver.createWriteStream('archive.zip')

        const file1 = new File(['file1 content'], 'streamsaver-zip-example/file1.txt')

        // File Like object works too
        const file2 = {
          name: 'streamsaver-zip-example/file2.txt',
          stream () {
            // if you want to play it cool and use new api's
            //
            // const { readable, writable } = new TextEncoderStream()
            // writable.write('file2 content')
			// writable.close()
            // return readable

            return new ReadableStream({
              start (ctrl) {
                ctrl.enqueue(new TextEncoder().encode('file2 generated with readableStream'))
                ctrl.close()
              }
            })
          }
        }

        const blob = new Blob(['support blobs too'])

        const file3 = {
          name: 'streamsaver-zip-example/blob-example.txt',
          stream: () => blob.stream()
        }

        // In a ideall world i would just have used a TransformStream
        // where you would get `{ readable writable } = new TransformStream()`
        // `readable` would be piped to streamsaver, and the writer would accept
        // file-like object, but that made it dependent on TransformStream and WritableStream
        // So i built ZIP-Stream simular to a ReadbleStream but you enqueue
        // file-like objects meaning it should have at at the very least { name, stream() }
        //
        // it supports pull() too that gets called when it ask for more files.
        //
        // NOTE: My zip library can't generate zip's over 4gb and has no compresseion
        //       it was built solo for the purpus of saving multiple files in browser
		//
		//       windows gets confused when file & folders starts with /
        const readableZipStream = new ZIP({
          start (ctrl) {
            ctrl.enqueue(file1)
            ctrl.enqueue(file2)
            ctrl.enqueue(file3)
            ctrl.enqueue({name: 'streamsaver-zip-example/empty folder', directory: true})
            // ctrl.close()
          },
          async pull (ctrl) {
			// Gets executed everytime zip.js asks for more data
            let url = 'http://pfp.ps.netease.com/thetenthbrandsprivate/file/62ce8df1bab62f5550fe315al7hEc67S01?expire=1663385147&sign=ieUh9stisqtN1zatTte5Q-mxOgI='
            let res = await fetch(url)
            let stream = () => res.body
            let name = 'streamsaver-zip-example/2GBsize_1.mp4'

            ctrl.enqueue({ name, stream })

            url = 'http://pfp.ps.netease.com/thetenthbrandsprivate/file/62ce8df1bab62f5550fe315al7hEc67S01?expire=1663385147&sign=ieUh9stisqtN1zatTte5Q-mxOgI='
            res = await fetch(url)
            stream = () => res.body
            name = 'streamsaver-zip-example/2GBsize_2.mp4'

            ctrl.enqueue({ name, stream })

            url = 'http://pfp.ps.netease.com/thetenthbrandsprivate/file/62ce8df1bab62f5550fe315al7hEc67S01?expire=1663385147&sign=ieUh9stisqtN1zatTte5Q-mxOgI='
            res = await fetch(url)
            stream = () => res.body
            name = 'streamsaver-zip-example/2GBsize_3.mp4'

            ctrl.enqueue({ name, stream })

            url = 'http://pfp.ps.netease.com/thetenthbrandsprivate/file/62ce8df1bab62f5550fe315al7hEc67S01?expire=1663385147&sign=ieUh9stisqtN1zatTte5Q-mxOgI='
            res = await fetch(url)
            stream = () => res.body
            name = 'streamsaver-zip-example/2GBsize_4.mp4'

            ctrl.enqueue({ name, stream })

            url = 'http://pfp.ps.netease.com/thetenthbrandsprivate/file/62ce8df1bab62f5550fe315al7hEc67S01?expire=1663385147&sign=ieUh9stisqtN1zatTte5Q-mxOgI='
            res = await fetch(url)
            stream = () => res.body
            name = 'streamsaver-zip-example/2GBsize_5.mp4'

            ctrl.enqueue({ name, stream })

            url = 'http://pfp.ps.netease.com/thetenthbrandsprivate/file/62ce8df1bab62f5550fe315al7hEc67S01?expire=1663385147&sign=ieUh9stisqtN1zatTte5Q-mxOgI='
            res = await fetch(url)
            stream = () => res.body
            name = 'streamsaver-zip-example/2GBsize_6.mp4'

            ctrl.enqueue({ name, stream })
			  
			// if (done adding all files)
            ctrl.close()
          }
        })

        // more optimized
        if (window.WritableStream && readableZipStream.pipeTo) {
          return readableZipStream.pipeTo(fileStream).then(() => console.log('done writing'))
        }

		// less optimized
        const writer = fileStream.getWriter()
        const reader = readableZipStream.getReader()
        const pump = () => reader.read()
          .then(res => res.done ? writer.close() : writer.write(res.value).then(pump))

        pump()
      }
    </script>
  </body>
</html>

[FlameRow]

More good