From eeae03eaa150a4257e3c70b1944d341073c62146 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 30 Aug 2021 17:16:08 +0200 Subject: [PATCH] Add options to brotli and implement clean brotli .br support - allow to specify brotli window size - parameter -m0=brotli:long=n, BROTLI_MAX_WINDOW_BITS (24) used by default in brotli-mt, smaller == faster - note that :long can be set up to BROTLI_LARGE_MAX_WINDOW_BITS (30), whereas :wlog can be set up to BROTLI_MAX_WINDOW_BITS (24) only... - todo: check whether set of BROTLI_PARAM_LARGE_WINDOW to BROTLI_TRUE is needed if (lgwin > BROTLI_MAX_WINDOW_BITS) - implementation of single-threaded brotli compression / decompression for .br data Signed-off-by: Sergey G. Brester Reviewed-by: Tino Reichardt --- C/zstdmt/brotli-mt.h | 2 +- C/zstdmt/brotli-mt_compress.c | 174 ++++++++++++-- C/zstdmt/brotli-mt_decompress.c | 120 ++++++++-- CPP/7zip/Archive/BrotliHandler.cpp | 351 ++++++++++++++++++++++++++++ CPP/7zip/Bundles/Format7zF/Arc.mak | 1 + CPP/7zip/Compress/BrotliDecoder.cpp | 2 +- CPP/7zip/Compress/BrotliEncoder.cpp | 44 +++- CPP/7zip/Compress/BrotliEncoder.h | 6 + 8 files changed, 660 insertions(+), 40 deletions(-) create mode 100644 CPP/7zip/Archive/BrotliHandler.cpp diff --git a/C/zstdmt/brotli-mt.h b/C/zstdmt/brotli-mt.h index b32ffd62..3280384f 100644 --- a/C/zstdmt/brotli-mt.h +++ b/C/zstdmt/brotli-mt.h @@ -101,7 +101,7 @@ typedef struct BROTLIMT_CCtx_s BROTLIMT_CCtx; * @inputsize - if zero, becomes some optimal value for the level * - if nonzero, the given value is taken */ -BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize); +BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, uint64_t unpackSize, int level, int inputsize, int lgwin); /** * 2) threaded compression diff --git a/C/zstdmt/brotli-mt_compress.c b/C/zstdmt/brotli-mt_compress.c index 664f1ccb..736821ae 100644 --- a/C/zstdmt/brotli-mt_compress.c +++ b/C/zstdmt/brotli-mt_compress.c @@ -15,6 +15,7 @@ #include #include "encode.h" +#include "common/constants.h" #include "brotli-mt.h" #include "memmt.h" @@ -52,12 +53,17 @@ struct BROTLIMT_CCtx_s { /* levels: 1..BROTLIMT_LEVEL_MAX */ int level; - /* threads: 1..BROTLIMT_THREAD_MAX */ + /* threads: 0..BROTLIMT_THREAD_MAX */ int threads; + /* size of file/stream to compress if known */ + uint64_t unpackSize; + /* should be used for read from input */ int inputsize; + int lgwin; + /* statistic */ size_t insize; size_t outsize; @@ -87,7 +93,7 @@ struct BROTLIMT_CCtx_s { * Compression ****************************************/ -BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize) +BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, uint64_t unpackSize, int level, int inputsize, int lgwin) { BROTLIMT_CCtx *ctx; int t; @@ -98,7 +104,7 @@ BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize) return 0; /* check threads value */ - if (threads < 1 || threads > BROTLIMT_THREAD_MAX) + if (threads < 0 || threads > BROTLIMT_THREAD_MAX) return 0; /* check level */ @@ -111,30 +117,37 @@ BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize) else ctx->inputsize = 1024 * 1024 * (level ? level : 1); + ctx->lgwin = lgwin; + /* setup ctx */ ctx->level = level; ctx->threads = threads; + ctx->unpackSize = unpackSize; ctx->insize = 0; ctx->outsize = 0; ctx->frames = 0; ctx->curframe = 0; - pthread_mutex_init(&ctx->read_mutex, NULL); - pthread_mutex_init(&ctx->write_mutex, NULL); + if (threads) { + pthread_mutex_init(&ctx->read_mutex, NULL); + pthread_mutex_init(&ctx->write_mutex, NULL); - /* free -> busy -> out -> free -> ... */ - INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */ - INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */ - INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */ + /* free -> busy -> out -> free -> ... */ + INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */ + INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */ + INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */ - ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); - if (!ctx->cwork) - goto err_cwork; + ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); + if (!ctx->cwork) + goto err_cwork; - for (t = 0; t < threads; t++) { - cwork_t *w = &ctx->cwork[t]; - w->ctx = ctx; - } + for (t = 0; t < threads; t++) { + cwork_t *w = &ctx->cwork[t]; + w->ctx = ctx; + } + } else { + ctx->cwork = NULL; + } return ctx; @@ -269,7 +282,7 @@ static void *pt_compress(void *arg) uint8_t *obuf = (uint8_t*)wl->out.buf + 16; wl->out.size -= 16; rv = BrotliEncoderCompress(ctx->level, - BROTLI_MAX_WINDOW_BITS, + ctx->lgwin, BROTLI_MODE_GENERIC, in.size, ibuf, &wl->out.size, obuf); @@ -319,6 +332,119 @@ static void *pt_compress(void *arg) return 0; } +/* single threaded (standard brotli stream, without header/mt-frames) */ +static size_t st_compress(void *arg) +{ + BROTLIMT_CCtx *ctx = (BROTLIMT_CCtx *) arg; + BrotliEncoderOperation brop = BROTLI_OPERATION_PROCESS; + BROTLIMT_Buffer Out; + BROTLIMT_Buffer *out = &Out; + BROTLIMT_Buffer In; + BROTLIMT_Buffer *in = &In; + BrotliEncoderState *state; + const uint8_t* next_in; + uint8_t* next_out; + int rv; + size_t retval = 0; + + /* allocate space for input buffer (default 1M * level) */ + in->allocated = ctx->inputsize; + in->buf = malloc(in->allocated); + if (!in->buf) + return MT_ERROR(memory_allocation); + next_in = in->buf; + in->size = 0; + + /* allocate space for output buffer */ + out->allocated = out->size = ctx->inputsize / 4; + out->buf = malloc(out->size); + if (!out->buf) { + free(in->buf); + return MT_ERROR(memory_allocation); + } + next_out = out->buf; + + state = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if (!state) { + free(in->buf); + free(out->buf); + return MT_ERROR(memory_allocation); + } + + BrotliEncoderSetParameter(state, BROTLI_PARAM_QUALITY, (uint32_t)ctx->level); + if (ctx->lgwin > 0) { + /* Specified by user. */ + /* Do not enable "large-window" extension, if not required. */ + if (ctx->lgwin > BROTLI_MAX_WINDOW_BITS) { + BrotliEncoderSetParameter(state, BROTLI_PARAM_LARGE_WINDOW, 1u); + } + BrotliEncoderSetParameter(state, BROTLI_PARAM_LGWIN, (uint32_t)ctx->lgwin); + } else { + /* 0, or not specified by user; could be chosen by compressor. */ + uint32_t lgwin = 24 /* DEFAULT_LGWIN */; + /* Use file size to limit lgwin. */ + if (ctx->unpackSize >= 0) { + lgwin = BROTLI_MIN_WINDOW_BITS; + while (BROTLI_MAX_BACKWARD_LIMIT(lgwin) < + (uint64_t)ctx->unpackSize) { + lgwin++; + if (lgwin == BROTLI_MAX_WINDOW_BITS) break; + } + } + BrotliEncoderSetParameter(state, BROTLI_PARAM_LGWIN, lgwin); + } + if (ctx->unpackSize > 0) { + uint32_t size_hint = ctx->unpackSize < (1 << 30) ? + (uint32_t)ctx->unpackSize : (1u << 30); + BrotliEncoderSetParameter(state, BROTLI_PARAM_SIZE_HINT, size_hint); + } + + while (1) { + if (in->size == 0 && brop != BROTLI_OPERATION_FINISH) { + in->size = in->allocated; + rv = ctx->fn_read(ctx->arg_read, in); + if (rv != 0) { + retval = mt_error(rv); + goto done; + } + if (in->size == 0) { + brop = BROTLI_OPERATION_FINISH; // eof + } + next_in = in->buf; + } + + if (!BrotliEncoderCompressStream(state, brop, &in->size, &next_in, &out->size, &next_out, 0)) { + retval = MT_ERROR(frame_compress); + goto done; + } + if (out->size == 0) { + out->size = next_out - (uint8_t*)out->buf; + rv = ctx->fn_write(ctx->arg_write, out); + if (rv != 0) { + retval = mt_error(rv); + goto done; + } + next_out = out->buf; + out->size = out->allocated; + } + if (BrotliEncoderIsFinished(state)) { + out->size = next_out - (uint8_t*)out->buf; + rv = ctx->fn_write(ctx->arg_write, out); + if (rv != 0) { + retval = mt_error(rv); + goto done; + } + break; + } + } + + done: + free(in->buf); + free(out->buf); + BrotliEncoderDestroyInstance(state); + return retval; +} + size_t BROTLIMT_compressCCtx(BROTLIMT_CCtx * ctx, BROTLIMT_RdWr_t * rdwr) { int t; @@ -333,6 +459,12 @@ size_t BROTLIMT_compressCCtx(BROTLIMT_CCtx * ctx, BROTLIMT_RdWr_t * rdwr) ctx->arg_read = rdwr->arg_read; ctx->arg_write = rdwr->arg_write; + /* single threaded brotli (no header, no mt-frames) */ + if (ctx->threads == 0) { + /* decompress single threaded */ + return st_compress(ctx); + } + /* start all workers */ for (t = 0; t < ctx->threads; t++) { cwork_t *w = &ctx->cwork[t]; @@ -394,9 +526,11 @@ void BROTLIMT_freeCCtx(BROTLIMT_CCtx * ctx) if (!ctx) return; - pthread_mutex_destroy(&ctx->read_mutex); - pthread_mutex_destroy(&ctx->write_mutex); - free(ctx->cwork); + if (ctx->threads) { + pthread_mutex_destroy(&ctx->read_mutex); + pthread_mutex_destroy(&ctx->write_mutex); + free(ctx->cwork); + } free(ctx); ctx = 0; diff --git a/C/zstdmt/brotli-mt_decompress.c b/C/zstdmt/brotli-mt_decompress.c index 483fe5a0..6538cd14 100644 --- a/C/zstdmt/brotli-mt_decompress.c +++ b/C/zstdmt/brotli-mt_decompress.c @@ -50,7 +50,7 @@ struct writelist { struct BROTLIMT_DCtx_s { - /* threads: 1..BROTLIMT_THREAD_MAX */ + /* threads: 0, 1..BROTLIMT_THREAD_MAX */ int threads; /* should be used for read from input */ @@ -96,7 +96,7 @@ BROTLIMT_DCtx *BROTLIMT_createDCtx(int threads, int inputsize) return 0; /* check threads value */ - if (threads < 1 || threads > BROTLIMT_THREAD_MAX) + if (threads < 0 || threads > BROTLIMT_THREAD_MAX) return 0; /* setup ctx */ @@ -112,12 +112,16 @@ BROTLIMT_DCtx *BROTLIMT_createDCtx(int threads, int inputsize) else ctx->inputsize = 1024 * 64; /* 64K buffer */ - pthread_mutex_init(&ctx->read_mutex, NULL); - pthread_mutex_init(&ctx->write_mutex, NULL); + if (threads) { + pthread_mutex_init(&ctx->read_mutex, NULL); + pthread_mutex_init(&ctx->write_mutex, NULL); - INIT_LIST_HEAD(&ctx->writelist_free); - INIT_LIST_HEAD(&ctx->writelist_busy); - INIT_LIST_HEAD(&ctx->writelist_done); + INIT_LIST_HEAD(&ctx->writelist_free); + INIT_LIST_HEAD(&ctx->writelist_busy); + INIT_LIST_HEAD(&ctx->writelist_done); + } else { + threads = 1; + } ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); if (!ctx->cwork) @@ -376,6 +380,90 @@ static void *pt_decompress(void *arg) return (void *)result; } +/* single threaded (standard brotli stream, without header/mt-frames) */ +static size_t st_decompress(void *arg) +{ + BROTLIMT_DCtx *ctx = (BROTLIMT_DCtx *) arg; + BrotliDecoderResult bres = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT; + cwork_t *w = &ctx->cwork[0]; + BROTLIMT_Buffer Out; + BROTLIMT_Buffer *out = &Out; + BROTLIMT_Buffer *in = &w->in; + BrotliDecoderState *state; + const uint8_t* next_in; + uint8_t* next_out; + int rv; + size_t retval = 0; + + /* allocate space for input buffer */ + in->allocated = in->size = ctx->inputsize; + in->buf = malloc(in->size); + if (!in->buf) + return MT_ERROR(memory_allocation); + next_in = in->buf; + + /* allocate space for output buffer */ + out->allocated = out->size = ctx->inputsize * 4; + out->buf = malloc(out->size); + if (!out->buf) { + free(in->buf); + return MT_ERROR(memory_allocation); + } + next_out = out->buf; + + state = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!state) { + free(in->buf); + free(out->buf); + return MT_ERROR(memory_allocation); + } + BrotliDecoderSetParameter(state, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1); + + while (1) { + if (bres == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + in->size = in->allocated; + rv = ctx->fn_read(ctx->arg_read, in); + if (in->size == 0) break; + if (rv != 0) { + retval = mt_error(rv); + goto done; + } + next_in = in->buf; + } else if (bres == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + out->size = next_out - (uint8_t*)out->buf; + rv = ctx->fn_write(ctx->arg_write, out); + if (rv != 0) { + retval = mt_error(rv); + goto done; + } + next_out = out->buf; + } else { + break; + } + + bres = BrotliDecoderDecompressStream(state, &in->size, &next_in, &out->size, &next_out, 0); + } + + if (bres != BROTLI_DECODER_RESULT_SUCCESS) { + retval = MT_ERROR(data_error); // corrupt input + goto done; + } + out->size = next_out - (uint8_t*)out->buf; + if (out->size != 0) { + rv = ctx->fn_write(ctx->arg_write, out); + if (rv != 0) { + retval = mt_error(rv); + goto done; + } + } + + done: + free(in->buf); + free(out->buf); + BrotliDecoderDestroyInstance(state); + return retval; +} + size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr) { unsigned char buf[4]; @@ -393,6 +481,14 @@ size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr) ctx->arg_read = rdwr->arg_read; ctx->arg_write = rdwr->arg_write; + /* For single threaded brotli (no header, no mt-frames), don't check for + BROTLIMT_MAGIC_SKIPPABLE here, because stdandard brotli stream may also + start with it, so will be mistakenly considered as brotli-mt stream. */ + if (ctx->threads == 0) { + /* decompress single threaded */ + return st_decompress(ctx); + } + /* check for BROTLIMT_MAGIC_SKIPPABLE */ in->buf = buf; in->size = 4; @@ -402,10 +498,6 @@ size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr) if (in->size != 4) return MT_ERROR(data_error); - /* single threaded with unknown sizes */ - if (MEM_readLE32(buf) != BROTLIMT_MAGIC_SKIPPABLE) - return MT_ERROR(data_error); - /* mark unused */ in->buf = 0; in->size = 0; @@ -485,8 +577,10 @@ void BROTLIMT_freeDCtx(BROTLIMT_DCtx * ctx) if (!ctx) return; - pthread_mutex_destroy(&ctx->read_mutex); - pthread_mutex_destroy(&ctx->write_mutex); + if (ctx->threads) { + pthread_mutex_destroy(&ctx->read_mutex); + pthread_mutex_destroy(&ctx->write_mutex); + } free(ctx->cwork); free(ctx); ctx = 0; diff --git a/CPP/7zip/Archive/BrotliHandler.cpp b/CPP/7zip/Archive/BrotliHandler.cpp new file mode 100644 index 00000000..734ac5e8 --- /dev/null +++ b/CPP/7zip/Archive/BrotliHandler.cpp @@ -0,0 +1,351 @@ +// BrotliHandler.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" +#include "../../Common/ComTry.h" +#include "../../Common/Defs.h" + +#include "../Common/ProgressUtils.h" +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" + +#include "../Compress/BrotliDecoder.h" +#include "../Compress/BrotliEncoder.h" +#include "../Compress/CopyCoder.h" + +#include "Common/DummyOutStream.h" +#include "Common/HandlerOut.h" + +using namespace NWindows; + +namespace NArchive { +namespace NBROTLI { + +class CHandler: + public IInArchive, + public IArchiveOpenSeq, + public IOutArchive, + public ISetProperties, + public CMyUnknownImp +{ + CMyComPtr _stream; + CMyComPtr _seqStream; + + bool _isArc; + bool _dataAfterEnd; + bool _needMoreInput; + + bool _packSize_Defined; + bool _unpackSize_Defined; + + UInt64 _packSize; + UInt64 _unpackSize; + UInt64 _numStreams; + UInt64 _numBlocks; + + CSingleMethodProps _props; + +public: + MY_UNKNOWN_IMP4( + IInArchive, + IArchiveOpenSeq, + IOutArchive, + ISetProperties) + INTERFACE_IInArchive(;) + INTERFACE_IOutArchive(;) + STDMETHOD(OpenSeq)(ISequentialInStream *stream); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); + + CHandler() { } +}; + +static const Byte kProps[] = +{ + kpidSize, + kpidPackSize +}; + +static const Byte kArcProps[] = +{ + kpidNumStreams, + kpidNumBlocks +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/) +{ + return S_OK; +} + +STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) +{ + *numItems = 1; + return S_OK; +} + +STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidPackSize: if (_packSize_Defined) prop = _packSize; break; + case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break; + } + prop.Detach(value); + return S_OK; +} + +// brotli format has no signature +// brotli stream can't be veriefied unless errors by unpack +#if 0 +API_FUNC_static_IsArc IsArc_Brotli(const Byte *p, size_t size) +{ + return k_IsArc_Res_YES; +} +} +#endif + +STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *) +{ + COM_TRY_BEGIN + Close(); + { + _isArc = true; + _stream = stream; + _seqStream = stream; + } + return S_OK; + COM_TRY_END +} + +STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) +{ + Close(); + _isArc = true; + _seqStream = stream; + return S_OK; +} + +STDMETHODIMP CHandler::Close() +{ + _isArc = false; + _dataAfterEnd = false; + _needMoreInput = false; + + _packSize_Defined = false; + _unpackSize_Defined = false; + + _packSize = 0; + + _seqStream.Release(); + _stream.Release(); + return S_OK; +} + +STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback) +{ + COM_TRY_BEGIN + if (numItems == 0) + return S_OK; + if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) + return E_INVALIDARG; + + if (_packSize_Defined) + extractCallback->SetTotal(_packSize); + + CMyComPtr realOutStream; + Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + RINOK(extractCallback->GetStream(0, &realOutStream, askMode)); + if (!testMode && !realOutStream) + return S_OK; + + extractCallback->PrepareOperation(askMode); + + Int32 opRes; + + { + + NCompress::NBROTLI::CDecoder *decoderSpec = new NCompress::NBROTLI::CDecoder; + decoderSpec->SetNumberOfThreads(0); /* .br - single threaded processing (without header/mt-frames) */ + CMyComPtr decoder = decoderSpec; + decoderSpec->SetInStream(_seqStream); + + CDummyOutStream *outStreamSpec = new CDummyOutStream; + CMyComPtr outStream(outStreamSpec); + outStreamSpec->SetStream(realOutStream); + outStreamSpec->Init(); + + realOutStream.Release(); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(extractCallback, true); + + UInt64 packSize = 0; + UInt64 unpackedSize = 0; + + HRESULT result = S_OK; + + for (;;) + { + lps->InSize = packSize; + lps->OutSize = unpackedSize; + + RINOK(lps->SetCur()); + result = decoderSpec->CodeResume(outStream, &unpackedSize, progress); + UInt64 streamSize = decoderSpec->GetInputProcessedSize(); + + if (result != S_FALSE && result != S_OK) + return result; + + if (unpackedSize == 0) + break; + + if (streamSize == packSize) + { + // no new bytes in input stream, So it's good end of archive. + result = S_OK; + break; + } + + if (packSize > streamSize) + return E_FAIL; + + if (result != S_OK) + break; + } + + decoderSpec->ReleaseInStream(); + outStream.Release(); + + if (!_isArc) + opRes = NExtract::NOperationResult::kIsNotArc; + else if (_needMoreInput) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else if (_dataAfterEnd) + opRes = NExtract::NOperationResult::kDataAfterEnd; + else if (result == S_FALSE) + opRes = NExtract::NOperationResult::kDataError; + else if (result == S_OK) + opRes = NExtract::NOperationResult::kOK; + else + return result; + + } + + return extractCallback->SetOperationResult(opRes); + + COM_TRY_END +} + +static HRESULT UpdateArchive( + UInt64 unpackSize, + ISequentialOutStream *outStream, + const CProps &props, + IArchiveUpdateCallback *updateCallback) +{ + RINOK(updateCallback->SetTotal(unpackSize)); + CMyComPtr fileInStream; + RINOK(updateCallback->GetStream(0, &fileInStream)); + CLocalProgress *localProgressSpec = new CLocalProgress; + CMyComPtr localProgress = localProgressSpec; + localProgressSpec->Init(updateCallback, true); + NCompress::NBROTLI::CEncoder *encoderSpec = new NCompress::NBROTLI::CEncoder; + encoderSpec->unpackSize = unpackSize; + encoderSpec->SetNumberOfThreads(0); /* .br - single threaded processing (without header/mt-frames) */ + CMyComPtr encoder = encoderSpec; + RINOK(props.SetCoderProps(encoderSpec, NULL)); + RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress)); + return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); +} + +STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) +{ + *type = NFileTimeType::kUnix; + return S_OK; +} + +STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *updateCallback) +{ + COM_TRY_BEGIN + + if (numItems != 1) + return E_INVALIDARG; + + Int32 newData, newProps; + UInt32 indexInArchive; + if (!updateCallback) + return E_FAIL; + RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive)); + + if ((newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop)); + if (prop.vt != VT_EMPTY) + if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE) + return E_INVALIDARG; + } + } + + if ((newData)) + { + UInt64 size; + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidSize, &prop)); + if (prop.vt != VT_UI8) + return E_INVALIDARG; + size = prop.uhVal.QuadPart; + } + return UpdateArchive(size, outStream, _props, updateCallback); + } + + if (indexInArchive != 0) + return E_INVALIDARG; + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(updateCallback, true); + + CMyComPtr opCallback; + updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback); + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kInArcIndex, 0, + NUpdateNotifyOp::kReplicate)) + } + + if (_stream) + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + + return NCompress::CopyStream(_stream, outStream, progress); + + COM_TRY_END +} + +STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) +{ + return _props.SetProperties(names, values, numProps); +} + +IMP_CreateArcIn +IMP_CreateArcOut +REGISTER_ARC_R( + "brotli", "br brotli tbr", 0, 0x1F, + 0, NULL, + 0, + NArcInfoFlags::kKeepName | NArcInfoFlags::kPureStartOpen | NArcInfoFlags::kByExtOnlyOpen, + 0, + CreateArc, CreateArcOut, + /* IsArc_Brotli */) +}} diff --git a/CPP/7zip/Bundles/Format7zF/Arc.mak b/CPP/7zip/Bundles/Format7zF/Arc.mak index f50966aa..f66cfd55 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -70,6 +70,7 @@ AR_OBJS = \ $O\ArHandler.obj \ $O\ArjHandler.obj \ $O\Base64Handler.obj \ + $O\BrotliHandler.obj \ $O\Bz2Handler.obj \ $O\ComHandler.obj \ $O\CpioHandler.obj \ diff --git a/CPP/7zip/Compress/BrotliDecoder.cpp b/CPP/7zip/Compress/BrotliDecoder.cpp index 3e04db2c..5027c089 100644 --- a/CPP/7zip/Compress/BrotliDecoder.cpp +++ b/CPP/7zip/Compress/BrotliDecoder.cpp @@ -98,7 +98,7 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) { const UInt32 kNumThreadsMax = BROTLIMT_THREAD_MAX; - if (numThreads < 1) numThreads = 1; + if (numThreads < 0) numThreads = 0; if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; _numThreads = numThreads; return S_OK; diff --git a/CPP/7zip/Compress/BrotliEncoder.cpp b/CPP/7zip/Compress/BrotliEncoder.cpp index 5db7afc7..b1dcb613 100644 --- a/CPP/7zip/Compress/BrotliEncoder.cpp +++ b/CPP/7zip/Compress/BrotliEncoder.cpp @@ -12,8 +12,10 @@ CEncoder::CEncoder(): _processedIn(0), _processedOut(0), _inputSize(0), - _ctx(NULL), - _numThreads(NWindows::NSystem::GetNumberOfProcessors()) + _numThreads(NWindows::NSystem::GetNumberOfProcessors()), + _Long(-1), + _WindowLog(-1), + _ctx(NULL) { _props.clear(); } @@ -52,6 +54,37 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI SetNumberOfThreads(v); break; } + case NCoderPropID::kLong: + { + /* like --long in zstd cli program */ + _Long = 1; + if (v == 0) { + if (prop.vt == VT_EMPTY) { + // m0=brotli:long -> long=default + _WindowLog = BROTLI_MAX_WINDOW_BITS; //BROTLI_DEFAULT_WINDOW; + } else { + // m0=brotli:long=0 -> long=auto + _WindowLog = 0; + } + } else if (v < BROTLI_MIN_WINDOW_BITS) { + // m0=brotli:long=9 -> long=10 + _WindowLog = BROTLI_MIN_WINDOW_BITS; + } else if (v > BROTLI_LARGE_MAX_WINDOW_BITS) { + // m0=brotli:long=33 -> long=max + _WindowLog = BROTLI_LARGE_MAX_WINDOW_BITS; + } else { + // m0=brotli:long=15 -> long=value + _WindowLog = v; + } + break; + } + case NCoderPropID::kWindowLog: + { + if (v < BROTLI_MIN_WINDOW_BITS) v = BROTLI_MIN_WINDOW_BITS; + if (v > BROTLI_MAX_WINDOW_BITS) v = BROTLI_MAX_WINDOW_BITS; + _WindowLog = v; + break; + } default: { break; @@ -102,11 +135,12 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, /* 2) create compression context, if needed */ if (!_ctx) - _ctx = BROTLIMT_createCCtx(_numThreads, _props._level, _inputSize); + _ctx = BROTLIMT_createCCtx(_numThreads, unpackSize, _props._level, _inputSize, + _WindowLog >= 0 ? _WindowLog : BROTLI_MAX_WINDOW_BITS); if (!_ctx) return S_FALSE; - /* 3) compress */ + /* 4) compress */ result = BROTLIMT_compressCCtx(_ctx, &rdwr); if (BROTLIMT_isError(result)) { if (result == (size_t)-BROTLIMT_error_canceled) @@ -120,7 +154,7 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads) { const UInt32 kNumThreadsMax = BROTLIMT_THREAD_MAX; - if (numThreads < 1) numThreads = 1; + if (numThreads < 0) numThreads = 0; if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; _numThreads = numThreads; return S_OK; diff --git a/CPP/7zip/Compress/BrotliEncoder.h b/CPP/7zip/Compress/BrotliEncoder.h index 239bea67..c207069a 100644 --- a/CPP/7zip/Compress/BrotliEncoder.h +++ b/CPP/7zip/Compress/BrotliEncoder.h @@ -45,9 +45,15 @@ class CEncoder: UInt32 _inputSize; UInt32 _numThreads; + Int32 _Long; + Int32 _WindowLog; + BROTLIMT_CCtx *_ctx; public: + + UInt64 unpackSize; + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) MY_QUERYINTERFACE_ENTRY(ICompressSetCoderProperties)