From ef790b520928e80d59b2143f668dd7579e1ab3fd Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Mon, 28 Aug 2017 16:34:04 +0200 Subject: [PATCH] Update to 7-Zip 17.01 Beta from Igor Pavlov - Minor speed optimization for LZMA2 (xz and 7z) multi-threading compression. 7-Zip now uses additional memory buffers for multi-block LZMA2 compression. CPU utilization was slightly improved. - 7-zip now creates multi-block xz archives by default. Block size can be specified with -ms[Size]{m|g} switch. - xz decoder now can unpack random block from multi-block xz archives. 7-Zip File Manager now can open nested multi-block xz archives (for example, image.iso.xz) without full unpacking of xz archive. - 7-Zip now can create zip archives from stdin to stdout. - 7-Zip command line: @listfile now doesn't work after -- switch. Use -i@listfile before -- switch instead. fixed bugs: - 7-Zip could add unrequired alternate file streams to WIM archives, for commands that contain filename wildcards and -sns switch. - 7-Zip 17.00 beta crashed for commands that write anti-item to 7z archive. - 7-Zip 17.00 beta ignored "Use large memory pages" option. --- C/7zCrc.c | 8 +- C/7zTypes.h | 12 +- C/7zVersion.h | 6 +- C/7zVersionTr.h | 2 +- C/AesOpt.c | 4 +- C/Alloc.c | 49 +- C/CpuArch.h | 22 +- C/LzFind.c | 47 +- C/LzFind.h | 8 +- C/LzFindMt.c | 39 +- C/Lzma2Enc.c | 658 ++++++--- C/Lzma2Enc.h | 43 +- C/LzmaEnc.c | 47 +- C/LzmaEnc.h | 37 +- C/MtCoder.c | 721 +++++++--- C/MtCoder.h | 151 ++- C/Threads.c | 14 +- C/Threads.h | 5 +- C/Util/7z/7zMain.c | 11 +- C/Util/7zipInstall/7zipInstall.c | 4 +- C/Util/Lzma/LzmaUtil.dsp | 8 +- C/Xz.c | 12 +- C/Xz.h | 35 +- C/XzCrc64.c | 16 +- C/XzCrc64Opt.c | 14 +- C/XzDec.c | 136 +- C/XzEnc.c | 1178 ++++++++++++++--- C/XzEnc.h | 27 +- C/XzIn.c | 32 +- CPP/7zip/Archive/7z/7zEncode.cpp | 13 +- CPP/7zip/Archive/7z/7zFolderInStream.cpp | 15 +- CPP/7zip/Archive/7z/7zOut.cpp | 2 +- CPP/7zip/Archive/7z/7zUpdate.cpp | 5 +- CPP/7zip/Archive/ExtHandler.cpp | 4 +- CPP/7zip/Archive/Nsis/NsisIn.cpp | 8 +- CPP/7zip/Archive/NtfsHandler.cpp | 88 +- CPP/7zip/Archive/Rar/Rar5Handler.cpp | 2 +- CPP/7zip/Archive/SquashfsHandler.cpp | 5 +- CPP/7zip/Archive/Tar/TarIn.cpp | 17 +- CPP/7zip/Archive/XzHandler.cpp | 631 +++++++-- CPP/7zip/Archive/Zip/ZipAddCommon.cpp | 94 +- CPP/7zip/Archive/Zip/ZipAddCommon.h | 8 +- CPP/7zip/Archive/Zip/ZipHandler.cpp | 3 +- CPP/7zip/Archive/Zip/ZipIn.cpp | 11 +- CPP/7zip/Archive/Zip/ZipItem.cpp | 18 + CPP/7zip/Archive/Zip/ZipOut.cpp | 3 + CPP/7zip/Archive/Zip/ZipUpdate.cpp | 232 ++-- CPP/7zip/Common/CWrappers.cpp | 8 +- CPP/7zip/Common/MethodProps.cpp | 36 +- CPP/7zip/Common/MethodProps.h | 47 +- CPP/7zip/Compress/Lzma2Encoder.cpp | 53 +- CPP/7zip/Compress/Lzma2Encoder.h | 8 +- CPP/7zip/Compress/LzmaEncoder.cpp | 39 +- CPP/7zip/Compress/LzmaEncoder.h | 8 +- CPP/7zip/Compress/Rar3Decoder.cpp | 3 +- CPP/7zip/Compress/Rar3Vm.cpp | 67 +- CPP/7zip/Compress/Rar3Vm.h | 2 +- CPP/7zip/Compress/Rar5Decoder.cpp | 8 +- CPP/7zip/Compress/XzDecoder.cpp | 31 +- CPP/7zip/Compress/XzDecoder.h | 6 +- CPP/7zip/Compress/XzEncoder.cpp | 305 +++-- CPP/7zip/Compress/XzEncoder.h | 14 +- CPP/7zip/Guid.txt | 1 + CPP/7zip/ICoder.h | 47 +- CPP/7zip/UI/Agent/Agent.cpp | 14 +- CPP/7zip/UI/Client7z/Client7z.cpp | 2 +- CPP/7zip/UI/Common/ArchiveExtractCallback.cpp | 4 +- CPP/7zip/UI/Common/ArchiveExtractCallback.h | 5 +- CPP/7zip/UI/Common/Bench.cpp | 113 +- CPP/7zip/UI/Common/CompressCall.cpp | 6 +- CPP/7zip/UI/Common/DirItem.h | 26 +- CPP/7zip/UI/Common/EnumDirItems.cpp | 17 +- CPP/7zip/UI/Common/Extract.cpp | 4 +- CPP/7zip/UI/Common/ExtractingFilePath.cpp | 23 +- CPP/7zip/UI/Common/ExtractingFilePath.h | 14 +- CPP/7zip/UI/Common/PropIDUtils.cpp | 72 +- CPP/7zip/UI/Common/Update.cpp | 88 +- CPP/7zip/UI/Common/UpdateCallback.h | 8 +- CPP/7zip/UI/Common/ZipRegistry.h | 7 +- .../UI/Console/ExtractCallbackConsole.cpp | 33 + CPP/7zip/UI/Console/Main.cpp | 3 +- CPP/7zip/UI/Console/UpdateCallbackConsole.cpp | 5 +- CPP/7zip/UI/Far/Messages.h | 2 +- CPP/7zip/UI/FileManager/App.cpp | 44 +- CPP/7zip/UI/FileManager/ExtractCallback.cpp | 2 +- CPP/7zip/UI/FileManager/FM.cpp | 3 + CPP/7zip/UI/FileManager/FSFolder.h | 2 +- CPP/7zip/UI/FileManager/FSFolderCopy.cpp | 9 +- CPP/7zip/UI/FileManager/LinkDialog.cpp | 4 +- CPP/7zip/UI/FileManager/Panel.cpp | 78 +- CPP/7zip/UI/FileManager/Panel.h | 35 +- CPP/7zip/UI/FileManager/PanelCopy.cpp | 6 +- CPP/7zip/UI/FileManager/PanelCrc.cpp | 2 +- CPP/7zip/UI/FileManager/PanelDrag.cpp | 29 +- CPP/7zip/UI/FileManager/PanelFolderChange.cpp | 15 +- CPP/7zip/UI/FileManager/PanelItemOpen.cpp | 38 +- CPP/7zip/UI/FileManager/PanelItems.cpp | 66 +- CPP/7zip/UI/FileManager/PanelMenu.cpp | 4 +- CPP/7zip/UI/FileManager/PanelOperations.cpp | 27 +- CPP/7zip/UI/FileManager/PanelSplitFile.cpp | 28 +- CPP/7zip/UI/GUI/CompressDialog.cpp | 145 +- CPP/7zip/UI/GUI/CompressDialog.h | 3 +- CPP/7zip/UI/GUI/GUI.cpp | 11 +- CPP/7zip/UI/GUI/GUI.dsp | 8 +- CPP/Build.mak | 4 + CPP/Windows/FileDir.cpp | 9 + CPP/Windows/FileDir.h | 13 + DOC/7zip.inf | 4 +- DOC/7zip.nsi | 2 +- DOC/7zip.wxs | 2 +- DOC/Methods.txt | 8 +- README.md | 8 +- 112 files changed, 4625 insertions(+), 1618 deletions(-) diff --git a/C/7zCrc.c b/C/7zCrc.c index 8cc006bd..40ab7595 100644 --- a/C/7zCrc.c +++ b/C/7zCrc.c @@ -1,5 +1,5 @@ /* 7zCrc.c -- CRC32 init -2017-04-03 : Igor Pavlov : Public domain */ +2017-06-06 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -86,8 +86,8 @@ void MY_FAST_CALL CrcGenerateTable() #ifdef MY_CPU_X86_OR_AMD64 if (!CPU_Is_InOrder()) - g_CrcUpdate = CrcUpdateT8; #endif + g_CrcUpdate = CrcUpdateT8; #endif #else @@ -101,7 +101,7 @@ void MY_FAST_CALL CrcGenerateTable() g_CrcUpdate = CrcUpdateT4; #if CRC_NUM_TABLES >= 8 g_CrcUpdateT8 = CrcUpdateT8; - // g_CrcUpdate = CrcUpdateT8; + g_CrcUpdate = CrcUpdateT8; #endif } else if (p[0] != 1 || p[1] != 2) @@ -118,7 +118,7 @@ void MY_FAST_CALL CrcGenerateTable() g_CrcUpdate = CrcUpdateT1_BeT4; #if CRC_NUM_TABLES >= 8 g_CrcUpdateT8 = CrcUpdateT1_BeT8; - // g_CrcUpdate = CrcUpdateT1_BeT8; + g_CrcUpdate = CrcUpdateT1_BeT8; #endif } } diff --git a/C/7zTypes.h b/C/7zTypes.h index 7b41682c..4977cdaa 100644 --- a/C/7zTypes.h +++ b/C/7zTypes.h @@ -1,5 +1,5 @@ /* 7zTypes.h -- Basic types -2017-04-03 : Igor Pavlov : Public domain */ +2017-07-17 : Igor Pavlov : Public domain */ #ifndef __7Z_TYPES_H #define __7Z_TYPES_H @@ -42,13 +42,23 @@ EXTERN_C_BEGIN typedef int SRes; + #ifdef _WIN32 + /* typedef DWORD WRes; */ typedef unsigned WRes; +#define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x) + #else + typedef int WRes; +#define MY__FACILITY_WIN32 7 +#define MY__FACILITY__WRes MY__FACILITY_WIN32 +#define MY_SRes_HRESULT_FROM_WRes(x) ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : ((HRESULT) (((x) & 0x0000FFFF) | (MY__FACILITY__WRes << 16) | 0x80000000))) + #endif + #ifndef RINOK #define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } #endif diff --git a/C/7zVersion.h b/C/7zVersion.h index 38ab5d0d..57ee33a5 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ #define MY_VER_MAJOR 17 -#define MY_VER_MINOR 00 +#define MY_VER_MINOR 01 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "17.00 ZS v1.3.1 R1" +#define MY_VERSION_NUMBERS "17.01 ZS v1.3.1 R1" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,7 +10,7 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2017-08-20" +#define MY_DATE "2017-08-28" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt" diff --git a/C/7zVersionTr.h b/C/7zVersionTr.h index 732b5efc..5751dc66 100644 --- a/C/7zVersionTr.h +++ b/C/7zVersionTr.h @@ -3,7 +3,7 @@ #define MY_VER_BUILD 1 #define MY_VERSION_NUMBERS "1.3.1 R1" #define MY_VERSION MY_VERSION_NUMBERS -#define MY_DATE "2017-08-20" +#define MY_DATE "2017-08-28" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Tino Reichardt" diff --git a/C/AesOpt.c b/C/AesOpt.c index e5d4d261..0e7f49a1 100644 --- a/C/AesOpt.c +++ b/C/AesOpt.c @@ -1,12 +1,12 @@ /* AesOpt.c -- Intel's AES -2013-11-12 : Igor Pavlov : Public domain */ +2017-06-08 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "CpuArch.h" #ifdef MY_CPU_X86_OR_AMD64 -#if _MSC_VER >= 1500 +#if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729) #define USE_INTEL_AES #endif #endif diff --git a/C/Alloc.c b/C/Alloc.c index ccab6efb..9f7cae3d 100644 --- a/C/Alloc.c +++ b/C/Alloc.c @@ -1,5 +1,5 @@ /* Alloc.c -- Memory allocation functions -2017-04-03 : Igor Pavlov : Public domain */ +2017-06-15 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -23,11 +23,11 @@ int g_allocCountBig = 0; void *MyAlloc(size_t size) { if (size == 0) - return 0; + return NULL; #ifdef _SZ_ALLOC_DEBUG { void *p = malloc(size); - fprintf(stderr, "\nAlloc %10d bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p); + fprintf(stderr, "\nAlloc %10u bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p); return p; } #else @@ -38,7 +38,7 @@ void *MyAlloc(size_t size) void MyFree(void *address) { #ifdef _SZ_ALLOC_DEBUG - if (address != 0) + if (address) fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address); #endif free(address); @@ -49,20 +49,20 @@ void MyFree(void *address) void *MidAlloc(size_t size) { if (size == 0) - return 0; + return NULL; #ifdef _SZ_ALLOC_DEBUG fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++); #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void MidFree(void *address) { #ifdef _SZ_ALLOC_DEBUG - if (address != 0) + if (address) fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid); #endif - if (address == 0) + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); } @@ -79,10 +79,10 @@ typedef SIZE_T (WINAPI *GetLargePageMinimumP)(); void SetLargePageSize() { #ifdef _7ZIP_LARGE_PAGES - SIZE_T size = 0; + SIZE_T size; GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP) GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); - if (largePageMinimum == 0) + if (!largePageMinimum) return; size = largePageMinimum(); if (size == 0 || (size & (size - 1)) != 0) @@ -95,31 +95,40 @@ void SetLargePageSize() void *BigAlloc(size_t size) { if (size == 0) - return 0; + return NULL; #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Big %10d bytes; count = %10d", size, g_allocCountBig++); + fprintf(stderr, "\nAlloc_Big %10u bytes; count = %10d", size, g_allocCountBig++); #endif #ifdef _7ZIP_LARGE_PAGES - if (g_LargePageSize != 0 && g_LargePageSize <= (1 << 30) && size >= (1 << 18)) { - void *res = VirtualAlloc(0, (size + g_LargePageSize - 1) & (~(g_LargePageSize - 1)), - MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); - if (res != 0) - return res; + SIZE_T ps = g_LargePageSize; + if (ps != 0 && ps <= (1 << 30) && size > (ps / 2)) + { + size_t size2; + ps--; + size2 = (size + ps) & ~ps; + if (size2 >= size) + { + void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); + if (res) + return res; + } + } } #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); + + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void BigFree(void *address) { #ifdef _SZ_ALLOC_DEBUG - if (address != 0) + if (address) fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); #endif - if (address == 0) + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); } diff --git a/C/CpuArch.h b/C/CpuArch.h index dc0ec675..998c2a7e 100644 --- a/C/CpuArch.h +++ b/C/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2017-04-03 : Igor Pavlov : Public domain */ +2017-06-30 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H @@ -190,8 +190,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #ifdef MY_CPU_LE #if defined(MY_CPU_X86_OR_AMD64) \ || defined(MY_CPU_ARM64) \ - || defined(__ARM_FEATURE_UNALIGNED) \ - || defined(__AARCH64EL__) + || defined(__ARM_FEATURE_UNALIGNED) #define MY_CPU_LE_UNALIGN #endif #endif @@ -237,6 +236,11 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #endif +#ifdef __has_builtin + #define MY__has_builtin(x) __has_builtin(x) +#else + #define MY__has_builtin(x) 0 +#endif #if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300) @@ -244,15 +248,21 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #include +#pragma intrinsic(_byteswap_ushort) #pragma intrinsic(_byteswap_ulong) #pragma intrinsic(_byteswap_uint64) + +/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */ #define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p)) #define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p)) #define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v) -#elif defined(MY_CPU_LE_UNALIGN) && defined (__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +#elif defined(MY_CPU_LE_UNALIGN) && ( \ + (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ + || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) ) +/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const Byte *)(p)) */ #define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p)) #define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p)) @@ -277,10 +287,14 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #endif +#ifndef GetBe16 + #define GetBe16(p) ( (UInt16) ( \ ((UInt16)((const Byte *)(p))[0] << 8) | \ ((const Byte *)(p))[1] )) +#endif + #ifdef MY_CPU_X86_OR_AMD64 diff --git a/C/LzFind.c b/C/LzFind.c index 25c2a22f..6ea82a9b 100644 --- a/C/LzFind.c +++ b/C/LzFind.c @@ -1,5 +1,5 @@ /* LzFind.c -- Match finder for LZ algorithms -2017-04-03 : Igor Pavlov : Public domain */ +2017-06-10 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -142,6 +142,7 @@ void MatchFinder_Construct(CMatchFinder *p) p->bufferBase = NULL; p->directInput = 0; p->hash = NULL; + p->expectedDataSize = (UInt64)(Int64)-1; MatchFinder_SetDefaultSettings(p); for (i = 0; i < 256; i++) @@ -208,7 +209,11 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, hs = (1 << 16) - 1; else { - hs = historySize - 1; + hs = historySize; + if (hs > p->expectedDataSize) + hs = (UInt32)p->expectedDataSize; + if (hs != 0) + hs--; hs |= (hs >> 1); hs |= (hs >> 2); hs |= (hs >> 4); @@ -292,17 +297,33 @@ static void MatchFinder_SetLimits(CMatchFinder *p) p->posLimit = p->pos + limit; } -void MatchFinder_Init_2(CMatchFinder *p, int readData) + +void MatchFinder_Init_LowHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash; + size_t numItems = p->fixedHashSize; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_HighHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash + p->fixedHashSize; + size_t numItems = (size_t)p->hashMask + 1; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_3(CMatchFinder *p, int readData) { - UInt32 i; - UInt32 *hash = p->hash; - UInt32 num = p->hashSizeSum; - for (i = 0; i < num; i++) - hash[i] = kEmptyHashValue; - p->cyclicBufferPos = 0; p->buffer = p->bufferBase; - p->pos = p->streamPos = p->cyclicBufferSize; + p->pos = + p->streamPos = p->cyclicBufferSize; p->result = SZ_OK; p->streamEndWasReached = 0; @@ -312,10 +333,14 @@ void MatchFinder_Init_2(CMatchFinder *p, int readData) MatchFinder_SetLimits(p); } + void MatchFinder_Init(CMatchFinder *p) { - MatchFinder_Init_2(p, True); + MatchFinder_Init_HighHash(p); + MatchFinder_Init_LowHash(p); + MatchFinder_Init_3(p, True); } + static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) { diff --git a/C/LzFind.h b/C/LzFind.h index 1d4f5902..c77added 100644 --- a/C/LzFind.h +++ b/C/LzFind.h @@ -1,5 +1,5 @@ /* LzFind.h -- Match finder for LZ algorithms -2017-04-03 : Igor Pavlov : Public domain */ +2017-06-10 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_H #define __LZ_FIND_H @@ -47,6 +47,8 @@ typedef struct _CMatchFinder SRes result; UInt32 crc[256]; size_t numRefs; + + UInt64 expectedDataSize; } CMatchFinder; #define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer) @@ -103,7 +105,9 @@ typedef struct _IMatchFinder void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable); -void MatchFinder_Init_2(CMatchFinder *p, int readData); +void MatchFinder_Init_LowHash(CMatchFinder *p); +void MatchFinder_Init_HighHash(CMatchFinder *p); +void MatchFinder_Init_3(CMatchFinder *p, int readData); void MatchFinder_Init(CMatchFinder *p); UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); diff --git a/C/LzFindMt.c b/C/LzFindMt.c index eebb62a0..2563824f 100644 --- a/C/LzFindMt.c +++ b/C/LzFindMt.c @@ -1,5 +1,5 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2017-04-03 : Igor Pavlov : Public domain */ +2017-06-10 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -33,6 +33,8 @@ static void MtSync_GetNextBlock(CMtSync *p) Event_Set(&p->canStart); Event_Wait(&p->wasStarted); + + // if (mt) MatchFinder_Init_LowHash(mt->MatchFinder); } else { @@ -155,6 +157,9 @@ static void HashThreadFunc(CMatchFinderMt *mt) UInt32 numProcessedBlocks = 0; Event_Wait(&p->canStart); Event_Set(&p->wasStarted); + + MatchFinder_Init_HighHash(mt->MatchFinder); + for (;;) { if (p->exit) @@ -205,7 +210,7 @@ static void HashThreadFunc(CMatchFinderMt *mt) if (num > kMtHashBlockSize - 2) num = kMtHashBlockSize - 2; mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc); - heads[0] += num; + heads[0] = 2 + num; } mf->pos += num; mf->buffer += num; @@ -496,14 +501,18 @@ SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddB } /* Call it after ReleaseStream / SetStream */ -void MatchFinderMt_Init(CMatchFinderMt *p) +static void MatchFinderMt_Init(CMatchFinderMt *p) { CMatchFinder *mf = p->MatchFinder; - p->btBufPos = p->btBufPosLimit = 0; - p->hashBufPos = p->hashBufPosLimit = 0; + + p->btBufPos = + p->btBufPosLimit = 0; + p->hashBufPos = + p->hashBufPosLimit = 0; /* Init without data reading. We don't want to read data in this thread */ - MatchFinder_Init_2(mf, False); + MatchFinder_Init_3(mf, False); + MatchFinder_Init_LowHash(mf); p->pointerToCurPos = Inline_MatchFinder_GetPointerToCurrentPos(mf); p->btNumAvailBytes = 0; @@ -684,8 +693,12 @@ static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *distances) UInt32 i; for (i = 0; i < len; i += 2) { - *distances++ = *btBuf++; - *distances++ = *btBuf++; + UInt32 v0 = btBuf[0]; + UInt32 v1 = btBuf[1]; + btBuf += 2; + distances[0] = v0; + distances[1] = v1; + distances += 2; } } INCREASE_LZ_POS @@ -712,8 +725,12 @@ static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances) distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances); do { - *distances2++ = *btBuf++; - *distances2++ = *btBuf++; + UInt32 v0 = btBuf[0]; + UInt32 v1 = btBuf[1]; + btBuf += 2; + distances2[0] = v0; + distances2[1] = v1; + distances2 += 2; } while ((len -= 2) != 0); len = (UInt32)(distances2 - (distances)); @@ -777,7 +794,7 @@ void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable) { case 2: p->GetHeadsFunc = GetHeads2; - p->MixMatchesFunc = (Mf_Mix_Matches)0; + p->MixMatchesFunc = (Mf_Mix_Matches)NULL; vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip; vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches; break; diff --git a/C/Lzma2Enc.c b/C/Lzma2Enc.c index 18a8677e..25e54af5 100644 --- a/C/Lzma2Enc.c +++ b/C/Lzma2Enc.c @@ -1,9 +1,8 @@ /* Lzma2Enc.c -- LZMA2 Encoder -2017-04-03 : Igor Pavlov : Public domain */ +2017-08-28 : Igor Pavlov : Public domain */ #include "Precomp.h" -/* #include */ #include /* #define _7ZIP_ST */ @@ -13,7 +12,7 @@ #ifndef _7ZIP_ST #include "MtCoder.h" #else -#define NUM_MT_CODER_THREADS_MAX 1 +#define MTCODER__THREADS_MAX 1 #endif #define LZMA2_CONTROL_LZMA (1 << 7) @@ -35,30 +34,83 @@ #define PRF(x) /* x */ + +/* ---------- CLimitedSeqInStream ---------- */ + +typedef struct +{ + ISeqInStream vt; + ISeqInStream *realStream; + UInt64 limit; + UInt64 processed; + int finished; +} CLimitedSeqInStream; + +static void LimitedSeqInStream_Init(CLimitedSeqInStream *p) +{ + p->limit = (UInt64)(Int64)-1; + p->processed = 0; + p->finished = 0; +} + +static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt); + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->finished = (size2 == 0 ? 1 : 0); + p->processed += size2; + } + *size = size2; + return res; +} + + /* ---------- CLzma2EncInt ---------- */ typedef struct { CLzmaEncHandle enc; + Byte propsAreSet; + Byte propsByte; + Byte needInitState; + Byte needInitProp; UInt64 srcPos; - Byte props; - Bool needInitState; - Bool needInitProp; } CLzma2EncInt; -static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props) + +static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props) +{ + if (!p->propsAreSet) + { + SizeT propsSize = LZMA_PROPS_SIZE; + Byte propsEncoded[LZMA_PROPS_SIZE]; + RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); + RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); + p->propsByte = propsEncoded[0]; + p->propsAreSet = True; + } + return SZ_OK; +} + +static void Lzma2EncInt_InitBlock(CLzma2EncInt *p) { - Byte propsEncoded[LZMA_PROPS_SIZE]; - SizeT propsSize = LZMA_PROPS_SIZE; - RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); - RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); p->srcPos = 0; - p->props = propsEncoded[0]; p->needInitState = True; p->needInitProp = True; - return SZ_OK; } + SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, @@ -70,6 +122,9 @@ void LzmaEnc_Finish(CLzmaEncHandle pp); void LzmaEnc_SaveState(CLzmaEncHandle pp); void LzmaEnc_RestoreState(CLzmaEncHandle pp); +/* +UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp); +*/ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, size_t *packSizeRes, ISeqOutStream *outStream) @@ -154,7 +209,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, outBuf[destPos++] = (Byte)pm; if (p->needInitProp) - outBuf[destPos++] = p->props; + outBuf[destPos++] = p->propsByte; p->needInitProp = False; p->needInitState = False; @@ -176,14 +231,16 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, void Lzma2EncProps_Init(CLzma2EncProps *p) { LzmaEncProps_Init(&p->lzmaProps); + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; p->numTotalThreads = -1; - p->numBlockThreads = -1; - p->blockSize = 0; } void Lzma2EncProps_Normalize(CLzma2EncProps *p) { - int t1, t1n, t2, t3; + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; { CLzmaEncProps lzmaProps = p->lzmaProps; LzmaEncProps_Normalize(&lzmaProps); @@ -191,11 +248,11 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) } t1 = p->lzmaProps.numThreads; - t2 = p->numBlockThreads; + t2 = p->numBlockThreads_Max; t3 = p->numTotalThreads; - if (t2 > NUM_MT_CODER_THREADS_MAX) - t2 = NUM_MT_CODER_THREADS_MAX; + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; if (t3 <= 0) { @@ -211,8 +268,8 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) t1 = 1; t2 = t3; } - if (t2 > NUM_MT_CODER_THREADS_MAX) - t2 = NUM_MT_CODER_THREADS_MAX; + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; } else if (t1 <= 0) { @@ -225,39 +282,64 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) p->lzmaProps.numThreads = t1; + t2r = t2; + + fileSize = p->lzmaProps.reduceSize; + + if ( p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO + && (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzmaProps.reduceSize = p->blockSize; + LzmaEncProps_Normalize(&p->lzmaProps); + p->lzmaProps.reduceSize = fileSize; + t1 = p->lzmaProps.numThreads; - if (p->blockSize == 0) + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) { - UInt32 dictSize = p->lzmaProps.dictSize; - UInt64 blockSize = (UInt64)dictSize << 2; - const UInt32 kMinSize = (UInt32)1 << 20; - const UInt32 kMaxSize = (UInt32)1 << 28; - if (blockSize < kMinSize) blockSize = kMinSize; - if (blockSize > kMaxSize) blockSize = kMaxSize; - if (blockSize < dictSize) blockSize = dictSize; - p->blockSize = (size_t)blockSize; + t2r = t2 = 1; + t3 = t1; } - - if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1) + else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1) + { + /* if there is no block multi-threading, we use SOLID block */ + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + } + else { - UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1; - if (temp > p->lzmaProps.reduceSize) + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + { + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + const UInt32 dictSize = p->lzmaProps.dictSize; + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + p->blockSize = blockSize; + } + + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) { - UInt64 numBlocks = temp / p->blockSize; + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; if (numBlocks < (unsigned)t2) { - t2 = (unsigned)numBlocks; - if (t2 == 0) - t2 = 1; - t3 = t1 * t2; + t2r = (unsigned)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; } } } - p->numBlockThreads = t2; + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; p->numTotalThreads = t3; } @@ -274,180 +356,110 @@ typedef struct { Byte propEncoded; CLzma2EncProps props; + UInt64 expectedDataSize; - Byte *outBuf; + Byte *tempBufLzma; ISzAllocPtr alloc; ISzAllocPtr allocBig; - CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX]; + CLzma2EncInt coders[MTCODER__THREADS_MAX]; #ifndef _7ZIP_ST + + ISeqOutStream *outStream; + Byte *outBuf; + size_t outBufSize; + size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; + Bool mtCoder_WasConstructed; CMtCoder mtCoder; + Byte *outBufs[MTCODER__BLOCKS_MAX]; + #endif } CLzma2Enc; -/* ---------- Lzma2EncThread ---------- */ -static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) +CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt64 packTotal = 0; - SRes res = SZ_OK; - - if (!mainEncoder->outBuf) + CLzma2Enc *p = (CLzma2Enc *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Enc)); + if (!p) + return NULL; + Lzma2EncProps_Init(&p->props); + Lzma2EncProps_Normalize(&p->props); + p->expectedDataSize = (UInt64)(Int64)-1; + p->tempBufLzma = NULL; + p->alloc = alloc; + p->allocBig = allocBig; { - mainEncoder->outBuf = (Byte *)ISzAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); - if (!mainEncoder->outBuf) - return SZ_ERROR_MEM; + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].enc = NULL; } - RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); - RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE, - mainEncoder->alloc, mainEncoder->allocBig)); - - for (;;) + #ifndef _7ZIP_ST + p->mtCoder_WasConstructed = False; { - size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; - res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream); - if (res != SZ_OK) - break; - packTotal += packSize; - res = Progress(progress, p->srcPos, packTotal); - if (res != SZ_OK) - break; - if (packSize == 0) - break; + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; } - - LzmaEnc_Finish(p->enc); + #endif - if (res == SZ_OK) - { - Byte b = 0; - if (ISeqOutStream_Write(outStream, &b, 1) != 1) - return SZ_ERROR_WRITE; - } - - return res; + return p; } #ifndef _7ZIP_ST -typedef struct +static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p) { - IMtCoderCallback funcTable; - CLzma2Enc *lzma2Enc; -} CMtCallbackImp; - -static SRes MtCallbackImp_Code(const IMtCoderCallback *pp, unsigned index, Byte *dest, size_t *destSize, - const Byte *src, size_t srcSize, int finished) -{ - CMtCallbackImp *imp = CONTAINER_FROM_VTBL(pp, CMtCallbackImp, funcTable); - CLzma2Enc *mainEncoder = imp->lzma2Enc; - CLzma2EncInt *p = &mainEncoder->coders[index]; - - SRes res = SZ_OK; - { - size_t destLim = *destSize; - *destSize = 0; - - if (srcSize != 0) - { - RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); - - RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE, - mainEncoder->alloc, mainEncoder->allocBig)); - - while (p->srcPos < srcSize) - { - size_t packSize = destLim - *destSize; - res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL); - if (res != SZ_OK) - break; - *destSize += packSize; - - if (packSize == 0) - { - res = SZ_ERROR_FAIL; - break; - } - - if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK) - { - res = SZ_ERROR_PROGRESS; - break; - } - } - - LzmaEnc_Finish(p->enc); - if (res != SZ_OK) - return res; - } - - if (finished) + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) { - if (*destSize == destLim) - return SZ_ERROR_OUTPUT_EOF; - dest[(*destSize)++] = 0; + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; } - } - return res; + p->outBufSize = 0; } #endif -/* ---------- Lzma2Enc ---------- */ - -CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - CLzma2Enc *p = (CLzma2Enc *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Enc)); - if (!p) - return NULL; - Lzma2EncProps_Init(&p->props); - Lzma2EncProps_Normalize(&p->props); - p->outBuf = 0; - p->alloc = alloc; - p->allocBig = allocBig; - { - unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - p->coders[i].enc = 0; - } - - #ifndef _7ZIP_ST - MtCoder_Construct(&p->mtCoder); - #endif - - return p; -} - void Lzma2Enc_Destroy(CLzma2EncHandle pp) { CLzma2Enc *p = (CLzma2Enc *)pp; unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) + for (i = 0; i < MTCODER__THREADS_MAX; i++) { CLzma2EncInt *t = &p->coders[i]; if (t->enc) { LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig); - t->enc = 0; + t->enc = NULL; } } + #ifndef _7ZIP_ST - MtCoder_Destruct(&p->mtCoder); + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + Lzma2Enc_FreeOutBufs(p); #endif - ISzAlloc_Free(p->alloc, p->outBuf); + ISzAlloc_Free(p->alloc, p->tempBufLzma); + p->tempBufLzma = NULL; + ISzAlloc_Free(p->alloc, pp); } + SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) { CLzma2Enc *p = (CLzma2Enc *)pp; @@ -460,6 +472,14 @@ SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) return SZ_OK; } + +void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) { CLzma2Enc *p = (CLzma2Enc *)pp; @@ -471,50 +491,310 @@ Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) return (Byte)i; } -SRes Lzma2Enc_Encode(CLzma2EncHandle pp, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) + +static SRes Lzma2Enc_EncodeMt1( + CLzma2Enc *me, + CLzma2EncInt *p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + int finished, + ICompressProgress *progress) { - CLzma2Enc *p = (CLzma2Enc *)pp; - int i; + UInt64 unpackTotal = 0; + UInt64 packTotal = 0; + size_t outLim = 0; + CLimitedSeqInStream limitedInStream; - for (i = 0; i < p->props.numBlockThreads; i++) + if (outBuf) { - CLzma2EncInt *t = &p->coders[(unsigned)i]; - if (!t->enc) + outLim = *outBufSize; + *outBufSize = 0; + } + + if (!p->enc) + { + p->propsAreSet = False; + p->enc = LzmaEnc_Create(me->alloc); + if (!p->enc) + return SZ_ERROR_MEM; + } + + limitedInStream.realStream = inStream; + if (inStream) + { + limitedInStream.vt.Read = LimitedSeqInStream_Read; + } + + if (!outBuf) + { + // outStream version works only in one thread. So we use CLzma2Enc::tempBufLzma + if (!me->tempBufLzma) { - t->enc = LzmaEnc_Create(p->alloc); - if (!t->enc) + me->tempBufLzma = (Byte *)ISzAlloc_Alloc(me->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); + if (!me->tempBufLzma) return SZ_ERROR_MEM; } } - #ifndef _7ZIP_ST - if (p->props.numBlockThreads > 1) + RINOK(Lzma2EncInt_InitStream(p, &me->props)); + + for (;;) { - CMtCallbackImp mtCallback; + SRes res = SZ_OK; + size_t inSizeCur = 0; + + Lzma2EncInt_InitBlock(p); + + LimitedSeqInStream_Init(&limitedInStream); + limitedInStream.limit = me->props.blockSize; + + if (inStream) + { + UInt64 expected = (UInt64)(Int64)-1; + // inStream version works only in one thread. So we use CLzma2Enc::expectedDataSize + if (me->expectedDataSize != (UInt64)(Int64)-1 + && me->expectedDataSize >= unpackTotal) + expected = me->expectedDataSize - unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && expected > me->props.blockSize) + expected = (size_t)me->props.blockSize; + + LzmaEnc_SetDataSize(p->enc, expected); + + RINOK(LzmaEnc_PrepareForLzma2(p->enc, + &limitedInStream.vt, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } + else + { + inSizeCur = inDataSize - (size_t)unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && inSizeCur > me->props.blockSize) + inSizeCur = (size_t)me->props.blockSize; + + // LzmaEnc_SetDataSize(p->enc, inSizeCur); + + RINOK(LzmaEnc_MemPrepare(p->enc, + inData + (size_t)unpackTotal, inSizeCur, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } - mtCallback.funcTable.Code = MtCallbackImp_Code; - mtCallback.lzma2Enc = p; + for (;;) + { + size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; + if (outBuf) + packSize = outLim - (size_t)packTotal; + + res = Lzma2EncInt_EncodeSubblock(p, + outBuf ? outBuf + (size_t)packTotal : me->tempBufLzma, &packSize, + outBuf ? NULL : outStream); + + if (res != SZ_OK) + break; + + packTotal += packSize; + if (outBuf) + *outBufSize = (size_t)packTotal; + + res = Progress(progress, unpackTotal + p->srcPos, packTotal); + if (res != SZ_OK) + break; + + /* + if (LzmaEnc_GetNumAvailableBytes(p->enc) == 0) + break; + */ + + if (packSize == 0) + break; + } + + LzmaEnc_Finish(p->enc); + unpackTotal += p->srcPos; + + RINOK(res); + + if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur)) + return SZ_ERROR_FAIL; + + if (inStream ? limitedInStream.finished : (unpackTotal == inDataSize)) + { + if (finished) + { + if (outBuf) + { + size_t destPos = *outBufSize; + if (destPos >= outLim) + return SZ_ERROR_OUTPUT_EOF; + outBuf[destPos] = 0; + *outBufSize = destPos + 1; + } + else + { + Byte b = 0; + if (ISeqOutStream_Write(outStream, &b, 1) != 1) + return SZ_ERROR_WRITE; + } + } + return SZ_OK; + } + } +} + + + +#ifndef _7ZIP_ST + +static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t destSize = me->outBufSize; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + me->outBufsDataSizes[outBufIndex] = 0; + + if (!dest) + { + dest = ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + progressThunk.index = coderIndex; + + res = Lzma2Enc_EncodeMt1(me, + &me->coders[coderIndex], + NULL, dest, &destSize, + NULL, src, srcSize, + finished, + &progressThunk.vt); + + me->outBufsDataSizes[outBufIndex] = destSize; + + return res; +} + + +static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t size = me->outBufsDataSizes[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + if (me->outStream) + return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE; + + if (size > me->outBufSize) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBufSize -= size; + me->outBuf += size; + return SZ_OK; +} + +#endif + + + +SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + + if (inStream && inData) + return E_INVALIDARG; + + if (outStream && outBuf) + return E_INVALIDARG; + + { + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].propsAreSet = False; + } + + #ifndef _7ZIP_ST + + if (p->props.numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) + { + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); + } + + vt.Code = Lzma2Enc_MtCallback_Code; + vt.Write = Lzma2Enc_MtCallback_Write; + + p->outStream = outStream; + p->outBuf = NULL; + p->outBufSize = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBufSize = *outBufSize; + *outBufSize = 0; + } + + p->mtCoder.allocBig = p->allocBig; p->mtCoder.progress = progress; p->mtCoder.inStream = inStream; - p->mtCoder.outStream = outStream; - p->mtCoder.alloc = p->alloc; - p->mtCoder.mtCallback = &mtCallback.funcTable; + p->mtCoder.inData = inData; + p->mtCoder.inDataSize = inDataSize; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + p->mtCoder.blockSize = (size_t)p->props.blockSize; + if (p->mtCoder.blockSize != p->props.blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ - p->mtCoder.blockSize = p->props.blockSize; - p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16; - if (p->mtCoder.destBlockSize < p->props.blockSize) { - p->mtCoder.destBlockSize = (size_t)0 - 1; - if (p->mtCoder.destBlockSize < p->props.blockSize) - return SZ_ERROR_FAIL; + size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16; + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + Lzma2Enc_FreeOutBufs(p); + p->outBufSize = destBlockSize; } - p->mtCoder.numThreads = p->props.numBlockThreads; + + p->mtCoder.numThreadsMax = p->props.numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; - return MtCoder_Code(&p->mtCoder); + { + SRes res = MtCoder_Code(&p->mtCoder); + if (!outStream) + *outBufSize = p->outBuf - outBuf; + return res; + } } + #endif - return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress); + + return Lzma2Enc_EncodeMt1(p, + &p->coders[0], + outStream, outBuf, outBufSize, + inStream, inData, inDataSize, + True, /* finished */ + progress); } diff --git a/C/Lzma2Enc.h b/C/Lzma2Enc.h index 9557cb29..65f2dd14 100644 --- a/C/Lzma2Enc.h +++ b/C/Lzma2Enc.h @@ -1,5 +1,5 @@ /* Lzma2Enc.h -- LZMA2 Encoder -2017-04-03 : Igor Pavlov : Public domain */ +2017-07-27 : Igor Pavlov : Public domain */ #ifndef __LZMA2_ENC_H #define __LZMA2_ENC_H @@ -8,11 +8,15 @@ EXTERN_C_BEGIN +#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0 +#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1) + typedef struct { CLzmaEncProps lzmaProps; - size_t blockSize; - int numBlockThreads; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; int numTotalThreads; } CLzma2EncProps; @@ -22,13 +26,14 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p); /* ---------- CLzmaEnc2Handle Interface ---------- */ /* Lzma2Enc_* functions can return the following exit codes: -Returns: +SRes: SZ_OK - OK SZ_ERROR_MEM - Memory allocation error SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ typedef void * CLzma2EncHandle; @@ -36,26 +41,14 @@ typedef void * CLzma2EncHandle; CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); void Lzma2Enc_Destroy(CLzma2EncHandle p); SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props); +void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize); Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p); -SRes Lzma2Enc_Encode(CLzma2EncHandle p, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); - -/* ---------- One Call Interface ---------- */ - -/* Lzma2Encode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - -/* -SRes Lzma2Encode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - const CLzmaEncProps *props, Byte *propsEncoded, int writeEndMark, - ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); -*/ +SRes Lzma2Enc_Encode2(CLzma2EncHandle p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress); EXTERN_C_END diff --git a/C/LzmaEnc.c b/C/LzmaEnc.c index 73dfcb84..9c45a214 100644 --- a/C/LzmaEnc.c +++ b/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2017-04-03 : Igor Pavlov : Public domain */ +2017-06-22 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -23,8 +23,8 @@ static unsigned g_STAT_OFFSET = 0; #endif -#define kMaxHistorySize ((UInt32)3 << 29) -/* #define kMaxHistorySize ((UInt32)7 << 29) */ +#define kLzmaMaxHistorySize ((UInt32)3 << 29) +/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */ #define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1) @@ -62,14 +62,15 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p) if (level < 0) level = 5; p->level = level; - if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26))); + if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26))); if (p->dictSize > p->reduceSize) { unsigned i; + UInt32 reduceSize = (UInt32)p->reduceSize; for (i = 11; i <= 30; i++) { - if ((UInt32)p->reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } - if ((UInt32)p->reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } + if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } + if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } } } @@ -445,7 +446,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) || props.lp > LZMA_LP_MAX || props.pb > LZMA_PB_MAX || props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress) - || props.dictSize > kMaxHistorySize) + || props.dictSize > kLzmaMaxHistorySize) return SZ_ERROR_PARAM; p->dictSize = props.dictSize; @@ -492,6 +493,15 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) return SZ_OK; } + +void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + p->matchFinderBase.expectedDataSize = expectedDataSiize; +} + + + static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; @@ -1996,6 +2006,8 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, { RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); p->matchFinderObj = &p->matchFinderMt; + p->matchFinderBase.bigHash = (Byte)( + (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0); MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); } else @@ -2135,6 +2147,7 @@ SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, LzmaEnc_SetInputBuf(p, src, srcLen); p->needInit = 1; + LzmaEnc_SetDataSize(pp, srcLen); return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); } @@ -2152,15 +2165,15 @@ void LzmaEnc_Finish(CLzmaEncHandle pp) typedef struct { - ISeqOutStream funcTable; + ISeqOutStream vt; Byte *data; SizeT rem; Bool overflow; -} CSeqOutStreamBuf; +} CLzmaEnc_SeqOutStreamBuf; -static size_t MyWrite(const ISeqOutStream *pp, const void *data, size_t size) +static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CSeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CSeqOutStreamBuf, funcTable); + CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt); if (p->rem < size) { size = p->rem; @@ -2193,9 +2206,9 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, CLzmaEnc *p = (CLzmaEnc *)pp; UInt64 nowPos64; SRes res; - CSeqOutStreamBuf outStream; + CLzmaEnc_SeqOutStreamBuf outStream; - outStream.funcTable.Write = MyWrite; + outStream.vt.Write = SeqOutStreamBuf_Write; outStream.data = dest; outStream.rem = *destLen; outStream.overflow = False; @@ -2209,7 +2222,7 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, LzmaEnc_InitPrices(p); nowPos64 = p->nowPos64; RangeEnc_Init(&p->rc); - p->rc.outStream = &outStream.funcTable; + p->rc.outStream = &outStream.vt; res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); @@ -2308,15 +2321,15 @@ SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte SRes res; CLzmaEnc *p = (CLzmaEnc *)pp; - CSeqOutStreamBuf outStream; + CLzmaEnc_SeqOutStreamBuf outStream; - outStream.funcTable.Write = MyWrite; + outStream.vt.Write = SeqOutStreamBuf_Write; outStream.data = dest; outStream.rem = *destLen; outStream.overflow = False; p->writeEndMark = writeEndMark; - p->rc.outStream = &outStream.funcTable; + p->rc.outStream = &outStream.vt; res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig); diff --git a/C/LzmaEnc.h b/C/LzmaEnc.h index 9f8d0f27..c9938f04 100644 --- a/C/LzmaEnc.h +++ b/C/LzmaEnc.h @@ -1,5 +1,5 @@ /* LzmaEnc.h -- LZMA Encoder -2017-04-03 : Igor Pavlov : Public domain */ +2017-07-27 : Igor Pavlov : Public domain */ #ifndef __LZMA_ENC_H #define __LZMA_ENC_H @@ -12,12 +12,10 @@ EXTERN_C_BEGIN typedef struct _CLzmaEncProps { - int level; /* 0 <= level <= 9 */ + int level; /* 0 <= level <= 9 */ UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version - (1 << 12) <= dictSize <= (1 << 30) for 64-bit version - default = (1 << 24) */ - UInt64 reduceSize; /* estimated size of data that will be compressed. default = 0xFFFFFFFF. - Encoder uses this value to reduce dictionary size */ + (1 << 12) <= dictSize <= (3 << 29) for 64-bit version + default = (1 << 24) */ int lc; /* 0 <= lc <= 8, default = 3 */ int lp; /* 0 <= lp <= 4, default = 0 */ int pb; /* 0 <= pb <= 4, default = 2 */ @@ -25,9 +23,12 @@ typedef struct _CLzmaEncProps int fb; /* 5 <= fb <= 273, default = 32 */ int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ int numHashBytes; /* 2, 3 or 4, default = 4 */ - UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ + UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ int numThreads; /* 1 or 2, default = 2 */ + + UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. + Encoder uses this value to reduce dictionary size */ } CLzmaEncProps; void LzmaEncProps_Init(CLzmaEncProps *p); @@ -37,38 +38,34 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2); /* ---------- CLzmaEncHandle Interface ---------- */ -/* LzmaEnc_* functions can return the following exit codes: -Returns: +/* LzmaEnc* functions can return the following exit codes: +SRes: SZ_OK - OK SZ_ERROR_MEM - Memory allocation error SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error. + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ typedef void * CLzmaEncHandle; CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc); void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig); + SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props); +void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize); SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p); + SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); -/* ---------- One Call Interface ---------- */ -/* LzmaEncode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ +/* ---------- One Call Interface ---------- */ SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, diff --git a/C/MtCoder.c b/C/MtCoder.c index 5b76203d..9cf53007 100644 --- a/C/MtCoder.c +++ b/C/MtCoder.c @@ -1,104 +1,73 @@ /* MtCoder.c -- Multi-thread Coder -2017-04-03 : Igor Pavlov : Public domain */ +2017-07-17 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "MtCoder.h" -void LoopThread_Construct(CLoopThread *p) -{ - Thread_Construct(&p->thread); - Event_Construct(&p->startEvent); - Event_Construct(&p->finishedEvent); -} - -void LoopThread_Close(CLoopThread *p) -{ - Thread_Close(&p->thread); - Event_Close(&p->startEvent); - Event_Close(&p->finishedEvent); -} - -static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE LoopThreadFunc(void *pp) -{ - CLoopThread *p = (CLoopThread *)pp; - for (;;) - { - if (Event_Wait(&p->startEvent) != 0) - return SZ_ERROR_THREAD; - if (p->stop) - return 0; - p->res = p->func(p->param); - if (Event_Set(&p->finishedEvent) != 0) - return SZ_ERROR_THREAD; - } -} - -WRes LoopThread_Create(CLoopThread *p) -{ - p->stop = 0; - RINOK(AutoResetEvent_CreateNotSignaled(&p->startEvent)); - RINOK(AutoResetEvent_CreateNotSignaled(&p->finishedEvent)); - return Thread_Create(&p->thread, LoopThreadFunc, p); -} - -WRes LoopThread_StopAndWait(CLoopThread *p) -{ - p->stop = 1; - if (Event_Set(&p->startEvent) != 0) - return SZ_ERROR_THREAD; - return Thread_Wait(&p->thread); -} - -WRes LoopThread_StartSubThread(CLoopThread *p) { return Event_Set(&p->startEvent); } -WRes LoopThread_WaitSubThread(CLoopThread *p) { return Event_Wait(&p->finishedEvent); } - -static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) -{ - return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; -} - static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) { unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - p->inSizes[i] = p->outSizes[i] = 0; - p->totalInSize = p->totalOutSize = 0; + p->progress = progress; p->res = SZ_OK; + p->totalInSize = 0; + p->totalOutSize = 0; + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + { + CMtProgressSizes *pair = &p->sizes[i]; + pair->inSize = 0; + pair->outSize = 0; + } } + static void MtProgress_Reinit(CMtProgress *p, unsigned index) { - p->inSizes[index] = 0; - p->outSizes[index] = 0; + CMtProgressSizes *pair = &p->sizes[index]; + pair->inSize = 0; + pair->outSize = 0; } + #define UPDATE_PROGRESS(size, prev, total) \ if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; } SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize) { SRes res; + CMtProgressSizes *pair; + CriticalSection_Enter(&p->cs); - UPDATE_PROGRESS(inSize, p->inSizes[index], p->totalInSize) - UPDATE_PROGRESS(outSize, p->outSizes[index], p->totalOutSize) - if (p->res == SZ_OK) - p->res = Progress(p->progress, p->totalInSize, p->totalOutSize); + + pair = &p->sizes[index]; + UPDATE_PROGRESS(inSize, pair->inSize, p->totalInSize) + UPDATE_PROGRESS(outSize, pair->outSize, p->totalOutSize) + if (p->res == SZ_OK && p->progress) + { + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + } res = p->res; + CriticalSection_Leave(&p->cs); + return res; } -static void MtProgress_SetError(CMtProgress *p, SRes res) + +static SRes MtProgress_GetError(CMtProgress *p) { + SRes res; CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; + res = p->res; CriticalSection_Leave(&p->cs); + return res; } -static void MtCoder_SetError(CMtCoder* p, SRes res) + +static void MtProgress_SetError(CMtProgress *p, SRes res) { CriticalSection_Enter(&p->cs); if (p->res == SZ_OK) @@ -106,222 +75,582 @@ static void MtCoder_SetError(CMtCoder* p, SRes res) CriticalSection_Leave(&p->cs); } -/* ---------- MtThread ---------- */ -void CMtThread_Construct(CMtThread *p, CMtCoder *mtCoder) +static SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) { - p->mtCoder = mtCoder; - p->outBuf = 0; - p->inBuf = 0; - Event_Construct(&p->canRead); - Event_Construct(&p->canWrite); - LoopThread_Construct(&p->thread); + CMtProgressThunk *p = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); + return MtProgress_Set(p->mtProgress, p->index, inSize, outSize); } -#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } -static void CMtThread_CloseEvents(CMtThread *p) +void MtProgressThunk_CreateVTable(CMtProgressThunk *p) { - Event_Close(&p->canRead); - Event_Close(&p->canWrite); + p->vt.Progress = MtProgressThunk_Progress; } -static void CMtThread_Destruct(CMtThread *p) + + +#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } + + +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) { - CMtThread_CloseEvents(p); + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); +} - if (Thread_WasCreated(&p->thread.thread)) - { - LoopThread_StopAndWait(&p->thread); - LoopThread_Close(&p->thread); - } - if (p->mtCoder->alloc) - ISzAlloc_Free(p->mtCoder->alloc, p->outBuf); - p->outBuf = 0; +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp); + - if (p->mtCoder->alloc) - ISzAlloc_Free(p->mtCoder->alloc, p->inBuf); - p->inBuf = 0; +static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t) +{ + WRes wres = ArEvent_OptCreate_And_Reset(&t->startEvent); + if (wres == 0) + { + t->stop = False; + if (!Thread_WasCreated(&t->thread)) + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + wres = Event_Set(&t->startEvent); + } + if (wres == 0) + return SZ_OK; + return MY_SRes_HRESULT_FROM_WRes(wres); } -#define MY_BUF_ALLOC(buf, size, newSize) \ - if (buf == 0 || size != newSize) \ - { ISzAlloc_Free(p->mtCoder->alloc, buf); \ - size = newSize; buf = (Byte *)ISzAlloc_Alloc(p->mtCoder->alloc, size); \ - if (buf == 0) return SZ_ERROR_MEM; } -static SRes CMtThread_Prepare(CMtThread *p) +static void MtCoderThread_Destruct(CMtCoderThread *t) { - MY_BUF_ALLOC(p->inBuf, p->inBufSize, p->mtCoder->blockSize) - MY_BUF_ALLOC(p->outBuf, p->outBufSize, p->mtCoder->destBlockSize) + if (Thread_WasCreated(&t->thread)) + { + t->stop = 1; + Event_Set(&t->startEvent); + Thread_Wait(&t->thread); + Thread_Close(&t->thread); + } - p->stopReading = False; - p->stopWriting = False; - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canRead)); - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canWrite)); + Event_Close(&t->startEvent); - return SZ_OK; + if (t->inBuf) + { + ISzAlloc_Free(t->mtCoder->allocBig, t->inBuf); + t->inBuf = NULL; + } } + + static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) { size_t size = *processedSize; *processedSize = 0; while (size != 0) { - size_t curSize = size; - SRes res = ISeqInStream_Read(stream, data, &curSize); - *processedSize += curSize; - data += curSize; - size -= curSize; + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; RINOK(res); - if (curSize == 0) + if (cur == 0) return SZ_OK; } return SZ_OK; } -#define GET_NEXT_THREAD(p) &p->mtCoder->threads[p->index == p->mtCoder->numThreads - 1 ? 0 : p->index + 1] -static SRes MtThread_Process(CMtThread *p, Bool *stop) +/* + ThreadFunc2() returns: + SZ_OK - in all normal cases (even for stream error or memory allocation error) + SZ_ERROR_THREAD - in case of failure in system synch function +*/ + +static SRes ThreadFunc2(CMtCoderThread *t) { - CMtThread *next; - *stop = True; - if (Event_Wait(&p->canRead) != 0) - return SZ_ERROR_THREAD; - - next = GET_NEXT_THREAD(p); - - if (p->stopReading) - { - next->stopReading = True; - return Event_Set(&next->canRead) == 0 ? SZ_OK : SZ_ERROR_THREAD; - } + CMtCoder *mtc = t->mtCoder; + for (;;) { - size_t size = p->mtCoder->blockSize; - size_t destSize = p->outBufSize; + unsigned bi; + SRes res; + SRes res2; + Bool finished; + unsigned bufIndex; + size_t size; + const Byte *inData; + UInt64 readProcessed = 0; + + RINOK_THREAD(Event_Wait(&mtc->readEvent)) + + /* after Event_Wait(&mtc->readEvent) we must call Event_Set(&mtc->readEvent) in any case to unlock another threads */ + + if (mtc->stopReading) + { + return Event_Set(&mtc->readEvent) == 0 ? SZ_OK : SZ_ERROR_THREAD; + } - RINOK(FullRead(p->mtCoder->inStream, p->inBuf, &size)); - next->stopReading = *stop = (size != p->mtCoder->blockSize); - if (Event_Set(&next->canRead) != 0) - return SZ_ERROR_THREAD; + res = MtProgress_GetError(&mtc->mtProgress); + + size = 0; + inData = NULL; + finished = True; + + if (res == SZ_OK) + { + size = mtc->blockSize; + if (mtc->inStream) + { + if (!t->inBuf) + { + t->inBuf = (Byte *)ISzAlloc_Alloc(mtc->allocBig, mtc->blockSize); + if (!t->inBuf) + res = SZ_ERROR_MEM; + } + if (res == SZ_OK) + { + res = FullRead(mtc->inStream, t->inBuf, &size); + readProcessed = mtc->readProcessed + size; + mtc->readProcessed = readProcessed; + } + if (res != SZ_OK) + { + mtc->readRes = res; + /* after reading error - we can stop encoding of previous blocks */ + MtProgress_SetError(&mtc->mtProgress, res); + } + else + finished = (size != mtc->blockSize); + } + else + { + size_t rem; + readProcessed = mtc->readProcessed; + rem = mtc->inDataSize - (size_t)readProcessed; + if (size > rem) + size = rem; + inData = mtc->inData + (size_t)readProcessed; + readProcessed += size; + mtc->readProcessed = readProcessed; + finished = (mtc->inDataSize == (size_t)readProcessed); + } + } - RINOK(IMtCoderCallback_Code(p->mtCoder->mtCallback, p->index, - p->outBuf, &destSize, p->inBuf, size, *stop)); + /* we must get some block from blocksSemaphore before Event_Set(&mtc->readEvent) */ - MtProgress_Reinit(&p->mtCoder->mtProgress, p->index); + res2 = SZ_OK; - if (Event_Wait(&p->canWrite) != 0) - return SZ_ERROR_THREAD; - if (p->stopWriting) - return SZ_ERROR_FAIL; - if (ISeqOutStream_Write(p->mtCoder->outStream, p->outBuf, destSize) != destSize) - return SZ_ERROR_WRITE; - return Event_Set(&next->canWrite) == 0 ? SZ_OK : SZ_ERROR_THREAD; + if (Semaphore_Wait(&mtc->blocksSemaphore) != 0) + { + res2 = SZ_ERROR_THREAD; + if (res == SZ_OK) + { + res = res2; + // MtProgress_SetError(&mtc->mtProgress, res); + } + } + + bi = mtc->blockIndex; + + if (++mtc->blockIndex >= mtc->numBlocksMax) + mtc->blockIndex = 0; + + bufIndex = (unsigned)(int)-1; + + if (res == SZ_OK) + res = MtProgress_GetError(&mtc->mtProgress); + + if (res != SZ_OK) + finished = True; + + if (!finished) + { + if (mtc->numStartedThreads < mtc->numStartedThreadsLimit + && mtc->expectedDataSize != readProcessed) + { + res = MtCoderThread_CreateAndStart(&mtc->threads[mtc->numStartedThreads]); + if (res == SZ_OK) + mtc->numStartedThreads++; + else + { + MtProgress_SetError(&mtc->mtProgress, res); + finished = True; + } + } + } + + if (finished) + mtc->stopReading = True; + + RINOK_THREAD(Event_Set(&mtc->readEvent)) + + if (res2 != SZ_OK) + return res2; + + if (res == SZ_OK) + { + CriticalSection_Enter(&mtc->cs); + bufIndex = mtc->freeBlockHead; + mtc->freeBlockHead = mtc->freeBlockList[bufIndex]; + CriticalSection_Leave(&mtc->cs); + + res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex, + mtc->inStream ? t->inBuf : inData, size, finished); + + MtProgress_Reinit(&mtc->mtProgress, t->index); + + if (res != SZ_OK) + MtProgress_SetError(&mtc->mtProgress, res); + } + + { + CMtCoderBlock *block = &mtc->blocks[bi]; + block->res = res; + block->bufIndex = bufIndex; + block->finished = finished; + } + + #ifdef MTCODER__USE_WRITE_THREAD + RINOK_THREAD(Event_Set(&mtc->writeEvents[bi])) + #else + { + unsigned wi; + { + CriticalSection_Enter(&mtc->cs); + wi = mtc->writeIndex; + if (wi == bi) + mtc->writeIndex = (unsigned)(int)-1; + else + mtc->ReadyBlocks[bi] = True; + CriticalSection_Leave(&mtc->cs); + } + + if (wi != bi) + { + if (res != SZ_OK || finished) + return 0; + continue; + } + + if (mtc->writeRes != SZ_OK) + res = mtc->writeRes; + + for (;;) + { + if (res == SZ_OK && bufIndex != (unsigned)(int)-1) + { + res = mtc->mtCallback->Write(mtc->mtCallbackObject, bufIndex); + if (res != SZ_OK) + { + mtc->writeRes = res; + MtProgress_SetError(&mtc->mtProgress, res); + } + } + + if (++wi >= mtc->numBlocksMax) + wi = 0; + { + Bool isReady; + + CriticalSection_Enter(&mtc->cs); + + if (bufIndex != (unsigned)(int)-1) + { + mtc->freeBlockList[bufIndex] = mtc->freeBlockHead; + mtc->freeBlockHead = bufIndex; + } + + isReady = mtc->ReadyBlocks[wi]; + + if (isReady) + mtc->ReadyBlocks[wi] = False; + else + mtc->writeIndex = wi; + + CriticalSection_Leave(&mtc->cs); + + RINOK_THREAD(Semaphore_Release1(&mtc->blocksSemaphore)) + + if (!isReady) + break; + } + + { + CMtCoderBlock *block = &mtc->blocks[wi]; + if (res == SZ_OK && block->res != SZ_OK) + res = block->res; + bufIndex = block->bufIndex; + finished = block->finished; + } + } + } + #endif + + if (finished || res != SZ_OK) + return 0; } } + static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) { - CMtThread *p = (CMtThread *)pp; + CMtCoderThread *t = (CMtCoderThread *)pp; for (;;) { - Bool stop; - CMtThread *next = GET_NEXT_THREAD(p); - SRes res = MtThread_Process(p, &stop); - if (res != SZ_OK) + if (Event_Wait(&t->startEvent) != 0) + return SZ_ERROR_THREAD; + if (t->stop) + return 0; { - MtCoder_SetError(p->mtCoder, res); - MtProgress_SetError(&p->mtCoder->mtProgress, res); - next->stopReading = True; - next->stopWriting = True; - Event_Set(&next->canRead); - Event_Set(&next->canWrite); - return res; + SRes res = ThreadFunc2(t); + CMtCoder *mtc = t->mtCoder; + if (res != SZ_OK) + { + MtProgress_SetError(&mtc->mtProgress, res); + } + + #ifndef MTCODER__USE_WRITE_THREAD + { + unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads); + if (numFinished == mtc->numStartedThreads) + if (Event_Set(&mtc->finishedEvent) != 0) + return SZ_ERROR_THREAD; + } + #endif } - if (stop) - return 0; } } -void MtCoder_Construct(CMtCoder* p) + + +void MtCoder_Construct(CMtCoder *p) { unsigned i; - p->alloc = 0; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) + + p->blockSize = 0; + p->numThreadsMax = 0; + p->expectedDataSize = (UInt64)(Int64)-1; + + p->inStream = NULL; + p->inData = NULL; + p->inDataSize = 0; + + p->progress = NULL; + p->allocBig = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + Event_Construct(&p->readEvent); + Semaphore_Construct(&p->blocksSemaphore); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) { - CMtThread *t = &p->threads[i]; + CMtCoderThread *t = &p->threads[i]; + t->mtCoder = p; t->index = i; - CMtThread_Construct(t, p); + t->inBuf = NULL; + t->stop = False; + Event_Construct(&t->startEvent); + Thread_Construct(&t->thread); } + + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + Event_Construct(&p->writeEvents[i]); + #else + Event_Construct(&p->finishedEvent); + #endif + CriticalSection_Init(&p->cs); CriticalSection_Init(&p->mtProgress.cs); } -void MtCoder_Destruct(CMtCoder* p) + + + +static void MtCoder_Free(CMtCoder *p) { unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - CMtThread_Destruct(&p->threads[i]); + + /* + p->stopReading = True; + if (Event_IsCreated(&p->readEvent)) + Event_Set(&p->readEvent); + */ + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + MtCoderThread_Destruct(&p->threads[i]); + + Event_Close(&p->readEvent); + Semaphore_Close(&p->blocksSemaphore); + + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + Event_Close(&p->writeEvents[i]); + #else + Event_Close(&p->finishedEvent); + #endif +} + + +void MtCoder_Destruct(CMtCoder *p) +{ + MtCoder_Free(p); + CriticalSection_Delete(&p->cs); CriticalSection_Delete(&p->mtProgress.cs); } + SRes MtCoder_Code(CMtCoder *p) { - unsigned i, numThreads = p->numThreads; + unsigned numThreads = p->numThreadsMax; + unsigned numBlocksMax; + unsigned i; SRes res = SZ_OK; - p->res = SZ_OK; - MtProgress_Init(&p->mtProgress, p->progress); + if (numThreads > MTCODER__THREADS_MAX) + numThreads = MTCODER__THREADS_MAX; + numBlocksMax = MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads); + + if (p->blockSize < ((UInt32)1 << 26)) numBlocksMax++; + if (p->blockSize < ((UInt32)1 << 24)) numBlocksMax++; + if (p->blockSize < ((UInt32)1 << 22)) numBlocksMax++; + + if (numBlocksMax > MTCODER__BLOCKS_MAX) + numBlocksMax = MTCODER__BLOCKS_MAX; - for (i = 0; i < numThreads; i++) + if (p->blockSize != p->allocatedBufsSize) { - RINOK(CMtThread_Prepare(&p->threads[i])); + for (i = 0; i < MTCODER__THREADS_MAX; i++) + { + CMtCoderThread *t = &p->threads[i]; + if (t->inBuf) + { + ISzAlloc_Free(p->allocBig, t->inBuf); + t->inBuf = NULL; + } + } + p->allocatedBufsSize = p->blockSize; } - for (i = 0; i < numThreads; i++) - { - CMtThread *t = &p->threads[i]; - CLoopThread *lt = &t->thread; + p->readRes = SZ_OK; + + MtProgress_Init(&p->mtProgress, p->progress); - if (!Thread_WasCreated(<->thread)) + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < numBlocksMax; i++) { - lt->func = ThreadFunc; - lt->param = t; + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->writeEvents[i])); + } + #else + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + #endif - if (LoopThread_Create(lt) != SZ_OK) - { - res = SZ_ERROR_THREAD; - break; - } + { + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->readEvent)); + + if (Semaphore_IsCreated(&p->blocksSemaphore)) + { + RINOK_THREAD(Semaphore_Close(&p->blocksSemaphore)); } + RINOK_THREAD(Semaphore_Create(&p->blocksSemaphore, numBlocksMax, numBlocksMax)); } - if (res == SZ_OK) + for (i = 0; i < MTCODER__BLOCKS_MAX - 1; i++) + p->freeBlockList[i] = i + 1; + p->freeBlockList[MTCODER__BLOCKS_MAX - 1] = (unsigned)(int)-1; + p->freeBlockHead = 0; + + p->readProcessed = 0; + p->blockIndex = 0; + p->numBlocksMax = numBlocksMax; + p->stopReading = False; + + #ifndef MTCODER__USE_WRITE_THREAD + p->writeIndex = 0; + p->writeRes = SZ_OK; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->ReadyBlocks[i] = False; + p->numFinishedThreads = 0; + #endif + + p->numStartedThreadsLimit = numThreads; + p->numStartedThreads = 0; + + // for (i = 0; i < numThreads; i++) + { + CMtCoderThread *nextThread = &p->threads[p->numStartedThreads++]; + RINOK(MtCoderThread_CreateAndStart(nextThread)); + } + + RINOK_THREAD(Event_Set(&p->readEvent)) + + #ifdef MTCODER__USE_WRITE_THREAD { - unsigned j; - for (i = 0; i < numThreads; i++) + unsigned bi = 0; + + for (;; bi++) { - CMtThread *t = &p->threads[i]; - if (LoopThread_StartSubThread(&t->thread) != SZ_OK) + if (bi >= numBlocksMax) + bi = 0; + + RINOK_THREAD(Event_Wait(&p->writeEvents[bi])) + { - res = SZ_ERROR_THREAD; - p->threads[0].stopReading = True; - break; + const CMtCoderBlock *block = &p->blocks[bi]; + unsigned bufIndex = block->bufIndex; + Bool finished = block->finished; + if (res == SZ_OK && block->res != SZ_OK) + res = block->res; + + if (bufIndex != (unsigned)(int)-1) + { + if (res == SZ_OK) + { + res = p->mtCallback->Write(p->mtCallbackObject, bufIndex); + if (res != SZ_OK) + MtProgress_SetError(&p->mtProgress, res); + } + + CriticalSection_Enter(&p->cs); + { + p->freeBlockList[bufIndex] = p->freeBlockHead; + p->freeBlockHead = bufIndex; + } + CriticalSection_Leave(&p->cs); + } + + RINOK_THREAD(Semaphore_Release1(&p->blocksSemaphore)) + + if (finished) + break; } } + } + #else + { + WRes wres = Event_Wait(&p->finishedEvent); + res = MY_SRes_HRESULT_FROM_WRes(wres); + } + #endif - Event_Set(&p->threads[0].canWrite); - Event_Set(&p->threads[0].canRead); + if (res == SZ_OK) + res = p->readRes; - for (j = 0; j < i; j++) - LoopThread_WaitSubThread(&p->threads[j].thread); - } + if (res == SZ_OK) + res = p->mtProgress.res; + + #ifndef MTCODER__USE_WRITE_THREAD + if (res == SZ_OK) + res = p->writeRes; + #endif - for (i = 0; i < numThreads; i++) - CMtThread_CloseEvents(&p->threads[i]); - return (res == SZ_OK) ? p->res : res; + if (res != SZ_OK) + MtCoder_Free(p); + return res; } diff --git a/C/MtCoder.h b/C/MtCoder.h index ce3ef2df..46674a31 100644 --- a/C/MtCoder.h +++ b/C/MtCoder.h @@ -1,5 +1,5 @@ /* MtCoder.h -- Multi-thread Coder -2017-04-03 : Igor Pavlov : Public domain */ +2017-06-18 : Igor Pavlov : Public domain */ #ifndef __MT_CODER_H #define __MT_CODER_H @@ -8,95 +8,144 @@ EXTERN_C_BEGIN -typedef struct -{ - CThread thread; - CAutoResetEvent startEvent; - CAutoResetEvent finishedEvent; - int stop; - - THREAD_FUNC_TYPE func; - LPVOID param; - THREAD_FUNC_RET_TYPE res; -} CLoopThread; - -void LoopThread_Construct(CLoopThread *p); -void LoopThread_Close(CLoopThread *p); -WRes LoopThread_Create(CLoopThread *p); -WRes LoopThread_StopAndWait(CLoopThread *p); -WRes LoopThread_StartSubThread(CLoopThread *p); -WRes LoopThread_WaitSubThread(CLoopThread *p); +/* + if ( defined MTCODER__USE_WRITE_THREAD) : main thread writes all data blocks to output stream + if (not defined MTCODER__USE_WRITE_THREAD) : any coder thread can write data blocks to output stream +*/ +/* #define MTCODER__USE_WRITE_THREAD */ #ifndef _7ZIP_ST -#define NUM_MT_CODER_THREADS_MAX 32 + #define MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1) + #define MTCODER__THREADS_MAX 64 + #define MTCODER__BLOCKS_MAX (MTCODER__GET_NUM_BLOCKS_FROM_THREADS(MTCODER__THREADS_MAX) + 3) #else -#define NUM_MT_CODER_THREADS_MAX 1 + #define MTCODER__THREADS_MAX 1 + #define MTCODER__BLOCKS_MAX 1 #endif + +typedef struct +{ + UInt64 inSize; + UInt64 outSize; +} CMtProgressSizes; + + typedef struct { - UInt64 totalInSize; - UInt64 totalOutSize; ICompressProgress *progress; SRes res; + UInt64 totalInSize; + UInt64 totalOutSize; CCriticalSection cs; - UInt64 inSizes[NUM_MT_CODER_THREADS_MAX]; - UInt64 outSizes[NUM_MT_CODER_THREADS_MAX]; + CMtProgressSizes sizes[MTCODER__THREADS_MAX]; } CMtProgress; -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize); + +typedef struct +{ + ICompressProgress vt; + CMtProgress *mtProgress; + unsigned index; +} CMtProgressThunk; + +void MtProgressThunk_CreateVTable(CMtProgressThunk *p); + + struct _CMtCoder; + typedef struct { struct _CMtCoder *mtCoder; - Byte *outBuf; - size_t outBufSize; - Byte *inBuf; - size_t inBufSize; unsigned index; - CLoopThread thread; + int stop; + Byte *inBuf; - Bool stopReading; - Bool stopWriting; - CAutoResetEvent canRead; - CAutoResetEvent canWrite; -} CMtThread; + CAutoResetEvent startEvent; + CThread thread; +} CMtCoderThread; -typedef struct IMtCoderCallback IMtCoderCallback; -struct IMtCoderCallback +typedef struct { - SRes (*Code)(const IMtCoderCallback *p, unsigned index, Byte *dest, size_t *destSize, + SRes (*Code)(void *p, unsigned coderIndex, unsigned outBufIndex, const Byte *src, size_t srcSize, int finished); -}; -#define IMtCoderCallback_Code(p, index, dest, destSize, src, srcSize, finished) (p)->Code(p, index, dest, destSize, src, srcSize, finished) + SRes (*Write)(void *p, unsigned outBufIndex); +} IMtCoderCallback2; + + +typedef struct +{ + SRes res; + unsigned bufIndex; + Bool finished; +} CMtCoderBlock; typedef struct _CMtCoder { - size_t blockSize; - size_t destBlockSize; - unsigned numThreads; + /* input variables */ + size_t blockSize; /* size of input block */ + unsigned numThreadsMax; + UInt64 expectedDataSize; + ISeqInStream *inStream; - ISeqOutStream *outStream; + const Byte *inData; + size_t inDataSize; + ICompressProgress *progress; - ISzAllocPtr alloc; + ISzAllocPtr allocBig; + + IMtCoderCallback2 *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + CAutoResetEvent readEvent; + CSemaphore blocksSemaphore; + + Bool stopReading; + SRes readRes; + + #ifdef MTCODER__USE_WRITE_THREAD + CAutoResetEvent writeEvents[MTCODER__BLOCKS_MAX]; + #else + CAutoResetEvent finishedEvent; + SRes writeRes; + unsigned writeIndex; + Byte ReadyBlocks[MTCODER__BLOCKS_MAX]; + LONG numFinishedThreads; + #endif + + unsigned numStartedThreadsLimit; + unsigned numStartedThreads; + + unsigned numBlocksMax; + unsigned blockIndex; + UInt64 readProcessed; - IMtCoderCallback *mtCallback; CCriticalSection cs; - SRes res; + + unsigned freeBlockHead; + unsigned freeBlockList[MTCODER__BLOCKS_MAX]; CMtProgress mtProgress; - CMtThread threads[NUM_MT_CODER_THREADS_MAX]; + CMtCoderBlock blocks[MTCODER__BLOCKS_MAX]; + CMtCoderThread threads[MTCODER__THREADS_MAX]; } CMtCoder; -void MtCoder_Construct(CMtCoder* p); -void MtCoder_Destruct(CMtCoder* p); + +void MtCoder_Construct(CMtCoder *p); +void MtCoder_Destruct(CMtCoder *p); SRes MtCoder_Code(CMtCoder *p); + EXTERN_C_END #endif diff --git a/C/Threads.c b/C/Threads.c index ece07e61..8fd86f22 100644 --- a/C/Threads.c +++ b/C/Threads.c @@ -1,5 +1,5 @@ /* Threads.c -- multithreading library -2014-09-21 : Igor Pavlov : Public domain */ +2017-06-26 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,18 +12,20 @@ static WRes GetError() { DWORD res = GetLastError(); - return (res) ? (WRes)(res) : 1; + return res ? (WRes)res : 1; } -WRes HandleToWRes(HANDLE h) { return (h != 0) ? 0 : GetError(); } -WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); } +static WRes HandleToWRes(HANDLE h) { return (h != NULL) ? 0 : GetError(); } +static WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); } WRes HandlePtr_Close(HANDLE *p) { if (*p != NULL) + { if (!CloseHandle(*p)) return GetError(); - *p = NULL; + *p = NULL; + } return 0; } @@ -49,7 +51,7 @@ WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param) return HandleToWRes(*p); } -WRes Event_Create(CEvent *p, BOOL manualReset, int signaled) +static WRes Event_Create(CEvent *p, BOOL manualReset, int signaled) { *p = CreateEvent(NULL, manualReset, (signaled ? TRUE : FALSE), NULL); return HandleToWRes(*p); diff --git a/C/Threads.h b/C/Threads.h index e927208d..f913241a 100644 --- a/C/Threads.h +++ b/C/Threads.h @@ -1,5 +1,5 @@ /* Threads.h -- multithreading library -2013-11-12 : Igor Pavlov : Public domain */ +2017-06-18 : Igor Pavlov : Public domain */ #ifndef __7Z_THREADS_H #define __7Z_THREADS_H @@ -49,7 +49,8 @@ WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled); WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p); typedef HANDLE CSemaphore; -#define Semaphore_Construct(p) (*p) = NULL +#define Semaphore_Construct(p) *(p) = NULL +#define Semaphore_IsCreated(p) (*(p) != NULL) #define Semaphore_Close(p) HandlePtr_Close(p) #define Semaphore_Wait(p) Handle_WaitObject(*(p)) WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount); diff --git a/C/Util/7z/7zMain.c b/C/Util/7z/7zMain.c index fff63b8a..d12ff79e 100644 --- a/C/Util/7z/7zMain.c +++ b/C/Util/7z/7zMain.c @@ -1,5 +1,5 @@ /* 7zMain.c - Test application for 7z Decoder -2017-04-05 : Igor Pavlov : Public domain */ +2017-08-26 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -616,7 +616,14 @@ int MY_CDECL main(int numargs, char *args[]) #ifdef USE_WINDOWS_FILE if (SzBitWithVals_Check(&db.Attribs, i)) - SetFileAttributesW(destPath, db.Attribs.Vals[i]); + { + UInt32 attrib = db.Attribs.Vals[i]; + /* p7zip stores posix attributes in high 16 bits and adds 0x8000 as marker. + We remove posix bits, if we detect posix mode field */ + if ((attrib & 0xF0000000) != 0) + attrib &= 0x7FFF; + SetFileAttributesW(destPath, attrib); + } #endif } PrintLF(); diff --git a/C/Util/7zipInstall/7zipInstall.c b/C/Util/7zipInstall/7zipInstall.c index 28427933..f624b31d 100644 --- a/C/Util/7zipInstall/7zipInstall.c +++ b/C/Util/7zipInstall/7zipInstall.c @@ -1,5 +1,5 @@ /* 7zipInstall.c - 7-Zip Installer -2017-04-04 : Igor Pavlov : Public domain */ +2017-08-28 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -154,7 +154,7 @@ static WRes MyCreateDir(LPCWSTR name) } #define IS_SEPAR(c) (c == WCHAR_PATH_SEPARATOR) -#define IS_LETTER_CHAR(c) ((c) >= 'a' && (c) <= 'z' || (c) >= 'A' && (c) <= 'Z') +#define IS_LETTER_CHAR(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) #define IS_DRIVE_PATH(s) (IS_LETTER_CHAR(s[0]) && s[1] == ':' && IS_SEPAR(s[2])) static int ReverseFind_PathSepar(const wchar_t *s) diff --git a/C/Util/Lzma/LzmaUtil.dsp b/C/Util/Lzma/LzmaUtil.dsp index df1f0cb0..eedde07d 100644 --- a/C/Util/Lzma/LzmaUtil.dsp +++ b/C/Util/Lzma/LzmaUtil.dsp @@ -98,6 +98,10 @@ SOURCE=..\..\7zStream.c # End Source File # Begin Source File +SOURCE=..\..\7zTypes.h +# End Source File +# Begin Source File + SOURCE=..\..\7zVersion.h # End Source File # Begin Source File @@ -160,9 +164,5 @@ SOURCE=..\..\Threads.c SOURCE=..\..\Threads.h # End Source File -# Begin Source File - -SOURCE=..\..\Types.h -# End Source File # End Target # End Project diff --git a/C/Xz.c b/C/Xz.c index 2039a5dc..7e061d6e 100644 --- a/C/Xz.c +++ b/C/Xz.c @@ -1,5 +1,5 @@ /* Xz.c - Xz -2017-04-03 : Igor Pavlov : Public domain */ +2017-05-12 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,7 +9,7 @@ #include "XzCrc64.h" const Byte XZ_SIG[XZ_SIG_SIZE] = { 0xFD, '7', 'z', 'X', 'Z', 0 }; -const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; +/* const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; */ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v) { @@ -26,16 +26,16 @@ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v) void Xz_Construct(CXzStream *p) { - p->numBlocks = p->numBlocksAllocated = 0; - p->blocks = 0; + p->numBlocks = 0; + p->blocks = NULL; p->flags = 0; } void Xz_Free(CXzStream *p, ISzAllocPtr alloc) { ISzAlloc_Free(alloc, p->blocks); - p->numBlocks = p->numBlocksAllocated = 0; - p->blocks = 0; + p->numBlocks = 0; + p->blocks = NULL; } unsigned XzFlags_GetCheckSize(CXzStreamFlags f) diff --git a/C/Xz.h b/C/Xz.h index 7f87937e..ba455c13 100644 --- a/C/Xz.h +++ b/C/Xz.h @@ -1,5 +1,5 @@ /* Xz.h - Xz interface -2017-04-03 : Igor Pavlov : Public domain */ +2017-07-27 : Igor Pavlov : Public domain */ #ifndef __XZ_H #define __XZ_H @@ -50,6 +50,7 @@ typedef struct #define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1) #define XzBlock_HasPackSize(p) (((p)->flags & XZ_BF_PACK_SIZE) != 0) #define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0) +#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) SRes XzBlock_Parse(CXzBlock *p, const Byte *header); SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes); @@ -60,7 +61,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt #define XZ_FOOTER_SIG_SIZE 2 extern const Byte XZ_SIG[XZ_SIG_SIZE]; + +/* extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE]; +*/ + +#define XZ_FOOTER_SIG_0 'Y' +#define XZ_FOOTER_SIG_1 'Z' #define XZ_STREAM_FLAGS_SIZE 2 #define XZ_STREAM_CRC_SIZE 4 @@ -106,7 +113,6 @@ typedef struct { CXzStreamFlags flags; size_t numBlocks; - size_t numBlocksAllocated; CXzBlockSizes *blocks; UInt64 startOffset; } CXzStream; @@ -218,6 +224,9 @@ typedef struct CXzBlock block; CXzCheck check; CSha256 sha; + + unsigned decodeOnlyOneBlock; + Byte shaDigest[SHA256_DIGEST_SIZE]; Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; } CXzUnpacker; @@ -258,7 +267,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status); -Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p); +Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); /* Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of @@ -268,7 +277,25 @@ XzUnpacker_Code() returns: res == SZ_ERROR_NO_ARCHIVE */ -UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p); +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p); + + +/* + for random block decoding: + XzUnpacker_Init(); + set CXzUnpacker::streamFlags + XzUnpacker_PrepareToRandomBlockDecoding() + loop + { + XzUnpacker_Code() + XzUnpacker_IsBlockFinished() + } +*/ + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p); +Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p); + +#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) EXTERN_C_END diff --git a/C/XzCrc64.c b/C/XzCrc64.c index 3ea0a4c7..e9ca9ec2 100644 --- a/C/XzCrc64.c +++ b/C/XzCrc64.c @@ -1,5 +1,5 @@ /* XzCrc64.c -- CRC64 calculation -2017-04-03 : Igor Pavlov : Public domain */ +2017-05-23 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,9 +9,9 @@ #define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42) #ifdef MY_CPU_LE - #define CRC_NUM_TABLES 4 + #define CRC64_NUM_TABLES 4 #else - #define CRC_NUM_TABLES 5 + #define CRC64_NUM_TABLES 5 #define CRC_UINT64_SWAP(v) \ ((v >> 56) \ | ((v >> 40) & ((UInt64)0xFF << 8)) \ @@ -29,10 +29,10 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); #endif -typedef UInt64 (MY_FAST_CALL *CRC_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); +typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); -static CRC_FUNC g_Crc64Update; -UInt64 g_Crc64Table[256 * CRC_NUM_TABLES]; +static CRC64_FUNC g_Crc64Update; +UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES]; UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size) { @@ -55,7 +55,7 @@ void MY_FAST_CALL Crc64GenerateTable() r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1))); g_Crc64Table[i] = r; } - for (i = 256; i < 256 * CRC_NUM_TABLES; i++) + for (i = 256; i < 256 * CRC64_NUM_TABLES; i++) { UInt64 r = g_Crc64Table[(size_t)i - 256]; g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8); @@ -74,7 +74,7 @@ void MY_FAST_CALL Crc64GenerateTable() else #endif { - for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) + for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--) { UInt64 x = g_Crc64Table[(size_t)i - 256]; g_Crc64Table[i] = CRC_UINT64_SWAP(x); diff --git a/C/XzCrc64Opt.c b/C/XzCrc64Opt.c index 3b793f80..9273465d 100644 --- a/C/XzCrc64Opt.c +++ b/C/XzCrc64Opt.c @@ -1,5 +1,5 @@ /* XzCrc64Opt.c -- CRC64 calculation -2017-04-03 : Igor Pavlov : Public domain */ +2017-06-30 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,13 +7,13 @@ #ifndef MY_CPU_BE -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table) { const Byte *p = (const Byte *)data; for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); + v = CRC64_UPDATE_BYTE_2(v, *p); for (; size >= 4; size -= 4, p += 4) { UInt32 d = (UInt32)v ^ *(const UInt32 *)p; @@ -24,7 +24,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con ^ (table + 0x000)[((d >> 24))]; } for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); + v = CRC64_UPDATE_BYTE_2(v, *p); return v; } @@ -43,7 +43,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con | ((v << 40) & ((UInt64)0xFF << 48)) \ | ((v << 56))) -#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) +#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table) { @@ -51,7 +51,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size table += 0x100; v = CRC_UINT64_SWAP(v); for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); + v = CRC64_UPDATE_BYTE_2_BE(v, *p); for (; size >= 4; size -= 4, p += 4) { UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p; @@ -62,7 +62,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size ^ (table + 0x300)[((d >> 24))]; } for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); + v = CRC64_UPDATE_BYTE_2_BE(v, *p); return CRC_UINT64_SWAP(v); } diff --git a/C/XzDec.c b/C/XzDec.c index ce29afc2..7747ba44 100644 --- a/C/XzDec.c +++ b/C/XzDec.c @@ -1,5 +1,5 @@ /* XzDec.c -- Xz Decode -2017-04-03 : Igor Pavlov : Public domain */ +2017-07-27 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -393,7 +393,7 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; - Bool allFinished = True; + *destLen = 0; *srcLen = 0; *status = CODER_STATUS_NOT_FINISHED; @@ -411,6 +411,7 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, for (;;) { Bool processed = False; + Bool allFinished = True; unsigned i; /* if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY) @@ -487,14 +488,17 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, if (destLenCur != 0 || srcLenCur != 0) processed = True; } + if (!processed) - break; + { + if (allFinished) + *status = CODER_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } } - if (allFinished) - *status = CODER_STATUS_FINISHED_WITH_MARK; - return SZ_OK; } + SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) { *p = (CXzStreamFlags)GetBe16(buf + XZ_SIG_SIZE); @@ -506,11 +510,11 @@ SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) { - return - indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) && - (GetUi32(buf) == CrcCalc(buf + 4, 6) && - flags == GetBe16(buf + 8) && - memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0); + return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) + && GetUi32(buf) == CrcCalc(buf + 4, 6) + && flags == GetBe16(buf + 8) + && buf[10] == XZ_FOOTER_SIG_0 + && buf[11] == XZ_FOOTER_SIG_1; } #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ @@ -524,14 +528,15 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) unsigned numFilters, i; unsigned headerSize = (unsigned)header[0] << 2; + /* (headerSize != 0) : another code checks */ + if (CrcCalc(header, headerSize) != GetUi32(header + headerSize)) return SZ_ERROR_ARCHIVE; pos = 1; - if (pos == headerSize) - return SZ_ERROR_ARCHIVE; p->flags = header[pos++]; + p->packSize = (UInt64)(Int64)-1; if (XzBlock_HasPackSize(p)) { READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize); @@ -539,6 +544,7 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) return SZ_ERROR_ARCHIVE; } + p->unpackSize = (UInt64)(Int64)-1; if (XzBlock_HasUnpackSize(p)) READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize); @@ -565,6 +571,9 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) #endif } + if (XzBlock_HasUnsupportedFlags(p)) + return SZ_ERROR_UNSUPPORTED; + while (pos < headerSize) if (header[pos++] != 0) return SZ_ERROR_ARCHIVE; @@ -615,6 +624,7 @@ void XzUnpacker_Init(CXzUnpacker *p) p->numFinishedStreams = 0; p->numTotalBlocks = 0; p->padSize = 0; + p->decodeOnlyOneBlock = 0; } void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc) @@ -628,6 +638,18 @@ void XzUnpacker_Free(CXzUnpacker *p) MixCoder_Free(&p->decoder); } + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p) +{ + p->indexSize = 0; + p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; + p->pos = 0; + p->decodeOnlyOneBlock = 1; +} + + SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status) { @@ -638,20 +660,44 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, *status = CODER_STATUS_NOT_SPECIFIED; for (;;) { - SizeT srcRem = srcLenOrig - *srcLen; + SizeT srcRem; if (p->state == XZ_STATE_BLOCK) { SizeT destLen2 = destLenOrig - *destLen; SizeT srcLen2 = srcLenOrig - *srcLen; SRes res; + + ECoderFinishMode finishMode2 = finishMode; + + if (p->block.packSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.packSize - p->packSize; + if (srcLen2 > rem) + srcLen2 = (SizeT)rem; + if (rem == 0 && p->block.unpackSize == p->unpackSize) + return SZ_ERROR_DATA; + } + + if (p->block.unpackSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.unpackSize - p->unpackSize; + if (destLen2 >= rem) + { + finishMode2 = CODER_FINISH_END; + destLen2 = (SizeT)rem; + } + } + + /* if (srcLen2 == 0 && destLen2 == 0) { *status = CODER_STATUS_NOT_FINISHED; return SZ_OK; } + */ - res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode, status); + res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode2, status); XzCheck_Update(&p->check, dest, destLen2); (*srcLen) += srcLen2; @@ -661,28 +707,41 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, (*destLen) += destLen2; dest += destLen2; p->unpackSize += destLen2; - + RINOK(res); - - if (*status == CODER_STATUS_FINISHED_WITH_MARK) + + if (*status != CODER_STATUS_FINISHED_WITH_MARK) + { + if (p->block.packSize == p->packSize + && *status == CODER_STATUS_NEEDS_MORE_INPUT) + { + *status = CODER_STATUS_NOT_SPECIFIED; + return SZ_ERROR_DATA; + } + + // if (srcLen2 == 0 && destLen2 == 0) + return SZ_OK; + } { Byte temp[32]; - unsigned num = Xz_WriteVarInt(temp, p->packSize + p->blockHeaderSize + XzFlags_GetCheckSize(p->streamFlags)); + unsigned num = Xz_WriteVarInt(temp, XzUnpacker_GetPackSizeForIndex(p)); num += Xz_WriteVarInt(temp + num, p->unpackSize); Sha256_Update(&p->sha, temp, num); p->indexSize += num; p->numBlocks++; - p->state = XZ_STATE_BLOCK_FOOTER; p->pos = 0; p->alignPos = 0; + + if (p->block.unpackSize != (UInt64)(Int64)-1) + if (p->block.unpackSize != p->unpackSize) + return SZ_ERROR_DATA; } - else if (srcLen2 == 0 && destLen2 == 0) - return SZ_OK; - - continue; + // continue; } + srcRem = srcLenOrig - *srcLen; + if (srcRem == 0) { *status = CODER_STATUS_NEEDS_MORE_INPUT; @@ -704,10 +763,10 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, { RINOK(Xz_ParseHeader(&p->streamFlags, p->buf)); p->numStartedStreams++; - p->state = XZ_STATE_BLOCK_HEADER; - Sha256_Init(&p->sha); p->indexSize = 0; p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; p->pos = 0; } break; @@ -721,6 +780,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, (*srcLen)++; if (p->buf[0] == 0) { + if (p->decodeOnlyOneBlock) + return SZ_ERROR_DATA; p->indexPreSize = 1 + Xz_WriteVarInt(p->buf + 1, p->numBlocks); p->indexPos = p->indexPreSize; p->indexSize += p->indexPreSize; @@ -728,10 +789,13 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, Sha256_Init(&p->sha); p->crc = CrcUpdate(CRC_INIT_VAL, p->buf, p->indexPreSize); p->state = XZ_STATE_STREAM_INDEX; + break; } p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4; + break; } - else if (p->pos != p->blockHeaderSize) + + if (p->pos != p->blockHeaderSize) { UInt32 cur = p->blockHeaderSize - p->pos; if (cur > srcRem) @@ -775,14 +839,20 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, p->pos += cur; (*srcLen) += cur; src += cur; + if (checkSize != p->pos) + break; } - else { Byte digest[XZ_CHECK_SIZE_MAX]; p->state = XZ_STATE_BLOCK_HEADER; p->pos = 0; if (XzCheck_Final(&p->check, digest) && memcmp(digest, p->buf, checkSize) != 0) return SZ_ERROR_CRC; + if (p->decodeOnlyOneBlock) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } } } break; @@ -898,12 +968,18 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, */ } -Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p) + +Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p) +{ + return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); +} + +Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) { return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0); } -UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p) +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) { UInt64 num = 0; if (p->state == XZ_STATE_STREAM_PADDING) diff --git a/C/XzEnc.c b/C/XzEnc.c index 811635be..6abde77d 100644 --- a/C/XzEnc.c +++ b/C/XzEnc.c @@ -1,11 +1,12 @@ /* XzEnc.c -- Xz Encode -2017-04-03 : Igor Pavlov : Public domain */ +2017-08-25 : Igor Pavlov : Public domain */ + +#include "Precomp.h" #include #include #include "7zCrc.h" -#include "Alloc.h" #include "Bra.h" #include "CpuArch.h" @@ -17,22 +18,41 @@ #include "XzEnc.h" +// #define _7ZIP_ST + +#ifndef _7ZIP_ST +#include "MtCoder.h" +#else +#define MTCODER__THREADS_MAX 1 +#define MTCODER__BLOCKS_MAX 1 +#endif + +#define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3) + +/* max pack size for LZMA2 block + check-64bytrs: */ +#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64) + +#define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize)) + + #define XzBlock_ClearFlags(p) (p)->flags = 0; #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1); #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; -static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size) + +static SRes WriteBytes(ISeqOutStream *s, const void *buf, size_t size) { return (ISeqOutStream_Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; } -static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc) +static SRes WriteBytesUpdateCrc(ISeqOutStream *s, const void *buf, size_t size, UInt32 *crc) { *crc = CrcUpdate(*crc, buf, size); return WriteBytes(s, buf, size); } + static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) { UInt32 crc; @@ -76,98 +96,154 @@ static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s) } -static SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s) + + +typedef struct { - Byte buf[32]; - UInt64 globalPos; - { - UInt32 crc = CRC_INIT_VAL; - unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); - size_t i; + size_t numBlocks; + size_t size; + size_t allocated; + Byte *blocks; +} CXzEncIndex; - globalPos = pos; - buf[0] = 0; - RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); - for (i = 0; i < p->numBlocks; i++) - { - const CXzBlockSizes *block = &p->blocks[i]; - pos = Xz_WriteVarInt(buf, block->totalSize); - pos += Xz_WriteVarInt(buf + pos, block->unpackSize); - globalPos += pos; - RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); - } - - pos = ((unsigned)globalPos & 3); - - if (pos != 0) - { - buf[0] = buf[1] = buf[2] = 0; - RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc)); - globalPos += 4 - pos; - } - { - SetUi32(buf, CRC_GET_DIGEST(crc)); - RINOK(WriteBytes(s, buf, 4)); - globalPos += 4; - } - } +static void XzEncIndex_Construct(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; + p->blocks = NULL; +} +static void XzEncIndex_Init(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; +} + +static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) +{ + if (p->blocks) { - UInt32 indexSize = (UInt32)((globalPos >> 2) - 1); - SetUi32(buf + 4, indexSize); - buf[8] = (Byte)(p->flags >> 8); - buf[9] = (Byte)(p->flags & 0xFF); - SetUi32(buf, CrcCalc(buf + 4, 6)); - memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE); - return WriteBytes(s, buf, 12); + ISzAlloc_Free(alloc, p->blocks); + p->blocks = NULL; } + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; +} + + +static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) +{ + Byte *blocks = ISzAlloc_Alloc(alloc, newSize); + if (!blocks) + return SZ_ERROR_MEM; + if (p->size != 0) + memcpy(blocks, p->blocks, p->size); + if (p->blocks) + ISzAlloc_Free(alloc, p->blocks); + p->blocks = blocks; + p->allocated = newSize; + return SZ_OK; } -static SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) +static SRes XzEncIndex_PreAlloc(CXzEncIndex *p, UInt64 numBlocks, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) { - if (!p->blocks || p->numBlocksAllocated == p->numBlocks) + UInt64 pos; { - size_t num = p->numBlocks * 2 + 1; - size_t newSize = sizeof(CXzBlockSizes) * num; - CXzBlockSizes *blocks; - if (newSize / sizeof(CXzBlockSizes) != num) - return SZ_ERROR_MEM; - blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, newSize); - if (!blocks) + Byte buf[32]; + unsigned pos2 = Xz_WriteVarInt(buf, totalSize); + pos2 += Xz_WriteVarInt(buf + pos2, unpackSize); + pos = numBlocks * pos2; + } + + if (pos <= p->allocated - p->size) + return SZ_OK; + { + UInt64 newSize64 = p->size + pos; + size_t newSize = (size_t)newSize64; + if (newSize != newSize64) return SZ_ERROR_MEM; - if (p->numBlocks != 0) - { - memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes)); - ISzAlloc_Free(alloc, p->blocks); - } - p->blocks = blocks; - p->numBlocksAllocated = num; + return XzEncIndex_ReAlloc(p, newSize, alloc); } +} + + +static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) +{ + Byte buf[32]; + unsigned pos = Xz_WriteVarInt(buf, totalSize); + pos += Xz_WriteVarInt(buf + pos, unpackSize); + + if (pos > p->allocated - p->size) { - CXzBlockSizes *block = &p->blocks[p->numBlocks++]; - block->unpackSize = unpackSize; - block->totalSize = totalSize; + size_t newSize = p->allocated * 2 + 16 * 2; + if (newSize < p->size + pos) + return SZ_ERROR_MEM; + RINOK(XzEncIndex_ReAlloc(p, newSize, alloc)); } + memcpy(p->blocks + p->size, buf, pos); + p->size += pos; + p->numBlocks++; return SZ_OK; } +static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStream *s) +{ + Byte buf[32]; + UInt64 globalPos; + UInt32 crc = CRC_INIT_VAL; + unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); + + globalPos = pos; + buf[0] = 0; + RINOK(WriteBytesUpdateCrc(s, buf, pos, &crc)); + RINOK(WriteBytesUpdateCrc(s, p->blocks, p->size, &crc)); + globalPos += p->size; + + pos = XZ_GET_PAD_SIZE(globalPos); + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + globalPos += pos; + + crc = CrcUpdate(crc, buf + 4 - pos, pos); + SetUi32(buf + 4, CRC_GET_DIGEST(crc)); + + SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2)); + buf[8 + 8] = (Byte)(flags >> 8); + buf[8 + 9] = (Byte)(flags & 0xFF); + SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6)); + buf[8 + 10] = XZ_FOOTER_SIG_0; + buf[8 + 11] = XZ_FOOTER_SIG_1; + + return WriteBytes(s, buf + 4 - pos, pos + 4 + 12); +} + + + /* ---------- CSeqCheckInStream ---------- */ typedef struct { ISeqInStream vt; ISeqInStream *realStream; + const Byte *data; + UInt64 limit; UInt64 processed; + int realStreamFinished; CXzCheck check; } CSeqCheckInStream; -static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned mode) +static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned checkMode) { + p->limit = (UInt64)(Int64)-1; p->processed = 0; - XzCheck_Init(&p->check, mode); + p->realStreamFinished = 0; + XzCheck_Init(&p->check, checkMode); } static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) @@ -178,9 +254,28 @@ static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) static SRes SeqCheckInStream_Read(const ISeqInStream *pp, void *data, size_t *size) { CSeqCheckInStream *p = CONTAINER_FROM_VTBL(pp, CSeqCheckInStream, vt); - SRes res = ISeqInStream_Read(p->realStream, data, size); - XzCheck_Update(&p->check, data, *size); - p->processed += *size; + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + if (p->realStream) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->realStreamFinished = (size2 == 0) ? 1 : 0; + } + else + memcpy(data, p->data + (size_t)p->processed, size2); + XzCheck_Update(&p->check, data, size2); + p->processed += size2; + } + *size = size2; return res; } @@ -191,13 +286,22 @@ typedef struct { ISeqOutStream vt; ISeqOutStream *realStream; + Byte *outBuf; + size_t outBufLimit; UInt64 processed; } CSeqSizeOutStream; -static size_t MyWrite(const ISeqOutStream *pp, const void *data, size_t size) +static size_t SeqSizeOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) { CSeqSizeOutStream *p = CONTAINER_FROM_VTBL(pp, CSeqSizeOutStream, vt); - size = ISeqOutStream_Write(p->realStream, data, size); + if (p->realStream) + size = ISeqOutStream_Write(p->realStream, data, size); + else + { + if (size > p->outBufLimit - (size_t)p->processed) + return 0; + memcpy(p->outBuf + (size_t)p->processed, data, size); + } p->processed += size; return size; } @@ -218,6 +322,26 @@ typedef struct int srcWasFinished; } CSeqInFilter; + +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); + +static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPtr alloc) +{ + if (!p->buf) + { + p->buf = ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); + if (!p->buf) + return SZ_ERROR_MEM; + } + p->curPos = p->endPos = 0; + p->srcWasFinished = 0; + RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, alloc)); + RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc)); + p->StateCoder.Init(p->StateCoder.p); + return SZ_OK; +} + + static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) { CSeqInFilter *p = CONTAINER_FROM_VTBL(pp, CSeqInFilter, p); @@ -256,33 +380,15 @@ static void SeqInFilter_Construct(CSeqInFilter *p) p->p.Read = SeqInFilter_Read; } -static void SeqInFilter_Free(CSeqInFilter *p) +static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) { if (p->buf) { - ISzAlloc_Free(&g_Alloc, p->buf); + ISzAlloc_Free(alloc, p->buf); p->buf = NULL; } } -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); - -static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props) -{ - if (!p->buf) - { - p->buf = ISzAlloc_Alloc(&g_Alloc, FILTER_BUF_SIZE); - if (!p->buf) - return SZ_ERROR_MEM; - } - p->curPos = p->endPos = 0; - p->srcWasFinished = 0; - RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc)); - RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc)); - p->StateCoder.Init(p->StateCoder.p); - return SZ_OK; -} - /* ---------- CSbEncInStream ---------- */ @@ -343,43 +449,233 @@ void SbEncInStream_Free(CSbEncInStream *p) #endif + +/* ---------- CXzProps ---------- */ + + +void XzFilterProps_Init(CXzFilterProps *p) +{ + p->id = 0; + p->delta = 0; + p->ip = 0; + p->ipDefined = False; +} + +void XzProps_Init(CXzProps *p) +{ + p->checkId = XZ_CHECK_CRC32; + p->blockSize = XZ_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; + p->numTotalThreads = -1; + p->reduceSize = (UInt64)(Int64)-1; + p->forceWriteSizesInHeader = 0; + // p->forceWriteSizesInHeader = 1; + + XzFilterProps_Init(&p->filterProps); + Lzma2EncProps_Init(&p->lzma2Props); +} + + +static void XzEncProps_Normalize_Fixed(CXzProps *p) +{ + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; + { + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + Lzma2EncProps_Normalize(&tp); + t1n = tp.numTotalThreads; + } + + t1 = p->lzma2Props.numTotalThreads; + t2 = p->numBlockThreads_Max; + t3 = p->numTotalThreads; + + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + + if (t3 <= 0) + { + if (t2 <= 0) + t2 = 1; + t3 = t1n * t2; + } + else if (t2 <= 0) + { + t2 = t3 / t1n; + if (t2 == 0) + { + t1 = 1; + t2 = t3; + } + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + } + else if (t1 <= 0) + { + t1 = t3 / t2; + if (t1 == 0) + t1 = 1; + } + else + t3 = t1n * t2; + + p->lzma2Props.numTotalThreads = t1; + + t2r = t2; + + fileSize = p->reduceSize; + + if ((p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzma2Props.lzmaProps.reduceSize = p->blockSize; + + Lzma2EncProps_Normalize(&p->lzma2Props); + + t1 = p->lzma2Props.numTotalThreads; + + { + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) + { + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; + if (numBlocks < (unsigned)t2) + { + t2r = (unsigned)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; + } + } + } + + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; + p->numTotalThreads = t3; +} + + +static void XzProps_Normalize(CXzProps *p) +{ + /* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties. + Lzma2Enc_SetProps() will normalize lzma2Props later. */ + + if (p->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID) + { + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + return; + } + else + { + CLzma2EncProps *lzma2 = &p->lzma2Props; + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + { + // xz-auto + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + { + // if (xz-auto && lzma2-solid) - we use solid for both + p->blockSize = XZ_PROPS__BLOCK_SIZE__SOLID; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + } + else + { + // if (xz-auto && (lzma2-auto || lzma2-fixed_) + // we calculate block size for lzma2 and use that block size for xz, lzma2 uses single-chunk per block + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + + Lzma2EncProps_Normalize(&tp); + + p->blockSize = tp.blockSize; // fixed or solid + p->numBlockThreads_Reduced = tp.numBlockThreads_Reduced; + p->numBlockThreads_Max = tp.numBlockThreads_Max; + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->lzmaProps.reduceSize = tp.blockSize; + lzma2->numBlockThreads_Reduced = 1; + lzma2->numBlockThreads_Max = 1; + return; + } + } + else + { + // xz-fixed + // we can use xz::reduceSize or xz::blockSize as base for lzmaProps::reduceSize + + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + { + UInt64 r = p->reduceSize; + if (r > p->blockSize || r == (UInt64)(Int64)-1) + r = p->blockSize; + lzma2->lzmaProps.reduceSize = r; + } + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->blockSize = p->blockSize; + + XzEncProps_Normalize_Fixed(p); + } + } +} + + +/* ---------- CLzma2WithFilters ---------- */ + typedef struct { CLzma2EncHandle lzma2; + CSeqInFilter filter; + #ifdef USE_SUBBLOCK CSbEncInStream sb; #endif - CSeqInFilter filter; - ISzAllocPtr alloc; - ISzAllocPtr bigAlloc; } CLzma2WithFilters; -static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAllocPtr alloc, ISzAllocPtr bigAlloc) +static void Lzma2WithFilters_Construct(CLzma2WithFilters *p) { - p->alloc = alloc; - p->bigAlloc = bigAlloc; p->lzma2 = NULL; + SeqInFilter_Construct(&p->filter); + #ifdef USE_SUBBLOCK SbEncInStream_Construct(&p->sb, alloc); #endif - SeqInFilter_Construct(&p->filter); } -static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p) + +static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p, ISzAllocPtr alloc, ISzAllocPtr bigAlloc) { - p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc); if (!p->lzma2) - return SZ_ERROR_MEM; + { + p->lzma2 = Lzma2Enc_Create(alloc, bigAlloc); + if (!p->lzma2) + return SZ_ERROR_MEM; + } return SZ_OK; } -static void Lzma2WithFilters_Free(CLzma2WithFilters *p) + +static void Lzma2WithFilters_Free(CLzma2WithFilters *p, ISzAllocPtr alloc) { - SeqInFilter_Free(&p->filter); #ifdef USE_SUBBLOCK SbEncInStream_Free(&p->sb); #endif + + SeqInFilter_Free(&p->filter, alloc); if (p->lzma2) { Lzma2Enc_Destroy(p->lzma2); @@ -388,137 +684,623 @@ static void Lzma2WithFilters_Free(CLzma2WithFilters *p) } -void XzProps_Init(CXzProps *p) +typedef struct { - p->lzma2Props = NULL; - p->filterProps = NULL; - p->checkId = XZ_CHECK_CRC32; + UInt64 unpackSize; + UInt64 totalSize; + size_t headerSize; +} CXzEncBlockInfo; + + +static SRes Xz_CompressBlock( + CLzma2WithFilters *lzmaf, + + ISeqOutStream *outStream, + Byte *outBufHeader, + Byte *outBufData, size_t outBufDataLimit, + + ISeqInStream *inStream, + // UInt64 expectedSize, + const Byte *inBuf, // used if (!inStream) + size_t inBufSize, // used if (!inStream), it's block size, props->blockSize is ignored + + const CXzProps *props, + ICompressProgress *progress, + int *inStreamFinished, /* only for inStream version */ + CXzEncBlockInfo *blockSizes, + ISzAllocPtr alloc, + ISzAllocPtr allocBig) +{ + CSeqCheckInStream checkInStream; + CSeqSizeOutStream seqSizeOutStream; + CXzBlock block; + unsigned filterIndex = 0; + CXzFilter *filter = NULL; + const CXzFilterProps *fp = &props->filterProps; + if (fp->id == 0) + fp = NULL; + + *inStreamFinished = False; + + RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig)); + + RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props)); + + XzBlock_ClearFlags(&block); + XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); + + if (fp) + { + filter = &block.filters[filterIndex++]; + filter->id = fp->id; + filter->propsSize = 0; + + if (fp->id == XZ_ID_Delta) + { + filter->props[0] = (Byte)(fp->delta - 1); + filter->propsSize = 1; + } + else if (fp->ipDefined) + { + SetUi32(filter->props, fp->ip); + filter->propsSize = 4; + } + } + + { + CXzFilter *f = &block.filters[filterIndex++]; + f->id = XZ_ID_LZMA2; + f->propsSize = 1; + f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); + } + + seqSizeOutStream.vt.Write = SeqSizeOutStream_Write; + seqSizeOutStream.realStream = outStream; + seqSizeOutStream.outBuf = outBufData; + seqSizeOutStream.outBufLimit = outBufDataLimit; + seqSizeOutStream.processed = 0; + + /* + if (expectedSize != (UInt64)(Int64)-1) + { + block.unpackSize = expectedSize; + if (props->blockSize != (UInt64)(Int64)-1) + if (expectedSize > props->blockSize) + block.unpackSize = props->blockSize; + XzBlock_SetHasUnpackSize(&block); + } + */ + + if (outStream) + { + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + } + + checkInStream.vt.Read = SeqCheckInStream_Read; + SeqCheckInStream_Init(&checkInStream, props->checkId); + + checkInStream.realStream = inStream; + checkInStream.data = inBuf; + checkInStream.limit = props->blockSize; + if (!inStream) + checkInStream.limit = inBufSize; + + if (fp) + { + #ifdef USE_SUBBLOCK + if (fp->id == XZ_ID_Subblock) + { + lzmaf->sb.inStream = &checkInStream.vt; + RINOK(SbEncInStream_Init(&lzmaf->sb)); + } + else + #endif + { + lzmaf->filter.realStream = &checkInStream.vt; + RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc)); + } + } + + { + SRes res; + Byte *outBuf = NULL; + size_t outSize = 0; + Bool useStream = (fp || inStream); + // useStream = True; + + if (!useStream) + { + XzCheck_Update(&checkInStream.check, inBuf, inBufSize); + checkInStream.processed = inBufSize; + } + + if (!outStream) + { + outBuf = seqSizeOutStream.outBuf; // + (size_t)seqSizeOutStream.processed; + outSize = seqSizeOutStream.outBufLimit; // - (size_t)seqSizeOutStream.processed; + } + + res = Lzma2Enc_Encode2(lzmaf->lzma2, + outBuf ? NULL : &seqSizeOutStream.vt, + outBuf, + outBuf ? &outSize : NULL, + + useStream ? + (fp ? + ( + #ifdef USE_SUBBLOCK + (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt: + #endif + &lzmaf->filter.p) : + &checkInStream.vt) : NULL, + + useStream ? NULL : inBuf, + useStream ? 0 : inBufSize, + + progress); + + if (outBuf) + seqSizeOutStream.processed += outSize; + + RINOK(res); + blockSizes->unpackSize = checkInStream.processed; + } + { + Byte buf[4 + 64]; + unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); + UInt64 packSize = seqSizeOutStream.processed; + + buf[0] = 0; + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + + SeqCheckInStream_GetDigest(&checkInStream, buf + 4); + RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))); + + blockSizes->totalSize = seqSizeOutStream.processed - padSize; + + if (!outStream) + { + seqSizeOutStream.outBuf = outBufHeader; + seqSizeOutStream.outBufLimit = XZ_BLOCK_HEADER_SIZE_MAX; + seqSizeOutStream.processed = 0; + + block.unpackSize = blockSizes->unpackSize; + XzBlock_SetHasUnpackSize(&block); + + block.packSize = packSize; + XzBlock_SetHasPackSize(&block); + + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + + blockSizes->headerSize = (size_t)seqSizeOutStream.processed; + blockSizes->totalSize += seqSizeOutStream.processed; + } + } + + if (inStream) + *inStreamFinished = checkInStream.realStreamFinished; + else + { + *inStreamFinished = False; + if (checkInStream.processed != inBufSize) + return SZ_ERROR_FAIL; + } + + return SZ_OK; } -void XzFilterProps_Init(CXzFilterProps *p) + + +typedef struct { - p->id = 0; - p->delta = 0; - p->ip = 0; - p->ipDefined = False; + ICompressProgress vt; + ICompressProgress *progress; + UInt64 inOffset; + UInt64 outOffset; +} CCompressProgress_XzEncOffset; + + +static SRes CompressProgress_XzEncOffset_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +{ + const CCompressProgress_XzEncOffset *p = CONTAINER_FROM_VTBL(pp, CCompressProgress_XzEncOffset, vt); + inSize += p->inOffset; + outSize += p->outOffset; + return ICompressProgress_Progress(p->progress, inSize, outSize); } -static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf, - ISeqOutStream *outStream, ISeqInStream *inStream, - const CXzProps *props, ICompressProgress *progress) + + +typedef struct { - xz->flags = (Byte)props->checkId; + ISzAllocPtr alloc; + ISzAllocPtr allocBig; - RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props)); - RINOK(Xz_WriteHeader(xz->flags, outStream)); + CXzProps xzProps; + UInt64 expectedDataSize; + CXzEncIndex xzIndex; + + CLzma2WithFilters lzmaf_Items[MTCODER__THREADS_MAX]; + + size_t outBufSize; /* size of allocated outBufs[i] */ + Byte *outBufs[MTCODER__BLOCKS_MAX]; + + #ifndef _7ZIP_ST + unsigned checkType; + ISeqOutStream *outStream; + Bool mtCoder_WasConstructed; + CMtCoder mtCoder; + CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX]; + #endif + +} CXzEnc; + + +static void XzEnc_Construct(CXzEnc *p) +{ + unsigned i; + + XzEncIndex_Construct(&p->xzIndex); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Construct(&p->lzmaf_Items[i]); + + #ifndef _7ZIP_ST + p->mtCoder_WasConstructed = False; { - CSeqCheckInStream checkInStream; - CSeqSizeOutStream seqSizeOutStream; - CXzBlock block; - unsigned filterIndex = 0; - CXzFilter *filter = NULL; - const CXzFilterProps *fp = props->filterProps; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; + } + #endif +} + + +static void XzEnc_FreeOutBufs(CXzEnc *p) +{ + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) + { + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; + } + p->outBufSize = 0; +} + + +static void XzEnc_Free(CXzEnc *p, ISzAllocPtr alloc) +{ + unsigned i; + + XzEncIndex_Free(&p->xzIndex, alloc); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Free(&p->lzmaf_Items[i], alloc); + + #ifndef _7ZIP_ST + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + XzEnc_FreeOutBufs(p); + #endif +} + + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CXzEnc *p = (CXzEnc *)ISzAlloc_Alloc(alloc, sizeof(CXzEnc)); + if (!p) + return NULL; + XzEnc_Construct(p); + XzProps_Init(&p->xzProps); + XzProps_Normalize(&p->xzProps); + p->expectedDataSize = (UInt64)(Int64)-1; + p->alloc = alloc; + p->allocBig = allocBig; + return p; +} + + +void XzEnc_Destroy(CXzEncHandle pp) +{ + CXzEnc *p = (CXzEnc *)pp; + XzEnc_Free(p, p->alloc); + ISzAlloc_Free(p->alloc, p); +} + + +SRes XzEnc_SetProps(CXzEncHandle pp, const CXzProps *props) +{ + CXzEnc *p = (CXzEnc *)pp; + p->xzProps = *props; + XzProps_Normalize(&p->xzProps); + return SZ_OK; +} + + +void XzEnc_SetDataSize(CXzEncHandle pp, UInt64 expectedDataSiize) +{ + CXzEnc *p = (CXzEnc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + + + +#ifndef _7ZIP_ST + +static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CXzEnc *me = (CXzEnc *)pp; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + UNUSED_VAR(finished) + + { + CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + bInfo->totalSize = 0; + bInfo->unpackSize = 0; + bInfo->headerSize = 0; + } + + if (!dest) + { + dest = ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + progressThunk.index = coderIndex; + + { + CXzEncBlockInfo blockSizes; + int inStreamFinished; + + res = Xz_CompressBlock( + &me->lzmaf_Items[coderIndex], + + NULL, + dest, + dest + XZ_BLOCK_HEADER_SIZE_MAX, me->outBufSize - XZ_BLOCK_HEADER_SIZE_MAX, + + NULL, + // srcSize, // expectedSize + src, srcSize, + + &me->xzProps, + &progressThunk.vt, + &inStreamFinished, + &blockSizes, + me->alloc, + me->allocBig); + + if (res == SZ_OK) + me->EncBlocks[outBufIndex] = blockSizes; + + return res; + } +} + + +static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex) +{ + CXzEnc *me = (CXzEnc *)pp; + + const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)); + + { + UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); + RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)); + } + + return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); +} + +#endif + + + +SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) +{ + CXzEnc *p = (CXzEnc *)pp; + + const CXzProps *props = &p->xzProps; + + XzEncIndex_Init(&p->xzIndex); + { + UInt64 numBlocks = 1; + UInt64 blockSize = props->blockSize; - XzBlock_ClearFlags(&block); - XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); + if (blockSize != XZ_PROPS__BLOCK_SIZE__SOLID + && props->reduceSize != (UInt64)(Int64)-1) + { + numBlocks = props->reduceSize / blockSize; + if (numBlocks * blockSize != props->reduceSize) + numBlocks++; + } + else + blockSize = (UInt64)1 << 62; - if (fp) + RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc)); + } + + RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream)); + + + #ifndef _7ZIP_ST + if (props->numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) { - filter = &block.filters[filterIndex++]; - filter->id = fp->id; - filter->propsSize = 0; - - if (fp->id == XZ_ID_Delta) - { - filter->props[0] = (Byte)(fp->delta - 1); - filter->propsSize = 1; - } - else if (fp->ipDefined) - { - SetUi32(filter->props, fp->ip); - filter->propsSize = 4; - } + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); } + vt.Code = XzEnc_MtCallback_Code; + vt.Write = XzEnc_MtCallback_Write; + + p->checkType = props->checkId; + p->xzProps = *props; + + p->outStream = outStream; + + p->mtCoder.allocBig = p->allocBig; + p->mtCoder.progress = progress; + p->mtCoder.inStream = inStream; + p->mtCoder.inData = NULL; + p->mtCoder.inDataSize = 0; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + if ( props->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID + || props->blockSize == XZ_PROPS__BLOCK_SIZE__AUTO) + return SZ_ERROR_FAIL; + + p->mtCoder.blockSize = (size_t)props->blockSize; + if (p->mtCoder.blockSize != props->blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ + { - CXzFilter *f = &block.filters[filterIndex++]; - f->id = XZ_ID_LZMA2; - f->propsSize = 1; - f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); + size_t destBlockSize = XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(p->mtCoder.blockSize); + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + XzEnc_FreeOutBufs(p); + p->outBufSize = destBlockSize; } - seqSizeOutStream.vt.Write = MyWrite; - seqSizeOutStream.realStream = outStream; - seqSizeOutStream.processed = 0; + p->mtCoder.numThreadsMax = props->numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; - RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + RINOK(MtCoder_Code(&p->mtCoder)); + } + else + #endif + { + int writeStartSizes; + CCompressProgress_XzEncOffset progress2; + Byte *bufData = NULL; + size_t bufSize = 0; + + progress2.vt.Progress = CompressProgress_XzEncOffset_Progress; + progress2.inOffset = 0; + progress2.outOffset = 0; + progress2.progress = progress; - checkInStream.vt.Read = SeqCheckInStream_Read; - checkInStream.realStream = inStream; - SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags)); + writeStartSizes = 0; - if (fp) + if (props->blockSize != XZ_PROPS__BLOCK_SIZE__SOLID) { - #ifdef USE_SUBBLOCK - if (fp->id == XZ_ID_Subblock) - { - lzmaf->sb.inStream = &checkInStream.vt; - RINOK(SbEncInStream_Init(&lzmaf->sb)); - } - else - #endif + writeStartSizes = (props->forceWriteSizesInHeader > 0); + + if (writeStartSizes) { - lzmaf->filter.realStream = &checkInStream.vt; - RINOK(SeqInFilter_Init(&lzmaf->filter, filter)); + size_t t2; + size_t t = (size_t)props->blockSize; + if (t != props->blockSize) + return SZ_ERROR_PARAM; + t = XZ_GET_MAX_BLOCK_PACK_SIZE(t); + if (t < props->blockSize) + return SZ_ERROR_PARAM; + t2 = XZ_BLOCK_HEADER_SIZE_MAX + t; + if (!p->outBufs[0] || t2 != p->outBufSize) + { + XzEnc_FreeOutBufs(p); + p->outBufs[0] = ISzAlloc_Alloc(p->alloc, t2); + if (!p->outBufs[0]) + return SZ_ERROR_MEM; + p->outBufSize = t2; + } + bufData = p->outBufs[0] + XZ_BLOCK_HEADER_SIZE_MAX; + bufSize = t; } } - + + for (;;) { - UInt64 packPos = seqSizeOutStream.processed; + CXzEncBlockInfo blockSizes; + int inStreamFinished; - SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.vt, - fp ? - #ifdef USE_SUBBLOCK - (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt: - #endif - &lzmaf->filter.p: - &checkInStream.vt, - progress); + /* + UInt64 rem = (UInt64)(Int64)-1; + if (props->reduceSize != (UInt64)(Int64)-1 + && props->reduceSize >= progress2.inOffset) + rem = props->reduceSize - progress2.inOffset; + */ - RINOK(res); - block.unpackSize = checkInStream.processed; - block.packSize = seqSizeOutStream.processed - packPos; - } + RINOK(Xz_CompressBlock( + &p->lzmaf_Items[0], + + writeStartSizes ? NULL : outStream, + writeStartSizes ? p->outBufs[0] : NULL, + bufData, bufSize, + + inStream, + // rem, + NULL, 0, + + props, + progress ? &progress2.vt : NULL, + &inStreamFinished, + &blockSizes, + p->alloc, + p->allocBig)); - { - unsigned padSize = 0; - Byte buf[128]; - while ((((unsigned)block.packSize + padSize) & 3) != 0) - buf[padSize++] = 0; - SeqCheckInStream_GetDigest(&checkInStream, buf + padSize); - RINOK(WriteBytes(&seqSizeOutStream.vt, buf, padSize + XzFlags_GetCheckSize(xz->flags))); - RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc)); + { + UInt64 totalPackFull = blockSizes.totalSize + XZ_GET_PAD_SIZE(blockSizes.totalSize); + + if (writeStartSizes) + { + RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize)); + RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize)); + } + + RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc)); + + progress2.inOffset += blockSizes.unpackSize; + progress2.outOffset += totalPackFull; + } + + if (inStreamFinished) + break; } } - return Xz_WriteFooter(xz, outStream); + + return XzEncIndex_WriteFooter(&p->xzIndex, (CXzStreamFlags)props->checkId, outStream); } +#include "Alloc.h" + SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, const CXzProps *props, ICompressProgress *progress) { SRes res; - CXzStream xz; - CLzma2WithFilters lzmaf; - Xz_Construct(&xz); - Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc); - res = Lzma2WithFilters_Create(&lzmaf); + CXzEncHandle xz = XzEnc_Create(&g_Alloc, &g_BigAlloc); + if (!xz) + return SZ_ERROR_MEM; + res = XzEnc_SetProps(xz, props); if (res == SZ_OK) - res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress); - Lzma2WithFilters_Free(&lzmaf); - Xz_Free(&xz, &g_Alloc); + res = XzEnc_Encode(xz, outStream, inStream, progress); + XzEnc_Destroy(xz); return res; } @@ -526,11 +1308,11 @@ SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, SRes Xz_EncodeEmpty(ISeqOutStream *outStream) { SRes res; - CXzStream xz; - Xz_Construct(&xz); - res = Xz_WriteHeader(xz.flags, outStream); + CXzEncIndex xzIndex; + XzEncIndex_Construct(&xzIndex); + res = Xz_WriteHeader((CXzStreamFlags)0, outStream); if (res == SZ_OK) - res = Xz_WriteFooter(&xz, outStream); - Xz_Free(&xz, &g_Alloc); + res = XzEncIndex_WriteFooter(&xzIndex, (CXzStreamFlags)0, outStream); + XzEncIndex_Free(&xzIndex, NULL); // g_Alloc return res; } diff --git a/C/XzEnc.h b/C/XzEnc.h index e9cea34f..529ac3fd 100644 --- a/C/XzEnc.h +++ b/C/XzEnc.h @@ -1,5 +1,5 @@ /* XzEnc.h -- Xz Encode -2011-02-07 : Igor Pavlov : Public domain */ +2017-06-27 : Igor Pavlov : Public domain */ #ifndef __XZ_ENC_H #define __XZ_ENC_H @@ -10,6 +10,11 @@ EXTERN_C_BEGIN + +#define XZ_PROPS__BLOCK_SIZE__AUTO LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO +#define XZ_PROPS__BLOCK_SIZE__SOLID LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + + typedef struct { UInt32 id; @@ -20,15 +25,31 @@ typedef struct void XzFilterProps_Init(CXzFilterProps *p); + typedef struct { - const CLzma2EncProps *lzma2Props; - const CXzFilterProps *filterProps; + CLzma2EncProps lzma2Props; + CXzFilterProps filterProps; unsigned checkId; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; + int numTotalThreads; + int forceWriteSizesInHeader; + UInt64 reduceSize; } CXzProps; void XzProps_Init(CXzProps *p); + +typedef void * CXzEncHandle; + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); +void XzEnc_Destroy(CXzEncHandle p); +SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props); +void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize); +SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); + SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, const CXzProps *props, ICompressProgress *progress); diff --git a/C/XzIn.c b/C/XzIn.c index c6c2a040..30054c4a 100644 --- a/C/XzIn.c +++ b/C/XzIn.c @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2017-04-03 : Igor Pavlov : Public domain */ +2017-05-11 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,6 +9,12 @@ #include "CpuArch.h" #include "Xz.h" +/* +#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0) +*/ +#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1) + + SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) { Byte sig[XZ_STREAM_HEADER_SIZE]; @@ -28,7 +34,7 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt unsigned headerSize; *headerSizeRes = 0; RINOK(SeqInStream_ReadByte(inStream, &header[0])); - headerSize = ((unsigned)header[0] << 2) + 4; + headerSize = (unsigned)header[0]; if (headerSize == 0) { *headerSizeRes = 1; @@ -37,12 +43,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt } *isIndex = False; + headerSize = (headerSize << 2) + 4; *headerSizeRes = headerSize; RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1)); return XzBlock_Parse(p, header); } -#define ADD_SIZE_CHECH(size, val) \ +#define ADD_SIZE_CHECK(size, val) \ { UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } UInt64 Xz_GetUnpackSize(const CXzStream *p) @@ -50,7 +57,7 @@ UInt64 Xz_GetUnpackSize(const CXzStream *p) UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECH(size, p->blocks[i].unpackSize); + ADD_SIZE_CHECK(size, p->blocks[i].unpackSize); return size; } @@ -59,7 +66,7 @@ UInt64 Xz_GetPackSize(const CXzStream *p) UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECH(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); + ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); return size; } @@ -96,9 +103,8 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt { size_t i; p->numBlocks = numBlocks; - p->numBlocksAllocated = numBlocks; p->blocks = ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); - if (p->blocks == 0) + if (!p->blocks) return SZ_ERROR_MEM; for (i = 0; i < numBlocks; i++) { @@ -126,7 +132,7 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, if (size != indexSize) return SZ_ERROR_UNSUPPORTED; buf = ISzAlloc_Alloc(alloc, size); - if (buf == 0) + if (!buf) return SZ_ERROR_MEM; res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED); if (res == SZ_OK) @@ -154,7 +160,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff pos -= XZ_STREAM_FOOTER_SIZE; RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0) + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) { UInt32 total = 0; pos += XZ_STREAM_FOOTER_SIZE; @@ -187,7 +193,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff return SZ_ERROR_NO_ARCHIVE; pos -= XZ_STREAM_FOOTER_SIZE; RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0) + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) return SZ_ERROR_NO_ARCHIVE; } @@ -262,7 +268,7 @@ UInt64 Xzs_GetUnpackSize(const CXzs *p) UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECH(size, Xz_GetUnpackSize(&p->streams[i])); + ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i])); return size; } @@ -272,7 +278,7 @@ UInt64 Xzs_GetPackSize(const CXzs *p) UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECH(size, Xz_GetTotalSize(&p->streams[i])); + ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i])); return size; } */ @@ -294,7 +300,7 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr { size_t newNum = p->num + p->num / 4 + 1; Byte *data = (Byte *)ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream)); - if (data == 0) + if (!data) return SZ_ERROR_MEM; p->numAllocated = newNum; if (p->num != 0) diff --git a/CPP/7zip/Archive/7z/7zEncode.cpp b/CPP/7zip/Archive/7z/7zEncode.cpp index 60b1ba6b..0222831c 100644 --- a/CPP/7zip/Archive/7z/7zEncode.cpp +++ b/CPP/7zip/Archive/7z/7zEncode.cpp @@ -370,6 +370,17 @@ HRESULT CEncoder::Encode( resetInitVector->ResetInitVector(); } + { + CMyComPtr optProps; + coder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void **)&optProps); + if (optProps) + { + PROPID propID = NCoderPropID::kExpectedDataSize; + NWindows::NCOM::CPropVariant prop = (UInt64)unpackSize; + RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1)); + } + } + CMyComPtr writeCoderProperties; coder->QueryInterface(IID_ICompressWriteCoderProperties, (void **)&writeCoderProperties); @@ -380,7 +391,7 @@ HRESULT CEncoder::Encode( CDynBufSeqOutStream *outStreamSpec = new CDynBufSeqOutStream; CMyComPtr dynOutStream(outStreamSpec); outStreamSpec->Init(); - writeCoderProperties->WriteCoderProperties(dynOutStream); + RINOK(writeCoderProperties->WriteCoderProperties(dynOutStream)); outStreamSpec->CopyToBuffer(props); } else diff --git a/CPP/7zip/Archive/7z/7zFolderInStream.cpp b/CPP/7zip/Archive/7z/7zFolderInStream.cpp index 51a6f689..eee11a08 100644 --- a/CPP/7zip/Archive/7z/7zFolderInStream.cpp +++ b/CPP/7zip/Archive/7z/7zFolderInStream.cpp @@ -80,14 +80,17 @@ STDMETHODIMP CFolderInStream::Read(void *data, UInt32 size, UInt32 *processedSiz { if (_stream) { - UInt32 processed2; - RINOK(_stream->Read(data, size, &processed2)); - if (processed2 != 0) + UInt32 cur = size; + const UInt32 kMax = (UInt32)1 << 20; + if (cur > kMax) + cur = kMax; + RINOK(_stream->Read(data, cur, &cur)); + if (cur != 0) { - _crc = CrcUpdate(_crc, data, processed2); - _pos += processed2; + _crc = CrcUpdate(_crc, data, cur); + _pos += cur; if (processedSize) - *processedSize = processed2; + *processedSize = cur; return S_OK; } diff --git a/CPP/7zip/Archive/7z/7zOut.cpp b/CPP/7zip/Archive/7z/7zOut.cpp index 20b1ef77..5bd3a417 100644 --- a/CPP/7zip/Archive/7z/7zOut.cpp +++ b/CPP/7zip/Archive/7z/7zOut.cpp @@ -515,7 +515,7 @@ HRESULT COutArchive::EncodeStream( outFolders.FolderUnpackCRCs.Vals.Add(CrcCalc(data, data.Size())); // outFolders.NumUnpackStreamsVector.Add(1); UInt64 dataSize64 = data.Size(); - UInt64 unpackSize; + UInt64 unpackSize = data.Size(); RINOK(encoder.Encode( EXTERNAL_CODECS_LOC_VARS stream, diff --git a/CPP/7zip/Archive/7z/7zUpdate.cpp b/CPP/7zip/Archive/7z/7zUpdate.cpp index 3630f146..c416b35f 100644 --- a/CPP/7zip/Archive/7z/7zUpdate.cpp +++ b/CPP/7zip/Archive/7z/7zUpdate.cpp @@ -2171,6 +2171,7 @@ HRESULT Update( #endif } + curUnpackSize = sizeToEncode; HRESULT encodeRes = encoder.Encode( EXTERNAL_CODECS_LOC_VARS @@ -2358,7 +2359,9 @@ HRESULT Update( inStreamSpec->Init(updateCallback, &indices[i], numSubFiles); unsigned startPackIndex = newDatabase.PackSizes.Size(); - UInt64 curFolderUnpackSize; + UInt64 curFolderUnpackSize = totalSize; + // curFolderUnpackSize = (UInt64)(Int64)-1; + RINOK(encoder.Encode( EXTERNAL_CODECS_LOC_VARS solidInStream, diff --git a/CPP/7zip/Archive/ExtHandler.cpp b/CPP/7zip/Archive/ExtHandler.cpp index dea12c55..0d46cd63 100644 --- a/CPP/7zip/Archive/ExtHandler.cpp +++ b/CPP/7zip/Archive/ExtHandler.cpp @@ -461,8 +461,8 @@ bool CHeader::Parse(const Byte *p) if (Is64Bit()) { - HI_32(150, NumBlocks); - // HI_32(154, NumBlocksSuper); + HI_32(0x150, NumBlocks); + // HI_32(0x154, NumBlocksSuper); HI_32(0x158, NumFreeBlocks); } diff --git a/CPP/7zip/Archive/Nsis/NsisIn.cpp b/CPP/7zip/Archive/Nsis/NsisIn.cpp index de4a3c31..64463ceb 100644 --- a/CPP/7zip/Archive/Nsis/NsisIn.cpp +++ b/CPP/7zip/Archive/Nsis/NsisIn.cpp @@ -2489,7 +2489,7 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p) if (c2 == NS_3_CODE_VAR) // if (c2 <= NS_3_CODE_SKIP && c2 != NS_3_CODE_SHELL && c2 != 1) { - if ((strData[i+ 2] & 0x80) != 0) + if ((strData[i + 2] & 0x80) != 0) { // const char *p2 = (const char *)(strData + i + 1); // p2 = p2; @@ -5048,6 +5048,12 @@ HRESULT CInArchive::Parse() DetectNsisType(bhEntries, _data + bhEntries.Offset); Decoder.IsNsisDeflate = (NsisType != k_NsisType_Nsis3); + + // some NSIS files (that are not detected as k_NsisType_Nsis3) + // use original (non-NSIS) Deflate + // How to detect these cases? + + // Decoder.IsNsisDeflate = false; #ifdef NSIS_SCRIPT diff --git a/CPP/7zip/Archive/NtfsHandler.cpp b/CPP/7zip/Archive/NtfsHandler.cpp index b66c620c..288da156 100644 --- a/CPP/7zip/Archive/NtfsHandler.cpp +++ b/CPP/7zip/Archive/NtfsHandler.cpp @@ -341,15 +341,19 @@ bool CVolInfo::Parse(const Byte *p, unsigned size) struct CAttr { UInt32 Type; + + Byte NonResident; + + // Non-Resident + Byte CompressionUnit; + // UInt32 Len; UString2 Name; // UInt16 Flags; // UInt16 Instance; CByteBuffer Data; - Byte NonResident; // Non-Resident - Byte CompressionUnit; UInt64 LowVcn; UInt64 HighVcn; UInt64 AllocatedSize; @@ -408,15 +412,16 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size) return 8; // required size is 4, but attributes are 8 bytes aligned. So we return 8 if (size < 0x18) return 0; + PRF(printf(" T=%2X", Type)); - UInt32 len = Get32(p + 0x04); + UInt32 len = Get32(p + 4); PRF(printf(" L=%3d", len)); if (len > size) return 0; if ((len & 7) != 0) return 0; - NonResident = p[0x08]; + NonResident = p[8]; { unsigned nameLen = p[9]; UInt32 nameOffset = Get16(p + 0x0A); @@ -437,6 +442,7 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size) UInt32 dataSize; UInt32 offs; + if (NonResident) { if (len < 0x40) @@ -472,16 +478,19 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size) { if (len < 0x18) return 0; + G32(p + 0x10, dataSize); + G16(p + 0x14, offs); + // G16(p + 0x16, ResidentFlags); PRF(printf(" RES")); - dataSize = Get32(p + 0x10); PRF(printf(" dataSize=%3d", dataSize)); - offs = Get16(p + 0x14); - // G16(p + 0x16, ResidentFlags); // PRF(printf(" ResFlags=%4X", ResidentFlags)); } + if (offs > len || dataSize > len || len - dataSize < offs) return 0; + Data.CopyFrom(p + offs, dataSize); + #ifdef SHOW_DEBUG_INFO PRF(printf(" : ")); for (unsigned i = 0; i < Data.Size(); i++) @@ -489,16 +498,19 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size) PRF(printf(" %02X", (unsigned)Data[i])); } #endif + return len; } + bool CAttr::ParseExtents(CRecordVector &extents, UInt64 numClustersMax, unsigned compressionUnit) const { const Byte *p = Data; unsigned size = (unsigned)Data.Size(); UInt64 vcn = LowVcn; UInt64 lcn = 0; - UInt64 highVcn1 = HighVcn + 1; + const UInt64 highVcn1 = HighVcn + 1; + if (LowVcn != extents.Back().Virt || highVcn1 > (UInt64)1 << 63) return false; @@ -528,15 +540,30 @@ bool CAttr::ParseExtents(CRecordVector &extents, UInt64 numClustersMax, if ((highVcn1 - vcn) < vSize) return false; + CExtent e; + e.Virt = vcn; + vcn += vSize; + num = (b >> 4) & 0xF; if (num > 8 || num > size) return false; - CExtent e; - e.Virt = vcn; + if (num == 0) { + // Sparse + + /* if Unit is compressed, it can have many Elements for each compressed Unit: + and last Element for unit MUST be without LCN. + Element 0: numCompressedClusters2, LCN_0 + Element 1: numCompressedClusters2, LCN_1 + ... + Last Element : (16 - total_clusters_in_previous_elements), no LCN + */ + + // sparse is not allowed for (compressionUnit == 0) ? Why ? if (compressionUnit == 0) return false; + e.Phy = kEmptyExtent; } else @@ -553,9 +580,10 @@ bool CAttr::ParseExtents(CRecordVector &extents, UInt64 numClustersMax, return false; e.Phy = lcn; } + extents.Add(e); - vcn += vSize; } + CExtent e; e.Phy = kEmptyExtent; e.Virt = vcn; @@ -563,10 +591,11 @@ bool CAttr::ParseExtents(CRecordVector &extents, UInt64 numClustersMax, return (highVcn1 == vcn); } + static const UInt64 kEmptyTag = (UInt64)(Int64)-1; static const unsigned kNumCacheChunksLog = 1; -static const size_t kNumCacheChunks = (1 << kNumCacheChunksLog); +static const size_t kNumCacheChunks = (size_t)1 << kNumCacheChunksLog; class CInStream: public IInStream, @@ -734,24 +763,27 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) while (_curRem == 0) { - UInt64 cacheTag = _virtPos >> _chunkSizeLog; - UInt32 cacheIndex = (UInt32)cacheTag & (kNumCacheChunks - 1); + const UInt64 cacheTag = _virtPos >> _chunkSizeLog; + const size_t cacheIndex = (size_t)cacheTag & (kNumCacheChunks - 1); + if (_tags[cacheIndex] == cacheTag) { - UInt32 chunkSize = (UInt32)1 << _chunkSizeLog; - UInt32 offset = (UInt32)_virtPos & (chunkSize - 1); - UInt32 cur = MyMin(chunkSize - offset, size); + const size_t chunkSize = (size_t)1 << _chunkSizeLog; + const size_t offset = (size_t)_virtPos & (chunkSize - 1); + size_t cur = chunkSize - offset; + if (cur > size) + cur = size; memcpy(data, _outBuf + (cacheIndex << _chunkSizeLog) + offset, cur); - *processedSize = cur; + *processedSize = (UInt32)cur; _virtPos += cur; return S_OK; } PRF2(printf("\nVirtPos = %6d", _virtPos)); - UInt32 comprUnitSize = (UInt32)1 << CompressionUnit; - UInt64 virtBlock = _virtPos >> BlockSizeLog; - UInt64 virtBlock2 = virtBlock & ~((UInt64)comprUnitSize - 1); + const UInt32 comprUnitSize = (UInt32)1 << CompressionUnit; + const UInt64 virtBlock = _virtPos >> BlockSizeLog; + const UInt64 virtBlock2 = virtBlock & ~((UInt64)comprUnitSize - 1); unsigned left = 0, right = Extents.Size(); for (;;) @@ -766,7 +798,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) } bool isCompressed = false; - UInt64 virtBlock2End = virtBlock2 + comprUnitSize; + const UInt64 virtBlock2End = virtBlock2 + comprUnitSize; if (CompressionUnit != 0) for (unsigned i = left; i < Extents.Size(); i++) { @@ -802,7 +834,9 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) _curRem = next - _virtPos; break; } + bool thereArePhy = false; + for (unsigned i2 = left; i2 < Extents.Size(); i2++) { const CExtent &e = Extents[i2]; @@ -814,6 +848,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) break; } } + if (!thereArePhy) { _curRem = (Extents[i + 1].Virt << BlockSizeLog) - _virtPos; @@ -823,6 +858,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) size_t offs = 0; UInt64 curVirt = virtBlock2; + for (i = left; i < Extents.Size(); i++) { const CExtent &e = Extents[i]; @@ -845,6 +881,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) _physPos += compressed; offs += compressed; } + size_t destLenMax = GetCuSize(); size_t destLen = destLenMax; const UInt64 rem = Size - (virtBlock2 << BlockSizeLog); @@ -863,6 +900,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) return S_FALSE; } } + if (size > _curRem) size = (UInt32)_curRem; HRESULT res = S_OK; @@ -913,6 +951,12 @@ static HRESULT DataParseExtents(unsigned clusterSizeLog, const CObjectVector> clusterSizeLog)) + { + } + */ + if (attr0.AllocatedSize < attr0.Size || (attrs[attrIndexLim - 1].HighVcn + 1) != (attr0.AllocatedSize >> clusterSizeLog) || (attr0.AllocatedSize & ((1 << clusterSizeLog) - 1)) != 0) diff --git a/CPP/7zip/Archive/Rar/Rar5Handler.cpp b/CPP/7zip/Archive/Rar/Rar5Handler.cpp index cf834f33..637af8ce 100644 --- a/CPP/7zip/Archive/Rar/Rar5Handler.cpp +++ b/CPP/7zip/Archive/Rar/Rar5Handler.cpp @@ -1363,7 +1363,7 @@ static const Byte kProps[] = kpidCharacts, kpidSymLink, kpidHardLink, - kpidCopyLink, + kpidCopyLink }; diff --git a/CPP/7zip/Archive/SquashfsHandler.cpp b/CPP/7zip/Archive/SquashfsHandler.cpp index 4f4e5848..77cd64a7 100644 --- a/CPP/7zip/Archive/SquashfsHandler.cpp +++ b/CPP/7zip/Archive/SquashfsHandler.cpp @@ -60,6 +60,7 @@ UInt64 Get64b(const Byte *p, bool be) { return be ? GetBe64(p) : GetUi64(p); } static const UInt32 kSignature32_LE = 0x73717368; static const UInt32 kSignature32_BE = 0x68737173; static const UInt32 kSignature32_LZ = 0x71736873; +static const UInt32 kSignature32_B2 = 0x73687371; #define kMethod_ZLIB 1 #define kMethod_LZMA 2 @@ -225,6 +226,7 @@ struct CHeader case kSignature32_LE: break; case kSignature32_BE: be = true; break; case kSignature32_LZ: SeveralMethods = true; break; + case kSignature32_B2: SeveralMethods = true; be = true; break; default: return false; } GET_32 (4, NumInodes); @@ -2258,7 +2260,8 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) static const Byte k_Signature[] = { 4, 'h', 's', 'q', 's', 4, 's', 'q', 's', 'h', - 4, 's', 'h', 's', 'q' }; + 4, 's', 'h', 's', 'q', + 4, 'q', 's', 'h', 's' }; REGISTER_ARC_I( "SquashFS", "squashfs", 0, 0xD2, diff --git a/CPP/7zip/Archive/Tar/TarIn.cpp b/CPP/7zip/Archive/Tar/TarIn.cpp index 7d41d4e6..a830a0da 100644 --- a/CPP/7zip/Archive/Tar/TarIn.cpp +++ b/CPP/7zip/Archive/Tar/TarIn.cpp @@ -26,24 +26,25 @@ static void MyStrNCpy(char *dest, const char *src, unsigned size) } } -static bool OctalToNumber(const char *srcString, unsigned size, UInt64 &res) +static bool OctalToNumber(const char *srcString, unsigned size, UInt64 &res, bool allowEmpty = false) { + res = 0; char sz[32]; MyStrNCpy(sz, srcString, size); sz[size] = 0; const char *end; unsigned i; for (i = 0; sz[i] == ' '; i++); + if (sz[i] == 0) + return allowEmpty; res = ConvertOctStringToUInt64(sz + i, &end); - if (end == sz + i) - return false; return (*end == ' ' || *end == 0); } -static bool OctalToNumber32(const char *srcString, unsigned size, UInt32 &res) +static bool OctalToNumber32(const char *srcString, unsigned size, UInt32 &res, bool allowEmpty = false) { UInt64 res64; - if (!OctalToNumber(srcString, size, res64)) + if (!OctalToNumber(srcString, size, res64, allowEmpty)) return false; res = (UInt32)res64; return (res64 <= 0xFFFFFFFF); @@ -123,7 +124,8 @@ API_FUNC_IsArc IsArc_Tar(const Byte *p2, size_t size) p += NFileHeader::kNameSize; UInt32 mode; - CHECK(OctalToNumber32(p, 8, mode)); p += 8; + // we allow empty Mode value for LongName prefix items + CHECK(OctalToNumber32(p, 8, mode, true)); p += 8; // if (!OctalToNumber32(p, 8, item.UID)) item.UID = 0; p += 8; @@ -194,7 +196,8 @@ static HRESULT GetNextItemReal(ISequentialInStream *stream, bool &filled, CItemE (item.Name.Len() == NFileHeader::kNameSize || item.Name.Len() == NFileHeader::kNameSize - 1); - RIF(OctalToNumber32(p, 8, item.Mode)); p += 8; + // we allow empty Mode value for LongName prefix items + RIF(OctalToNumber32(p, 8, item.Mode, true)); p += 8; if (!OctalToNumber32(p, 8, item.UID)) item.UID = 0; p += 8; if (!OctalToNumber32(p, 8, item.GID)) item.GID = 0; p += 8; diff --git a/CPP/7zip/Archive/XzHandler.cpp b/CPP/7zip/Archive/XzHandler.cpp index 26d1e10b..59d1d753 100644 --- a/CPP/7zip/Archive/XzHandler.cpp +++ b/CPP/7zip/Archive/XzHandler.cpp @@ -7,8 +7,11 @@ #include "../../Common/ComTry.h" #include "../../Common/Defs.h" #include "../../Common/IntToString.h" +#include "../../Common/MyBuffer.h" +#include "../../Common/StringToInt.h" #include "../../Windows/PropVariant.h" +#include "../../Windows/System.h" #include "../Common/CWrappers.h" #include "../Common/ProgressUtils.h" @@ -33,9 +36,19 @@ namespace NXz { #define k_LZMA2_Name "LZMA2" +struct CBlockInfo +{ + unsigned StreamFlags; + UInt64 PackPos; + UInt64 PackSize; // pure value from Index record, it doesn't include pad zeros + UInt64 UnpackPos; +}; + + class CHandler: public IInArchive, public IArchiveOpenSeq, + public IInArchiveGetStream, #ifndef EXTRACT_ONLY public IOutArchive, public ISetProperties, @@ -48,9 +61,7 @@ class CHandler: bool _isArc; bool _needSeekToStart; bool _phySize_Defined; - - CMyComPtr _stream; - CMyComPtr _seqStream; + bool _firstBlockWasRead; AString _methodsString; @@ -58,8 +69,20 @@ class CHandler: UInt32 _filterId; + UInt64 _numSolidBytes; + + HRESULT SetSolidFromString(const UString &s); + HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value); + HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); + + void InitSolid() + { + _numSolidBytes = XZ_PROPS__BLOCK_SIZE__AUTO; + } + void Init() { + InitSolid(); _filterId = 0; CMultiMethodProps::Init(); } @@ -83,6 +106,7 @@ class CHandler: public: MY_QUERYINTERFACE_BEGIN2(IInArchive) MY_QUERYINTERFACE_ENTRY(IArchiveOpenSeq) + MY_QUERYINTERFACE_ENTRY(IInArchiveGetStream) #ifndef EXTRACT_ONLY MY_QUERYINTERFACE_ENTRY(IOutArchive) MY_QUERYINTERFACE_ENTRY(ISetProperties) @@ -92,22 +116,45 @@ class CHandler: INTERFACE_IInArchive(;) STDMETHOD(OpenSeq)(ISequentialInStream *stream); + STDMETHOD(GetStream)(UInt32 index, ISequentialInStream **stream); #ifndef EXTRACT_ONLY INTERFACE_IOutArchive(;) STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); #endif + size_t _blocksArraySize; + CBlockInfo *_blocks; + UInt64 _maxBlocksSize; + CMyComPtr _stream; + CMyComPtr _seqStream; + + CXzBlock _firstBlock; + CHandler(); + ~CHandler(); + + HRESULT SeekToPackPos(UInt64 pos) + { + return _stream->Seek(pos, STREAM_SEEK_SET, NULL); + } }; -CHandler::CHandler() + +CHandler::CHandler(): + _blocks(NULL), + _blocksArraySize(0) { #ifndef EXTRACT_ONLY Init(); #endif } +CHandler::~CHandler() +{ + MyFree(_blocks); +} + static const Byte kProps[] = { @@ -120,7 +167,9 @@ static const Byte kArcProps[] = { kpidMethod, kpidNumStreams, - kpidNumBlocks + kpidNumBlocks, + kpidClusterSize, + kpidCharacts }; IMP_IInArchive_Props @@ -177,7 +226,7 @@ static const CMethodNamePair g_NamePairs[] = { XZ_ID_LZMA2, "LZMA2" } }; -static AString GetMethodString(const CXzFilter &f) +static void AddMethodString(AString &s, const CXzFilter &f) { const char *p = NULL; for (unsigned i = 0; i < ARRAY_SIZE(g_NamePairs); i++) @@ -193,7 +242,7 @@ static AString GetMethodString(const CXzFilter &f) p = temp; } - AString s (p); + s += p; if (f.propsSize > 0) { @@ -210,13 +259,6 @@ static AString GetMethodString(const CXzFilter &f) s += ']'; } } - return s; -} - -static void AddString(AString &dest, const AString &src) -{ - dest.Add_Space_if_NotEmpty(); - dest += src; } static const char * const kChecks[] = @@ -239,27 +281,24 @@ static const char * const kChecks[] = , NULL }; -static AString GetCheckString(const CXzs &xzs) +static void AddCheckString(AString &s, const CXzs &xzs) { size_t i; UInt32 mask = 0; for (i = 0; i < xzs.num; i++) mask |= ((UInt32)1 << XzFlags_GetCheckType(xzs.streams[i].flags)); - AString s; for (i = 0; i <= XZ_CHECK_MASK; i++) if (((mask >> i) & 1) != 0) { - AString s2; + s.Add_Space_if_NotEmpty(); if (kChecks[i]) - s2 = kChecks[i]; + s += kChecks[i]; else { - s2 = "Check-"; - s2.Add_UInt32((UInt32)i); + s += "Check-"; + s.Add_UInt32((UInt32)i); } - AddString(s, s2); } - return s; } STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) @@ -272,11 +311,26 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) case kpidNumStreams: if (_stat.NumStreams_Defined) prop = _stat.NumStreams; break; case kpidNumBlocks: if (_stat.NumBlocks_Defined) prop = _stat.NumBlocks; break; case kpidUnpackSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; + case kpidClusterSize: if (_stat.NumBlocks_Defined && _stat.NumBlocks > 1) prop = _maxBlocksSize; break; + case kpidCharacts: + if (_firstBlockWasRead) + { + AString s; + if (XzBlock_HasPackSize(&_firstBlock)) + s.Add_OptSpaced("BlockPackSize"); + if (XzBlock_HasUnpackSize(&_firstBlock)) + s.Add_OptSpaced("BlockUnpackSize"); + if (!s.IsEmpty()) + prop = s; + } + break; + + case kpidMethod: if (!_methodsString.IsEmpty()) prop = _methodsString; break; case kpidErrorFlags: { UInt32 v = 0; - if (!_isArc) v |= kpv_ErrorFlags_IsNotArc;; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; if (_stat.UnexpectedEnd) v |= kpv_ErrorFlags_UnexpectedEnd; if (_stat.DataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; if (_stat.HeadersError) v |= kpv_ErrorFlags_HeadersError; @@ -284,6 +338,13 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) if (_stat.DataError) v |= kpv_ErrorFlags_DataError; if (_stat.CrcError) v |= kpv_ErrorFlags_CrcError; prop = v; + break; + } + + case kpidMainSubfile: + { + // if (_blocks) prop = (UInt32)0; + break; } } prop.Detach(value); @@ -409,9 +470,15 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal SRes res2 = XzBlock_ReadHeader(&block, &inStreamWrap.vt, &isIndex, &headerSizeRes); if (res2 == SZ_OK && !isIndex) { + _firstBlockWasRead = true; + _firstBlock = block; + unsigned numFilters = XzBlock_GetNumFilters(&block); for (unsigned i = 0; i < numFilters; i++) - AddString(_methodsString, GetMethodString(block.filters[i])); + { + _methodsString.Add_Space_if_NotEmpty(); + AddMethodString(_methodsString, block.filters[i]); + } } } } @@ -422,7 +489,7 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal RINOK(callback->SetTotal(NULL, &_stat.PhySize)); } - CSeekInStreamWrap inStreamImp;; + CSeekInStreamWrap inStreamImp; inStreamImp.Init(inStream); @@ -460,7 +527,64 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal _stat.NumBlocks = Xzs_GetNumBlocks(&xzs.p); _stat.NumBlocks_Defined = true; - AddString(_methodsString, GetCheckString(xzs.p)); + AddCheckString(_methodsString, xzs.p); + + const size_t numBlocks = (size_t)_stat.NumBlocks + 1; + const size_t bytesAlloc = numBlocks * sizeof(CBlockInfo); + + if (bytesAlloc / sizeof(CBlockInfo) == _stat.NumBlocks + 1) + { + _blocks = (CBlockInfo *)MyAlloc(bytesAlloc); + if (_blocks) + { + unsigned blockIndex = 0; + UInt64 unpackPos = 0; + + for (size_t si = xzs.p.num; si != 0;) + { + si--; + const CXzStream &str = xzs.p.streams[si]; + UInt64 packPos = str.startOffset + XZ_STREAM_HEADER_SIZE; + + for (size_t bi = 0; bi < str.numBlocks; bi++) + { + const CXzBlockSizes &bs = str.blocks[bi]; + const UInt64 packSizeAligned = bs.totalSize + ((0 - (unsigned)bs.totalSize) & 3); + + if (bs.unpackSize != 0) + { + if (blockIndex >= _stat.NumBlocks) + return E_FAIL; + + CBlockInfo &block = _blocks[blockIndex++]; + block.StreamFlags = str.flags; + block.PackSize = bs.totalSize; // packSizeAligned; + block.PackPos = packPos; + block.UnpackPos = unpackPos; + } + packPos += packSizeAligned; + unpackPos += bs.unpackSize; + if (_maxBlocksSize < bs.unpackSize) + _maxBlocksSize = bs.unpackSize; + } + } + + /* + if (blockIndex != _stat.NumBlocks) + { + // there are Empty blocks; + } + */ + if (_stat.OutSize != unpackPos) + return E_FAIL; + CBlockInfo &block = _blocks[blockIndex++]; + block.StreamFlags = 0; + block.PackSize = 0; + block.PackPos = 0; + block.UnpackPos = unpackPos; + _blocksArraySize = blockIndex; + } + } } else { @@ -474,6 +598,8 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal return S_OK; } + + STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCallback *callback) { COM_TRY_BEGIN @@ -499,38 +625,296 @@ STDMETHODIMP CHandler::Close() _isArc = false; _needSeekToStart = false; - _phySize_Defined = false; + _firstBlockWasRead = false; _methodsString.Empty(); _stream.Release(); _seqStream.Release(); + + MyFree(_blocks); + _blocks = NULL; + _blocksArraySize = 0; + _maxBlocksSize = 0; + return S_OK; } +struct CXzUnpackerCPP2 +{ + Byte *InBuf; + // Byte *OutBuf; + CXzUnpacker p; + + CXzUnpackerCPP2(); + ~CXzUnpackerCPP2(); +}; + +CXzUnpackerCPP2::CXzUnpackerCPP2(): InBuf(NULL) + // , OutBuf(NULL) +{ + XzUnpacker_Construct(&p, &g_Alloc); +} -class CSeekToSeqStream: +CXzUnpackerCPP2::~CXzUnpackerCPP2() +{ + XzUnpacker_Free(&p); + MidFree(InBuf); + // MidFree(OutBuf); +} + + +class CInStream: public IInStream, public CMyUnknownImp { public: - CMyComPtr Stream; + UInt64 _virtPos; + UInt64 Size; + UInt64 _cacheStartPos; + size_t _cacheSize; + CByteBuffer _cache; + // UInt64 _startPos; + CXzUnpackerCPP2 xz; + + void InitAndSeek() + { + _virtPos = 0; + _cacheStartPos = 0; + _cacheSize = 0; + // _startPos = startPos; + } + + CHandler *_handlerSpec; + CMyComPtr _handler; + MY_UNKNOWN_IMP1(IInStream) STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); + + ~CInStream(); }; -STDMETHODIMP CSeekToSeqStream::Read(void *data, UInt32 size, UInt32 *processedSize) + +CInStream::~CInStream() { - return Stream->Read(data, size, processedSize); + // _cache.Free(); +} + + +size_t FindBlock(const CBlockInfo *blocks, size_t numBlocks, UInt64 pos) +{ + size_t left = 0, right = numBlocks; + for (;;) + { + size_t mid = (left + right) / 2; + if (mid == left) + return left; + if (pos < blocks[mid].UnpackPos) + right = mid; + else + left = mid; + } } -STDMETHODIMP CSeekToSeqStream::Seek(Int64, UInt32, UInt64 *) { return E_NOTIMPL; } +static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, + ISequentialInStream *seqInStream, + unsigned streamFlags, + UInt64 packSize, // pure size from Index record, it doesn't include pad zeros + size_t unpackSize, Byte *dest + // , ICompressProgressInfo *progress + ) +{ + const size_t kInBufSize = (size_t)1 << 16; + XzUnpacker_Init(&xzu.p); + + if (!xzu.InBuf) + { + xzu.InBuf = (Byte *)MidAlloc(kInBufSize); + if (!xzu.InBuf) + return E_OUTOFMEMORY; + } + + xzu.p.streamFlags = (UInt16)streamFlags; + XzUnpacker_PrepareToRandomBlockDecoding(&xzu.p); + + const UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + UInt64 packRem = packSizeAligned; + + UInt32 inSize = 0; + SizeT inPos = 0; + SizeT outPos = 0; + + HRESULT readRes = S_OK; + + for (;;) + { + if (inPos == inSize && readRes == S_OK) + { + inPos = 0; + inSize = 0; + UInt32 rem = kInBufSize; + if (rem > packRem) + rem = (UInt32)packRem; + if (rem != 0) + readRes = seqInStream->Read(xzu.InBuf, rem, &inSize); + } + + SizeT inLen = inSize - inPos; + SizeT outLen = unpackSize - outPos; + + ECoderStatus status; + + SRes res = XzUnpacker_Code(&xzu.p, + dest + outPos, &outLen, + xzu.InBuf + inPos, &inLen, + CODER_FINISH_END, &status); + + // return E_OUTOFMEMORY; + // res = SZ_ERROR_CRC; + + if (res != SZ_OK) + { + if (res == SZ_ERROR_CRC) + return S_FALSE; + return SResToHRESULT(res); + } + + inPos += inLen; + outPos += outLen; + + packRem -= inLen; + + Bool blockFinished = XzUnpacker_IsBlockFinished(&xzu.p); + + if ((inLen == 0 && outLen == 0) || blockFinished) + { + if (packRem != 0 || !blockFinished || unpackSize != outPos) + return S_FALSE; + if (XzUnpacker_GetPackSizeForIndex(&xzu.p) != packSize) + return S_FALSE; + return S_OK; + } + } +} + + +STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) +{ + COM_TRY_BEGIN + + if (processedSize) + *processedSize = 0; + if (size == 0) + return S_OK; + + { + if (_virtPos >= Size) + return S_OK; // (Size == _virtPos) ? S_OK: E_FAIL; + { + UInt64 rem = Size - _virtPos; + if (size > rem) + size = (UInt32)rem; + } + } + + if (size == 0) + return S_OK; + + if (_virtPos < _cacheStartPos || _virtPos >= _cacheStartPos + _cacheSize) + { + size_t bi = FindBlock(_handlerSpec->_blocks, _handlerSpec->_blocksArraySize, _virtPos); + const CBlockInfo &block = _handlerSpec->_blocks[bi]; + const UInt64 unpackSize = _handlerSpec->_blocks[bi + 1].UnpackPos - block.UnpackPos; + if (_cache.Size() < unpackSize) + return E_FAIL; + + _cacheSize = 0; + + RINOK(_handlerSpec->SeekToPackPos(block.PackPos)); + RINOK(DecodeBlock(xz, _handlerSpec->_seqStream, block.StreamFlags, block.PackSize, + (size_t)unpackSize, _cache)); + _cacheStartPos = block.UnpackPos; + _cacheSize = (size_t)unpackSize; + } + + { + size_t offset = (size_t)(_virtPos - _cacheStartPos); + size_t rem = _cacheSize - offset; + if (size > rem) + size = (UInt32)rem; + memcpy(data, _cache + offset, size); + _virtPos += size; + if (processedSize) + *processedSize = size; + return S_OK; + } + + COM_TRY_END +} + + +STDMETHODIMP CInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition) +{ + switch (seekOrigin) + { + case STREAM_SEEK_SET: break; + case STREAM_SEEK_CUR: offset += _virtPos; break; + case STREAM_SEEK_END: offset += Size; break; + default: return STG_E_INVALIDFUNCTION; + } + if (offset < 0) + return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; + _virtPos = offset; + if (newPosition) + *newPosition = offset; + return S_OK; +} + + + +static const UInt64 kMaxBlockSize_for_GetStream = (UInt64)1 << 40; + +STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) +{ + COM_TRY_BEGIN + + *stream = NULL; + + if (index != 0) + return E_INVALIDARG; + + if (!_stat.UnpackSize_Defined + || _maxBlocksSize > kMaxBlockSize_for_GetStream + || _maxBlocksSize != (size_t)_maxBlocksSize) + return S_FALSE; + + UInt64 physSize = (UInt64)(sizeof(size_t)) << 29; + bool ramSize_Defined = NSystem::GetRamSize(physSize); + if (ramSize_Defined) + { + if (_maxBlocksSize > physSize / 4) + return S_FALSE; + } + + CInStream *spec = new CInStream; + CMyComPtr specStream = spec; + spec->_cache.Alloc((size_t)_maxBlocksSize); + spec->_handlerSpec = this; + spec->_handler = (IInArchive *)this; + spec->Size = _stat.OutSize; + spec->InitAndSeek(); + + *stream = specStream.Detach(); + return S_OK; + + COM_TRY_END +} @@ -584,6 +968,8 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, COM_TRY_END } + + #ifndef EXTRACT_ONLY STDMETHODIMP CHandler::GetFileTimeType(UInt32 *timeType) @@ -592,16 +978,15 @@ STDMETHODIMP CHandler::GetFileTimeType(UInt32 *timeType) return S_OK; } + STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, IArchiveUpdateCallback *updateCallback) { COM_TRY_BEGIN - if (numItems == 0) { CSeqOutStreamWrap seqOutStream; - seqOutStream.Init(outStream); SRes res = Xz_EncodeEmpty(&seqOutStream.vt); return SResToHRESULT(res); @@ -641,82 +1026,76 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt NCompress::NXz::CEncoder *encoderSpec = new NCompress::NXz::CEncoder; CMyComPtr encoder = encoderSpec; - CLzma2EncProps &lzma2Props = encoderSpec->_lzma2Props; - lzma2Props.lzmaProps.level = GetLevel(); + CXzProps &xzProps = encoderSpec->xzProps; + CLzma2EncProps &lzma2Props = xzProps.lzma2Props; - CMyComPtr fileInStream; - RINOK(updateCallback->GetStream(0, &fileInStream)); + lzma2Props.lzmaProps.level = GetLevel(); + xzProps.reduceSize = size; + /* { NCOM::CPropVariant prop = (UInt64)size; RINOK(encoderSpec->SetCoderProp(NCoderPropID::kReduceSize, prop)); } + */ - FOR_VECTOR (i, _methods) + #ifndef _7ZIP_ST + xzProps.numTotalThreads = _numThreads; + #endif + + xzProps.blockSize = _numSolidBytes; + if (_numSolidBytes == XZ_PROPS__BLOCK_SIZE__SOLID) { - COneMethodInfo &m = _methods[i]; + xzProps.lzma2Props.blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + } - /* - SetGlobalLevelTo(m); - #ifndef _7ZIP_ST - CMultiMethodProps::SetMethodThreads(m, _numThreads); - #endif - */ + RINOK(encoderSpec->SetCheckSize(_crcSize)); + { + CXzFilterProps &filter = xzProps.filterProps; + + if (_filterId == XZ_ID_Delta) { - FOR_VECTOR (j, m.Props) + bool deltaDefined = false; + FOR_VECTOR (j, _filterMethod.Props) { - const CProp &prop = m.Props[j]; - RINOK(encoderSpec->SetCoderProp(prop.Id, prop.Value)); + const CProp &prop = _filterMethod.Props[j]; + if (prop.Id == NCoderPropID::kDefaultProp && prop.Value.vt == VT_UI4) + { + UInt32 delta = (UInt32)prop.Value.ulVal; + if (delta < 1 || delta > 256) + return E_INVALIDARG; + filter.delta = delta; + deltaDefined = true; + } + else + return E_INVALIDARG; } + if (!deltaDefined) + return E_INVALIDARG; } + filter.id = _filterId; } - #ifndef _7ZIP_ST - lzma2Props.numTotalThreads = _numThreads; - #endif - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; - lps->Init(updateCallback, true); - - CXzProps &xzProps = encoderSpec->xzProps; - CXzFilterProps &filter = encoderSpec->filter; - - XzProps_Init(&xzProps); - XzFilterProps_Init(&filter); - - xzProps.lzma2Props = &lzma2Props; - xzProps.filterProps = (_filterId != 0 ? &filter : NULL); - switch (_crcSize) - { - case 0: xzProps.checkId = XZ_CHECK_NO; break; - case 4: xzProps.checkId = XZ_CHECK_CRC32; break; - case 8: xzProps.checkId = XZ_CHECK_CRC64; break; - case 32: xzProps.checkId = XZ_CHECK_SHA256; break; - default: return E_INVALIDARG; - } - filter.id = _filterId; - if (_filterId == XZ_ID_Delta) + FOR_VECTOR (i, _methods) { - bool deltaDefined = false; - FOR_VECTOR (j, _filterMethod.Props) + COneMethodInfo &m = _methods[i]; + + FOR_VECTOR (j, m.Props) { - const CProp &prop = _filterMethod.Props[j]; - if (prop.Id == NCoderPropID::kDefaultProp && prop.Value.vt == VT_UI4) - { - UInt32 delta = (UInt32)prop.Value.ulVal; - if (delta < 1 || delta > 256) - return E_INVALIDARG; - filter.delta = delta; - deltaDefined = true; - } + const CProp &prop = m.Props[j]; + RINOK(encoderSpec->SetCoderProp(prop.Id, prop.Value)); } - if (!deltaDefined) - return E_INVALIDARG; } + CMyComPtr fileInStream; + RINOK(updateCallback->GetStream(0, &fileInStream)); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(updateCallback, true); + return encoderSpec->Code(fileInStream, outStream, NULL, NULL, progress); } @@ -747,11 +1126,83 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt } +HRESULT CHandler::SetSolidFromString(const UString &s) +{ + UString s2 = s; + s2.MakeLower_Ascii(); + + { + const wchar_t *start = ((const wchar_t *)s2); + const wchar_t *end; + UInt64 v = ConvertStringToUInt64(start, &end); + if (start == end) + return E_INVALIDARG; + if ((unsigned)(end - start) + 1 != s2.Len()) + return E_INVALIDARG; + wchar_t c = *end; + { + unsigned numBits; + switch (c) + { + case 'b': numBits = 0; break; + case 'k': numBits = 10; break; + case 'm': numBits = 20; break; + case 'g': numBits = 30; break; + case 't': numBits = 40; break; + default: return E_INVALIDARG; + } + _numSolidBytes = (v << numBits); + } + } + return S_OK; +} + + +HRESULT CHandler::SetSolidFromPROPVARIANT(const PROPVARIANT &value) +{ + bool isSolid; + switch (value.vt) + { + case VT_EMPTY: isSolid = true; break; + case VT_BOOL: isSolid = (value.boolVal != VARIANT_FALSE); break; + case VT_BSTR: + if (StringToBool(value.bstrVal, isSolid)) + break; + return SetSolidFromString(value.bstrVal); + default: return E_INVALIDARG; + } + _numSolidBytes = (isSolid ? XZ_PROPS__BLOCK_SIZE__SOLID : XZ_PROPS__BLOCK_SIZE__AUTO); + return S_OK; +} + + +HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) +{ + UString name = nameSpec; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + if (name[0] == L's') + { + name.Delete(0); + if (name.IsEmpty()) + return SetSolidFromPROPVARIANT(value); + if (value.vt != VT_EMPTY) + return E_INVALIDARG; + return SetSolidFromString(name); + } + + return CMultiMethodProps::SetProperty(name, value); +} + + STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { COM_TRY_BEGIN Init(); + for (UInt32 i = 0; i < numProps; i++) { RINOK(SetProperty(names[i], values[i])); @@ -781,7 +1232,9 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR AString &methodName = _methods[0].MethodName; if (methodName.IsEmpty()) methodName = k_LZMA2_Name; - else if (!methodName.IsEqualTo_Ascii_NoCase(k_LZMA2_Name)) + else if ( + !methodName.IsEqualTo_Ascii_NoCase(k_LZMA2_Name) + && !methodName.IsEqualTo_Ascii_NoCase("xz")) return E_INVALIDARG; } diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp index 0a97793b..a0beab26 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp @@ -36,6 +36,7 @@ static const UInt32 kLzmaHeaderSize = 4 + kLzmaPropsSize; class CLzmaEncoder: public ICompressCoder, public ICompressSetCoderProperties, + public ICompressSetCoderPropertiesOpt, public CMyUnknownImp { public: @@ -46,8 +47,11 @@ class CLzmaEncoder: STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); - MY_UNKNOWN_IMP1(ICompressSetCoderProperties) + MY_UNKNOWN_IMP2( + ICompressSetCoderProperties, + ICompressSetCoderPropertiesOpt) }; STDMETHODIMP CLzmaEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps) @@ -71,6 +75,11 @@ STDMETHODIMP CLzmaEncoder::SetCoderProperties(const PROPID *propIDs, const PROPV return S_OK; } +STDMETHODIMP CLzmaEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps) +{ + return EncoderSpec->SetCoderPropertiesOpt(propIDs, props, numProps); +} + STDMETHODIMP CLzmaEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { @@ -118,7 +127,8 @@ HRESULT CAddCommon::CalcStreamCRC(ISequentialInStream *inStream, UInt32 &resultC } -HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, CCompressingResult &opRes) const +HRESULT CAddCommon::Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize, + CCompressingResult &opRes) const { // We use Zip64, if unPackSize size is larger than 0xF8000000 to support // cases when compressed size can be about 3% larger than uncompressed size @@ -144,7 +154,7 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, C opRes.LzmaEos = false; opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default; - opRes.FileTimeWasUsed = false; + opRes.DescriptorMode = outSeqMode; if (_options.PasswordIsDefined) { @@ -153,8 +163,8 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, C opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Aes; else { - if (seqMode) - opRes.FileTimeWasUsed = true; + if (inSeqMode) + opRes.DescriptorMode = true; } } @@ -187,7 +197,8 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, C HRESULT CAddCommon::Compress( DECL_EXTERNAL_CODECS_LOC_VARS ISequentialInStream *inStream, IOutStream *outStream, - bool seqMode, UInt32 fileTime, + bool inSeqMode, bool outSeqMode, + UInt32 fileTime, UInt64 expectedDataSize, ICompressProgressInfo *progress, CCompressingResult &opRes) { opRes.LzmaEos = false; @@ -202,16 +213,31 @@ HRESULT CAddCommon::Compress( CMyComPtr inCrcStream = inSecCrcStreamSpec; CMyComPtr inStream2; - if (!seqMode) + if (!inSeqMode) + { inStream->QueryInterface(IID_IInStream, (void **)&inStream2); + if (!inStream2) + { + // inSeqMode = true; + // inSeqMode must be correct before + return E_FAIL; + } + } inSecCrcStreamSpec->SetStream(inStream); inSecCrcStreamSpec->Init(); unsigned numTestMethods = _options.MethodSequence.Size(); - - if (seqMode || (numTestMethods > 1 && !inStream2)) - numTestMethods = 1; + + bool descriptorMode = outSeqMode; + if (!outSeqMode) + if (inSeqMode && _options.PasswordIsDefined && !_options.IsAesMode) + descriptorMode = true; + opRes.DescriptorMode = descriptorMode; + + if (numTestMethods > 1) + if (inSeqMode || outSeqMode || !inStream2) + numTestMethods = 1; UInt32 crc = 0; bool crc_IsCalculated = false; @@ -219,19 +245,23 @@ HRESULT CAddCommon::Compress( Byte method = 0; CFilterCoder::C_OutStream_Releaser outStreamReleaser; opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default; - opRes.FileTimeWasUsed = false; for (unsigned i = 0; i < numTestMethods; i++) { opRes.LzmaEos = false; opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default; - if (inStream2 && i != 0) + + if (i != 0) { - inSecCrcStreamSpec->Init(); - RINOK(inStream2->Seek(0, STREAM_SEEK_SET, NULL)); + if (inStream2) + { + inSecCrcStreamSpec->Init(); + RINOK(inStream2->Seek(0, STREAM_SEEK_SET, NULL)); + } + + RINOK(outStream->SetSize(0)); + RINOK(outStream->Seek(0, STREAM_SEEK_SET, NULL)); } - RINOK(outStream->SetSize(0)); - RINOK(outStream->Seek(0, STREAM_SEEK_SET, NULL)); if (_options.PasswordIsDefined) { @@ -264,7 +294,12 @@ HRESULT CAddCommon::Compress( UInt32 check; - if (inStream2) + if (descriptorMode) + { + // it's Info-ZIP modification for stream_mode descriptor_mode (bit 3 of the general purpose bit flag is set) + check = (fileTime & 0xFFFF); + } + else { if (!crc_IsCalculated) { @@ -275,11 +310,6 @@ HRESULT CAddCommon::Compress( } check = (crc >> 16); } - else - { - opRes.FileTimeWasUsed = true; - check = (fileTime & 0xFFFF); - } RINOK(_filterSpec->WriteHeader_Check16(outStream, (UInt16)check)); } @@ -295,7 +325,13 @@ HRESULT CAddCommon::Compress( { case NCompressionMethod::kStore: { - if (_copyCoderSpec == NULL) + if (descriptorMode) + { + // we still can create descriptor_mode archives with "Store" method, but they are not good for 100% + return E_NOTIMPL; + } + + if (!_copyCoderSpec) { _copyCoderSpec = new NCompress::CCopyCoder; _copyCoder = _copyCoderSpec; @@ -390,6 +426,18 @@ HRESULT CAddCommon::Compress( outStreamNew = outStream; if (_compressExtractVersion > opRes.ExtractVersion) opRes.ExtractVersion = _compressExtractVersion; + + { + CMyComPtr optProps; + _compressEncoder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void **)&optProps); + if (optProps) + { + PROPID propID = NCoderPropID::kExpectedDataSize; + NWindows::NCOM::CPropVariant prop = (UInt64)expectedDataSize; + RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1)); + } + } + RINOK(_compressEncoder->Code(inCrcStream, outStreamNew, NULL, NULL, progress)); break; } diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.h b/CPP/7zip/Archive/Zip/ZipAddCommon.h index d59fcc08..6f3b0f65 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.h +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.h @@ -26,7 +26,7 @@ struct CCompressingResult UInt32 CRC; UInt16 Method; Byte ExtractVersion; - bool FileTimeWasUsed; + bool DescriptorMode; bool LzmaEos; }; @@ -53,12 +53,14 @@ class CAddCommon CAddCommon(const CCompressionMethodMode &options); ~CAddCommon(); - HRESULT Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, CCompressingResult &opRes) const; + HRESULT Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize, + CCompressingResult &opRes) const; HRESULT Compress( DECL_EXTERNAL_CODECS_LOC_VARS ISequentialInStream *inStream, IOutStream *outStream, - bool seqMode, UInt32 fileTime, + bool inSeqMode, bool outSeqMode, + UInt32 fileTime, UInt64 expectedDataSize, ICompressProgressInfo *progress, CCompressingResult &opRes); }; diff --git a/CPP/7zip/Archive/Zip/ZipHandler.cpp b/CPP/7zip/Archive/Zip/ZipHandler.cpp index ad15cbf7..fda0250c 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -564,7 +564,8 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val AString s2 = FlagsToString(g_HeaderCharacts, ARRAY_SIZE(g_HeaderCharacts), flags); if (!s2.IsEmpty()) { - s.Add_OptSpaced(":"); + if (!s.IsEmpty()) + s.Add_OptSpaced(":"); s.Add_OptSpaced(s2); } } diff --git a/CPP/7zip/Archive/Zip/ZipIn.cpp b/CPP/7zip/Archive/Zip/ZipIn.cpp index 5af52279..0e4fd240 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.cpp +++ b/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -459,7 +459,16 @@ API_FUNC_IsArc IsArc_Zip(const Byte *p, size_t size) extraSize -= 4; p += 4; if (dataSize > extraSize) - return k_IsArc_Res_NO; + { + // It can be error on header. + // We want to support such rare case bad archives. + // We use additional checks to reduce false-positive probability. + if (nameSize == 0 + || nameSize > (1 << 9) + || extraSize > (1 << 9)) + return k_IsArc_Res_NO; + return k_IsArc_Res_YES; + } if (dataSize > size) return k_IsArc_Res_NEED_MORE; size -= dataSize; diff --git a/CPP/7zip/Archive/Zip/ZipItem.cpp b/CPP/7zip/Archive/Zip/ZipItem.cpp index 2f999da3..330dad40 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.cpp +++ b/CPP/7zip/Archive/Zip/ZipItem.cpp @@ -269,7 +269,25 @@ UInt32 CItem::GetWinAttrib() const case NHostOS::kUnix: // do we need to clear 16 low bits in this case? if (FromCentral) + { + /* + Some programs write posix attributes in high 16 bits of ExternalAttrib + Also some programs can write additional marker flag: + 0x8000 - p7zip + 0x4000 - Zip in MacOS + no marker - Info-Zip + + Client code has two options to detect posix field: + 1) check 0x8000 marker. In that case we must add 0x8000 marker here. + 2) check that high 4 bits (file type bits in posix field) of attributes are not zero. + */ + winAttrib = ExternalAttrib & 0xFFFF0000; + + // #ifndef _WIN32 + winAttrib |= 0x8000; // add posix mode marker + // #endif + } break; } if (IsDir()) // test it; diff --git a/CPP/7zip/Archive/Zip/ZipOut.cpp b/CPP/7zip/Archive/Zip/ZipOut.cpp index f1430646..f0573576 100644 --- a/CPP/7zip/Archive/Zip/ZipOut.cpp +++ b/CPP/7zip/Archive/Zip/ZipOut.cpp @@ -174,6 +174,9 @@ void COutArchive::WriteLocalHeader_Replace(CItemOut &item) { WriteDescriptor(item); m_OutBuffer.FlushWithCheck(); + return; + // we don't replace local header, if we write Descriptor. + // so local header with Descriptor flag must be written to local header before. } const UInt64 nextPos = m_CurPos; diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/CPP/7zip/Archive/Zip/ZipUpdate.cpp index 4a05e4b7..25518ddc 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.cpp +++ b/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -62,7 +62,7 @@ static void AddAesExtra(CItem &item, Byte aesKeyMode, UInt16 method) static void SetFileHeader( const CCompressionMethodMode &options, const CUpdateItem &ui, - // bool isSeqMode, + bool useDescriptor, CItemOut &item) { item.Size = ui.Size; @@ -94,7 +94,7 @@ static void SetFileHeader( item.InternalAttrib = 0; // test it item.SetEncrypted(!isDir && options.PasswordIsDefined); - // item.SetDescriptorMode(isSeqMode); + item.SetDescriptorMode(useDescriptor); if (isDir) { @@ -166,18 +166,22 @@ struct CThreadInfo HRESULT Result; CCompressingResult CompressingResult; - bool SeqMode; + bool InSeqMode; + bool OutSeqMode; bool IsFree; UInt32 UpdateIndex; UInt32 FileTime; + UInt64 ExpectedDataSize; CThreadInfo(const CCompressionMethodMode &options): ExitThread(false), ProgressSpec(0), OutStreamSpec(0), Coder(options), - SeqMode(false), - FileTime(0) + InSeqMode(false), + OutSeqMode(false), + FileTime(0), + ExpectedDataSize((UInt64)(Int64)-1) {} HRESULT CreateEvents() @@ -210,7 +214,9 @@ void CThreadInfo::WaitAndCode() Result = Coder.Compress( EXTERNAL_CODECS_LOC_VARS - InStream, OutStream, SeqMode, FileTime, Progress, CompressingResult); + InStream, OutStream, + InSeqMode, OutSeqMode, FileTime, ExpectedDataSize, + Progress, CompressingResult); if (Result == S_OK && Progress) Result = Progress->SetRatioInfo(&CompressingResult.UnpackSize, &CompressingResult.PackSize); @@ -237,10 +243,13 @@ class CThreads struct CMemBlocks2: public CMemLockBlocks { - CCompressingResult CompressingResult; - bool Defined; bool Skip; - CMemBlocks2(): Defined(false), Skip(false) {} + bool InSeqMode; + bool PreDescriptorMode; + bool Finished; + CCompressingResult CompressingResult; + + CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false) {} }; class CMemRefs @@ -412,7 +421,7 @@ static HRESULT UpdateItemOldData( static void WriteDirHeader(COutArchive &archive, const CCompressionMethodMode *options, const CUpdateItem &ui, CItemOut &item) { - SetFileHeader(*options, ui, item); + SetFileHeader(*options, ui, false, item); archive.WriteLocalHeader(item); } @@ -471,7 +480,7 @@ static HRESULT Update2St( CInArchive *inArchive, const CObjectVector &inputItems, CObjectVector &updateItems, - const CCompressionMethodMode *options, + const CCompressionMethodMode *options, bool outSeqMode, const CByteBuffer *comment, IArchiveUpdateCallback *updateCallback, UInt64 &totalComplexity, @@ -527,28 +536,28 @@ static HRESULT Update2St( if (!fileInStream) return E_INVALIDARG; - bool seqMode; + bool inSeqMode = false; + if (!inSeqMode) { CMyComPtr inStream2; fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2); - seqMode = (inStream2 == NULL); + inSeqMode = (inStream2 == NULL); } // seqMode = true; // to test seqMode UpdatePropsFromStream(ui, fileInStream, updateCallback, totalComplexity); - SetFileHeader(*options, ui, item); - - item.SetDescriptorMode(seqMode); - - // file Size can be 64-bit !!! CCompressingResult compressingResult; RINOK(compressor.Set_Pre_CompressionResult( - seqMode, + inSeqMode, outSeqMode, ui.Size, compressingResult)); + SetFileHeader(*options, ui, compressingResult.DescriptorMode, item); + + // file Size can be 64-bit !!! + SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item); archive.WriteLocalHeader(item); @@ -559,17 +568,12 @@ static HRESULT Update2St( RINOK(compressor.Compress( EXTERNAL_CODECS_LOC_VARS fileInStream, outStream, - seqMode, ui.Time, + inSeqMode, outSeqMode, + ui.Time, ui.Size, progress, compressingResult)); - if (compressingResult.FileTimeWasUsed) - { - /* - if (!item.HasDescriptor()) - return E_FAIL; - */ - item.SetDescriptorMode(true); - } + if (item.HasDescriptor() != compressingResult.DescriptorMode) + return E_FAIL; SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item); @@ -609,7 +613,7 @@ static HRESULT Update2( CInArchive *inArchive, const CObjectVector &inputItems, CObjectVector &updateItems, - const CCompressionMethodMode &options, + const CCompressionMethodMode &options, bool outSeqMode, const CByteBuffer *comment, IArchiveUpdateCallback *updateCallback) { @@ -743,32 +747,38 @@ static HRESULT Update2( const UInt64 averageSize = numBytesToCompress / numFilesToCompress; const UInt32 blockSize = oneMethodMain->Get_BZip2_BlockSize(); const UInt64 averageNumberOfBlocks = averageSize / blockSize + 1; - numBZip2Threads = 32; + numBZip2Threads = 64; if (numBZip2Threads > averageNumberOfBlocks) numBZip2Threads = (UInt32)averageNumberOfBlocks; + if (numBZip2Threads > numThreads) + numBZip2Threads = numThreads; oneMethodMain->AddProp_NumThreads(numBZip2Threads); } numThreads /= numBZip2Threads; } - - if (method == NFileHeader::NCompressionMethod::kXz) + else if (method == NFileHeader::NCompressionMethod::kXz) { - bool fixedNumber; - UInt32 numLzma2Threads = oneMethodMain->Get_Lzma2_NumThreads(fixedNumber); - if (!fixedNumber) + UInt32 numLzmaThreads = 1; + int numXzThreads = oneMethodMain->Get_Xz_NumThreads(numLzmaThreads); + if (numXzThreads < 0) { const UInt64 averageSize = numBytesToCompress / numFilesToCompress; - const UInt64 blockSize = oneMethodMain->Get_Lzma2_BlockSize(); - const UInt64 averageNumberOfBlocks = averageSize / blockSize + 1; - numLzma2Threads = 2; - if (numLzma2Threads > averageNumberOfBlocks) - numLzma2Threads = (UInt32)averageNumberOfBlocks; - oneMethodMain->AddProp_NumThreads(numLzma2Threads); + const UInt64 blockSize = oneMethodMain->Get_Xz_BlockSize(); + UInt64 averageNumberOfBlocks = 1; + if (blockSize != (UInt64)(Int64)-1) + averageNumberOfBlocks = averageSize / blockSize + 1; + UInt32 t = 256; + if (t > averageNumberOfBlocks) + t = (UInt32)averageNumberOfBlocks; + t *= numLzmaThreads; + if (t > numThreads) + t = numThreads; + oneMethodMain->AddProp_NumThreads(t); + numXzThreads = t; } - numThreads /= numLzma2Threads; + numThreads /= (unsigned)numXzThreads; } - - if (method == NFileHeader::NCompressionMethod::kLZMA) + else if (method == NFileHeader::NCompressionMethod::kLZMA) { // we suppose that default LZMA is 2 thread. So we don't change it UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads(); @@ -782,12 +792,14 @@ static HRESULT Update2( mtMode = false; } + // mtMode = true; // to test mtMode for seqMode + if (!mtMode) #endif return Update2St( EXTERNAL_CODECS_LOC_VARS archive, inArchive, - inputItems, updateItems, &options2, comment, updateCallback, totalComplexity, opCallback); + inputItems, updateItems, &options2, outSeqMode, comment, updateCallback, totalComplexity, opCallback); #ifndef _7ZIP_ST @@ -830,8 +842,10 @@ static HRESULT Update2( threadInfo.ProgressSpec = new CMtCompressProgress(); threadInfo.Progress = threadInfo.ProgressSpec; threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, (int)i); - threadInfo.SeqMode = false; // fix it ! - threadInfo.FileTime = 0; // fix it ! + threadInfo.InSeqMode = false; + threadInfo.OutSeqMode = false; + threadInfo.FileTime = 0; + threadInfo.ExpectedDataSize = (UInt64)(Int64)-1; RINOK(threadInfo.CreateThread()); } } @@ -840,10 +854,15 @@ static HRESULT Update2( unsigned itemIndex = 0; int lastRealStreamItemIndex = -1; + while (itemIndex < updateItems.Size()) { if (threadIndices.Size() < numThreads && mtItemIndex < updateItems.Size()) { + // we start ahead the threads for compressing + // also we set refs.Refs[itemIndex].SeqMode that is used later + // don't move that code block + CUpdateItem &ui = updateItems[mtItemIndex++]; if (!ui.NewData) continue; @@ -869,6 +888,8 @@ static HRESULT Update2( CMyComPtr fileInStream; + CMemBlocks2 &memRef2 = refs.Refs[mtItemIndex - 1]; + { NWindows::NSynchronization::CCriticalSectionLock lock(mtProgressMixerSpec->Mixer2->CriticalSection); HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream); @@ -878,7 +899,7 @@ static HRESULT Update2( complexity += kLocalHeaderSize; mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity); RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)); - refs.Refs[mtItemIndex - 1].Skip = true; + memRef2.Skip = true; continue; } RINOK(res); @@ -888,26 +909,46 @@ static HRESULT Update2( RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)); } - for (UInt32 k = 0; k < numThreads; k++) + UInt32 k; + for (k = 0; k < numThreads; k++) + if (threads.Threads[k].IsFree) + break; + + if (k == numThreads) + return E_FAIL; { - CThreadInfo &threadInfo = threads.Threads[k]; - if (threadInfo.IsFree) { + CThreadInfo &threadInfo = threads.Threads[k]; threadInfo.IsFree = false; threadInfo.InStream = fileInStream; + bool inSeqMode = false; + + if (!inSeqMode) + { + CMyComPtr inStream2; + fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2); + inSeqMode = (inStream2 == NULL); + } + memRef2.InSeqMode = inSeqMode; + // !!!!! we must release ref before sending event // BUG was here in v4.43 and v4.44. It could change ref counter in two threads in same time fileInStream.Release(); threadInfo.OutStreamSpec->Init(); threadInfo.ProgressSpec->Reinit(); - threadInfo.CompressEvent.Set(); + threadInfo.UpdateIndex = mtItemIndex - 1; - + threadInfo.InSeqMode = inSeqMode; + threadInfo.OutSeqMode = outSeqMode; + threadInfo.FileTime = ui.Time; // FileTime is used for ZipCrypto only in seqMode + threadInfo.ExpectedDataSize = ui.Size; + + threadInfo.CompressEvent.Set(); + compressingCompletedEvents.Add(threadInfo.CompressionCompletedEvent); threadIndices.Add(k); - break; } } @@ -946,15 +987,17 @@ static HRESULT Update2( { CMemBlocks2 &memRef = refs.Refs[itemIndex]; - if (memRef.Defined) + if (memRef.Finished) { if (lastRealStreamItemIndex < (int)itemIndex) lastRealStreamItemIndex = itemIndex; - SetFileHeader(options, ui, item); + SetFileHeader(options, ui, memRef.CompressingResult.DescriptorMode, item); + // the BUG was fixed in 9.26: // SetItemInfoFromCompressingResult must be after SetFileHeader // to write correct Size. + SetItemInfoFromCompressingResult(memRef.CompressingResult, options.IsRealAesMode(), options.AesKeyMode, item); archive.WriteLocalHeader(item); @@ -967,16 +1010,25 @@ static HRESULT Update2( } else { + // current file was not finished + if (lastRealStreamItemIndex < (int)itemIndex) { + // LocalHeader was not written for current itemIndex still + lastRealStreamItemIndex = itemIndex; - SetFileHeader(options, ui, item); + + // thread was started before for that item already, and memRef.SeqMode was set CCompressingResult compressingResult; RINOK(compressor.Set_Pre_CompressionResult( - false, // seqMode + memRef.InSeqMode, outSeqMode, ui.Size, compressingResult)); + + memRef.PreDescriptorMode = compressingResult.DescriptorMode; + SetFileHeader(options, ui, compressingResult.DescriptorMode, item); + SetItemInfoFromCompressingResult(compressingResult, options.IsRealAesMode(), options.AesKeyMode, item); // file Size can be 64-bit !!! @@ -1015,19 +1067,29 @@ static HRESULT Update2( if (t == 0) { + // if thread for current file was finished. + if (threadInfo.UpdateIndex != itemIndex) + return E_FAIL; + + if (memRef.PreDescriptorMode != threadInfo.CompressingResult.DescriptorMode) + return E_FAIL; + RINOK(threadInfo.OutStreamSpec->WriteToRealStream()); threadInfo.OutStreamSpec->ReleaseOutStream(); - SetFileHeader(options, ui, item); + SetFileHeader(options, ui, threadInfo.CompressingResult.DescriptorMode, item); SetItemInfoFromCompressingResult(threadInfo.CompressingResult, options.IsRealAesMode(), options.AesKeyMode, item); + archive.WriteLocalHeader_Replace(item); } else { + // it's not current file. So we must store information in array CMemBlocks2 &memRef2 = refs.Refs[threadInfo.UpdateIndex]; threadInfo.OutStreamSpec->DetachData(memRef2); memRef2.CompressingResult = threadInfo.CompressingResult; - memRef2.Defined = true; + // memRef2.SeqMode = threadInfo.SeqMode; // it was set before + memRef2.Finished = true; continue; } } @@ -1060,6 +1122,7 @@ class CCacheOutStream: public CMyUnknownImp { CMyComPtr _stream; + CMyComPtr _seqStream; Byte *_cache; UInt64 _virtPos; UInt64 _virtSize; @@ -1075,10 +1138,10 @@ class CCacheOutStream: } HRESULT FlushCache(); public: - CCacheOutStream(): _cache(0) {} + CCacheOutStream(): _cache(NULL) {} ~CCacheOutStream(); bool Allocate(); - HRESULT Init(IOutStream *stream); + HRESULT Init(ISequentialOutStream *seqStream, IOutStream *stream); MY_UNKNOWN_IMP @@ -1094,13 +1157,19 @@ bool CCacheOutStream::Allocate() return (_cache != NULL); } -HRESULT CCacheOutStream::Init(IOutStream *stream) +HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *stream) { - _virtPos = _phyPos = 0; + _virtPos = 0; + _phyPos = 0; + _virtSize = 0; + _seqStream = seqStream; _stream = stream; - RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos)); - RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize)); - RINOK(_stream->Seek(_virtPos, STREAM_SEEK_SET, &_virtPos)); + if (_stream) + { + RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos)); + RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize)); + RINOK(_stream->Seek(_virtPos, STREAM_SEEK_SET, &_virtPos)); + } _phyPos = _virtPos; _phySize = _virtSize; _cachedPos = 0; @@ -1114,12 +1183,14 @@ HRESULT CCacheOutStream::MyWrite(size_t size) { if (_phyPos != _cachedPos) { + if (!_stream) + return E_FAIL; RINOK(_stream->Seek(_cachedPos, STREAM_SEEK_SET, &_phyPos)); } size_t pos = (size_t)_cachedPos & kCacheMask; size_t curSize = MyMin(kCacheSize - pos, _cachedSize); curSize = MyMin(curSize, size); - RINOK(WriteStream(_stream, _cache + pos, curSize)); + RINOK(WriteStream(_seqStream, _cache + pos, curSize)); _phyPos += curSize; if (_phySize < _phyPos) _phySize = _phyPos; @@ -1138,10 +1209,13 @@ HRESULT CCacheOutStream::FlushCache() CCacheOutStream::~CCacheOutStream() { FlushCache(); - if (_virtSize != _phySize) - _stream->SetSize(_virtSize); - if (_virtPos != _phyPos) - _stream->Seek(_virtPos, STREAM_SEEK_SET, NULL); + if (_stream) + { + if (_virtSize != _phySize) + _stream->SetSize(_virtSize); + if (_virtPos != _phyPos) + _stream->Seek(_virtPos, STREAM_SEEK_SET, NULL); + } ::MidFree(_cache); } @@ -1250,6 +1324,8 @@ STDMETHODIMP CCacheOutStream::SetSize(UInt64 newSize) _virtSize = newSize; if (newSize < _phySize) { + if (!_stream) + return E_NOTIMPL; RINOK(_stream->SetSize(newSize)); _phySize = newSize; } @@ -1281,11 +1357,14 @@ HRESULT Update( CMyComPtr outStream; + bool outSeqMode; { CMyComPtr outStreamReal; seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStreamReal); if (!outStreamReal) - return E_NOTIMPL; + { + // return E_NOTIMPL; + } if (inArchive) { @@ -1293,7 +1372,7 @@ HRESULT Update( { IInStream *baseStream = inArchive->GetBaseStream(); RINOK(baseStream->Seek(0, STREAM_SEEK_SET, NULL)); - RINOK(NCompress::CopyStream_ExactSize(baseStream, outStreamReal, inArchive->ArcInfo.Base, NULL)); + RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, inArchive->ArcInfo.Base, NULL)); } } @@ -1301,7 +1380,8 @@ HRESULT Update( outStream = cacheStream; if (!cacheStream->Allocate()) return E_OUTOFMEMORY; - RINOK(cacheStream->Init(outStreamReal)); + RINOK(cacheStream->Init(seqOutStream, outStreamReal)); + outSeqMode = (outStreamReal == NULL); } COutArchive outArchive; @@ -1323,7 +1403,7 @@ HRESULT Update( EXTERNAL_CODECS_LOC_VARS outArchive, inArchive, inputItems, updateItems, - compressionMethodMode, + compressionMethodMode, outSeqMode, inArchive ? &inArchive->ArcInfo.Comment : NULL, updateCallback); } diff --git a/CPP/7zip/Common/CWrappers.cpp b/CPP/7zip/Common/CWrappers.cpp index baf90e26..df914cd8 100644 --- a/CPP/7zip/Common/CWrappers.cpp +++ b/CPP/7zip/Common/CWrappers.cpp @@ -28,13 +28,17 @@ HRESULT SResToHRESULT(SRes res) throw() switch (res) { case SZ_OK: return S_OK; + case SZ_ERROR_DATA: return S_FALSE; + case SZ_ERROR_CRC: return S_FALSE; case SZ_ERROR_MEM: return E_OUTOFMEMORY; case SZ_ERROR_PARAM: return E_INVALIDARG; case SZ_ERROR_PROGRESS: return E_ABORT; - case SZ_ERROR_DATA: return S_FALSE; case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; - // case SZ_ERROR_READ: return E_NOTIMPL; + // case SZ_ERROR_THREAD: return E_FAIL; + // case SZ_ERROR_READ: return E_FAIL; } + if (res < 0) + return res; return E_FAIL; } diff --git a/CPP/7zip/Common/MethodProps.cpp b/CPP/7zip/Common/MethodProps.cpp index fd618c01..d4903809 100644 --- a/CPP/7zip/Common/MethodProps.cpp +++ b/CPP/7zip/Common/MethodProps.cpp @@ -42,6 +42,14 @@ unsigned ParseStringToUInt32(const UString &srcString, UInt32 &number) return (unsigned)(end - start); } +static unsigned ParseStringToUInt64(const UString &srcString, UInt64 &number) +{ + const wchar_t *start = srcString; + const wchar_t *end; + number = ConvertStringToUInt64(start, &end); + return (unsigned)(end - start); +} + HRESULT ParsePropToUInt32(const UString &name, const PROPVARIANT &prop, UInt32 &resValue) { // =VT_UI4 @@ -263,7 +271,11 @@ static const CNameToPropID g_NameToPropID[] = { VT_UI4, "mt" }, { VT_BOOL, "eos" }, { VT_UI4, "x" }, - { VT_UI4, "reduceSize" } + { VT_UI8, "reduce" }, + { VT_UI8, "expect" }, + { VT_UI4, "b" }, + { VT_UI4, "check" }, + { VT_BSTR, "filter" } }; static int FindPropIdExact(const UString &name) @@ -345,7 +357,8 @@ static bool IsLogSizeProp(PROPID propid) case NCoderPropID::kDictionarySize: case NCoderPropID::kUsedMemorySize: case NCoderPropID::kBlockSize: - case NCoderPropID::kReduceSize: + case NCoderPropID::kBlockSize2: + // case NCoderPropID::kReduceSize: return true; } return false; @@ -378,9 +391,22 @@ HRESULT CMethodProps::SetParam(const UString &name, const UString &value) } else if (!value.IsEmpty()) { - UInt32 number; - if (ParseStringToUInt32(value, number) == value.Len()) - propValue = number; + if (nameToPropID.VarType == VT_UI4) + { + UInt32 number; + if (ParseStringToUInt32(value, number) == value.Len()) + propValue = number; + else + propValue = value; + } + else if (nameToPropID.VarType == VT_UI8) + { + UInt64 number; + if (ParseStringToUInt64(value, number) == value.Len()) + propValue = number; + else + propValue = value; + } else propValue = value; } diff --git a/CPP/7zip/Common/MethodProps.h b/CPP/7zip/Common/MethodProps.h index efdc31bb..c8a3d0dc 100644 --- a/CPP/7zip/Common/MethodProps.h +++ b/CPP/7zip/Common/MethodProps.h @@ -4,6 +4,7 @@ #define __7Z_METHOD_PROPS_H #include "../../Common/MyString.h" +#include "../../Common/Defs.h" #include "../../Windows/Defs.h" @@ -133,38 +134,50 @@ class CMethodProps: public CProps return 2; } - UInt32 Get_Lzma2_NumThreads(bool &fixedNumber) const + int Get_Xz_NumThreads(UInt32 &lzmaThreads) const { - fixedNumber = false; + lzmaThreads = 1; int numThreads = Get_NumThreads(); - if (numThreads >= 0) - { - fixedNumber = true; - if (numThreads < 1) return 1; - const unsigned kNumLzma2ThreadsMax = 32; - if (numThreads > kNumLzma2ThreadsMax) return kNumLzma2ThreadsMax; - return numThreads; - } - return 1; + if (numThreads >= 0 && numThreads <= 1) + return 1; + if (Get_Lzma_Algo() != 0) + lzmaThreads = 2; + return numThreads; } - UInt64 Get_Lzma2_BlockSize() const + UInt64 GetProp_BlockSize(PROPID id) const { - int i = FindProp(NCoderPropID::kBlockSize); + int i = FindProp(id); if (i >= 0) { const NWindows::NCOM::CPropVariant &val = Props[i].Value; - if (val.vt == VT_UI4) return val.ulVal; - if (val.vt == VT_UI8) return val.uhVal.QuadPart; + if (val.vt == VT_UI4) { return val.ulVal; } + if (val.vt == VT_UI8) { return val.uhVal.QuadPart; } } + return 0; + } - UInt32 dictSize = Get_Lzma_DicSize(); - UInt64 blockSize = (UInt64)dictSize << 2; + UInt64 Get_Xz_BlockSize() const + { + { + UInt64 blockSize1 = GetProp_BlockSize(NCoderPropID::kBlockSize); + UInt64 blockSize2 = GetProp_BlockSize(NCoderPropID::kBlockSize2); + UInt64 minSize = MyMin(blockSize1, blockSize2); + if (minSize != 0) + return minSize; + UInt64 maxSize = MyMax(blockSize1, blockSize2); + if (maxSize != 0) + return maxSize; + } const UInt32 kMinSize = (UInt32)1 << 20; const UInt32 kMaxSize = (UInt32)1 << 28; + UInt32 dictSize = Get_Lzma_DicSize(); + UInt64 blockSize = (UInt64)dictSize << 2; if (blockSize < kMinSize) blockSize = kMinSize; if (blockSize > kMaxSize) blockSize = kMaxSize; if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); return blockSize; } diff --git a/CPP/7zip/Compress/Lzma2Encoder.cpp b/CPP/7zip/Compress/Lzma2Encoder.cpp index 0328f4de..a16f9c13 100644 --- a/CPP/7zip/Compress/Lzma2Encoder.cpp +++ b/CPP/7zip/Compress/Lzma2Encoder.cpp @@ -21,18 +21,19 @@ namespace NLzma2 { CEncoder::CEncoder() { - _encoder = 0; + _encoder = NULL; _encoder = Lzma2Enc_Create(&g_Alloc, &g_BigAlloc); - if (_encoder == 0) + if (!_encoder) throw 1; } CEncoder::~CEncoder() { - if (_encoder != 0) + if (_encoder) Lzma2Enc_Destroy(_encoder); } + HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzma2Props) { switch (propID) @@ -42,12 +43,7 @@ HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzm if (prop.vt == VT_UI4) lzma2Props.blockSize = prop.ulVal; else if (prop.vt == VT_UI8) - { - size_t v = (size_t)prop.uhVal.QuadPart; - if (v != prop.uhVal.QuadPart) - return E_INVALIDARG; - lzma2Props.blockSize = v; - } + lzma2Props.blockSize = prop.uhVal.QuadPart; else return E_INVALIDARG; break; @@ -60,6 +56,7 @@ HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzm return S_OK; } + STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) { @@ -73,30 +70,52 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, return SResToHRESULT(Lzma2Enc_SetProps(_encoder, &lzma2Props)); } + +STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, + const PROPVARIANT *coderProps, UInt32 numProps) +{ + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + PROPID propID = propIDs[i]; + if (propID == NCoderPropID::kExpectedDataSize) + if (prop.vt == VT_UI8) + Lzma2Enc_SetDataSize(_encoder, prop.uhVal.QuadPart); + } + return S_OK; +} + + STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream) { Byte prop = Lzma2Enc_WriteProperties(_encoder); return WriteStream(outStream, &prop, 1); } + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) { CSeqInStreamWrap inWrap; CSeqOutStreamWrap outWrap; CCompressProgressWrap progressWrap; - + inWrap.Init(inStream); outWrap.Init(outStream); progressWrap.Init(progress); - SRes res = Lzma2Enc_Encode(_encoder, &outWrap.vt, &inWrap.vt, progress ? &progressWrap.vt : NULL); - if (res == SZ_ERROR_READ && inWrap.Res != S_OK) - return inWrap.Res; - if (res == SZ_ERROR_WRITE && outWrap.Res != S_OK) - return outWrap.Res; - if (res == SZ_ERROR_PROGRESS && progressWrap.Res != S_OK) - return progressWrap.Res; + SRes res = Lzma2Enc_Encode2(_encoder, + &outWrap.vt, NULL, NULL, + &inWrap.vt, NULL, 0, + progress ? &progressWrap.vt : NULL); + + RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + return SResToHRESULT(res); } diff --git a/CPP/7zip/Compress/Lzma2Encoder.h b/CPP/7zip/Compress/Lzma2Encoder.h index 3b3d218c..6539e73a 100644 --- a/CPP/7zip/Compress/Lzma2Encoder.h +++ b/CPP/7zip/Compress/Lzma2Encoder.h @@ -16,16 +16,22 @@ class CEncoder: public ICompressCoder, public ICompressSetCoderProperties, public ICompressWriteCoderProperties, + public ICompressSetCoderPropertiesOpt, public CMyUnknownImp { CLzma2EncHandle _encoder; public: - MY_UNKNOWN_IMP3(ICompressCoder, ICompressSetCoderProperties, ICompressWriteCoderProperties) + MY_UNKNOWN_IMP4( + ICompressCoder, + ICompressSetCoderProperties, + ICompressWriteCoderProperties, + ICompressSetCoderPropertiesOpt) STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream); + STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); CEncoder(); virtual ~CEncoder(); diff --git a/CPP/7zip/Compress/LzmaEncoder.cpp b/CPP/7zip/Compress/LzmaEncoder.cpp index c2475a42..6ce89bae 100644 --- a/CPP/7zip/Compress/LzmaEncoder.cpp +++ b/CPP/7zip/Compress/LzmaEncoder.cpp @@ -74,14 +74,19 @@ HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep) return E_INVALIDARG; return ParseMatchFinder(prop.bstrVal, &ep.btMode, &ep.numHashBytes) ? S_OK : E_INVALIDARG; } + if (propID > NCoderPropID::kReduceSize) return S_OK; + if (propID == NCoderPropID::kReduceSize) { if (prop.vt == VT_UI8) ep.reduceSize = prop.uhVal.QuadPart; + else + return E_INVALIDARG; return S_OK; } + if (prop.vt != VT_UI4) return E_INVALIDARG; UInt32 v = prop.ulVal; @@ -123,6 +128,22 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props)); } + +STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, + const PROPVARIANT *coderProps, UInt32 numProps) +{ + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + PROPID propID = propIDs[i]; + if (propID == NCoderPropID::kExpectedDataSize) + if (prop.vt == VT_UI8) + LzmaEnc_SetDataSize(_encoder, prop.uhVal.QuadPart); + } + return S_OK; +} + + STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream) { Byte props[LZMA_PROPS_SIZE]; @@ -131,6 +152,10 @@ STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream) return WriteStream(outStream, props, size); } + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) { @@ -142,15 +167,15 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream outWrap.Init(outStream); progressWrap.Init(progress); - SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, progress ? &progressWrap.vt : NULL, &g_Alloc, &g_BigAlloc); + SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, + progress ? &progressWrap.vt : NULL, &g_Alloc, &g_BigAlloc); _inputProcessed = inWrap.Processed; - if (res == SZ_ERROR_READ && inWrap.Res != S_OK) - return inWrap.Res; - if (res == SZ_ERROR_WRITE && outWrap.Res != S_OK) - return outWrap.Res; - if (res == SZ_ERROR_PROGRESS && progressWrap.Res != S_OK) - return progressWrap.Res; + + RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + return SResToHRESULT(res); } diff --git a/CPP/7zip/Compress/LzmaEncoder.h b/CPP/7zip/Compress/LzmaEncoder.h index efcac2d1..7b31c66d 100644 --- a/CPP/7zip/Compress/LzmaEncoder.h +++ b/CPP/7zip/Compress/LzmaEncoder.h @@ -16,17 +16,23 @@ class CEncoder: public ICompressCoder, public ICompressSetCoderProperties, public ICompressWriteCoderProperties, + public ICompressSetCoderPropertiesOpt, public CMyUnknownImp { CLzmaEncHandle _encoder; UInt64 _inputProcessed; public: - MY_UNKNOWN_IMP3(ICompressCoder, ICompressSetCoderProperties, ICompressWriteCoderProperties) + MY_UNKNOWN_IMP4( + ICompressCoder, + ICompressSetCoderProperties, + ICompressWriteCoderProperties, + ICompressSetCoderPropertiesOpt) STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream); + STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); CEncoder(); virtual ~CEncoder(); diff --git a/CPP/7zip/Compress/Rar3Decoder.cpp b/CPP/7zip/Compress/Rar3Decoder.cpp index 53536013..78ac0bf9 100644 --- a/CPP/7zip/Compress/Rar3Decoder.cpp +++ b/CPP/7zip/Compress/Rar3Decoder.cpp @@ -144,7 +144,8 @@ void CDecoder::ExecuteFilter(int tempFilterIndex, NVm::CBlockRef &outBlockRef) CFilter *filter = _filters[tempFilter->FilterIndex]; if (!filter->IsSupported) _unsupportedFilter = true; - _vm.Execute(filter, tempFilter, outBlockRef, filter->GlobalData); + if (!_vm.Execute(filter, tempFilter, outBlockRef, filter->GlobalData)) + _unsupportedFilter = true; delete tempFilter; _tempFilters[tempFilterIndex] = 0; } diff --git a/CPP/7zip/Compress/Rar3Vm.cpp b/CPP/7zip/Compress/Rar3Vm.cpp index 5e3c40d3..0695a789 100644 --- a/CPP/7zip/Compress/Rar3Vm.cpp +++ b/CPP/7zip/Compress/Rar3Vm.cpp @@ -155,7 +155,7 @@ bool CVm::Execute(CProgram *prg, const CProgramInitState *initState, #ifdef RARVM_STANDARD_FILTERS if (prg->StandardFilterIndex >= 0) - ExecuteStandardFilter(prg->StandardFilterIndex); + res = ExecuteStandardFilter(prg->StandardFilterIndex); else #endif { @@ -880,10 +880,10 @@ static void E8E9Decode(Byte *data, UInt32 dataSize, UInt32 fileOffset, bool e9) if (((*data++) & cmpMask) == 0xE8) { UInt32 offset = curPos + fileOffset; - UInt32 addr = (Int32)GetValue32(data); + UInt32 addr = GetValue32(data); if (addr < kFileSize) SetValue32(data, addr - offset); - else if ((Int32)addr < 0 && (Int32)(addr + offset) >= 0) + else if ((addr & 0x80000000) != 0 && ((addr + offset) & 0x80000000) == 0) SetValue32(data, addr + kFileSize); data += 4; curPos += 4; @@ -935,7 +935,7 @@ static void ItaniumDecode(Byte *data, UInt32 dataSize, UInt32 fileOffset) static void DeltaDecode(Byte *data, UInt32 dataSize, UInt32 numChannels) { UInt32 srcPos = 0; - UInt32 border = dataSize * 2; + const UInt32 border = dataSize * 2; for (UInt32 curChannel = 0; curChannel < numChannels; curChannel++) { Byte prevByte = 0; @@ -947,12 +947,13 @@ static void DeltaDecode(Byte *data, UInt32 dataSize, UInt32 numChannels) static void RgbDecode(Byte *srcData, UInt32 dataSize, UInt32 width, UInt32 posR) { Byte *destData = srcData + dataSize; - const UInt32 numChannels = 3; - for (UInt32 curChannel = 0; curChannel < numChannels; curChannel++) + const UInt32 kNumChannels = 3; + + for (UInt32 curChannel = 0; curChannel < kNumChannels; curChannel++) { Byte prevByte = 0; - for (UInt32 i = curChannel; i < dataSize; i+= numChannels) + for (UInt32 i = curChannel; i < dataSize; i += kNumChannels) { unsigned int predicted; if (i < width) @@ -978,7 +979,8 @@ static void RgbDecode(Byte *srcData, UInt32 dataSize, UInt32 width, UInt32 posR) } if (dataSize < 3) return; - for (UInt32 i = posR, border = dataSize - 2; i < border; i += 3) + const UInt32 border = dataSize - 2; + for (UInt32 i = posR; i < border; i += 3) { Byte g = destData[i + 1]; destData[i ] = (Byte)(destData[i ] + g); @@ -1064,11 +1066,11 @@ static UInt32 UpCaseDecode(Byte *data, UInt32 dataSize) } */ -void CVm::ExecuteStandardFilter(unsigned filterIndex) +bool CVm::ExecuteStandardFilter(unsigned filterIndex) { UInt32 dataSize = R[4]; if (dataSize >= kGlobalOffset) - return; + return false; EStandardFilter filterType = kStdFilters[filterIndex].Type; switch (filterType) @@ -1077,42 +1079,59 @@ void CVm::ExecuteStandardFilter(unsigned filterIndex) case SF_E8E9: E8E9Decode(Mem, dataSize, R[6], (filterType == SF_E8E9)); break; + case SF_ITANIUM: ItaniumDecode(Mem, dataSize, R[6]); break; + case SF_DELTA: + { if (dataSize >= kGlobalOffset / 2) - break; + return false; + UInt32 numChannels = R[0]; + if (numChannels == 0 || numChannels > 1024) // unrar 5.5.5 + return false; SetBlockPos(dataSize); - DeltaDecode(Mem, dataSize, R[0]); + DeltaDecode(Mem, dataSize, numChannels); break; + } + case SF_RGB: - if (dataSize >= kGlobalOffset / 2) - break; - { - UInt32 width = R[0]; - if (width <= 3) - break; - SetBlockPos(dataSize); - RgbDecode(Mem, dataSize, width, R[1]); - } + { + if (dataSize >= kGlobalOffset / 2 || dataSize < 3) // unrar 5.5.5 + return false; + UInt32 width = R[0]; + UInt32 posR = R[1]; + if (width < 3 || width - 3 > dataSize || posR > 2) // unrar 5.5.5 + return false; + SetBlockPos(dataSize); + RgbDecode(Mem, dataSize, width, posR); break; + } + case SF_AUDIO: + { if (dataSize >= kGlobalOffset / 2) - break; + return false; + UInt32 numChannels = R[0]; + if (numChannels == 0 || numChannels > 128) // unrar 5.5.5 + return false; SetBlockPos(dataSize); - AudioDecode(Mem, dataSize, R[0]); + AudioDecode(Mem, dataSize, numChannels); break; + } + /* case SF_UPCASE: if (dataSize >= kGlobalOffset / 2) - break; + return false; UInt32 destSize = UpCaseDecode(Mem, dataSize); SetBlockSize(destSize); SetBlockPos(dataSize); break; */ } + return true; } #endif diff --git a/CPP/7zip/Compress/Rar3Vm.h b/CPP/7zip/Compress/Rar3Vm.h index a9fb7b62..ec115c5f 100644 --- a/CPP/7zip/Compress/Rar3Vm.h +++ b/CPP/7zip/Compress/Rar3Vm.h @@ -173,7 +173,7 @@ class CVm #endif #ifdef RARVM_STANDARD_FILTERS - void ExecuteStandardFilter(unsigned filterIndex); + bool ExecuteStandardFilter(unsigned filterIndex); #endif Byte *Mem; diff --git a/CPP/7zip/Compress/Rar5Decoder.cpp b/CPP/7zip/Compress/Rar5Decoder.cpp index d6a90fab..dbc94d63 100644 --- a/CPP/7zip/Compress/Rar5Decoder.cpp +++ b/CPP/7zip/Compress/Rar5Decoder.cpp @@ -196,6 +196,8 @@ HRESULT CDecoder::ExecuteFilter(const CFilter &f) default: _unsupportedFilter = true; + memset(_filterSrc, 0, f.Size); + // return S_OK; // unrar } return WriteData(useDest ? @@ -301,7 +303,11 @@ HRESULT CDecoder::AddFilter(CBitDecoder &_bitStream) UInt32 blockStart = ReadUInt32(_bitStream); f.Size = ReadUInt32(_bitStream); - // if (f.Size > ((UInt32)1 << 16)) _unsupportedFilter = true; + if (f.Size > ((UInt32)1 << 22)) + { + _unsupportedFilter = true; + f.Size = 0; // unrar 5.5.5 + } f.Type = (Byte)_bitStream.ReadBits9fix(3); f.Channels = 0; diff --git a/CPP/7zip/Compress/XzDecoder.cpp b/CPP/7zip/Compress/XzDecoder.cpp index 4cca0a2d..51ecc4ee 100644 --- a/CPP/7zip/Compress/XzDecoder.cpp +++ b/CPP/7zip/Compress/XzDecoder.cpp @@ -16,6 +16,19 @@ namespace NCompress { namespace NXz { +CXzUnpackerCPP::CXzUnpackerCPP(): InBuf(NULL), OutBuf(NULL) +{ + XzUnpacker_Construct(&p, &g_Alloc); +} + +CXzUnpackerCPP::~CXzUnpackerCPP() +{ + XzUnpacker_Free(&p); + MidFree(InBuf); + MidFree(OutBuf); +} + + void CStatInfo::Clear() { InSize = 0; @@ -40,20 +53,6 @@ void CStatInfo::Clear() } -CXzUnpackerCPP::CXzUnpackerCPP(): InBuf(0), OutBuf(0) -{ - XzUnpacker_Construct(&p, &g_Alloc); -} - -CXzUnpackerCPP::~CXzUnpackerCPP() -{ - XzUnpacker_Free(&p); - MidFree(InBuf); - MidFree(OutBuf); -} - - - HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *progress) { @@ -95,8 +94,12 @@ HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream SizeT inLen = inSize - inPos; SizeT outLen = kOutBufSize - outPos; ECoderFinishMode finishMode = CODER_FINISH_ANY; + + /* + // 17.01 : the code was disabled: if (inSize == 0) finishMode = CODER_FINISH_END; + */ if (outSizeLimit) { diff --git a/CPP/7zip/Compress/XzDecoder.h b/CPP/7zip/Compress/XzDecoder.h index ba7ddee7..9a3bf518 100644 --- a/CPP/7zip/Compress/XzDecoder.h +++ b/CPP/7zip/Compress/XzDecoder.h @@ -3,13 +3,11 @@ #ifndef __XZ_DECODER_H #define __XZ_DECODER_H -#include "../../Common/MyCom.h" -#include "../ICoder.h" - #include "../../../C/Xz.h" +#include "../../Common/MyCom.h" -// #include "../../Archive/XzHandler.h" +#include "../ICoder.h" namespace NCompress { namespace NXz { diff --git a/CPP/7zip/Compress/XzEncoder.cpp b/CPP/7zip/Compress/XzEncoder.cpp index 55b42e58..458a928c 100644 --- a/CPP/7zip/Compress/XzEncoder.cpp +++ b/CPP/7zip/Compress/XzEncoder.cpp @@ -4,6 +4,9 @@ #include "../../../C/Alloc.h" +#include "../../Common/MyString.h" +#include "../../Common/StringToInt.h" + #include "../Common/CWrappers.h" #include "../Common/StreamUtils.h" @@ -19,172 +22,224 @@ HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzm namespace NXz { -extern "C" { - -static void *SzBigAlloc(ISzAllocPtr, size_t size) { return BigAlloc(size); } -static void SzBigFree(ISzAllocPtr, void *address) { BigFree(address); } -static const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; - -static void *SzAlloc(ISzAllocPtr, size_t size) { return MyAlloc(size); } -static void SzFree(ISzAllocPtr, void *address) { MyFree(address); } -static const ISzAlloc g_Alloc = { SzAlloc, SzFree }; - -} - void CEncoder::InitCoderProps() { - Lzma2EncProps_Init(&_lzma2Props); XzProps_Init(&xzProps); - XzFilterProps_Init(&filter); - - xzProps.lzma2Props = &_lzma2Props; - // xzProps.filterProps = (_filterId != 0 ? &filter : NULL); - xzProps.filterProps = NULL; } CEncoder::CEncoder() { - InitCoderProps(); + XzProps_Init(&xzProps); + _encoder = NULL; + _encoder = XzEnc_Create(&g_Alloc, &g_BigAlloc); + if (!_encoder) + throw 1; } CEncoder::~CEncoder() { + if (_encoder) + XzEnc_Destroy(_encoder); } -HRESULT CEncoder::SetCoderProp(PROPID propID, const PROPVARIANT &prop) +struct CMethodNamePair { - return NLzma2::SetLzma2Prop(propID, prop, _lzma2Props); -} + UInt32 Id; + const char *Name; +}; -STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, - const PROPVARIANT *coderProps, UInt32 numProps) +static const CMethodNamePair g_NamePairs[] = { - Lzma2EncProps_Init(&_lzma2Props); - - for (UInt32 i = 0; i < numProps; i++) + { XZ_ID_Delta, "Delta" }, + { XZ_ID_X86, "BCJ" }, + { XZ_ID_PPC, "PPC" }, + { XZ_ID_IA64, "IA64" }, + { XZ_ID_ARM, "ARM" }, + { XZ_ID_ARMT, "ARMT" }, + { XZ_ID_SPARC, "SPARC" } + // { XZ_ID_LZMA2, "LZMA2" } +}; + +static int FilterIdFromName(const wchar_t *name) +{ + for (unsigned i = 0; i < ARRAY_SIZE(g_NamePairs); i++) { - RINOK(SetCoderProp(propIDs[i], coderProps[i])); + const CMethodNamePair &pair = g_NamePairs[i]; + if (StringsAreEqualNoCase_Ascii(name, pair.Name)) + return (int)pair.Id; } - return S_OK; - // return SResToHRESULT(Lzma2Enc_SetProps(_encoder, &lzma2Props)); + return -1; } - -STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, - ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 * /* outSize */, - ICompressProgressInfo *progress) +HRESULT CEncoder::SetCheckSize(UInt32 checkSizeInBytes) { - CSeqOutStreamWrap seqOutStream; - - seqOutStream.Init(outStream); - - // if (IntToBool(newData)) + unsigned id; + switch (checkSizeInBytes) { - /* - UInt64 size; - { - NCOM::CPropVariant prop; - RINOK(updateCallback->GetProperty(0, kpidSize, &prop)); - if (prop.vt != VT_UI8) - return E_INVALIDARG; - size = prop.uhVal.QuadPart; - RINOK(updateCallback->SetTotal(size)); - } - */ + case 0: id = XZ_CHECK_NO; break; + case 4: id = XZ_CHECK_CRC32; break; + case 8: id = XZ_CHECK_CRC64; break; + case 32: id = XZ_CHECK_SHA256; break; + default: return E_INVALIDARG; + } + xzProps.checkId = id; + return S_OK; +} - /* - CLzma2EncProps lzma2Props; - Lzma2EncProps_Init(&lzma2Props); - lzma2Props.lzmaProps.level = GetLevel(); - */ +HRESULT CEncoder::SetCoderProp(PROPID propID, const PROPVARIANT &prop) +{ + if (propID == NCoderPropID::kNumThreads) + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + xzProps.numTotalThreads = (int)(prop.ulVal); + return S_OK; + } - CSeqInStreamWrap seqInStream; - - seqInStream.Init(inStream); + if (propID == NCoderPropID::kCheckSize) + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + return SetCheckSize(prop.ulVal); + } + if (propID == NCoderPropID::kBlockSize2) + { + if (prop.vt == VT_UI4) + xzProps.blockSize = prop.ulVal; + else if (prop.vt == VT_UI8) + xzProps.blockSize = prop.uhVal.QuadPart; + else + return E_INVALIDARG; + return S_OK; + } - /* + if (propID == NCoderPropID::kReduceSize) + { + if (prop.vt == VT_UI8) + xzProps.reduceSize = prop.uhVal.QuadPart; + else + return E_INVALIDARG; + return S_OK; + } + + if (propID == NCoderPropID::kFilter) + { + if (prop.vt == VT_UI4) { - NCOM::CPropVariant prop = (UInt64)size; - RINOK(NCompress::NLzma2::SetLzma2Prop(NCoderPropID::kReduceSize, prop, lzma2Props)); + UInt32 id32 = prop.ulVal; + if (id32 == XZ_ID_Delta) + return E_INVALIDARG; + xzProps.filterProps.id = prop.ulVal; } - - FOR_VECTOR (i, _methods) + else { - COneMethodInfo &m = _methods[i]; - SetGlobalLevelAndThreads(m - #ifndef _7ZIP_ST - , _numThreads - #endif - ); + if (prop.vt != VT_BSTR) + return E_INVALIDARG; + + const wchar_t *name = prop.bstrVal; + const wchar_t *end; + + UInt32 id32 = ConvertStringToUInt32(name, &end); + + if (end != name) + name = end; + else { - FOR_VECTOR (j, m.Props) + if (IsString1PrefixedByString2_NoCase_Ascii(name, "Delta")) { - const CProp &prop = m.Props[j]; - RINOK(NCompress::NLzma2::SetLzma2Prop(prop.Id, prop.Value, lzma2Props)); + name += 5; // strlen("Delta"); + id32 = XZ_ID_Delta; } - } - } - - #ifndef _7ZIP_ST - lzma2Props.numTotalThreads = _numThreads; - #endif - - */ - - CCompressProgressWrap progressWrap; - - progressWrap.Init(progress); - - xzProps.checkId = XZ_CHECK_CRC32; - // xzProps.checkId = XZ_CHECK_CRC64; - /* - CXzProps xzProps; - CXzFilterProps filter; - XzProps_Init(&xzProps); - XzFilterProps_Init(&filter); - xzProps.lzma2Props = &_lzma2Props; - */ - /* - xzProps.filterProps = (_filterId != 0 ? &filter : NULL); - switch (_crcSize) - { - case 0: xzProps.checkId = XZ_CHECK_NO; break; - case 4: xzProps.checkId = XZ_CHECK_CRC32; break; - case 8: xzProps.checkId = XZ_CHECK_CRC64; break; - case 32: xzProps.checkId = XZ_CHECK_SHA256; break; - default: return E_INVALIDARG; - } - filter.id = _filterId; - if (_filterId == XZ_ID_Delta) - { - bool deltaDefined = false; - FOR_VECTOR (j, _filterMethod.Props) - { - const CProp &prop = _filterMethod.Props[j]; - if (prop.Id == NCoderPropID::kDefaultProp && prop.Value.vt == VT_UI4) + else { - UInt32 delta = (UInt32)prop.Value.ulVal; - if (delta < 1 || delta > 256) + int filterId = FilterIdFromName(prop.bstrVal); + if (filterId < 0 /* || filterId == XZ_ID_LZMA2 */) return E_INVALIDARG; - filter.delta = delta; - deltaDefined = true; + id32 = filterId; } } - if (!deltaDefined) - return E_INVALIDARG; + + if (id32 == XZ_ID_Delta) + { + wchar_t c = *name; + if (c != '-' && c != ':') + return E_INVALIDARG; + name++; + UInt32 delta = ConvertStringToUInt32(name, &end); + if (end == name || *end != 0 || delta == 0 || delta > 256) + return E_INVALIDARG; + xzProps.filterProps.delta = delta; + } + + xzProps.filterProps.id = id32; } - */ - SRes res = Xz_Encode(&seqOutStream.vt, &seqInStream.vt, &xzProps, progress ? &progressWrap.vt : NULL); - /* - if (res == SZ_OK) - return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); - */ - return SResToHRESULT(res); + + return S_OK; + } + + return NLzma2::SetLzma2Prop(propID, prop, xzProps.lzma2Props); +} + + +STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, + const PROPVARIANT *coderProps, UInt32 numProps) +{ + XzProps_Init(&xzProps); + + for (UInt32 i = 0; i < numProps; i++) + { + RINOK(SetCoderProp(propIDs[i], coderProps[i])); + } + + return S_OK; + // return SResToHRESULT(XzEnc_SetProps(_encoder, &xzProps)); +} + + +STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, + const PROPVARIANT *coderProps, UInt32 numProps) +{ + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + PROPID propID = propIDs[i]; + if (propID == NCoderPropID::kExpectedDataSize) + if (prop.vt == VT_UI8) + XzEnc_SetDataSize(_encoder, prop.uhVal.QuadPart); } + return S_OK; +} + + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + +STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) +{ + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; + + inWrap.Init(inStream); + outWrap.Init(outStream); + progressWrap.Init(progress); + + SRes res = XzEnc_SetProps(_encoder, &xzProps); + if (res == SZ_OK) + res = XzEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, progress ? &progressWrap.vt : NULL); + + // SRes res = Xz_Encode(&outWrap.vt, &inWrap.vt, &xzProps, progress ? &progressWrap.vt : NULL); + + RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + + return SResToHRESULT(res); } }} diff --git a/CPP/7zip/Compress/XzEncoder.h b/CPP/7zip/Compress/XzEncoder.h index 4db68c4c..79d81f79 100644 --- a/CPP/7zip/Compress/XzEncoder.h +++ b/CPP/7zip/Compress/XzEncoder.h @@ -16,24 +16,26 @@ namespace NXz { class CEncoder: public ICompressCoder, public ICompressSetCoderProperties, + public ICompressSetCoderPropertiesOpt, public CMyUnknownImp { - // CXzEncHandle _encoder; + CXzEncHandle _encoder; public: - CLzma2EncProps _lzma2Props; - CXzProps xzProps; - CXzFilterProps filter; - MY_UNKNOWN_IMP2(ICompressCoder, ICompressSetCoderProperties) + MY_UNKNOWN_IMP3( + ICompressCoder, + ICompressSetCoderProperties, + ICompressSetCoderPropertiesOpt) void InitCoderProps(); - + HRESULT SetCheckSize(UInt32 checkSizeInBytes); HRESULT SetCoderProp(PROPID propID, const PROPVARIANT &prop); STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); CEncoder(); virtual ~CEncoder(); diff --git a/CPP/7zip/Guid.txt b/CPP/7zip/Guid.txt index 6314bb34..e896fe65 100644 --- a/CPP/7zip/Guid.txt +++ b/CPP/7zip/Guid.txt @@ -40,6 +40,7 @@ 04 ICompressProgressInfo 05 ICompressCoder 18 ICompressCoder2 + 1F ICompressSetCoderPropertiesOpt 20 ICompressSetCoderProperties 21 ICompressSetDecoderProperties // 22 ICompressSetDecoderProperties2 diff --git a/CPP/7zip/ICoder.h b/CPP/7zip/ICoder.h index c1e1b838..c0fe27e9 100644 --- a/CPP/7zip/ICoder.h +++ b/CPP/7zip/ICoder.h @@ -104,25 +104,40 @@ namespace NCoderPropID enum EEnum { kDefaultProp = 0, - kDictionarySize, - kUsedMemorySize, - kOrder, - kBlockSize, - kPosStateBits, - kLitContextBits, - kLitPosBits, - kNumFastBytes, - kMatchFinder, - kMatchFinderCycles, - kNumPasses, - kAlgorithm, - kNumThreads, - kEndMarker, - kLevel, - kReduceSize // estimated size of data that will be compressed. Encoder can use this value to reduce dictionary size. + kDictionarySize, // VT_UI4 + kUsedMemorySize, // VT_UI4 + kOrder, // VT_UI4 + kBlockSize, // VT_UI4 or VT_UI8 + kPosStateBits, // VT_UI4 + kLitContextBits, // VT_UI4 + kLitPosBits, // VT_UI4 + kNumFastBytes, // VT_UI4 + kMatchFinder, // VT_BSTR + kMatchFinderCycles, // VT_UI4 + kNumPasses, // VT_UI4 + kAlgorithm, // VT_UI4 + kNumThreads, // VT_UI4 + kEndMarker, // VT_BOOL + kLevel, // VT_UI4 + kReduceSize, // VT_UI8 : it's estimated size of largest data stream that will be compressed + // encoder can use this value to reduce dictionary size and allocate data buffers + + kExpectedDataSize, // VT_UI8 : for ICompressSetCoderPropertiesOpt : + // it's estimated size of current data stream + // real data size can differ from that size + // encoder can use this value to optimize encoder initialization + + kBlockSize2, // VT_UI4 or VT_UI8 + kCheckSize, // VT_UI4 : size of digest in bytes + kFilter // VT_BSTR }; } +CODER_INTERFACE(ICompressSetCoderPropertiesOpt, 0x1F) +{ + STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps) PURE; +}; + CODER_INTERFACE(ICompressSetCoderProperties, 0x20) { STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps) PURE; diff --git a/CPP/7zip/UI/Agent/Agent.cpp b/CPP/7zip/UI/Agent/Agent.cpp index fef455ff..32ef8026 100644 --- a/CPP/7zip/UI/Agent/Agent.cpp +++ b/CPP/7zip/UI/Agent/Agent.cpp @@ -1453,7 +1453,12 @@ STDMETHODIMP CAgentFolder::Extract(const UInt32 *indices, pathMode = NExtract::NPathMode::kNoPathnames; */ - extractCallbackSpec->InitForMulti(false, pathMode, overwriteMode); + extractCallbackSpec->InitForMulti( + false, // multiArchives + pathMode, + overwriteMode, + true // keepEmptyDirPrefixes + ); if (extractCallback2) extractCallback2->SetTotal(_agentSpec->GetArc().GetEstmatedPhySize()); @@ -1717,7 +1722,12 @@ STDMETHODIMP CAgent::Extract( COM_TRY_BEGIN CArchiveExtractCallback *extractCallbackSpec = new CArchiveExtractCallback; CMyComPtr extractCallback = extractCallbackSpec; - extractCallbackSpec->InitForMulti(false, pathMode, overwriteMode); + extractCallbackSpec->InitForMulti( + false, // multiArchives + pathMode, + overwriteMode, + true // keepEmptyDirPrefixes + ); CExtractNtOptions extractNtOptions; extractNtOptions.AltStreams.Val = true; // change it!!! diff --git a/CPP/7zip/UI/Client7z/Client7z.cpp b/CPP/7zip/UI/Client7z/Client7z.cpp index 11f22758..775c8ef3 100644 --- a/CPP/7zip/UI/Client7z/Client7z.cpp +++ b/CPP/7zip/UI/Client7z/Client7z.cpp @@ -479,7 +479,7 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 operationResult) } _outFileStream.Release(); if (_extractMode && _processedFileInfo.AttribDefined) - SetFileAttrib(_diskFilePath, _processedFileInfo.Attrib); + SetFileAttrib_PosixHighDetect(_diskFilePath, _processedFileInfo.Attrib); PrintNewLine(); return S_OK; } diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp index 8c3f47ad..5dd06d3c 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp @@ -988,7 +988,7 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) || !pathParts.IsEmpty() || !(_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt)) #endif - Correct_FsPath(_pathMode == NExtract::NPathMode::kAbsPaths, pathParts, _item.MainIsDir); + Correct_FsPath(_pathMode == NExtract::NPathMode::kAbsPaths, _keepAndReplaceEmptyDirPrefixes, pathParts, _item.MainIsDir); #ifdef SUPPORT_ALT_STREAMS @@ -1542,7 +1542,7 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) NumFiles++; if (!_stdOutMode && _extractMode && _fi.AttribDefined) - SetFileAttrib(_diskFilePath, _fi.Attrib); + SetFileAttrib_PosixHighDetect(_diskFilePath, _fi.Attrib); RINOK(_extractCallback2->SetOperationResult(opRes, BoolToInt(_encrypted))); diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.h b/CPP/7zip/UI/Common/ArchiveExtractCallback.h index 75ca1296..ec857fa6 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.h +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.h @@ -172,6 +172,7 @@ class CArchiveExtractCallback: FString _dirPathPrefix_Full; NExtract::NPathMode::EEnum _pathMode; NExtract::NOverwriteMode::EEnum _overwriteMode; + bool _keepAndReplaceEmptyDirPrefixes; // replace them to "_"; #ifndef _SFX @@ -278,11 +279,13 @@ class CArchiveExtractCallback: void InitForMulti(bool multiArchives, NExtract::NPathMode::EEnum pathMode, - NExtract::NOverwriteMode::EEnum overwriteMode) + NExtract::NOverwriteMode::EEnum overwriteMode, + bool keepAndReplaceEmptyDirPrefixes) { _multiArchives = multiArchives; _pathMode = pathMode; _overwriteMode = overwriteMode; + _keepAndReplaceEmptyDirPrefixes = keepAndReplaceEmptyDirPrefixes; NumFolders = NumFiles = NumAltStreams = UnpackSize = AltStreams_UnpackSize = 0; } diff --git a/CPP/7zip/UI/Common/Bench.cpp b/CPP/7zip/UI/Common/Bench.cpp index 2da88b59..f86e6e1e 100644 --- a/CPP/7zip/UI/Common/Bench.cpp +++ b/CPP/7zip/UI/Common/Bench.cpp @@ -2376,15 +2376,7 @@ static UInt32 GetNumThreadsNext(unsigned i, UInt32 numThreads) static bool AreSameMethodNames(const char *fullName, const char *shortName) { - for (;;) - { - char c2 = *shortName++; - if (c2 == 0) - return true; - char c1 = *fullName++; - if (MyCharLower_Ascii(c1) != MyCharLower_Ascii(c2)) - return false; - } + return StringsAreEqualNoCase_Ascii(fullName, shortName); } @@ -2545,27 +2537,24 @@ static const char * const k_PF[] = #endif -static void PrintSize(AString &s, UInt64 ptr) +static void PrintSize(AString &s, UInt64 v) { - UInt64 v = ptr; - UInt64 t = v >> 40; - UInt64 g = v >> 30; - UInt64 m = v >> 20; - UInt64 k = v >> 10; - UInt32 d = (UInt32)v; char c = 0; - if (t >= 1000) { d = (UInt32)t; c = 'T'; } - else if (g >= 1000) { d = (UInt32)g; c = 'G'; } - else if (m >= 1000) { d = (UInt32)m; c = 'M'; } - else if (k >= 10) { d = (UInt32)k; c = 'K'; } - - s.Add_UInt32(d); - // s += ' '; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'K'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'M'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'G'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'T'; + }}}} + else + { + PrintHex(s, v); + return; + } + char temp[32]; + ConvertUInt64ToString(v, temp); + s += temp; if (c) s += c; - - - // PrintHex(s, (DWORD_PTR)v); } @@ -2631,12 +2620,19 @@ static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si) s += " "; DWORD_PTR minAdd = (DWORD_PTR)si.lpMinimumApplicationAddress; - if (minAdd != (1 << 16)) + UInt64 maxSize = (UInt64)(DWORD_PTR)si.lpMaximumApplicationAddress + 1; + const UInt32 kReserveSize = ((UInt32)1 << 16); + if (minAdd != kReserveSize) { PrintSize(s, minAdd); s += "-"; } - PrintSize(s, (UInt64)(DWORD_PTR)si.lpMaximumApplicationAddress + 1); + else + { + if ((maxSize & (kReserveSize - 1)) == 0) + maxSize += kReserveSize; + } + PrintSize(s, maxSize); } #ifndef _WIN64 @@ -3006,26 +3002,34 @@ HRESULT Bench( UInt32 complexity = 10000; const UInt32 *checkSum = NULL; { - for (unsigned i = 0; i < ARRAY_SIZE(g_Hash); i++) + unsigned i; + for (i = 0; i < ARRAY_SIZE(g_Hash); i++) { const CBenchHash &h = g_Hash[i]; - AString s (h.Name); - AString hProp; - int propPos = s.Find(':'); + AString benchMethod (h.Name); + AString benchProps; + int propPos = benchMethod.Find(':'); if (propPos >= 0) { - hProp = s.Ptr(propPos + 1); - s.DeleteFrom(propPos); + benchProps = benchMethod.Ptr(propPos + 1); + benchMethod.DeleteFrom(propPos); } - if (AreSameMethodNames(s, methodName)) + if (AreSameMethodNames(benchMethod, methodName)) { - complexity = h.Complex; - checkSum = &h.CheckSum; - if (method.PropsString.IsEqualTo_Ascii_NoCase(hProp)) - break; + if (benchProps.IsEmpty() + || benchMethod.IsEqualTo_Ascii_NoCase("crc32") && benchProps == "8" && method.PropsString.IsEmpty() + || method.PropsString.IsPrefixedBy_Ascii_NoCase(benchProps)) + { + complexity = h.Complex; + checkSum = &h.CheckSum; + if (method.PropsString.IsEqualTo_Ascii_NoCase(benchProps)) + break; + } } } + if (i == ARRAY_SIZE(g_Hash)) + return E_NOTIMPL; } f.NewLine(); @@ -3316,18 +3320,35 @@ HRESULT Bench( bool needSetComplexity = true; if (!methodName.IsEqualTo_Ascii_NoCase("LZMA")) { - for (unsigned i = 0; i < ARRAY_SIZE(g_Bench); i++) + unsigned i; + for (i = 0; i < ARRAY_SIZE(g_Bench); i++) { const CBenchMethod &h = g_Bench[i]; - if (AreSameMethodNames(h.Name, methodName)) + AString benchMethod (h.Name); + AString benchProps; + int propPos = benchMethod.Find(':'); + if (propPos >= 0) { - callback.BenchProps.EncComplex = h.EncComplex; - callback.BenchProps.DecComplexCompr = h.DecComplexCompr; - callback.BenchProps.DecComplexUnc = h.DecComplexUnc;; - needSetComplexity = false; - break; + benchProps = benchMethod.Ptr(propPos + 1); + benchMethod.DeleteFrom(propPos); + } + + if (AreSameMethodNames(benchMethod, methodName)) + { + if (benchProps.IsEmpty() + || benchProps == "x5" && method.PropsString.IsEmpty() + || method.PropsString.IsPrefixedBy_Ascii_NoCase(benchProps)) + { + callback.BenchProps.EncComplex = h.EncComplex; + callback.BenchProps.DecComplexCompr = h.DecComplexCompr; + callback.BenchProps.DecComplexUnc = h.DecComplexUnc;; + needSetComplexity = false; + break; + } } } + if (i == ARRAY_SIZE(g_Bench)) + return E_NOTIMPL; } if (needSetComplexity) callback.BenchProps.SetLzmaCompexity(); diff --git a/CPP/7zip/UI/Common/CompressCall.cpp b/CPP/7zip/UI/Common/CompressCall.cpp index eef0320e..d00923b1 100644 --- a/CPP/7zip/UI/Common/CompressCall.cpp +++ b/CPP/7zip/UI/Common/CompressCall.cpp @@ -40,7 +40,6 @@ using namespace NWindows; #define kArcIncludeSwitches " -an -ai" #define kHashIncludeSwitches " -i" #define kStopSwitchParsing " --" -#define kLargePagesDisable " -slp-" static NCompression::CInfo m_RegistryInfo; extern HWND g_HWND; @@ -96,8 +95,8 @@ static HRESULT Call7zGui(const UString ¶ms, static void AddLagePagesSwitch(UString ¶ms) { - if (!ReadLockMemoryEnable()) - params += kLargePagesDisable; + if (ReadLockMemoryEnable()) + params += " -slp"; } class CRandNameGenerator @@ -338,6 +337,7 @@ void Benchmark(bool totalMode) UString params ('b'); if (totalMode) params += " -mm=*"; + AddLagePagesSwitch(params); HRESULT result = Call7zGui(params, false, NULL); if (result != S_OK) ErrorMessageHRESULT(result); diff --git a/CPP/7zip/UI/Common/DirItem.h b/CPP/7zip/UI/Common/DirItem.h index 2c402edf..47485bec 100644 --- a/CPP/7zip/UI/Common/DirItem.h +++ b/CPP/7zip/UI/Common/DirItem.h @@ -21,7 +21,7 @@ struct CDirItemsStat UInt64 NumErrors; - UInt64 Get_NumItems() const { return NumDirs + NumFiles + NumAltStreams; } + // UInt64 Get_NumItems() const { return NumDirs + NumFiles + NumAltStreams; } UInt64 Get_NumDataItems() const { return NumFiles + NumAltStreams; } UInt64 GetTotalBytes() const { return FilesSize + AltStreamsSize; } @@ -43,6 +43,30 @@ struct CDirItemsStat {} }; + +struct CDirItemsStat2: public CDirItemsStat +{ + UInt64 Anti_NumDirs; + UInt64 Anti_NumFiles; + UInt64 Anti_NumAltStreams; + + // UInt64 Get_NumItems() const { return Anti_NumDirs + Anti_NumFiles + Anti_NumAltStreams + CDirItemsStat::Get_NumItems(); } + UInt64 Get_NumDataItems2() const { return Anti_NumFiles + Anti_NumAltStreams + CDirItemsStat::Get_NumDataItems(); } + + bool IsEmpty() const { return CDirItemsStat::IsEmpty() + && 0 == Anti_NumDirs + && 0 == Anti_NumFiles + && 0 == Anti_NumAltStreams; } + + CDirItemsStat2(): + Anti_NumDirs(0), + Anti_NumFiles(0), + Anti_NumAltStreams(0) + {} +}; + + + #define INTERFACE_IDirItemsCallback(x) \ virtual HRESULT ScanError(const FString &path, DWORD systemError) x; \ virtual HRESULT ScanProgress(const CDirItemsStat &st, const FString &path, bool isDir) x; \ diff --git a/CPP/7zip/UI/Common/EnumDirItems.cpp b/CPP/7zip/UI/Common/EnumDirItems.cpp index a10a31e9..722bda0e 100644 --- a/CPP/7zip/UI/Common/EnumDirItems.cpp +++ b/CPP/7zip/UI/Common/EnumDirItems.cpp @@ -343,6 +343,7 @@ static HRESULT EnumerateAltStreams( const NWildcard::CCensorNode &curNode, int phyParent, int logParent, const FString &fullPath, const UStringVector &addArchivePrefix, // prefix from curNode + bool addAllItems, CDirItems &dirItems) { NFind::CStreamEnumerator enumerator(fullPath); @@ -363,6 +364,10 @@ static HRESULT EnumerateAltStreams( addArchivePrefixNew.Back() += reducedName; if (curNode.CheckPathToRoot(false, addArchivePrefixNew, true)) continue; + if (!addAllItems) + if (!curNode.CheckPathToRoot(true, addArchivePrefixNew, true)) + continue; + NFind::CFileInfo fi2 = fi; fi2.Name += us2fs(reducedName); fi2.Size = si.Size; @@ -413,6 +418,8 @@ static HRESULT EnumerateForItem( } int dirItemIndex = -1; + bool addAllSubStreams = false; + if (curNode.CheckPathToRoot(true, addArchivePrefixNew, !fi.IsDir())) { int secureIndex = -1; @@ -427,6 +434,8 @@ static HRESULT EnumerateForItem( dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); if (fi.IsDir()) enterToSubFolders2 = true; + + addAllSubStreams = true; } #ifndef UNDER_CE @@ -434,7 +443,9 @@ static HRESULT EnumerateForItem( { RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, phyPrefix + fi.Name, - addArchivePrefixNew, dirItems)); + addArchivePrefixNew, + addAllSubStreams, + dirItems)); } if (dirItemIndex >= 0) @@ -643,7 +654,9 @@ static HRESULT EnumerateDirItems( UStringVector pathParts; pathParts.Add(fs2us(fi.Name)); RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, - fullPath, pathParts, dirItems)); + fullPath, pathParts, + true, /* addAllSubStreams */ + dirItems)); } #endif diff --git a/CPP/7zip/UI/Common/Extract.cpp b/CPP/7zip/UI/Common/Extract.cpp index d8e569b9..ad9dff32 100644 --- a/CPP/7zip/UI/Common/Extract.cpp +++ b/CPP/7zip/UI/Common/Extract.cpp @@ -279,7 +279,9 @@ HRESULT Extract( CArchiveExtractCallback *ecs = new CArchiveExtractCallback; CMyComPtr ec(ecs); bool multi = (numArcs > 1); - ecs->InitForMulti(multi, options.PathMode, options.OverwriteMode); + ecs->InitForMulti(multi, options.PathMode, options.OverwriteMode, + false // keepEmptyDirParts + ); #ifndef _SFX ecs->SetHashMethods(hash); #endif diff --git a/CPP/7zip/UI/Common/ExtractingFilePath.cpp b/CPP/7zip/UI/Common/ExtractingFilePath.cpp index 915f17b1..13665de5 100644 --- a/CPP/7zip/UI/Common/ExtractingFilePath.cpp +++ b/CPP/7zip/UI/Common/ExtractingFilePath.cpp @@ -146,6 +146,9 @@ static void CorrectUnsupportedName(UString &name) static void Correct_PathPart(UString &s) { // "." and ".." + if (s.IsEmpty()) + return; + if (s[0] == '.' && (s[1] == 0 || s[1] == '.' && s[2] == 0)) s.Empty(); #ifdef _WIN32 @@ -172,7 +175,7 @@ UString Get_Correct_FsFile_Name(const UString &name) } -void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) +void Correct_FsPath(bool absIsAllowed, bool keepAndReplaceEmptyPrefixes, UStringVector &parts, bool isDir) { unsigned i = 0; @@ -181,6 +184,7 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) #if defined(_WIN32) && !defined(UNDER_CE) bool isDrive = false; #endif + if (parts[0].IsEmpty()) { i = 1; @@ -191,7 +195,7 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) if (parts.Size() > 2 && parts[2] == L"?") { i = 3; - if (parts.Size() > 3 && NWindows::NFile::NName::IsDrivePath2(parts[3])) + if (parts.Size() > 3 && NWindows::NFile::NName::IsDrivePath2(parts[3])) { isDrive = true; i = 4; @@ -220,6 +224,9 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) #endif } + if (i != 0) + keepAndReplaceEmptyPrefixes = false; + for (; i < parts.Size();) { UString &s = parts[i]; @@ -228,15 +235,17 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) if (s.IsEmpty()) { - if (isDir || i != parts.Size() - 1) - { - parts.Delete(i); - continue; - } + if (!keepAndReplaceEmptyPrefixes) + if (isDir || i != parts.Size() - 1) + { + parts.Delete(i); + continue; + } s = k_EmptyReplaceName; } else { + keepAndReplaceEmptyPrefixes = false; #ifdef _WIN32 CorrectUnsupportedName(s); #endif diff --git a/CPP/7zip/UI/Common/ExtractingFilePath.h b/CPP/7zip/UI/Common/ExtractingFilePath.h index ce200b80..12eb0bad 100644 --- a/CPP/7zip/UI/Common/ExtractingFilePath.h +++ b/CPP/7zip/UI/Common/ExtractingFilePath.h @@ -12,7 +12,19 @@ void Correct_AltStream_Name(UString &s); // replaces unsuported characters, and replaces "." , ".." and "" to "[]" UString Get_Correct_FsFile_Name(const UString &name); -void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir); +/* + Correct_FsPath() corrects path parts to prepare it for File System operations. + It also corrects empty path parts like "\\\\": + - frontal empty path parts : it removes them or changes them to "_" + - another empty path parts : it removes them + if (absIsAllowed && path is absolute) : it removes empty path parts after start absolute path prefix marker + else + { + if (!keepAndReplaceEmptyPrefixes) : it removes empty path parts + if ( keepAndReplaceEmptyPrefixes) : it changes each empty frontal path part to "_" + } +*/ +void Correct_FsPath(bool absIsAllowed, bool keepAndReplaceEmptyPrefixes, UStringVector &parts, bool isDir); UString MakePathFromParts(const UStringVector &parts); diff --git a/CPP/7zip/UI/Common/PropIDUtils.cpp b/CPP/7zip/UI/Common/PropIDUtils.cpp index 3322686d..8527da86 100644 --- a/CPP/7zip/UI/Common/PropIDUtils.cpp +++ b/CPP/7zip/UI/Common/PropIDUtils.cpp @@ -19,12 +19,16 @@ using namespace NWindows; -static const char g_WinAttribChars[16 + 1] = "RHS8DAdNTsLCOnE_"; +static const unsigned kNumWinAtrribFlags = 21; +static const char g_WinAttribChars[kNumWinAtrribFlags + 1] = "RHS8DAdNTsLCOIEV.X.PU"; + /* +FILE_ATTRIBUTE_ + 0 READONLY 1 HIDDEN 2 SYSTEM - +3 (Volume label - obsolete) 4 DIRECTORY 5 ARCHIVE 6 DEVICE @@ -34,12 +38,19 @@ static const char g_WinAttribChars[16 + 1] = "RHS8DAdNTsLCOnE_"; 10 REPARSE_POINT 11 COMPRESSED 12 OFFLINE -13 NOT_CONTENT_INDEXED +13 NOT_CONTENT_INDEXED (I - Win10 attrib/Explorer) 14 ENCRYPTED - -16 VIRTUAL +15 INTEGRITY_STREAM (V - ReFS Win8/Win2012) +16 VIRTUAL (reserved) +17 NO_SCRUB_DATA (X - ReFS Win8/Win2012 attrib) +18 RECALL_ON_OPEN or EA +19 PINNED +20 UNPINNED +21 STRICTLY_SEQUENTIAL +22 RECALL_ON_DATA_ACCESS */ + static const char kPosixTypes[16] = { '0', 'p', 'c', '3', 'd', '5', 'b', '7', '-', '9', 'l', 'B', 's', 'D', 'E', 'F' }; #define MY_ATTR_CHAR(a, n, c) ((a) & (1 << (n))) ? c : '-'; @@ -65,24 +76,57 @@ static void ConvertPosixAttribToString(char *s, UInt32 a) throw() } } + void ConvertWinAttribToString(char *s, UInt32 wa) throw() { - for (int i = 0; i < 16; i++) - if ((wa & (1 << i)) && i != 7) - *s++ = g_WinAttribChars[i]; - *s = 0; + /* + some programs store posix attributes in high 16 bits. + p7zip - stores additional 0x8000 flag marker. + macos - stores additional 0x4000 flag marker. + info-zip - no additional marker. + */ + + bool isPosix = ((wa & 0xF0000000) != 0); + + UInt32 posix = 0; + if (isPosix) + { + posix = wa >> 16; + wa &= (UInt32)0x3FFF; + } - // we support p7zip trick that stores posix attributes in high 16 bits, and 0x8000 flag - // we also support ZIP archives created in Unix, that store posix attributes in high 16 bits without 0x8000 flag + for (unsigned i = 0; i < kNumWinAtrribFlags; i++) + { + UInt32 flag = (1 << i); + if ((wa & flag) != 0) + { + char c = g_WinAttribChars[i]; + if (c != '.') + { + wa &= ~flag; + // if (i != 7) // we can disable N (NORMAL) printing + *s++ = c; + } + } + } - // if (wa & 0x8000) - if ((wa >> 16) != 0) + if (wa != 0) { *s++ = ' '; - ConvertPosixAttribToString(s, wa >> 16); + ConvertUInt32ToHex8Digits(wa, s); + s += strlen(s); + } + + *s = 0; + + if (isPosix) + { + *s++ = ' '; + ConvertPosixAttribToString(s, posix); } } + void ConvertPropertyToShortString2(char *dest, const PROPVARIANT &prop, PROPID propID, int level) throw() { *dest = 0; diff --git a/CPP/7zip/UI/Common/Update.cpp b/CPP/7zip/UI/Common/Update.cpp index 1a2ec25a..76800f8f 100644 --- a/CPP/7zip/UI/Common/Update.cpp +++ b/CPP/7zip/UI/Common/Update.cpp @@ -657,38 +657,74 @@ static HRESULT Compress( FOR_VECTOR (i, updatePairs2) { const CUpdatePair2 &up = updatePairs2[i]; - if (up.NewData) + + // 17.01: anti-item is (up.NewData && (p.UseArcProps in most cases)) + + if (up.NewData && !up.UseArcProps) { - CDirItemsStat &stat = stat2.NewData; - const CDirItem &di = dirItems.Items[up.DirIndex]; - if (di.IsDir()) - stat.NumDirs++; - else if (di.IsAltStream) + if (up.ExistOnDisk()) { - stat.NumAltStreams++; - stat.AltStreamsSize += di.Size; - } - else - { - stat.NumFiles++; - stat.FilesSize += di.Size; + CDirItemsStat2 &stat = stat2.NewData; + const CDirItem &di = dirItems.Items[up.DirIndex]; + if (di.IsDir()) + { + if (up.IsAnti) + stat.Anti_NumDirs++; + else + stat.NumDirs++; + } + else if (di.IsAltStream) + { + if (up.IsAnti) + stat.Anti_NumAltStreams++; + else + { + stat.NumAltStreams++; + stat.AltStreamsSize += di.Size; + } + } + else + { + if (up.IsAnti) + stat.Anti_NumFiles++; + else + { + stat.NumFiles++; + stat.FilesSize += di.Size; + } + } } } else if (up.ArcIndex >= 0) { - CDirItemsStat &stat = stat2.OldData; + CDirItemsStat2 &stat = *(up.NewData ? &stat2.NewData : &stat2.OldData); const CArcItem &ai = arcItems[up.ArcIndex]; if (ai.IsDir) - stat.NumDirs++; + { + if (up.IsAnti) + stat.Anti_NumDirs++; + else + stat.NumDirs++; + } else if (ai.IsAltStream) { - stat.NumAltStreams++; - stat.AltStreamsSize += ai.Size; + if (up.IsAnti) + stat.Anti_NumAltStreams++; + else + { + stat.NumAltStreams++; + stat.AltStreamsSize += ai.Size; + } } else { - stat.NumFiles++; - stat.FilesSize += ai.Size; + if (up.IsAnti) + stat.Anti_NumFiles++; + else + { + stat.NumFiles++; + stat.FilesSize += ai.Size; + } } } } @@ -1592,8 +1628,18 @@ HRESULT UpdateArchive( { if (processedItems[i] != 0 || dirItem.Size == 0) { - RINOK(callback->DeletingAfterArchiving(phyPath, false)); - DeleteFileAlways(phyPath); + NFind::CFileInfo fileInfo; + if (fileInfo.Find(phyPath)) + { + // maybe we must exclude also files with archive name: "a a.7z * -sdel" + if (fileInfo.Size == dirItem.Size + && CompareFileTime(&fileInfo.MTime, &dirItem.MTime) == 0 + && CompareFileTime(&fileInfo.CTime, &dirItem.CTime) == 0) + { + RINOK(callback->DeletingAfterArchiving(phyPath, false)); + DeleteFileAlways(phyPath); + } + } } else { diff --git a/CPP/7zip/UI/Common/UpdateCallback.h b/CPP/7zip/UI/Common/UpdateCallback.h index 28789762..1cc35328 100644 --- a/CPP/7zip/UI/Common/UpdateCallback.h +++ b/CPP/7zip/UI/Common/UpdateCallback.h @@ -17,13 +17,13 @@ struct CArcToDoStat { - CDirItemsStat NewData; - CDirItemsStat OldData; - CDirItemsStat DeleteData; + CDirItemsStat2 NewData; + CDirItemsStat2 OldData; + CDirItemsStat2 DeleteData; UInt64 Get_NumDataItems_Total() const { - return NewData.Get_NumDataItems() + OldData.Get_NumDataItems(); + return NewData.Get_NumDataItems2() + OldData.Get_NumDataItems2(); } }; diff --git a/CPP/7zip/UI/Common/ZipRegistry.h b/CPP/7zip/UI/Common/ZipRegistry.h index d349faee..4c16b61c 100644 --- a/CPP/7zip/UI/Common/ZipRegistry.h +++ b/CPP/7zip/UI/Common/ZipRegistry.h @@ -48,9 +48,14 @@ namespace NCompression UString Options; UString EncryptionMethod; + void Reset_BlockLogSize() + { + BlockLogSize = (UInt32)(Int32)-1; + } + void ResetForLevelChange() { - BlockLogSize = NumThreads = Level = Dictionary = Order = UInt32(-1); + BlockLogSize = NumThreads = Level = Dictionary = Order = (UInt32)(Int32)-1; Method.Empty(); // Options.Empty(); // EncryptionMethod.Empty(); diff --git a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp index 56175b6b..c8d83221 100644 --- a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp @@ -114,6 +114,39 @@ void Print_DirItemsStat(AString &s, const CDirItemsStat &st) } } + +void Print_DirItemsStat2(AString &s, const CDirItemsStat2 &st) +{ + Print_DirItemsStat(s, (CDirItemsStat &)st); + bool needLF = true; + if (st.Anti_NumDirs != 0) + { + if (needLF) + s.Add_LF(); + needLF = false; + Print_UInt64_and_String(s, st.Anti_NumDirs, st.Anti_NumDirs == 1 ? "anti-folder" : "anti-folders"); + } + if (st.Anti_NumFiles != 0) + { + if (needLF) + s.Add_LF(); + else + s += ", "; + needLF = false; + Print_UInt64_and_String(s, st.Anti_NumFiles, st.Anti_NumFiles == 1 ? "anti-file" : "anti-files"); + } + if (st.Anti_NumAltStreams != 0) + { + if (needLF) + s.Add_LF(); + else + s += ", "; + needLF = false; + Print_UInt64_and_String(s, st.Anti_NumAltStreams, "anti-alternate-streams"); + } +} + + void CExtractScanConsole::PrintStat(const CDirItemsStat &st) { if (_so) diff --git a/CPP/7zip/UI/Console/Main.cpp b/CPP/7zip/UI/Console/Main.cpp index 68505697..e85364c3 100644 --- a/CPP/7zip/UI/Console/Main.cpp +++ b/CPP/7zip/UI/Console/Main.cpp @@ -97,7 +97,6 @@ static const char * const kHelpString = #endif #endif " [...] [...]\n" - " [<@listfiles...>]\n" "\n" "\n" " a : Add files to archive\n" @@ -114,6 +113,7 @@ static const char * const kHelpString = "\n" "\n" " -- : Stop switches parsing\n" + " @listfile : set path to listfile that contains file names\n" " -ai[r[-|0]]{@listfile|!wildcard} : Include archives\n" " -ax[r[-|0]]{@listfile|!wildcard} : eXclude archives\n" " -ao{a|s|t|u} : set Overwrite mode\n" @@ -548,6 +548,7 @@ int Main2( if (options.LargePages) { SetLargePageSize(); + // note: this process also can inherit that Privilege from parent process g_LargePagesMode = #if defined(_WIN32) && !defined(UNDER_CE) NSecurity::EnablePrivilege_LockMemory(); diff --git a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp index b1728a3a..5e1c914a 100644 --- a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp @@ -241,6 +241,7 @@ static void PrintPropPair(AString &s, const char *name, UInt64 val) void PrintSize_bytes_Smart(AString &s, UInt64 val); void Print_DirItemsStat(AString &s, const CDirItemsStat &st); +void Print_DirItemsStat2(AString &s, const CDirItemsStat2 &st); HRESULT CUpdateCallbackConsole::FinishScanning(const CDirItemsStat &st) { @@ -399,10 +400,10 @@ HRESULT CUpdateCallbackConsole::Finalize() */ -void static PrintToDoStat(CStdOutStream *_so, const CDirItemsStat &stat, const char *name) +void static PrintToDoStat(CStdOutStream *_so, const CDirItemsStat2 &stat, const char *name) { AString s; - Print_DirItemsStat(s, stat); + Print_DirItemsStat2(s, stat); *_so << name << ": " << s << endl; } diff --git a/CPP/7zip/UI/Far/Messages.h b/CPP/7zip/UI/Far/Messages.h index c099f104..4863d229 100644 --- a/CPP/7zip/UI/Far/Messages.h +++ b/CPP/7zip/UI/Far/Messages.h @@ -7,7 +7,7 @@ namespace NMessageID { -const unsigned k_Last_PropId_supported_by_plugin = kpidStreamId; +const unsigned k_Last_PropId_supported_by_plugin = kpidCopyLink; enum EEnum { diff --git a/CPP/7zip/UI/FileManager/App.cpp b/CPP/7zip/UI/FileManager/App.cpp index a0ee587d..aeae20cb 100644 --- a/CPP/7zip/UI/FileManager/App.cpp +++ b/CPP/7zip/UI/FileManager/App.cpp @@ -574,7 +574,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) } else if (!srcPanel.DoesItSupportOperations()) { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; } @@ -608,6 +608,8 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) UStringVector copyFolders; ReadCopyHistory(copyFolders); + bool useFullItemPaths = srcPanel.Is_IO_FS_Folder(); // maybe we need flat also here ?? + { CCopyDialog copyDialog; @@ -626,14 +628,14 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) { if (destPath.IsEmpty()) { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; } UString correctName; if (!srcPanel.CorrectFsPath(destPath, correctName)) { - srcPanel.MessageBoxError(E_INVALIDARG); + srcPanel.MessageBox_Error_HRESULT(E_INVALIDARG); return; } @@ -647,7 +649,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) if (destPath.Len() > 0 && destPath[0] == '\\') if (destPath.Len() == 1 || destPath[1] != '\\') { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; } #endif @@ -658,7 +660,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) { if (NumPanels == 1 || CompareFileNames(destPath, srcPanel.GetFsPath()) == 0) { - srcPanel.MessageBoxMyError(L"Can not copy files onto itself"); + srcPanel.MessageBox_Error(L"Can not copy files onto itself"); return; } @@ -674,7 +676,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) destIsFsPath = true; else if (destPanel.IsFSDrivesFolder() || destPanel.IsRootFolder()) { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; } } @@ -683,7 +685,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) if (IsAltPathPrefix(us2fs(destPath))) { // we allow alt streams dest only to alt stream folder in second panel - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; /* FString basePath = us2fs(destPath); @@ -704,7 +706,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) int pos = destPath.ReverseFind_PathSepar(); if (pos < 0) { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; } { @@ -713,7 +715,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) UString name = destPath.Ptr(pos + 1); if (name.Find(L':') >= 0) { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; } #endif @@ -721,7 +723,8 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) UString prefix = destPath.Left(pos + 1); if (!CreateComplexDir(us2fs(prefix))) { - srcPanel.MessageBoxError2Lines(prefix, GetLastError()); + DWORD lastError = ::GetLastError(); + srcPanel.MessageBox_Error_2Lines_Message_HRESULT(prefix, lastError); return; } } @@ -732,7 +735,8 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) NName::NormalizeDirPathPrefix(destPath); if (!CreateComplexDir(us2fs(destPath))) { - srcPanel.MessageBoxError2Lines(destPath, GetLastError()); + DWORD lastError = ::GetLastError(); + srcPanel.MessageBox_Error_2Lines_Message_HRESULT(destPath, lastError); return; } } @@ -754,7 +758,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) bool useTemp = useSrcPanel && useDestPanel; if (useTemp && NumPanels == 1) { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; } @@ -793,13 +797,25 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) { UStringVector filePaths; UString folderPrefix; + if (useTemp) folderPrefix = fs2us(tempDirPrefix); else folderPrefix = srcPanel.GetFsPath(); + filePaths.ClearAndReserve(indices.Size()); + FOR_VECTOR (i, indices) - filePaths.AddInReserved(srcPanel.GetItemRelPath2(indices[i])); + { + UInt32 index = indices[i]; + UString s; + if (useFullItemPaths) + s = srcPanel.GetItemRelPath2(index); + else + s = srcPanel.GetItemName_for_Copy(index); + filePaths.AddInReserved(s); + } + result = destPanel.CopyFrom(move, folderPrefix, filePaths, true, 0); } @@ -812,7 +828,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) // srcPanel.InvalidateList(NULL, true); if (result != E_ABORT) - srcPanel.MessageBoxError(result); + srcPanel.MessageBox_Error_HRESULT(result); // return; } diff --git a/CPP/7zip/UI/FileManager/ExtractCallback.cpp b/CPP/7zip/UI/FileManager/ExtractCallback.cpp index c4380b46..6433e917 100644 --- a/CPP/7zip/UI/FileManager/ExtractCallback.cpp +++ b/CPP/7zip/UI/FileManager/ExtractCallback.cpp @@ -1025,7 +1025,7 @@ HRESULT CVirtFileSystem::FlushToDisk(bool closeLast) _numFlushed++; _fileIsOpen = false; if (file.AttribDefined) - NDir::SetFileAttrib(path, file.Attrib); + NDir::SetFileAttrib_PosixHighDetect(path, file.Attrib); } return S_OK; } diff --git a/CPP/7zip/UI/FileManager/FM.cpp b/CPP/7zip/UI/FileManager/FM.cpp index f1fc174e..eb524254 100644 --- a/CPP/7zip/UI/FileManager/FM.cpp +++ b/CPP/7zip/UI/FileManager/FM.cpp @@ -383,7 +383,10 @@ static void SetMemoryLock() NSecurity::AddLockMemoryPrivilege(); if (ReadLockMemoryEnable()) + { + // note: child processes can inherit that Privilege g_LargePagesMode = NSecurity::EnablePrivilege_LockMemory(); + } } bool g_SymLink_Supported = false; diff --git a/CPP/7zip/UI/FileManager/FSFolder.h b/CPP/7zip/UI/FileManager/FSFolder.h index 6b6ba1fd..9c3fafaf 100644 --- a/CPP/7zip/UI/FileManager/FSFolder.h +++ b/CPP/7zip/UI/FileManager/FSFolder.h @@ -205,7 +205,7 @@ struct CCopyStateIO CCopyStateIO(): DeleteSrcFile(false), TotalSize(0), StartPos(0) {} - HRESULT MyCopyFile(CFSTR inPath, CFSTR outPath); + HRESULT MyCopyFile(CFSTR inPath, CFSTR outPath, DWORD attrib = INVALID_FILE_ATTRIBUTES); }; HRESULT SendLastErrorMessage(IFolderOperationsExtractCallback *callback, const FString &fileName); diff --git a/CPP/7zip/UI/FileManager/FSFolderCopy.cpp b/CPP/7zip/UI/FileManager/FSFolderCopy.cpp index 79cc08bb..cc06aef1 100644 --- a/CPP/7zip/UI/FileManager/FSFolderCopy.cpp +++ b/CPP/7zip/UI/FileManager/FSFolderCopy.cpp @@ -31,7 +31,7 @@ extern bool g_IsNT; namespace NFsFolder { -HRESULT CCopyStateIO::MyCopyFile(CFSTR inPath, CFSTR outPath) +HRESULT CCopyStateIO::MyCopyFile(CFSTR inPath, CFSTR outPath, DWORD attrib) { ErrorFileIndex = -1; ErrorMessage.Empty(); @@ -87,6 +87,9 @@ HRESULT CCopyStateIO::MyCopyFile(CFSTR inPath, CFSTR outPath) } } + if (attrib != INVALID_FILE_ATTRIBUTES) + SetFileAttrib(outPath, attrib); + if (DeleteSrcFile) { if (!DeleteFileAlways(inPath)) @@ -424,7 +427,9 @@ static HRESULT CopyFile_Ask( state2.DeleteSrcFile = state.MoveMode; state2.TotalSize = state.ProgressInfo.TotalSize; state2.StartPos = state.ProgressInfo.StartPos; - RINOK(state2.MyCopyFile(srcPath, destPathNew)); + + RINOK(state2.MyCopyFile(srcPath, destPathNew, srcFileInfo.Attrib)); + if (state2.ErrorFileIndex >= 0) { if (state2.ErrorMessage.IsEmpty()) diff --git a/CPP/7zip/UI/FileManager/LinkDialog.cpp b/CPP/7zip/UI/FileManager/LinkDialog.cpp index e0e3c918..9e69b522 100644 --- a/CPP/7zip/UI/FileManager/LinkDialog.cpp +++ b/CPP/7zip/UI/FileManager/LinkDialog.cpp @@ -314,7 +314,7 @@ void CApp::Link() CPanel &srcPanel = Panels[srcPanelIndex]; if (!srcPanel.IsFSFolder()) { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; } CRecordVector indices; @@ -323,7 +323,7 @@ void CApp::Link() return; if (indices.Size() != 1) { - srcPanel.MessageBoxErrorLang(IDS_SELECT_ONE_FILE); + srcPanel.MessageBox_Error_LangID(IDS_SELECT_ONE_FILE); return; } int index = indices[0]; diff --git a/CPP/7zip/UI/FileManager/Panel.cpp b/CPP/7zip/UI/FileManager/Panel.cpp index 92293cc0..8e126317 100644 --- a/CPP/7zip/UI/FileManager/Panel.cpp +++ b/CPP/7zip/UI/FileManager/Panel.cpp @@ -697,43 +697,57 @@ bool CPanel::OnCommand(int code, int itemID, LPARAM lParam, LRESULT &result) return CWindow2::OnCommand(code, itemID, lParam, result); } -void CPanel::MessageBoxInfo(LPCWSTR message, LPCWSTR caption) + + +/* +void CPanel::MessageBox_Info(LPCWSTR message, LPCWSTR caption) const { ::MessageBoxW((HWND)*this, message, caption, MB_OK); } -void CPanel::MessageBox(LPCWSTR message, LPCWSTR caption) +void CPanel::MessageBox_Warning(LPCWSTR message) const + { ::MessageBoxW((HWND)*this, message, L"7-Zip", MB_OK | MB_ICONWARNING); } +*/ + +void CPanel::MessageBox_Error_Caption(LPCWSTR message, LPCWSTR caption) const { ::MessageBoxW((HWND)*this, message, caption, MB_OK | MB_ICONSTOP); } -void CPanel::MessageBox(LPCWSTR message) - { MessageBox(message, L"7-Zip"); } -void CPanel::MessageBoxWarning(LPCWSTR message) - { ::MessageBoxW(NULL, message, L"7-Zip", MB_OK | MB_ICONWARNING); } -void CPanel::MessageBoxMyError(LPCWSTR message) - { MessageBox(message, L"7-Zip"); } +void CPanel::MessageBox_Error(LPCWSTR message) const + { MessageBox_Error_Caption(message, L"7-Zip"); } + +static UString ErrorHResult_To_Message(HRESULT errorCode) +{ + if (errorCode == 0) + errorCode = E_FAIL; + return HResultToMessage(errorCode); +} -void CPanel::MessageBoxError(HRESULT errorCode, LPCWSTR caption) +void CPanel::MessageBox_Error_HRESULT_Caption(HRESULT errorCode, LPCWSTR caption) const { - MessageBox(HResultToMessage(errorCode), caption); + MessageBox_Error_Caption(ErrorHResult_To_Message(errorCode), caption); } -void CPanel::MessageBoxError2Lines(LPCWSTR message, HRESULT errorCode) +void CPanel::MessageBox_Error_HRESULT(HRESULT errorCode) const + { MessageBox_Error_HRESULT_Caption(errorCode, L"7-Zip"); } + +void CPanel::MessageBox_Error_2Lines_Message_HRESULT(LPCWSTR message, HRESULT errorCode) const { UString m = message; - if (errorCode != 0) - { - m.Add_LF(); - m += HResultToMessage(errorCode); - } - MessageBoxMyError(m); + m.Add_LF(); + m += ErrorHResult_To_Message(errorCode); + MessageBox_Error(m); } -void CPanel::MessageBoxError(HRESULT errorCode) - { MessageBoxError(errorCode, L"7-Zip"); } -void CPanel::MessageBoxLastError(LPCWSTR caption) - { MessageBoxError(::GetLastError(), caption); } -void CPanel::MessageBoxLastError() - { MessageBoxLastError(L"7-Zip"); } +void CPanel::MessageBox_LastError(LPCWSTR caption) const + { MessageBox_Error_HRESULT_Caption(::GetLastError(), caption); } + +void CPanel::MessageBox_LastError() const + { MessageBox_LastError(L"7-Zip"); } + +void CPanel::MessageBox_Error_LangID(UINT resourceID) const + { MessageBox_Error(LangString(resourceID)); } + +void CPanel::MessageBox_Error_UnsupportOperation() const + { MessageBox_Error_LangID(IDS_OPERATION_IS_NOT_SUPPORTED); } + -void CPanel::MessageBoxErrorLang(UINT resourceID) - { MessageBox(LangString(resourceID)); } void CPanel::SetFocusToList() @@ -821,7 +835,7 @@ void CPanel::ChangeFlatMode() _flatModeForArc = _flatMode; else _flatModeForDisk = _flatMode; - RefreshListCtrlSaveFocused(); + RefreshListCtrl_SaveFocused(); } /* @@ -848,12 +862,12 @@ void CPanel::AddToArchive() GetOperatedItemIndices(indices); if (!Is_IO_FS_Folder()) { - MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + MessageBox_Error_UnsupportOperation(); return; } if (indices.Size() == 0) { - MessageBoxErrorLang(IDS_SELECT_FILES); + MessageBox_Error_LangID(IDS_SELECT_FILES); return; } UStringVector names; @@ -873,7 +887,7 @@ void CPanel::AddToArchive() if (res != S_OK) { if (destCurDirPrefix.Len() >= MAX_PATH) - MessageBoxErrorLang(IDS_MESSAGE_UNSUPPORTED_OPERATION_FOR_LONG_PATH_FOLDER); + MessageBox_Error_LangID(IDS_MESSAGE_UNSUPPORTED_OPERATION_FOR_LONG_PATH_FOLDER); } // KillSelection(); } @@ -910,7 +924,7 @@ void CPanel::GetFilePaths(const CRecordVector &indices, UStringVector &p } if (paths.Size() == 0) { - MessageBoxErrorLang(IDS_SELECT_FILES); + MessageBox_Error_LangID(IDS_SELECT_FILES); return; } } @@ -1026,7 +1040,7 @@ void CPanel::TestArchives() if (res != S_OK) { if (res != E_ABORT) - MessageBoxError(res); + MessageBox_Error_HRESULT(res); } return; @@ -1071,7 +1085,7 @@ void CPanel::TestArchives() if (!IsFSFolder()) { - MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + MessageBox_Error_UnsupportOperation(); return; } UStringVector paths; diff --git a/CPP/7zip/UI/FileManager/Panel.h b/CPP/7zip/UI/FileManager/Panel.h index 5671ee3f..dd217ec3 100644 --- a/CPP/7zip/UI/FileManager/Panel.h +++ b/CPP/7zip/UI/FileManager/Panel.h @@ -220,11 +220,12 @@ class CMyComboBoxEdit: public NWindows::NControl::CEdit struct CSelectedState { int FocusedItem; - UString FocusedName; bool SelectFocused; + bool FocusedName_Defined; + UString FocusedName; UStringVector SelectedNames; - CSelectedState(): FocusedItem(-1), SelectFocused(false) {} + CSelectedState(): FocusedItem(-1), FocusedName_Defined(false), SelectFocused(true) {} }; #ifdef UNDER_CE @@ -326,9 +327,7 @@ class CPanel: public NWindows::NControl::CWindow2 void AddColumn(const CPropColumn &prop); void SetFocusedSelectedItem(int index, bool select); - HRESULT RefreshListCtrl(const UString &focusedName, int focusedPos, bool selectFocused, - const UStringVector &selectedNames); - + void OnShiftSelectMessage(); void OnArrowWithShift(); @@ -463,7 +462,7 @@ class CPanel: public NWindows::NControl::CWindow2 void GetSelectedNames(UStringVector &selectedNames); void SaveSelectedState(CSelectedState &s); HRESULT RefreshListCtrl(const CSelectedState &s); - HRESULT RefreshListCtrlSaveFocused(); + HRESULT RefreshListCtrl_SaveFocused(); bool GetItem_BoolProp(UInt32 itemIndex, PROPID propID) const; bool IsItem_Deleted(int itemIndex) const; @@ -745,20 +744,20 @@ class CPanel: public NWindows::NControl::CWindow2 HRESULT RefreshListCtrl(); - void MessageBoxInfo(LPCWSTR message, LPCWSTR caption); - void MessageBox(LPCWSTR message); - void MessageBoxWarning(LPCWSTR message); - void MessageBox(LPCWSTR message, LPCWSTR caption); - void MessageBoxMyError(LPCWSTR message); - void MessageBoxError(HRESULT errorCode, LPCWSTR caption); - void MessageBoxError(HRESULT errorCode); - void MessageBoxError2Lines(LPCWSTR message, HRESULT errorCode); - void MessageBoxLastError(LPCWSTR caption); - void MessageBoxLastError(); - + + // void MessageBox_Info(LPCWSTR message, LPCWSTR caption) const; + // void MessageBox_Warning(LPCWSTR message) const; + void MessageBox_Error_Caption(LPCWSTR message, LPCWSTR caption) const; + void MessageBox_Error(LPCWSTR message) const; + void MessageBox_Error_HRESULT_Caption(HRESULT errorCode, LPCWSTR caption) const; + void MessageBox_Error_HRESULT(HRESULT errorCode) const; + void MessageBox_Error_2Lines_Message_HRESULT(LPCWSTR message, HRESULT errorCode) const; + void MessageBox_LastError(LPCWSTR caption) const; + void MessageBox_LastError() const; + void MessageBox_Error_LangID(UINT resourceID) const; + void MessageBox_Error_UnsupportOperation() const; // void MessageBoxErrorForUpdate(HRESULT errorCode, UINT resourceID); - void MessageBoxErrorLang(UINT resourceID); void OpenAltStreams(); diff --git a/CPP/7zip/UI/FileManager/PanelCopy.cpp b/CPP/7zip/UI/FileManager/PanelCopy.cpp index 55e17f11..6f7555cb 100644 --- a/CPP/7zip/UI/FileManager/PanelCopy.cpp +++ b/CPP/7zip/UI/FileManager/PanelCopy.cpp @@ -101,7 +101,7 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind { UString errorMessage = LangString(IDS_OPERATION_IS_NOT_SUPPORTED); if (options.showErrorMessages) - MessageBox(errorMessage); + MessageBox_Error(errorMessage); else if (messages != 0) messages->Add(errorMessage); return E_FAIL; @@ -307,7 +307,7 @@ HRESULT CPanel::CopyFrom(bool moveMode, const UString &folderPrefix, const UStri { UString errorMessage = LangString(IDS_OPERATION_IS_NOT_SUPPORTED); if (showErrorMessages) - MessageBox(errorMessage); + MessageBox_Error(errorMessage); else if (messages != 0) messages->Add(errorMessage); return E_ABORT; @@ -334,7 +334,7 @@ void CPanel::CopyFromNoAsk(const UStringVector &filePaths) // For Password: SetFocusToList(); if (result != E_ABORT) - MessageBoxError(result); + MessageBox_Error_HRESULT(result); return; } diff --git a/CPP/7zip/UI/FileManager/PanelCrc.cpp b/CPP/7zip/UI/FileManager/PanelCrc.cpp index ddb47885..90443078 100644 --- a/CPP/7zip/UI/FileManager/PanelCrc.cpp +++ b/CPP/7zip/UI/FileManager/PanelCrc.cpp @@ -382,6 +382,6 @@ void CApp::CalculateCrc(const char *methodName) { unsigned srcPanelIndex = GetFocusedPanelIndex(); CPanel &srcPanel = Panels[srcPanelIndex]; - srcPanel.MessageBoxError(res); + srcPanel.MessageBox_Error_HRESULT(res); } } diff --git a/CPP/7zip/UI/FileManager/PanelDrag.cpp b/CPP/7zip/UI/FileManager/PanelDrag.cpp index 936287b6..105a94b1 100644 --- a/CPP/7zip/UI/FileManager/PanelDrag.cpp +++ b/CPP/7zip/UI/FileManager/PanelDrag.cpp @@ -327,6 +327,7 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) // CSelectedState selState; // SaveSelectedState(selState); + // FString dirPrefix2; FString dirPrefix; CTempDir tempDirectory; @@ -337,6 +338,7 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) { tempDirectory.Create(kTempDirPrefix); dirPrefix = tempDirectory.GetPath(); + // dirPrefix2 = dirPrefix; NFile::NName::NormalizeDirPathPrefix(dirPrefix); } @@ -345,6 +347,10 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) { UStringVector names; + + // names variable is USED for drag and drop from 7-zip to Explorer or to 7-zip archive folder. + // names variable is NOT USED for drag and drop from 7-zip to 7-zip File System folder. + FOR_VECTOR (i, indices) { UInt32 index = indices[i]; @@ -354,6 +360,23 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) else { s = GetItemName(index); + /* + // We use (keepAndReplaceEmptyPrefixes = true) in CAgentFolder::Extract + // So the following code is not required. + // Maybe we also can change IFolder interface and send some flag also. + + if (s.IsEmpty()) + { + // Correct_FsFile_Name("") returns "_". + // If extracting code removes empty folder prefixes from path (as it was in old version), + // Explorer can't find "_" folder in temp folder. + // We can ask Explorer to copy parent temp folder "7zE" instead. + + names.Clear(); + names.Add(dirPrefix2); + break; + } + */ s = Get_Correct_FsFile_Name(s); } names.Add(fs2us(dirPrefix) + s); @@ -408,7 +431,7 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) // we ignore E_UNEXPECTED that is returned if we drag file to printer if (res != DRAGDROP_S_CANCEL && res != S_OK && res != E_UNEXPECTED) - MessageBoxError(res); + MessageBox_Error_HRESULT(res); res = dropSourceSpec->Result; } @@ -422,10 +445,10 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) if (res != S_OK && res != E_ABORT) { - // we restore Notify before MessageBoxError. So we will se files selection + // we restore Notify before MessageBox_Error_HRESULT. So we will se files selection disableNotify.Restore(); // SetFocusToList(); - MessageBoxError(res); + MessageBox_Error_HRESULT(res); } if (res == S_OK && dropSourceSpec->Messages.IsEmpty() && !canceled) KillSelection(); diff --git a/CPP/7zip/UI/FileManager/PanelFolderChange.cpp b/CPP/7zip/UI/FileManager/PanelFolderChange.cpp index 642797cd..ae6d420d 100644 --- a/CPP/7zip/UI/FileManager/PanelFolderChange.cpp +++ b/CPP/7zip/UI/FileManager/PanelFolderChange.cpp @@ -298,7 +298,7 @@ HRESULT CPanel::BindToPathAndRefresh(const UString &path) #endif HRESULT res = BindToPath(s, UString(), archiveIsOpened, encrypted); - RefreshListCtrl(UString(), -1, true, UStringVector()); + RefreshListCtrl(); return res; } @@ -740,17 +740,20 @@ void CPanel::OpenParentFolder() } } - UStringVector selectedItems; + CSelectedState state; + state.FocusedName = focusedName; + state.FocusedName_Defined = true; /* if (!focusedName.IsEmpty()) - selectedItems.Add(focusedName); + state.SelectedNames.Add(focusedName); */ LoadFullPath(); // ::SetCurrentDirectory(::_currentFolderPrefix); - RefreshListCtrl(focusedName, -1, true, selectedItems); + RefreshListCtrl(state); // _listView.EnsureVisible(_listView.GetFocusedItem(), false); } + void CPanel::CloseOneLevel() { ReleaseFolder(); @@ -783,7 +786,7 @@ void CPanel::OpenRootFolder() CDisableNotify disableNotify(*this); _parentFolders.Clear(); SetToRootFolder(); - RefreshListCtrl(UString(), -1, true, UStringVector()); + RefreshListCtrl(); // ::SetCurrentDirectory(::_currentFolderPrefix); /* BeforeChangeFolder(); @@ -847,7 +850,7 @@ void CPanel::OpenAltStreams() CDisableTimerProcessing disableTimerProcessing(*this); CDisableNotify disableNotify(*this); SetNewFolder(newFolder); - RefreshListCtrl(UString(), -1, true, UStringVector()); + RefreshListCtrl(); return; } return; diff --git a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp index f060e4db..8a83301c 100644 --- a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp +++ b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp @@ -559,11 +559,11 @@ HRESULT CPanel::OpenAsArc(IInStream *inStream, */ /* if (!s.IsEmpty()) - MessageBoxWarning(s); + MessageBox_Warning(s); else */ - // after MessageBoxWarning it throws exception in nested archives in Debug Mode. why ?. - // MessageBoxWarning(L"test error"); + // after MessageBox_Warning it throws exception in nested archives in Debug Mode. why ?. + // MessageBox_Warning(L"test error"); } } @@ -585,9 +585,10 @@ HRESULT CPanel::OpenAsArc_Msg(IInStream *inStream, if (res == E_ABORT) return res; - if (showErrorMessage && encrypted) + if (showErrorMessage) + if (encrypted || res != S_FALSE) // 17.01 : we show message also for (res != S_FALSE) { - UString message("Error"); + UString message; if (res == S_FALSE) { message = MyFormatNew( @@ -598,7 +599,7 @@ HRESULT CPanel::OpenAsArc_Msg(IInStream *inStream, } else message = HResultToMessage(res); - MessageBoxMyError(message); + MessageBox_Error(message); } return res; @@ -799,6 +800,12 @@ void CApp::DiffFiles() { const CPanel &panel = GetFocusedPanel(); + if (!panel.Is_IO_FS_Folder()) + { + panel.MessageBox_Error_UnsupportOperation(); + return; + } + CRecordVector indices; panel.GetSelectedItemsIndices(indices); @@ -811,6 +818,13 @@ void CApp::DiffFiles() else if (indices.Size() == 1 && NumPanels >= 2) { const CPanel &destPanel = Panels[1 - LastFocusedPanel]; + + if (!destPanel.Is_IO_FS_Folder()) + { + panel.MessageBox_Error_UnsupportOperation(); + return; + } + path1 = panel.GetItemFullPath(indices[0]); CRecordVector indices2; destPanel.GetSelectedItemsIndices(indices2); @@ -1061,7 +1075,7 @@ bool CPanel::IsVirus_Message(const UString &name) s.Add_LF(); s += name2; s.Add_LF(); s += name3; - MessageBoxMyError(s); + MessageBox_Error(s); return true; } @@ -1095,7 +1109,7 @@ void CPanel::OpenItem(int index, bool tryInternal, bool tryExternal, const wchar return; if (res != S_FALSE) { - MessageBoxError(res); + MessageBox_Error_HRESULT(res); return; } } @@ -1130,7 +1144,7 @@ HRESULT CPanel::OnOpenItemChanged(UInt32 index, const wchar_t *fullFilePath, { if (!_folderOperations) { - MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + MessageBox_Error_UnsupportOperation(); return E_FAIL; } @@ -1589,7 +1603,7 @@ void CPanel::OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bo if (!_folderOperations) { - MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + MessageBox_Error_UnsupportOperation(); return; } @@ -1600,7 +1614,7 @@ void CPanel::OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bo CTempDir tempDirectory; if (!tempDirectory.Create(kTempDirPrefix)) { - MessageBoxLastError(); + MessageBox_LastError(); return; } @@ -1729,7 +1743,7 @@ void CPanel::OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bo if (result != S_OK) { if (result != E_ABORT) - MessageBoxError(result); + MessageBox_Error_HRESULT(result); return; } diff --git a/CPP/7zip/UI/FileManager/PanelItems.cpp b/CPP/7zip/UI/FileManager/PanelItems.cpp index 05c39f3f..7fce2321 100644 --- a/CPP/7zip/UI/FileManager/PanelItems.cpp +++ b/CPP/7zip/UI/FileManager/PanelItems.cpp @@ -11,6 +11,8 @@ #include "../../PropID.h" +#include "../Common/ExtractingFilePath.h" + #include "resource.h" #include "LangUtils.h" @@ -317,7 +319,8 @@ void CPanel::AddColumn(const CPropColumn &prop) HRESULT CPanel::RefreshListCtrl() { - return RefreshListCtrl(UString(), -1, true, UStringVector()); + CSelectedState state; + return RefreshListCtrl(state); } int CALLBACK CompareItems(LPARAM lParam1, LPARAM lParam2, LPARAM lpData); @@ -356,7 +359,9 @@ void CPanel::GetSelectedNames(UStringVector &selectedNames) void CPanel::SaveSelectedState(CSelectedState &s) { + s.FocusedName_Defined = false; s.FocusedName.Empty(); + s.SelectFocused = true; // false; s.SelectedNames.Clear(); s.FocusedItem = _listView.GetFocusedItem(); { @@ -364,7 +369,12 @@ void CPanel::SaveSelectedState(CSelectedState &s) { int realIndex = GetRealItemIndex(s.FocusedItem); if (realIndex != kParentIndex) + { s.FocusedName = GetItemRelPath(realIndex); + s.FocusedName_Defined = true; + + s.SelectFocused = _listView.IsItemSelected(s.FocusedItem); + /* const int kSize = 1024; WCHAR name[kSize + 1]; @@ -376,21 +386,26 @@ void CPanel::SaveSelectedState(CSelectedState &s) item.mask = LVIF_TEXT; if (_listView.GetItem(&item)) focusedName = item.pszText; - */ + */ + } } } GetSelectedNames(s.SelectedNames); } +/* HRESULT CPanel::RefreshListCtrl(const CSelectedState &s) { bool selectFocused = s.SelectFocused; if (_mySelectMode) selectFocused = true; - return RefreshListCtrl(s.FocusedName, s.FocusedItem, selectFocused, s.SelectedNames); + return RefreshListCtrl2( + s.FocusedItem >= 0, // allowEmptyFocusedName + s.FocusedName, s.FocusedItem, selectFocused, s.SelectedNames); } +*/ -HRESULT CPanel::RefreshListCtrlSaveFocused() +HRESULT CPanel::RefreshListCtrl_SaveFocused() { CSelectedState state; SaveSelectedState(state); @@ -420,8 +435,7 @@ void CPanel::SetFocusedSelectedItem(int index, bool select) #endif -HRESULT CPanel::RefreshListCtrl(const UString &focusedName, int focusedPos, bool selectFocused, - const UStringVector &selectedNames) +HRESULT CPanel::RefreshListCtrl(const CSelectedState &state) { if (!_folder) return S_OK; @@ -433,6 +447,7 @@ HRESULT CPanel::RefreshListCtrl(const UString &focusedName, int focusedPos, bool CDisableTimerProcessing timerProcessing(*this); CDisableNotify disableNotify(*this); + int focusedPos = state.FocusedItem; if (focusedPos < 0) focusedPos = 0; @@ -534,8 +549,8 @@ HRESULT CPanel::RefreshListCtrl(const UString &focusedName, int focusedPos, bool { UString itemName (".."); item.iItem = listViewItemCount; - if (itemName == focusedName) - cursorIndex = item.iItem; + if (itemName == state.FocusedName) + cursorIndex = listViewItemCount; item.mask = LVIF_TEXT | LVIF_PARAM | LVIF_IMAGE; int subItem = 0; item.iSubItem = subItem++; @@ -573,7 +588,7 @@ HRESULT CPanel::RefreshListCtrl(const UString &focusedName, int focusedPos, bool bool selected = false; - if (!focusedName.IsEmpty() || !selectedNames.IsEmpty()) + if (state.FocusedName_Defined || !state.SelectedNames.IsEmpty()) { relPath.Empty(); @@ -599,9 +614,9 @@ HRESULT CPanel::RefreshListCtrl(const UString &focusedName, int focusedPos, bool } } relPath += name; - if (relPath == focusedName) + if (relPath == state.FocusedName) cursorIndex = listViewItemCount; - if (selectedNames.FindInSorted(relPath) >= 0) + if (state.SelectedNames.FindInSorted(relPath) >= 0) selected = true; } @@ -724,7 +739,7 @@ HRESULT CPanel::RefreshListCtrl(const UString &focusedName, int focusedPos, bool disableNotify.Restore(); if (_listView.GetItemCount() > 0 && cursorIndex >= 0) - SetFocusedSelectedItem(cursorIndex, selectFocused); + SetFocusedSelectedItem(cursorIndex, state.SelectFocused); Print_OnNotify("after SetFocusedSelectedItem"); @@ -738,7 +753,7 @@ HRESULT CPanel::RefreshListCtrl(const UString &focusedName, int focusedPos, bool if (focusedPos >= _listView.GetItemCount()) focusedPos = _listView.GetItemCount() - 1; // we select item only in showDots mode. - SetFocusedSelectedItem(focusedPos, showDots); + SetFocusedSelectedItem(focusedPos, showDots && (focusedPos == 0)); } // m_RedrawEnabled = true; @@ -789,6 +804,7 @@ HRESULT CPanel::RefreshListCtrl(const UString &focusedName, int focusedPos, bool return S_OK; } + void CPanel::GetSelectedItemsIndices(CRecordVector &indices) const { indices.Clear(); @@ -800,13 +816,16 @@ void CPanel::GetSelectedItemsIndices(CRecordVector &indices) const if (_listView.GetItemParam(itemIndex, param)) indices.Add(param); } + HeapSort(&indices.Front(), indices.Size()); */ - FOR_VECTOR (i, _selectedStatusVector) - if (_selectedStatusVector[i]) + const bool *v = &_selectedStatusVector.Front(); + unsigned size = _selectedStatusVector.Size(); + for (unsigned i = 0; i < size; i++) + if (v[i]) indices.Add(i); - // HeapSort(&indices.Front(), indices.Size()); } + void CPanel::GetOperatedItemIndices(CRecordVector &indices) const { GetSelectedItemsIndices(indices); @@ -922,7 +941,7 @@ void CPanel::OpenSelectedItems(bool tryInternal) GetOperatedItemIndices(indices); if (indices.Size() > 20) { - MessageBoxErrorLang(IDS_TOO_MANY_ITEMS); + MessageBox_Error_LangID(IDS_TOO_MANY_ITEMS); return; } @@ -974,17 +993,20 @@ UString CPanel::GetItemName_for_Copy(int itemIndex) const { if (itemIndex == kParentIndex) return L".."; + UString s; { NCOM::CPropVariant prop; if (_folder->GetProperty(itemIndex, kpidOutName, &prop) == S_OK) { if (prop.vt == VT_BSTR) - return prop.bstrVal; - if (prop.vt != VT_EMPTY) + s = prop.bstrVal; + else if (prop.vt != VT_EMPTY) throw 2723401; } + if (s.IsEmpty()) + s = GetItemName(itemIndex); } - return GetItemName(itemIndex); + return Get_Correct_FsFile_Name(s); } void CPanel::GetItemName(int itemIndex, UString &s) const @@ -1224,9 +1246,9 @@ void CPanel::ShowColumnsContextMenu(int x, int y) void CPanel::OnReload() { - HRESULT res = RefreshListCtrlSaveFocused(); + HRESULT res = RefreshListCtrl_SaveFocused(); if (res != S_OK) - MessageBoxError(res); + MessageBox_Error_HRESULT(res); } void CPanel::OnTimer() diff --git a/CPP/7zip/UI/FileManager/PanelMenu.cpp b/CPP/7zip/UI/FileManager/PanelMenu.cpp index 639ac273..7fd23486 100644 --- a/CPP/7zip/UI/FileManager/PanelMenu.cpp +++ b/CPP/7zip/UI/FileManager/PanelMenu.cpp @@ -661,7 +661,7 @@ bool CPanel::CheckBeforeUpdate(UINT resourceID) { if (!_folderOperations) { - MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + MessageBox_Error_UnsupportOperation(); // resourceID = resourceID; // MessageBoxErrorForUpdate(E_NOINTERFACE, resourceID); return false; @@ -689,7 +689,7 @@ bool CPanel::CheckBeforeUpdate(UINT resourceID) s += _parentFolders[i - 1].VirtualPath; s.Add_LF(); AddLangString(s, IDS_PROP_READ_ONLY); - MessageBoxMyError(s); + MessageBox_Error(s); return false; } diff --git a/CPP/7zip/UI/FileManager/PanelOperations.cpp b/CPP/7zip/UI/FileManager/PanelOperations.cpp index 218b3392..1db4b21c 100644 --- a/CPP/7zip/UI/FileManager/PanelOperations.cpp +++ b/CPP/7zip/UI/FileManager/PanelOperations.cpp @@ -111,9 +111,9 @@ typedef int (WINAPI * SHFileOperationWP)(LPSHFILEOPSTRUCTW lpFileOp); void CPanel::MessageBoxErrorForUpdate(HRESULT errorCode, UINT resourceID) { if (errorCode == E_NOINTERFACE) - MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + MessageBox_Error_UnsupportOperation(); else - MessageBoxError(errorCode, LangString(resourceID)); + MessageBox_Error_HRESULT_Caption(errorCode, LangString(resourceID)); } */ @@ -178,7 +178,7 @@ void CPanel::DeleteItems(bool NON_CE_VAR(toRecycleBin)) { if (toRecycleBin) { - MessageBoxErrorLang(IDS_ERROR_LONG_PATH_TO_RECYCLE); + MessageBox_Error_LangID(IDS_ERROR_LONG_PATH_TO_RECYCLE); return; } useInternalDelete = true; @@ -210,7 +210,7 @@ void CPanel::DeleteItems(bool NON_CE_VAR(toRecycleBin)) } /* if (fo.fAnyOperationsAborted) - MessageBoxError(result, LangString(IDS_ERROR_DELETING, 0x03020217)); + MessageBox_Error_HRESULT_Caption(result, LangString(IDS_ERROR_DELETING)); */ if (!useInternalDelete) { @@ -301,7 +301,7 @@ BOOL CPanel::OnEndLabelEdit(LV_DISPINFOW * lpnmh) UString newName = lpnmh->item.pszText; if (!IsCorrectFsName(newName)) { - MessageBoxError(E_INVALIDARG); + MessageBox_Error_HRESULT(E_INVALIDARG); return FALSE; } @@ -310,7 +310,7 @@ BOOL CPanel::OnEndLabelEdit(LV_DISPINFOW * lpnmh) UString correctName; if (!CorrectFsPath(newName, correctName)) { - MessageBoxError(E_INVALIDARG); + MessageBox_Error_HRESULT(E_INVALIDARG); return FALSE; } newName = correctName; @@ -344,6 +344,7 @@ BOOL CPanel::OnEndLabelEdit(LV_DISPINFOW * lpnmh) // Can't use RefreshListCtrl here. // RefreshListCtrlSaveFocused(); _selectedState.FocusedName = prefix + newName; + _selectedState.FocusedName_Defined = true; _selectedState.SelectFocused = true; // We need clear all items to disable GetText before Reload: @@ -375,7 +376,7 @@ void CPanel::CreateFolder() if (!IsCorrectFsName(newName)) { - MessageBoxError(E_INVALIDARG); + MessageBox_Error_HRESULT(E_INVALIDARG); return; } @@ -384,7 +385,7 @@ void CPanel::CreateFolder() UString correctName; if (!CorrectFsPath(newName, correctName)) { - MessageBoxError(E_INVALIDARG); + MessageBox_Error_HRESULT(E_INVALIDARG); return; } newName = correctName; @@ -413,6 +414,7 @@ void CPanel::CreateFolder() if (!_mySelectMode) state.SelectedNames.Clear(); state.FocusedName = newName; + state.FocusedName_Defined = true; state.SelectFocused = true; } RefreshTitleAlways(); @@ -444,7 +446,7 @@ void CPanel::CreateFile() UString correctName; if (!CorrectFsPath(newName, correctName)) { - MessageBoxError(E_INVALIDARG); + MessageBox_Error_HRESULT(E_INVALIDARG); return; } newName = correctName; @@ -453,7 +455,7 @@ void CPanel::CreateFile() HRESULT result = _folderOperations->CreateFile(newName, 0); if (result != S_OK) { - MessageBoxError(result, LangString(IDS_CREATE_FILE_ERROR)); + MessageBox_Error_HRESULT_Caption(result, LangString(IDS_CREATE_FILE_ERROR)); // MessageBoxErrorForUpdate(result, IDS_CREATE_FILE_ERROR); return; } @@ -463,6 +465,7 @@ void CPanel::CreateFile() if (!_mySelectMode) state.SelectedNames.Clear(); state.FocusedName = newName; + state.FocusedName_Defined = true; state.SelectFocused = true; RefreshListCtrl(state); } @@ -515,9 +518,9 @@ void CPanel::ChangeComment() if (result != S_OK) { if (result == E_NOINTERFACE) - MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + MessageBox_Error_UnsupportOperation(); else - MessageBoxError(result, L"Set Comment Error"); + MessageBox_Error_HRESULT_Caption(result, L"Set Comment Error"); } RefreshListCtrl(state); } diff --git a/CPP/7zip/UI/FileManager/PanelSplitFile.cpp b/CPP/7zip/UI/FileManager/PanelSplitFile.cpp index b91a51a9..d3d4b72e 100644 --- a/CPP/7zip/UI/FileManager/PanelSplitFile.cpp +++ b/CPP/7zip/UI/FileManager/PanelSplitFile.cpp @@ -167,7 +167,7 @@ void CApp::Split() CPanel &srcPanel = Panels[srcPanelIndex]; if (!srcPanel.Is_IO_FS_Folder()) { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_UnsupportOperation(); return; } CRecordVector indices; @@ -176,13 +176,13 @@ void CApp::Split() return; if (indices.Size() != 1) { - srcPanel.MessageBoxErrorLang(IDS_SELECT_ONE_FILE); + srcPanel.MessageBox_Error_LangID(IDS_SELECT_ONE_FILE); return; } int index = indices[0]; if (srcPanel.IsItem_Folder(index)) { - srcPanel.MessageBoxErrorLang(IDS_SELECT_ONE_FILE); + srcPanel.MessageBox_Error_LangID(IDS_SELECT_ONE_FILE); return; } const UString itemName = srcPanel.GetItemName(index); @@ -203,12 +203,12 @@ void CApp::Split() NFind::CFileInfo fileInfo; if (!fileInfo.Find(us2fs(srcPath + itemName))) { - srcPanel.MessageBoxMyError(L"Can not find file"); + srcPanel.MessageBox_Error(L"Can not find file"); return; } if (fileInfo.Size <= splitDialog.VolumeSizes.Front()) { - srcPanel.MessageBoxErrorLang(IDS_SPLIT_VOL_MUST_BE_SMALLER); + srcPanel.MessageBox_Error_LangID(IDS_SPLIT_VOL_MUST_BE_SMALLER); return; } const UInt64 numVolumes = GetNumberOfVolumes(fileInfo.Size, splitDialog.VolumeSizes); @@ -226,7 +226,8 @@ void CApp::Split() NName::NormalizeDirPathPrefix(path); if (!CreateComplexDir(us2fs(path))) { - srcPanel.MessageBoxError2Lines(MyFormatNew(IDS_CANNOT_CREATE_FOLDER, path), ::GetLastError()); + DWORD lastError = ::GetLastError(); + srcPanel.MessageBox_Error_2Lines_Message_HRESULT(MyFormatNew(IDS_CANNOT_CREATE_FOLDER, path), lastError); return; } @@ -350,7 +351,7 @@ void CApp::Combine() CPanel &srcPanel = Panels[srcPanelIndex]; if (!srcPanel.IsFSFolder()) { - srcPanel.MessageBoxErrorLang(IDS_OPERATION_IS_NOT_SUPPORTED); + srcPanel.MessageBox_Error_LangID(IDS_OPERATION_IS_NOT_SUPPORTED); return; } CRecordVector indices; @@ -360,7 +361,7 @@ void CApp::Combine() int index = indices[0]; if (indices.Size() != 1 || srcPanel.IsItem_Folder(index)) { - srcPanel.MessageBoxErrorLang(IDS_COMBINE_SELECT_ONE_FILE); + srcPanel.MessageBox_Error_LangID(IDS_COMBINE_SELECT_ONE_FILE); return; } const UString itemName = srcPanel.GetItemName(index); @@ -376,7 +377,7 @@ void CApp::Combine() CVolSeqName volSeqName; if (!volSeqName.ParseName(itemName)) { - srcPanel.MessageBoxErrorLang(IDS_COMBINE_CANT_DETECT_SPLIT_FILE); + srcPanel.MessageBox_Error_LangID(IDS_COMBINE_CANT_DETECT_SPLIT_FILE); return; } @@ -396,13 +397,13 @@ void CApp::Combine() } if (combiner.Names.Size() == 1) { - srcPanel.MessageBoxErrorLang(IDS_COMBINE_CANT_FIND_MORE_THAN_ONE_PART); + srcPanel.MessageBox_Error_LangID(IDS_COMBINE_CANT_FIND_MORE_THAN_ONE_PART); return; } if (combiner.TotalSize == 0) { - srcPanel.MessageBoxMyError(L"No data"); + srcPanel.MessageBox_Error(L"No data"); return; } @@ -438,7 +439,8 @@ void CApp::Combine() NName::NormalizeDirPathPrefix(path); if (!CreateComplexDir(us2fs(path))) { - srcPanel.MessageBoxError2Lines(MyFormatNew(IDS_CANNOT_CREATE_FOLDER, path), ::GetLastError()); + DWORD lastError = ::GetLastError(); + srcPanel.MessageBox_Error_2Lines_Message_HRESULT(MyFormatNew(IDS_CANNOT_CREATE_FOLDER, path), lastError); return; } @@ -457,7 +459,7 @@ void CApp::Combine() combiner.OutputPath = us2fs(destFilePath); if (fileInfo.Find(combiner.OutputPath)) { - srcPanel.MessageBoxMyError(MyFormatNew(IDS_FILE_EXIST, destFilePath)); + srcPanel.MessageBox_Error(MyFormatNew(IDS_FILE_EXIST, destFilePath)); return; } diff --git a/CPP/7zip/UI/GUI/CompressDialog.cpp b/CPP/7zip/UI/GUI/CompressDialog.cpp index 22b25834..e189e9b5 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.cpp +++ b/CPP/7zip/UI/GUI/CompressDialog.cpp @@ -80,6 +80,9 @@ using namespace NDir; static const unsigned kHistorySize = 20; +static const UInt32 kNoSolidBlockSize = 0; +static const UInt32 kSolidBlockSize = 64; + static LPCSTR const kExeExt = ".exe"; #define k7zFormat "7z" @@ -292,7 +295,7 @@ static const CFormatInfo g_Formats[] = "xz", (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), METHODS_PAIR(g_XzMethods), - false, false, true, false, false, false + false, true, true, false, false, false }, { "zstd", /* 6 */ @@ -838,10 +841,17 @@ void CCompressDialog::OnOK() Info.OrderMode = GetOrderMode(); Info.NumThreads = GetNumThreadsSpec(); - UInt32 solidLogSize = GetBlockSizeSpec(); - Info.SolidBlockSize = 0; - if (solidLogSize > 0 && solidLogSize != (UInt32)(Int32)-1) - Info.SolidBlockSize = (solidLogSize >= 64) ? (UInt64)(Int64)-1 : ((UInt64)1 << solidLogSize); + { + // Info.SolidIsSpecified = g_Formats[GetStaticFormatIndex()].Solid; + UInt32 solidLogSize = GetBlockSizeSpec(); + Info.SolidBlockSize = 0; + if (solidLogSize == (UInt32)(Int32)-1) + Info.SolidIsSpecified = false; + else if (solidLogSize > 0) + Info.SolidBlockSize = (solidLogSize >= 64) ? + (UInt64)(Int64)-1 : + ((UInt64)1 << solidLogSize); + } Info.Method = GetMethodSpec(); Info.EncryptionMethod = GetEncryptionMethodSpec(); @@ -949,6 +959,7 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) */ return true; } + case IDC_COMPRESS_FORMAT: { bool isSFX = IsSFX(); @@ -963,6 +974,7 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) SetMemoryUsage(); return true; } + case IDC_COMPRESS_LEVEL: { SetMethod(GetMethodID()); @@ -972,6 +984,7 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) SetMemoryUsage(); return true; } + case IDC_COMPRESS_METHOD: { SetLevel(); @@ -983,13 +996,34 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) SetMemoryUsage(); return true; } + case IDC_COMPRESS_DICTIONARY: + { + UInt32 blockSizeLog = GetBlockSizeSpec(); + if (blockSizeLog != (UInt32)(Int32)-1 + && blockSizeLog != kNoSolidBlockSize + && blockSizeLog != kSolidBlockSize) + { + const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + int index = FindRegistryFormatAlways(ai.Name); + NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; + fo.Reset_BlockLogSize(); + SetSolidBlockSize(true); + } + + SetMemoryUsage(); + return true; + } + case IDC_COMPRESS_ORDER: + return true; + + case IDC_COMPRESS_SOLID: { - SetSolidBlockSize(); SetMemoryUsage(); return true; } + case IDC_COMPRESS_THREADS: { SetMemoryUsage(); @@ -1282,6 +1316,12 @@ bool CCompressDialog::IsZipFormat() return ai.Name.IsEqualTo_Ascii_NoCase("zip"); } +bool CCompressDialog::IsXzFormat() +{ + const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + return ai.Name.IsEqualTo_Ascii_NoCase("xz"); +} + void CCompressDialog::SetEncryptionMethod() { _encryptionMethod.ResetContent(); @@ -1647,10 +1687,8 @@ bool CCompressDialog::GetOrderMode() return false; } -static const UInt32 kNoSolidBlockSize = 0; -static const UInt32 kSolidBlockSize = 64; -void CCompressDialog::SetSolidBlockSize() +void CCompressDialog::SetSolidBlockSize(bool useDictionary) { m_Solid.ResetContent(); const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; @@ -1668,6 +1706,10 @@ void CCompressDialog::SetSolidBlockSize() UInt32 defaultBlockSize = (UInt32)(Int32)-1; const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + + if (useDictionary) + defaultBlockSize = GetBlockSizeSpec(); + else { int index = FindRegistryFormat(ai.Name); if (index >= 0) @@ -1678,23 +1720,54 @@ void CCompressDialog::SetSolidBlockSize() } } + bool is7z = ai.Name.IsEqualTo_Ascii_NoCase("7z"); + { - int index = (int)m_Solid.AddString(LangString(IDS_COMPRESS_NON_SOLID)); + UString s ('-'); + if (is7z) + LangString(IDS_COMPRESS_NON_SOLID, s); + int index = (int)m_Solid.AddString(s); m_Solid.SetItemData(index, (UInt32)kNoSolidBlockSize); - m_Solid.SetCurSel(0); + if (defaultBlockSize == kNoSolidBlockSize) + m_Solid.SetCurSel(0); } + + UInt64 blockSize; - bool needSet = (defaultBlockSize == (UInt32)(Int32)-1); - + if (is7z) + { + blockSize = (UInt64)dict << 7; + const UInt32 kMinSize = (UInt32)1 << 24; + const UInt64 kMaxSize = (UInt64)1 << 32; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + } + else + { + blockSize = (UInt64)dict << 2; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dict) blockSize = dict; + } + for (unsigned i = 20; i <= 36; i++) { - if (needSet && dict >= (((UInt64)1 << (i - 7))) && i <= 32) + if (defaultBlockSize == (UInt32)(Int32)-1 && ((UInt64)1 << i) >= blockSize) defaultBlockSize = i; + TCHAR s[40]; + char post; ConvertUInt32ToString(1 << (i % 10), s); - if (i < 30) lstrcat(s, TEXT(" M")); - else lstrcat(s, TEXT(" G")); - lstrcat(s, TEXT("B")); + if (i < 20) post = 'K'; + else if (i < 30) post = 'M'; + else post = 'G'; + unsigned pos = (unsigned)lstrlen(s); + s[pos++] = ' '; + s[pos++] = post; + s[pos++] = 'B'; + s[pos] = 0; int index = (int)m_Solid.AddString(s); m_Solid.SetItemData(index, (UInt32)i); } @@ -1815,20 +1888,42 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) size1 += (2 << 20) + (4 << 20); numThreads1 = 2; } - + UInt32 numBlockThreads = numThreads / numThreads1; + + UInt64 chunkSize = 0; - if (methidId == kLZMA || numBlockThreads == 1) - size1 += (UInt64)dict * 3 / 2; - else + if (methidId != kLZMA && numBlockThreads != 1) { - UInt64 chunkSize = (UInt64)dict << 2; + chunkSize = (UInt64)dict << 2; chunkSize = MyMax(chunkSize, (UInt64)(1 << 20)); chunkSize = MyMin(chunkSize, (UInt64)(1 << 28)); chunkSize = MyMax(chunkSize, (UInt64)dict); - size1 += chunkSize * 2; + + if (IsXzFormat()) + { + UInt32 blockSizeLog = GetBlockSizeSpec(); + if (blockSizeLog != (UInt32)(Int32)-1) + { + if (blockSizeLog == kSolidBlockSize) + { + numBlockThreads = 1; + chunkSize = 0; + } + else if (blockSizeLog != kNoSolidBlockSize) + chunkSize = (UInt64)1 << blockSizeLog; + } + } + } + + if (chunkSize == 0) + size += numBlockThreads * (size1 + (UInt64)dict * 3 / 2); + else + { + size += numBlockThreads * (size1 + chunkSize); + UInt64 numPackChunks = numBlockThreads + (numBlockThreads / 4) + 2; + size += numPackChunks * chunkSize; } - size += size1 * numBlockThreads; decompressMemory = dict + (2 << 20); return size; @@ -1843,9 +1938,11 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) case kDeflate: case kDeflate64: { + /* UInt32 order = GetOrder(); if (order == (UInt32)(Int32)-1) order = 32; + */ if (level >= 7) size += (1 << 20); size += 3 << 20; diff --git a/CPP/7zip/UI/GUI/CompressDialog.h b/CPP/7zip/UI/GUI/CompressDialog.h index 1c0e6c61..41e836a0 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.h +++ b/CPP/7zip/UI/GUI/CompressDialog.h @@ -137,6 +137,7 @@ class CCompressDialog: public NWindows::NControl::CModalDialog UString GetEncryptionMethodSpec(); bool IsZipFormat(); + bool IsXzFormat(); void SetEncryptionMethod(); @@ -161,7 +162,7 @@ class CCompressDialog: public NWindows::NControl::CModalDialog void SetOrder(); bool GetOrderMode(); - void SetSolidBlockSize(); + void SetSolidBlockSize(bool useDictionary = false); void SetNumThreads(); UInt64 GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory); diff --git a/CPP/7zip/UI/GUI/GUI.cpp b/CPP/7zip/UI/GUI/GUI.cpp index 54924026..c5e43e79 100644 --- a/CPP/7zip/UI/GUI/GUI.cpp +++ b/CPP/7zip/UI/GUI/GUI.cpp @@ -128,14 +128,21 @@ static int Main2() #if defined(_WIN32) && !defined(UNDER_CE) NSecurity::EnablePrivilege_SymLink(); + #endif + #ifdef _7ZIP_LARGE_PAGES if (options.LargePages) { SetLargePageSize(); - g_LargePagesMode = NSecurity::EnablePrivilege_LockMemory(); + // note: this process also can inherit that Privilege from parent process + g_LargePagesMode = + #if defined(_WIN32) && !defined(UNDER_CE) + NSecurity::EnablePrivilege_LockMemory(); + #else + true; + #endif } #endif - #endif CREATE_CODECS_OBJECT diff --git a/CPP/7zip/UI/GUI/GUI.dsp b/CPP/7zip/UI/GUI/GUI.dsp index 242ddbc0..223b441f 100644 --- a/CPP/7zip/UI/GUI/GUI.dsp +++ b/CPP/7zip/UI/GUI/GUI.dsp @@ -45,7 +45,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /Yu"stdafx.h" /FD /c -# ADD CPP /nologo /Gr /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_WINDOWS" /D "LANG" /D "WIN_LONG_PATH" /D "EXTERNAL_CODECS" /D "SUPPORT_DEVICE_FILE" /FAcs /Yu"stdafx.h" /FD /c +# ADD CPP /nologo /Gr /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_WINDOWS" /D "LANG" /D "WIN_LONG_PATH" /D "EXTERNAL_CODECS" /D "SUPPORT_DEVICE_FILE" /D "_7ZIP_LARGE_PAGES" /FAcs /Yu"stdafx.h" /FD /c # ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "NDEBUG" @@ -72,7 +72,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /Yu"stdafx.h" /FD /GZ /c -# ADD CPP /nologo /Gr /MDd /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_WINDOWS" /D "LANG" /D "WIN_LONG_PATH" /D "EXTERNAL_CODECS" /D "SUPPORT_DEVICE_FILE" /Yu"stdafx.h" /FD /GZ /c +# ADD CPP /nologo /Gr /MDd /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_WINDOWS" /D "LANG" /D "WIN_LONG_PATH" /D "EXTERNAL_CODECS" /D "SUPPORT_DEVICE_FILE" /D "_7ZIP_LARGE_PAGES" /Yu"stdafx.h" /FD /GZ /c # ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "_DEBUG" @@ -99,7 +99,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /MD /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /Yu"stdafx.h" /FD /c -# ADD CPP /nologo /Gr /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_WINDOWS" /D "LANG" /D "WIN_LONG_PATH" /D "EXTERNAL_CODECS" /D "SUPPORT_DEVICE_FILE" /Yu"stdafx.h" /FD /c +# ADD CPP /nologo /Gr /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_WINDOWS" /D "LANG" /D "WIN_LONG_PATH" /D "EXTERNAL_CODECS" /D "SUPPORT_DEVICE_FILE" /D "_7ZIP_LARGE_PAGES" /Yu"stdafx.h" /FD /c # ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "NDEBUG" @@ -127,7 +127,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /MDd /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /Yu"stdafx.h" /FD /GZ /c -# ADD CPP /nologo /Gr /MDd /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_WINDOWS" /D "LANG" /D "WIN_LONG_PATH" /D "EXTERNAL_CODECS" /D "SUPPORT_DEVICE_FILE" /Yu"stdafx.h" /FD /GZ /c +# ADD CPP /nologo /Gr /MDd /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_WINDOWS" /D "LANG" /D "WIN_LONG_PATH" /D "EXTERNAL_CODECS" /D "SUPPORT_DEVICE_FILE" /D "_7ZIP_LARGE_PAGES" /Yu"stdafx.h" /FD /GZ /c # ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "_DEBUG" diff --git a/CPP/Build.mak b/CPP/Build.mak index 68a6e010..c090f758 100644 --- a/CPP/Build.mak +++ b/CPP/Build.mak @@ -30,7 +30,9 @@ LFLAGS = $(LFLAGS) /ENTRY:mainACRTStartup !IFNDEF NEW_COMPILER LFLAGS = $(LFLAGS) !ENDIF +# !IF "$(CPU)" != "ARM" CFLAGS = $(CFLAGS) -Gr +# !ENDIF LIBS = $(LIBS) user32.lib advapi32.lib shell32.lib !ENDIF @@ -78,7 +80,9 @@ LFLAGS = $(LFLAGS) /LTCG /LARGEADDRESSAWARE !IFDEF DEF_FILE LFLAGS = $(LFLAGS) -DLL -DEF:$(DEF_FILE) !ELSE +# !IF "$(CPU)" != "ARM" LFLAGS = $(LFLAGS) /FIXED +# !ENDIF # /BASE:0x400000 !ENDIF diff --git a/CPP/Windows/FileDir.cpp b/CPP/Windows/FileDir.cpp index 3adb5a02..91808b6d 100644 --- a/CPP/Windows/FileDir.cpp +++ b/CPP/Windows/FileDir.cpp @@ -128,6 +128,15 @@ bool SetFileAttrib(CFSTR path, DWORD attrib) return false; } + +bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib) +{ + if ((attrib & 0xF0000000) != 0) + attrib &= 0x3FFF; + return SetFileAttrib(path, attrib); +} + + bool RemoveDir(CFSTR path) { #ifndef _UNICODE diff --git a/CPP/Windows/FileDir.h b/CPP/Windows/FileDir.h index 35f69577..8d2b56a6 100644 --- a/CPP/Windows/FileDir.h +++ b/CPP/Windows/FileDir.h @@ -15,7 +15,20 @@ bool GetWindowsDir(FString &path); bool GetSystemDir(FString &path); bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime); + + bool SetFileAttrib(CFSTR path, DWORD attrib); + +/* + Some programs store posix attributes in high 16 bits of windows attributes field. + Also some programs use additional flag markers: 0x8000 or 0x4000. + SetFileAttrib_PosixHighDetect() tries to detect posix field, and it extracts only attribute + bits that are related to current system only. +*/ + +bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib); + + bool MyMoveFile(CFSTR existFileName, CFSTR newFileName); #ifndef UNDER_CE diff --git a/DOC/7zip.inf b/DOC/7zip.inf index 3f4f5194..2c5311b2 100644 --- a/DOC/7zip.inf +++ b/DOC/7zip.inf @@ -10,8 +10,8 @@ AppName = "7-Zip" InstallDir = %CE1%\%AppName% [Strings] -AppVer = "17.00" -AppDate = "2017-04-29" +AppVer = "17.01" +AppDate = "2017-08-28" [CEDevice] ; ProcessorType = 2577 ; ARM diff --git a/DOC/7zip.nsi b/DOC/7zip.nsi index 17cacd6f..f8e37859 100644 --- a/DOC/7zip.nsi +++ b/DOC/7zip.nsi @@ -2,7 +2,7 @@ ;Defines !define VERSION_MAJOR 17 -!define VERSION_MINOR 00 +!define VERSION_MINOR 01 !define VERSION_POSTFIX_FULL " ZS" !ifdef WIN64 !ifdef IA64 diff --git a/DOC/7zip.wxs b/DOC/7zip.wxs index 913b9749..bfc7e709 100644 --- a/DOC/7zip.wxs +++ b/DOC/7zip.wxs @@ -1,7 +1,7 @@ - + diff --git a/DOC/Methods.txt b/DOC/Methods.txt index 9ee38247..11adcb0b 100644 --- a/DOC/Methods.txt +++ b/DOC/Methods.txt @@ -1,8 +1,8 @@ 7-Zip method IDs for 7z and xz archives --------------------------------------- -Version: 17.00 -Date: 2017-01-01 +Version: 17.01 +Date: 2017-05-27 Each compression or crypto method in 7z is associated with unique binary value (ID). The length of ID in bytes is arbitrary but it can not exceed 63 bits (8 bytes). @@ -128,10 +128,10 @@ List of defined IDs 11 xx - reserved (Tino Reichardt) 01 - ZSTD - 02 - Brotli + 02 - BROTLI 04 - LZ4 05 - LZ5 - 06 - Lizard + 06 - LIZARD 06.. - Crypto diff --git a/README.md b/README.md index 956e8cb6..e1317d72 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ You can install it in two ways: The output should look like this: ``` -7-Zip 17.00 ZS v1.3.1 R1 (x64) : Copyright (c) 1999-2017 Igor Pavlov +7-Zip 17.01 ZS v1.3.1 R1 (x64) : Copyright (c) 1999-2017 Igor Pavlov Libs: @@ -139,7 +139,7 @@ Codecs: The output should look like this: ``` -7-Zip 17.00 beta (x64) : Copyright (c) 1999-2017 Igor Pavlov : 2017-04-29 +7-Zip 17.01 beta (x64) : Copyright (c) 1999-2017 Igor Pavlov : 2017-08-28 Libs: @@ -236,14 +236,14 @@ You find this project useful, maybe you consider a donation ;-) ## Version Information -- 7-Zip ZS Version 17.00 +- 7-Zip ZS Version 17.01 - [Brotli] Version 0.6.0 - [Lizard] Version 1.0 - [LZ4] Version 1.7.5 - [LZ5] Version 1.5 - [Zstandard] Version 1.3.1 -/TR 2017-08-20 +/TR 2017-08-28 [7-Zip]:http://www.7-zip.org/ [lzip]:http://www.nongnu.org/lzip/