Add zstd fast levels and update 7z property sizes

- add the "fast compression levels" of zstd via "fast" option (fast=1..64) - change the 7-Zip property sizes of LZ4, LZ5 and Zstandard to 3 - 3 and 5 byte header are valid now (default is 3) - update the Methods-Extern.md file, to reflect the property changes
mcmilk · Nov 25, 2018 · f98edef · f98edef
1 parent 4728ce2
commit f98edef
Show file tree

Hide file tree

Showing 15 changed files with 129 additions and 88 deletions.
diff --git a/C/7zVersion.h b/C/7zVersion.h
@@ -10,7 +10,7 @@
   #define MY_VERSION_CPU MY_VERSION
 #endif
 
-#define MY_DATE "2018-11-17"
+#define MY_DATE "2018-11-25"
 #undef MY_COPYRIGHT
 #undef MY_VERSION_COPYRIGHT_DATE
 #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt"

diff --git a/C/7zVersionTr.h b/C/7zVersionTr.h
@@ -1,11 +1,14 @@
-#define MY_VER_MAJOR 18
-#define MY_VER_MINOR 05
-#define MY_VER_BUILD 0
-#define MY_VERSION_NUMBERS "1.3.7 R3"
-#define MY_VERSION         MY_VERSION_NUMBERS
-#define MY_DATE            "2018-11-17"
-#undef MY_COPYRIGHT
-#undef MY_VERSION_COPYRIGHT_DATE
-#define MY_AUTHOR_NAME     "Tino Reichardt"
-#define MY_COPYRIGHT       "Copyright (c) 2016 - 2018 Tino Reichardt"
-#define MY_VERSION_COPYRIGHT_DATE MY_VERSION " : " MY_COPYRIGHT " : " MY_DATE
+
+#include "7zVersion.h"
+
+#undef  MY_AUTHOR_NAME
+#define MY_AUTHOR_NAME  "Tino Reichardt"
+
+#undef  MY_COPYRIGHT
+#define MY_COPYRIGHT    "Copyright (c) 2016 - 2018 Tino Reichardt"
+
+#undef  MY_COPYRIGHT_DATE
+#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE
+
+#undef  MY_VERSION_COPYRIGHT_DATE
+#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE
diff --git a/CPP/7zip/Archive/7z/7zHandler.cpp b/CPP/7zip/Archive/7z/7zHandler.cpp
@@ -507,7 +507,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const
       else if (id == k_LZ4)
       {
         name = "LZ4";
-        if (propsSize == 5)
+        if (propsSize == 3 || propsSize == 5)
         {
           char *dest = s;
           *dest++ = 'v';
@@ -525,7 +525,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const
       else if (id == k_LZ5)
       {
         name = "LZ5";
-        if (propsSize == 5)
+        if (propsSize == 3 || propsSize == 5)
         {
           char *dest = s;
           *dest++ = 'v';
@@ -543,18 +543,25 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const
       else if (id == k_ZSTD)
       {
         name = "ZSTD";
-        if (propsSize == 5)
+        if (propsSize == 3 || propsSize == 5)
         {
           char *dest = s;
+          UInt32 l = props[2];
           *dest++ = 'v';
           ConvertUInt32ToString(props[0], dest);
           dest += MyStringLen(dest);
           *dest++ = '.';
           ConvertUInt32ToString(props[1], dest);
           dest += MyStringLen(dest);
           *dest++ = ',';
-          *dest++ = 'l';
-          ConvertUInt32ToString(props[2], dest);
+          if (l <= 22) {
+            *dest++ = 'l';
+            ConvertUInt32ToString(l, dest);
+          } else {
+            *dest++ = 'f';
+            *dest++ = 'l';
+            ConvertUInt32ToString(l - 32, dest);
+          }
           dest += MyStringLen(dest);
         }
       }

diff --git a/CPP/7zip/Bundles/Codec_brotli/resource.rc b/CPP/7zip/Bundles/Codec_brotli/resource.rc
@@ -1,6 +1,6 @@
 #include "../../../../C/7zVersionTr.h"
 #include "../../../../C/7zVersion.rc"
 
-MY_VERSION_INFO_DLL("7-Zip Brotli Plugin v1.0.6", "Brotli")
+MY_VERSION_INFO_DLL("7-Zip Brotli Plugin v1.0.7", "Brotli")
 
 101  ICON  "../../Archive/Icons/7z.ico"
diff --git a/CPP/7zip/Common/MethodProps.cpp b/CPP/7zip/Common/MethodProps.cpp
@@ -281,6 +281,7 @@ static const CNameToPropID g_NameToPropID[] =
   { VT_BSTR, "filter" },
   { VT_UI8, "memuse" },
   { VT_UI4, "strat" },
+  { VT_UI4, "fast" },
   { VT_UI4, "long" },
   { VT_UI4, "wlog" },
   { VT_UI4, "hlog" },

diff --git a/CPP/7zip/Compress/Lz4Decoder.cpp b/CPP/7zip/Compress/Lz4Decoder.cpp
@@ -87,12 +87,16 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
 {
   DProps *pProps = (DProps *)prop;
 
-  if (size != sizeof(DProps))
+  switch (size) {
+  case 3:
+    memcpy(&_props, pProps, 3);
+    return S_OK;
+  case 5:
+    memcpy(&_props, pProps, 5);
+    return S_OK;
+  default:
     return E_NOTIMPL;
-
-  memcpy(&_props, pProps, sizeof (DProps));
-
-  return S_OK;
+  }
 }
 
 STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)

diff --git a/CPP/7zip/Compress/Lz4Encoder.h b/CPP/7zip/Compress/Lz4Encoder.h
@@ -29,7 +29,6 @@ struct CProps
   Byte _ver_major;
   Byte _ver_minor;
   Byte _level;
-  Byte _reserved[2];
 };
 
 class CEncoder:

diff --git a/CPP/7zip/Compress/Lz5Decoder.cpp b/CPP/7zip/Compress/Lz5Decoder.cpp
@@ -87,12 +87,16 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
 {
   DProps *pProps = (DProps *)prop;
 
-  if (size != sizeof(DProps))
+  switch (size) {
+  case 3:
+    memcpy(&_props, pProps, 3);
+    return S_OK;
+  case 5:
+    memcpy(&_props, pProps, 5);
+    return S_OK;
+  default:
     return E_NOTIMPL;
-
-  memcpy(&_props, pProps, sizeof (DProps));
-
-  return S_OK;
+  }
 }
 
 STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)

diff --git a/CPP/7zip/Compress/Lz5Encoder.h b/CPP/7zip/Compress/Lz5Encoder.h
@@ -29,7 +29,6 @@ struct CProps
   Byte _ver_major;
   Byte _ver_minor;
   Byte _level;
-  Byte _reserved[2];
 };
 
 class CEncoder:

diff --git a/CPP/7zip/Compress/ZstdDecoder.cpp b/CPP/7zip/Compress/ZstdDecoder.cpp
@@ -39,8 +39,10 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
 
   switch (size) {
   case 3:
+    memcpy(&_props, pProps, 3);
+    return S_OK;
   case 5:
-    memcpy(&_props, pProps, sizeof (DProps));
+    memcpy(&_props, pProps, 5);
     return S_OK;
   default:
     return E_NOTIMPL;

diff --git a/CPP/7zip/Compress/ZstdEncoder.cpp b/CPP/7zip/Compress/ZstdEncoder.cpp
@@ -23,8 +23,9 @@ CEncoder::CEncoder():
   _processedIn(0),
   _processedOut(0),
   _numThreads(NWindows::NSystem::GetNumberOfProcessors()),
-  _Long(-1),
   _Strategy(-1),
+  _Long(-1),
+  _Level(ZSTD_CLEVEL_DEFAULT),
   _WindowLog(-1),
   _HashLog(-1),
   _ChainLog(-1),
@@ -62,32 +63,68 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI
     UInt32 v = (UInt32)prop.ulVal;
     switch (propID)
     {
-    case NCoderPropID::kLevel:
-      {
-        if (prop.vt != VT_UI4)
-          return E_INVALIDARG;
-
-        /* level 1..22 */
-        _props._level = static_cast < Byte > (prop.ulVal);
-        Byte mylevel = static_cast < Byte > (ZSTD_LEVEL_MAX);
-        if (_props._level > mylevel)
-          _props._level = mylevel;
-
-        break;
-      }
     case NCoderPropID::kNumThreads:
       {
         SetNumberOfThreads(v);
         break;
       }
-
     case NCoderPropID::kStrategy:
       {
         if (v < 1) v = 1;
         if (v > 8) v = 8;
         _Strategy = v;
         break;
       }
+    case NCoderPropID::kLevel:
+      {
+        _Level = v;
+        if (v < 1) {
+          _Level = 1;
+        } else if ((Int32)v > ZSTD_maxCLevel()) {
+          _Level = ZSTD_maxCLevel();
+        }
+
+        /**
+         * zstd default levels: _Level => 1..ZSTD_maxCLevel()
+         */
+        _props._level = static_cast < Byte > (_Level);
+        break;
+      }
+    case NCoderPropID::kFast:
+      {
+        /* like --fast in zstd cli program */
+        UInt32 _Fast = v;
+
+        if (v < 1) {
+          _Fast = 1;
+        } else if (v > 64) {
+          _Fast = 64;
+        }
+
+        /**
+         * zstd fast levels:
+         * _Fast  => 1..ZSTD_minCLevel()  (_Level => _Fast + 32)
+         */
+        _props._level = static_cast < Byte > (_Fast + 32);
+
+        /* negative levels are the fast ones */
+        _Level = _Fast * -1;
+        break;
+      }
+    case NCoderPropID::kLong:
+      {
+        /* like --long in zstd cli program */
+        _Long = 1;
+        if (v == 0) {
+          // m0=zstd:long:tlen=x
+          _WindowLog = 27;
+        } else if (v < 10) {
+          _WindowLog = 10;
+        } else if (v > ZSTD_WINDOWLOG_MAX) {
+          _WindowLog = ZSTD_WINDOWLOG_MAX;
+        }
+        break;
+      }
     case NCoderPropID::kWindowLog:
       {
         if (v < ZSTD_WINDOWLOG_MIN) v = ZSTD_WINDOWLOG_MIN;
@@ -137,20 +174,6 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI
         _OverlapLog = v;
         break;
       }
-    case NCoderPropID::kLong:
-      {
-        /* exact like --long in zstd cli program */
-        _Long = 1;
-        if (v == 0) {
-          // m0=zstd:long:tlen=x
-          _WindowLog = 27;
-        } else if (v < 10) {
-          _WindowLog = 10;
-        } else if (v > ZSTD_WINDOWLOG_MAX) {
-          _WindowLog = ZSTD_WINDOWLOG_MAX;
-        }
-        break;
-      }
     case NCoderPropID::kLdmHashLog:
       {
         if (v < ZSTD_HASHLOG_MIN) v = ZSTD_HASHLOG_MIN;
@@ -217,7 +240,7 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
       return E_OUTOFMEMORY;
 
     /* setup level */
-    err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_compressionLevel, _props._level);
+    err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_compressionLevel, (UInt32)_Level);
     if (ZSTD_isError(err)) return E_INVALIDARG;
 
     /* setup thread count */
@@ -233,7 +256,7 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
       _Long = 1;
 
     /* set ldm */
-    if (_Long == 1) {
+    if (_Long != -1) {
       err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_enableLongDistanceMatching, _Long);
       if (ZSTD_isError(err)) return E_INVALIDARG;
     }

diff --git a/CPP/7zip/Compress/ZstdEncoder.h b/CPP/7zip/Compress/ZstdEncoder.h
@@ -28,7 +28,6 @@ struct CProps
   Byte _ver_major;
   Byte _ver_minor;
   Byte _level;
-  Byte _reserved[2];
 };
 
 class CEncoder:
@@ -53,14 +52,15 @@ class CEncoder:
 
   /* zstd advanced compression options */
   Int32 _Strategy;
+  Int32 _Long;
+  Int32 _Level;
   Int32 _WindowLog;
   Int32 _HashLog;
   Int32 _ChainLog;
   Int32 _SearchLog;
   Int32 _SearchLength;
   Int32 _TargetLen;
   Int32 _OverlapLog;
-  Int32 _Long;
   Int32 _LdmHashLog;
   Int32 _LdmSearchLength;
   Int32 _LdmBucketSizeLog;

diff --git a/CPP/7zip/ICoder.h b/CPP/7zip/ICoder.h
@@ -135,8 +135,9 @@ namespace NCoderPropID
 
     /* zstd props */
     kStrategy,          // VT_UI4 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra
-    kLong,              // VT_UI4 0=disable ldm (default: 27)
-    kWindowLog,         // VT_UI4 x32=10(1KiB)..30(1GiB)   x64=10(1KiB)..31(2GiB)
+    kFast,              // VT_UI4 The minimum fast is 1 and the maximum is 64 (default: unused)
+    kLong,              // VT_UI4 The minimum long is 10 (1KiB) and the maximum is 30 (1GiB) on x32 and 31 (2GiB) on x64
+    kWindowLog,         // VT_UI4 The minimum long is 10 (1KiB) and the maximum is 30 (1GiB) on x32 and 31 (2GiB) on x64
     kHashLog,           // VT_UI4 The minimum hlog is 6 (64 B) and the maximum is 26 (128 MiB).
     kChainLog,          // VT_UI4 The minimum clog is 6 (64 B) and the maximum is 28 (256 MiB)
     kSearchLog,         // VT_UI4 The minimum slog is 1 and the maximum is 26