Update Brotli to v1.0.1

- update README.md to the new versions (LZ4, Brotli)

C/7zVersion.h

@@ -1,7 +1,7 @@
 #define MY_VER_MAJOR 17
 #define MY_VER_MINOR 01
 #define MY_VER_BUILD 0
-#define MY_VERSION_NUMBERS "17.01 ZS v1.3.1 R1"
+#define MY_VERSION_NUMBERS "17.01 ZS v1.3.1 R2"
 #define MY_VERSION MY_VERSION_NUMBERS
 
 #ifdef MY_CPU_NAME
@@ -10,7 +10,7 @@
   #define MY_VERSION_CPU MY_VERSION
 #endif
 
-#define MY_DATE "2017-08-28"
+#define MY_DATE "2017-09-28"
 #undef MY_COPYRIGHT
 #undef MY_VERSION_COPYRIGHT_DATE
 #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt"

C/7zVersionTr.h

@@ -1,9 +1,9 @@
 #define MY_VER_MAJOR 1
 #define MY_VER_MINOR 3
 #define MY_VER_BUILD 1
-#define MY_VERSION_NUMBERS "1.3.1 R1"
+#define MY_VERSION_NUMBERS "1.3.1 R2"
 #define MY_VERSION         MY_VERSION_NUMBERS
-#define MY_DATE            "2017-08-28"
+#define MY_DATE            "2017-09-28"
 #undef MY_COPYRIGHT
 #undef MY_VERSION_COPYRIGHT_DATE
 #define MY_AUTHOR_NAME     "Tino Reichardt"

C/brotli/Brotli-Adjust.sh

@@ -34,3 +34,11 @@ cd ../enc
 sed -i 's|include "./|include "./enc/|g' *.c
 sed -i 's|include "../common|include "./common/|g' *.c
 for f in *.c; do mv $f ../br_$f; done
+
+exit
+# and also disable these msc warnings:
+
+#  pragma warning(disable : 4100)
+#  pragma warning(disable : 4127)
+#  pragma warning(disable : 4389)
+#  pragma warning(disable : 4701)

C/brotli/README.md

@@ -8,64 +8,76 @@ and 2nd order context modeling, with a compression ratio comparable to the best
 currently available general-purpose compression methods. It is similar in speed
 with deflate but offers more dense compression.
 
-The specification of the Brotli Compressed Data Format is defined in [RFC 7932](https://www.ietf.org/rfc/rfc7932.txt).
+The specification of the Brotli Compressed Data Format is defined in [RFC 7932](https://tools.ietf.org/html/rfc7932).
 
 Brotli is open-sourced under the MIT License, see the LICENSE file.
 
 Brotli mailing list:
 https://groups.google.com/forum/#!forum/brotli
 
-[![Build Status](https://travis-ci.org/google/brotli.svg?branch=master)](https://travis-ci.org/google/brotli)
+[![TravisCI Build Status](https://travis-ci.org/google/brotli.svg?branch=master)](https://travis-ci.org/google/brotli)
+[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/github/google/brotli?branch=master&svg=true)](https://ci.appveyor.com/project/szabadka/brotli)
 
 ### Build instructions
 
-#### Make
+#### Autotools-style CMake
 
-To build and run tests, simply do:
+[configure-cmake](https://github.com/nemequ/configure-cmake) is an
+autotools-style configure script for CMake-based projects (not supported on Windows).
 
-    $ ./configure && make
+The basic commands to build, test and install brotli are:
 
-If you want to install brotli, use one of the more advanced build systems below.
+    $ mkdir out && cd out
+    $ ../configure-cmake
+    $ make
+    $ make test
+    $ make install
+
+By default, debug binaries are built. To generate "release" `Makefile` specify `--disable-debug` option to `configure-cmake`.
 
 #### Bazel
 
 See [Bazel](http://www.bazel.build/)
 
 #### CMake
 
-The basic commands to build, test and install brotli are:
+The basic commands to build and install brotli are:
 
-    $ mkdir out && cd out && ../configure-cmake && make
-    $ make test
-    $ make install
+    $ mkdir out && cd out
+    $ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./installed ..
+    $ cmake --build . --config Release --target install
 
-You can use other [CMake](https://cmake.org/) configuration. For example, to
-build static libraries and use a custom installation directory:
-
-    $ mkdir out-static && \
-      cd out-static && \
-      ../configure-cmake --disable-shared-libs --prefix='/my/prefix/dir/'
-    $ make install
+You can use other [CMake](https://cmake.org/) configuration.
 
 #### Premake5
 
 See [Premake5](https://premake.github.io/)
 
 #### Python
 
-To install the Python module from source, run the following:
+To install the latest release of the Python module, run the following:
+
+    $ pip install brotli
+
+To install the tip-of-the-tree version, run:
 
-    $ python setup.py install
+    $ pip install --upgrade git+https://github.com/google/brotli
 
-See the [Python readme](python/README.md) for more details on testing
-and development.
+See the [Python readme](python/README.md) for more details on installing
+from source, development, and testing.
 
 ### Benchmarks
 * [Squash Compression Benchmark](https://quixdb.github.io/squash-benchmark/) / [Unstable Squash Compression Benchmark](https://quixdb.github.io/squash-benchmark/unstable/)
 * [Large Text Compression Benchmark](http://mattmahoney.net/dc/text.html)
 * [Lzturbo Benchmark](https://sites.google.com/site/powturbo/home/benchmark)
 
 ### Related projects
+> **Disclaimer:** Brotli authors take no responsibility for the third party projects mentioned in this section.
+
 Independent [decoder](https://github.com/madler/brotli) implementation by Mark Adler, based entirely on format specification.
 
 JavaScript port of brotli [decoder](https://github.com/devongovett/brotli.js). Could be used directly via `npm install brotli`
+
+Hand ported [decoder / encoder](https://github.com/dominikhlbg/BrotliHaxe) in haxe by Dominik Homberger. Output source code: JavaScript, PHP, Python, Java and C#
+
+7Zip [plugin](https://github.com/mcmilk/7-Zip-Zstd)

C/brotli/br_backward_references.c

@@ -48,6 +48,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
 #define EXPAND_CAT(a, b) CAT(a, b)
 #define CAT(a, b) a ## b
 #define FN(X) EXPAND_CAT(X, HASHER())
+#define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER()))
+#define PREFIX() N
 
 #define HASHER() H2
 /* NOLINTNEXTLINE(build/include) */
@@ -94,6 +96,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
 #include "./enc/backward_references_inc.h"
 #undef HASHER
 
+#undef PREFIX
+#undef EXPORT_FN
 #undef FN
 #undef CAT
 #undef EXPAND_CAT
@@ -113,11 +117,11 @@ void BrotliCreateBackwardReferences(const BrotliDictionary* dictionary,
   switch (params->hasher.type) {
 #define CASE_(N)                                                  \
     case N:                                                       \
-      CreateBackwardReferencesH ## N(dictionary,                  \
+      CreateBackwardReferencesNH ## N(dictionary,                 \
           kStaticDictionaryHash, num_bytes, position, ringbuffer, \
           ringbuffer_mask, params, hasher, dist_cache,            \
           last_insert_len, commands, num_commands, num_literals); \
-      break;
+      return;
     FOR_GENERIC_HASHERS(CASE_)
 #undef CASE_
     default:

C/brotli/br_backward_references_hq.c

@@ -300,6 +300,7 @@ static size_t ComputeMinimumCopyLength(const float start_cost,
 static uint32_t ComputeDistanceShortcut(const size_t block_start,
                                         const size_t pos,
                                         const size_t max_backward,
+                                        const size_t gap,
                                         const ZopfliNode* nodes) {
   const size_t clen = ZopfliNodeCopyLength(&nodes[pos]);
   const size_t ilen = nodes[pos].insert_length;
@@ -311,8 +312,8 @@ static uint32_t ComputeDistanceShortcut(const size_t block_start,
      does not update the last distances. */
   if (pos == 0) {
     return 0;
-  } else if (dist + clen <= block_start + pos &&
-             dist <= max_backward &&
+  } else if (dist + clen <= block_start + pos + gap &&
+             dist <= max_backward + gap &&
              ZopfliNodeDistanceCode(&nodes[pos]) > 0) {
     return (uint32_t)pos;
   } else {
@@ -350,12 +351,12 @@ static void ComputeDistanceCache(const size_t pos,
    is eligible. */
 static void EvaluateNode(
     const size_t block_start, const size_t pos, const size_t max_backward_limit,
-    const int* starting_dist_cache, const ZopfliCostModel* model,
-    StartPosQueue* queue, ZopfliNode* nodes) {
+    const size_t gap, const int* starting_dist_cache,
+    const ZopfliCostModel* model, StartPosQueue* queue, ZopfliNode* nodes) {
   /* Save cost, because ComputeDistanceCache invalidates it. */
   float node_cost = nodes[pos].u.cost;
   nodes[pos].u.shortcut = ComputeDistanceShortcut(
-      block_start, pos, max_backward_limit, nodes);
+      block_start, pos, max_backward_limit, gap, nodes);
   if (node_cost <= ZopfliCostModelGetLiteralCosts(model, 0, pos)) {
     PosData posdata;
     posdata.pos = pos;
@@ -385,9 +386,10 @@ static size_t UpdateNodes(
   size_t min_len;
   size_t result = 0;
   size_t k;
+  size_t gap = 0;
 
-  EvaluateNode(block_start, pos, max_backward_limit, starting_dist_cache, model,
-               queue, nodes);
+  EvaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache,
+      model, queue, nodes);
 
   {
     const PosData* posdata = StartPosQueueAt(queue, 0);
@@ -415,25 +417,31 @@ static size_t UpdateNodes(
       const size_t backward =
           (size_t)(posdata->distance_cache[idx] + kDistanceCacheOffset[j]);
       size_t prev_ix = cur_ix - backward;
-      if (prev_ix >= cur_ix) {
-        continue;
+      size_t len = 0;
+      uint8_t continuation = ringbuffer[cur_ix_masked + best_len];
+      if (cur_ix_masked + best_len > ringbuffer_mask) {
+        break;
       }
-      if (BROTLI_PREDICT_FALSE(backward > max_distance)) {
+      if (BROTLI_PREDICT_FALSE(backward > max_distance + gap)) {
         continue;
       }
-      prev_ix &= ringbuffer_mask;
+      if (backward <= max_distance) {
+        if (prev_ix >= cur_ix) {
+          continue;
+        }
 
-      if (cur_ix_masked + best_len > ringbuffer_mask ||
-          prev_ix + best_len > ringbuffer_mask ||
-          ringbuffer[cur_ix_masked + best_len] !=
-              ringbuffer[prev_ix + best_len]) {
+        prev_ix &= ringbuffer_mask;
+        if (prev_ix + best_len > ringbuffer_mask ||
+            continuation != ringbuffer[prev_ix + best_len]) {
+          continue;
+        }
+        len = FindMatchLengthWithLimit(&ringbuffer[prev_ix],
+                                       &ringbuffer[cur_ix_masked],
+                                       max_len);
+      } else {
         continue;
       }
       {
-        const size_t len =
-            FindMatchLengthWithLimit(&ringbuffer[prev_ix],
-                                     &ringbuffer[cur_ix_masked],
-                                     max_len);
         const float dist_cost = base_cost +
             ZopfliCostModelGetDistanceCost(model, j);
         size_t l;
@@ -464,7 +472,8 @@ static size_t UpdateNodes(
       for (j = 0; j < num_matches; ++j) {
         BackwardMatch match = matches[j];
         size_t dist = match.distance;
-        BROTLI_BOOL is_dictionary_match = TO_BROTLI_BOOL(dist > max_distance);
+        BROTLI_BOOL is_dictionary_match =
+            TO_BROTLI_BOOL(dist > max_distance + gap);
         /* We already tried all possible last distance matches, so we can use
            normal distance code here. */
         size_t dist_code = dist + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
@@ -526,11 +535,14 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
                                 const ZopfliNode* nodes,
                                 int* dist_cache,
                                 size_t* last_insert_len,
+                                const BrotliEncoderParams* params,
                                 Command* commands,
                                 size_t* num_literals) {
   size_t pos = 0;
   uint32_t offset = nodes[0].u.next;
   size_t i;
+  size_t gap = 0;
+  BROTLI_UNUSED(params);
   for (i = 0; offset != BROTLI_UINT32_MAX; i++) {
     const ZopfliNode* next = &nodes[pos + offset];
     size_t copy_length = ZopfliNodeCopyLength(next);
@@ -546,11 +558,11 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
       size_t len_code = ZopfliNodeLengthCode(next);
       size_t max_distance =
           BROTLI_MIN(size_t, block_start + pos, max_backward_limit);
-      BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance);
+      BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance + gap);
       size_t dist_code = ZopfliNodeDistanceCode(next);
 
-      InitCommand(
-          &commands[i], insert_length, copy_length, len_code, dist_code);
+      InitCommand(&commands[i], insert_length,
+          copy_length, (int)len_code - (int)copy_length, dist_code);
 
       if (!is_dictionary && dist_code > 0) {
         dist_cache[3] = dist_cache[2];
@@ -572,6 +584,7 @@ static size_t ZopfliIterate(size_t num_bytes,
                             size_t ringbuffer_mask,
                             const BrotliEncoderParams* params,
                             const size_t max_backward_limit,
+                            const size_t gap,
                             const int* dist_cache,
                             const ZopfliCostModel* model,
                             const uint32_t* num_matches,
@@ -600,8 +613,8 @@ static size_t ZopfliIterate(size_t num_bytes,
       while (skip) {
         i++;
         if (i + 3 >= num_bytes) break;
-        EvaluateNode(
-            position, i, max_backward_limit, dist_cache, model, &queue, nodes);
+        EvaluateNode(position, i, max_backward_limit, gap, dist_cache, model,
+            &queue, nodes);
         cur_match_pos += num_matches[i];
         skip--;
       }
@@ -629,6 +642,7 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
   const size_t store_end = num_bytes >= StoreLookaheadH10() ?
       position + num_bytes - StoreLookaheadH10() + 1 : position;
   size_t i;
+  size_t gap = 0;
   nodes[0].length = 0;
   nodes[0].u.cost = 0;
   InitZopfliCostModel(m, &model, num_bytes);
@@ -640,7 +654,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
     const size_t pos = position + i;
     const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
     size_t num_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
-        ringbuffer_mask, pos, num_bytes - i, max_distance, params, matches);
+        ringbuffer_mask, pos, num_bytes - i, max_distance, gap, params,
+        matches);
     size_t skip;
     if (num_matches > 0 &&
         BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
@@ -662,8 +677,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
       while (skip) {
         i++;
         if (i + HashTypeLengthH10() - 1 >= num_bytes) break;
-        EvaluateNode(
-            position, i, max_backward_limit, dist_cache, &model, &queue, nodes);
+        EvaluateNode(position, i, max_backward_limit, gap, dist_cache, &model,
+            &queue, nodes);
         skip--;
       }
     }
@@ -688,7 +703,7 @@ void BrotliCreateZopfliBackwardReferences(
       dist_cache, hasher, nodes);
   if (BROTLI_IS_OOM(m)) return;
   BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, nodes,
-      dist_cache, last_insert_len, commands, num_literals);
+      dist_cache, last_insert_len, params, commands, num_literals);
   BROTLI_FREE(m, nodes);
 }
 
@@ -712,6 +727,7 @@ void BrotliCreateHqZopfliBackwardReferences(
   ZopfliCostModel model;
   ZopfliNode* nodes;
   BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size);
+  size_t gap = 0;
   if (BROTLI_IS_OOM(m)) return;
   for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) {
     const size_t pos = position + i;
@@ -725,14 +741,12 @@ void BrotliCreateHqZopfliBackwardReferences(
         cur_match_pos + MAX_NUM_MATCHES_H10);
     if (BROTLI_IS_OOM(m)) return;
     num_found_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
-        ringbuffer_mask, pos, max_length, max_distance, params,
+        ringbuffer_mask, pos, max_length, max_distance, gap, params,
         &matches[cur_match_pos]);
     cur_match_end = cur_match_pos + num_found_matches;
     for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
-      assert(BackwardMatchLength(&matches[j]) <
+      assert(BackwardMatchLength(&matches[j]) <=
           BackwardMatchLength(&matches[j + 1]));
-      assert(matches[j].distance > max_distance ||
-             matches[j].distance <= matches[j + 1].distance);
     }
     num_matches[i] = (uint32_t)num_found_matches;
     if (num_found_matches > 0) {
@@ -774,10 +788,10 @@ void BrotliCreateHqZopfliBackwardReferences(
     *last_insert_len = orig_last_insert_len;
     memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
     *num_commands += ZopfliIterate(num_bytes, position, ringbuffer,
-        ringbuffer_mask, params, max_backward_limit, dist_cache,
+        ringbuffer_mask, params, max_backward_limit, gap, dist_cache,
         &model, num_matches, matches, nodes);
     BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit,
-        nodes, dist_cache, last_insert_len, commands, num_literals);
+        nodes, dist_cache, last_insert_len, params, commands, num_literals);
   }
   CleanupZopfliCostModel(m, &model);
   BROTLI_FREE(m, nodes);