allow dictionary usage in multithreaded scenarios

dropbox · Nov 1, 2018 · e2c8c02 · e2c8c02
1 parent df3236a
commit e2c8c02
Show file tree

Hide file tree

Showing 8 changed files with 29 additions and 12 deletions.
diff --git a/src/bin/brotli.rs b/src/bin/brotli.rs
@@ -515,6 +515,7 @@ fn main() {
       }
       if (argument == "-catable" || argument == "--catable") && !double_dash {
           params.catable = true;
+          params.use_dictionary = false;
           params.appendable = true;
           continue;
       }

diff --git a/src/bin/test_broccoli.rs b/src/bin/test_broccoli.rs
@@ -180,6 +180,7 @@ fn test_append_then_empty_works() {
     let mut params0 = BrotliEncoderParams::default();
     params0.appendable = first;
     params0.catable = !first;
+    params0.use_dictionary = first;
     super::compress(src, dst, 4096, &params0, &[], 1).unwrap();
     first = false;
   }
@@ -201,6 +202,7 @@ fn test_append_then_cat_works() {
     let mut params0 = BrotliEncoderParams::default();
     params0.appendable = first;
     params0.catable = !first;
+    params0.use_dictionary = first;
     super::compress(src, dst, 4096, &params0, &[], 1).unwrap();
     first = false;
   }
@@ -222,6 +224,7 @@ fn test_one_byte_works() {
     let mut params0 = BrotliEncoderParams::default();
     params0.appendable = first;
     params0.catable = !first;
+    params0.use_dictionary = first;
     super::compress(src, dst, 4096, &params0, &[], 1).unwrap();
     first = false;
   }
@@ -243,6 +246,7 @@ fn test_one_byte_before_works() {
     let mut params0 = BrotliEncoderParams::default();
     params0.appendable = first;
     params0.catable = !first;
+    params0.use_dictionary = first;
     super::compress(src, dst, 4096, &params0, &[], 1).unwrap();
     first = false;
   }
@@ -264,6 +268,7 @@ fn test_two_byte_works() {
     let mut params0 = BrotliEncoderParams::default();
     params0.appendable = first;
     params0.catable = !first;
+    params0.use_dictionary = first;
     super::compress(src, dst, 4096, &params0, &[], 1).unwrap();
     first = false;
   }
@@ -285,6 +290,7 @@ fn test_two_byte_before_works() {
     let mut params0 = BrotliEncoderParams::default();
     params0.appendable = first;
     params0.catable = !first;
+    params0.use_dictionary = first;
     super::compress(src, dst, 4096, &params0, &[], 1).unwrap();
     first = false;
   }
@@ -308,6 +314,7 @@ fn test_empty_then_cat_works() {
     let mut params0 = BrotliEncoderParams::default();
     params0.appendable = first;
     params0.catable = !first;
+    params0.use_dictionary = first;
     super::compress(src, dst, 4096, &params0, &[], 1).unwrap();
     first = false;
   }
@@ -381,6 +388,7 @@ fn test_concat() {
             } else {
                 option.appendable = false;
                 option.catable = true;
+                option.use_dictionary = false;
             }
             super::compress(src, dst, 4096, option, &[], 1).unwrap();
             src.reset_read();
@@ -402,6 +410,7 @@ fn test_concat() {
     let options_len = options.len();
     for (index, (src, dst)) in files.iter_mut().zip(ufiles.iter_mut()).enumerate() {
       options[core::cmp::min(index, options_len - 1)].catable = true;
+      options[core::cmp::min(index, options_len - 1)].use_dictionary = false;
       options[core::cmp::min(index, options_len - 1)].appendable = false;
       options[core::cmp::min(index, options_len - 1)].quality = core::cmp::max(
         2, options[core::cmp::min(index, options_len - 1)].quality);

diff --git a/src/bin/test_custom_dict.rs b/src/bin/test_custom_dict.rs
@@ -102,6 +102,7 @@ fn test_custom_dict_for_multithreading() {
         super::decompress(br, rt, 4096, Rebox::from(vec)).unwrap();
         assert_eq!(rt.data(), raw.data());
         params.catable = true;
+        params.use_dictionary = false;
     }
     let mut bro_cat_li = BroCatli::new();
     let mut output = UnlimitedBuffer::new(&[]);

diff --git a/src/bin/test_threading.rs b/src/bin/test_threading.rs
@@ -25,6 +25,7 @@ fn multi_threaded_split_compression_test(num_threads: usize, quality: i32, catab
     params.magic_number = true;
     if catable {
         params.catable = true;
+        params.use_dictionary = false;
     }
     let mut output = Rebox::from(vec![0u8;BrotliEncoderMaxCompressedSizeMulti(RANDOM_THEN_UNICODE.len(), num_threads)]);
     let mut alloc_per_thread = [
@@ -94,6 +95,7 @@ fn thread_spawn_per_job_split_compression_test(num_threads: usize, quality: i32,
     params.magic_number = true;
     if catable {
         params.catable = true;
+        params.use_dictionary = false;
     }
     let mut output = Rebox::from(vec![0u8;BrotliEncoderMaxCompressedSizeMulti(RANDOM_THEN_UNICODE.len(), num_threads)]);
     let mut alloc_per_thread = [

diff --git a/src/enc/backward_references.rs b/src/enc/backward_references.rs
@@ -89,6 +89,8 @@ pub struct BrotliEncoderParams {
   pub avoid_distance_prefix_search: bool,
   // construct brotli in such a way that it may be concatenated with another brotli file using appropriate bit ops
   pub catable: bool,
+  // can use the dictionary (default yes unless catable is set)
+  pub use_dictionary: bool,
   // construct brotli in such a way that another concatable brotli file may be appended
   pub appendable: bool,
   // include a magic number and version number and size_hint at the beginning
@@ -1644,7 +1646,7 @@ pub fn BrotliCreateBackwardReferences<Alloc: alloc::Allocator<u16> + alloc::Allo
       &mut UnionHasher::H10(ref mut hasher) => {
           if params.quality >= 11 {
               super::backward_references_hq::BrotliCreateHqZopfliBackwardReferences(
-                  alloc, if params.catable {None} else {Some(dictionary)},
+                  alloc, if params.use_dictionary {Some(dictionary)} else {None},
                   num_bytes,
                   position,
                   ringbuffer,
@@ -1659,7 +1661,7 @@ pub fn BrotliCreateBackwardReferences<Alloc: alloc::Allocator<u16> + alloc::Allo
           } else {
               super::backward_references_hq::BrotliCreateZopfliBackwardReferences(
                   alloc,
-                  if params.catable {None} else {Some(dictionary)},
+                  if params.use_dictionary {Some(dictionary)} else {None},
                   num_bytes,
                   position,
                   ringbuffer,
@@ -1674,7 +1676,7 @@ pub fn BrotliCreateBackwardReferences<Alloc: alloc::Allocator<u16> + alloc::Allo
           }
     }
     &mut UnionHasher::H2(ref mut hasher) => {
-      CreateBackwardReferences(if params.catable {None} else {Some(dictionary)},
+      CreateBackwardReferences(if params.use_dictionary {Some(dictionary)} else {None},
                                &kStaticDictionaryHash[..],
                                num_bytes,
                                position,
@@ -1689,7 +1691,7 @@ pub fn BrotliCreateBackwardReferences<Alloc: alloc::Allocator<u16> + alloc::Allo
                                num_literals)
     }
     &mut UnionHasher::H3(ref mut hasher) => {
-      CreateBackwardReferences(if params.catable {None} else {Some(dictionary)},
+      CreateBackwardReferences(if params.use_dictionary {Some(dictionary)} else {None},
                                &kStaticDictionaryHash[..],
                                num_bytes,
                                position,
@@ -1704,7 +1706,7 @@ pub fn BrotliCreateBackwardReferences<Alloc: alloc::Allocator<u16> + alloc::Allo
                                num_literals)
     }
     &mut UnionHasher::H4(ref mut hasher) => {
-      CreateBackwardReferences(if params.catable {None} else {Some(dictionary)},
+      CreateBackwardReferences(if params.use_dictionary {Some(dictionary)} else {None},
                                &kStaticDictionaryHash[..],
                                num_bytes,
                                position,
@@ -1719,7 +1721,7 @@ pub fn BrotliCreateBackwardReferences<Alloc: alloc::Allocator<u16> + alloc::Allo
                                num_literals)
     }
     &mut UnionHasher::H5(ref mut hasher) => {
-      CreateBackwardReferences(if params.catable {None} else {Some(dictionary)},
+      CreateBackwardReferences(if params.use_dictionary {Some(dictionary)} else {None},
                                &kStaticDictionaryHash[..],
                                num_bytes,
                                position,
@@ -1734,7 +1736,7 @@ pub fn BrotliCreateBackwardReferences<Alloc: alloc::Allocator<u16> + alloc::Allo
                                num_literals)
     }
     &mut UnionHasher::H6(ref mut hasher) => {
-      CreateBackwardReferences(if params.catable {None} else {Some(dictionary)},
+      CreateBackwardReferences(if params.use_dictionary {Some(dictionary)} else {None},
                                &kStaticDictionaryHash[..],
                                num_bytes,
                                position,
@@ -1749,7 +1751,7 @@ pub fn BrotliCreateBackwardReferences<Alloc: alloc::Allocator<u16> + alloc::Allo
                                num_literals)
     }
     &mut UnionHasher::H9(ref mut hasher) => {
-      CreateBackwardReferences(if params.catable {None} else {Some(dictionary)},
+      CreateBackwardReferences(if params.use_dictionary {Some(dictionary)} else {None},
                                &kStaticDictionaryHash[..],
                                num_bytes,
                                position,
@@ -1764,7 +1766,7 @@ pub fn BrotliCreateBackwardReferences<Alloc: alloc::Allocator<u16> + alloc::Allo
                                num_literals)
     }
     &mut UnionHasher::H54(ref mut hasher) => {
-      CreateBackwardReferences(if params.catable {None} else {Some(dictionary)},
+      CreateBackwardReferences(if params.use_dictionary {Some(dictionary)} else {None},
                                &kStaticDictionaryHash[..],
                                num_bytes,
                                position,

diff --git a/src/enc/backward_references_hq.rs b/src/enc/backward_references_hq.rs
@@ -497,7 +497,7 @@ fn FindAllMatchesH10<AllocU32:Allocator<u32>, Buckets: Allocable<u32, AllocU32>+
                max_length,
                &mut dict_matches[..],
         ) != 0 {
-            assert_eq!(params.catable, false);
+            assert_eq!(params.use_dictionary, true);
             let maxlen
                 : usize
                 = brotli_min_size_t(37usize,max_length);

diff --git a/src/enc/brotli_bit_stream.rs b/src/enc/brotli_bit_stream.rs
@@ -2786,7 +2786,7 @@ pub fn BrotliWriteMetadataMetaBlock(params: &BrotliEncoderParams, storage_ix: &m
 
     BrotliWriteBits(8u8, 3 + size_hint_bytes, storage_ix, storage); // 1 byte of data: writing 12 for the magic number header
     JumpToByteBoundary(storage_ix, storage);
-    let magic_number = if params.catable {
+    let magic_number = if params.catable && !params.use_dictionary {
         [0xe1, 0x97, 0x81]
     } else if params.appendable {
         [0xe1, 0x97, 0x82]

diff --git a/src/enc/encode.rs b/src/enc/encode.rs
@@ -324,6 +324,7 @@ value: u32) -> i32 {
     if !params.appendable {
       params.appendable = value != 0;
     }
+    params.use_dictionary = (value == 0);
     return 1i32;
   }
   if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_APPENDABLE as (i32) {
@@ -392,6 +393,7 @@ pub fn BrotliEncoderInitParams() -> BrotliEncoderParams {
            prior_bitmask_detection: 0,
            literal_adaptation: [(0,0);4],
            catable: false,
+           use_dictionary: true,
            appendable: false,
            magic_number: false,
            hasher: BrotliHasherParams {
@@ -613,7 +615,7 @@ fn SanitizeParams(params: &mut BrotliEncoderParams) {
     }
   }
   if params.catable {
-    params.appendable = true;
+      params.appendable = true;
   }
 }