3.0.0 candidate

.travis.yml

@@ -11,9 +11,9 @@ os:
 
 script:
   - rustc --version | grep nightly && cargo test --features=simd || ( echo skip && rustc --version | grep -v nightly )
-  - cargo test
-  - cargo test --features=no-stdlib
-  - cargo test --release
+  - cargo test --no-default-features
+  - cargo test --no-default-features --features=std
+  - cargo test --no-default-features --features=std --release
   - rustc --version | grep 1[.][89][.] || rustc --version | grep -v 1[.]2[789][.] | grep 1[.][12][0-9][.]||cargo build --features=validation
 
 matrix:

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "brotli"
-version = "2.5.1"
+version = "3.0.0"
 authors = ["Daniel Reiter Horn <danielrh@dropbox.com>", "The Brotli Authors"]
 description = "A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe."
 license = "BSD-3-Clause/MIT"
@@ -23,16 +23,17 @@ name = "catbrotli"
 lto=true
 
 [dependencies]
-"alloc-no-stdlib" = "~1.3"
-"brotli-decompressor" = {version="~1.5"}
-
+"alloc-no-stdlib" = {version="2.0"}
+"brotli-decompressor" = {version="~2.0"}
+"alloc-stdlib" = {version="~0.2", optional=true}
 "packed_simd" = {version="0.3", optional=true}
 "sha2" = {version="~0.8", optional=true}
 
 [features]
+default=["std"]
 validation=["sha2"]
 seccomp = ["brotli-decompressor/seccomp"]
-no-stdlib = ["alloc-no-stdlib/no-stdlib", "brotli-decompressor/no-stdlib"]
+std = ["alloc-stdlib", "brotli-decompressor/std"]
 external-literal-probability = []
 disable-timer = ["brotli-decompressor/disable-timer"]
 benchmark = ["brotli-decompressor/benchmark"]

README.md

@@ -3,14 +3,23 @@
 [![crates.io](http://meritbadge.herokuapp.com/brotli)](https://crates.io/crates/brotli)
 [![Build Status](https://travis-ci.org/dropbox/rust-brotli.svg?branch=master)](https://travis-ci.org/dropbox/rust-brotli)
 
+## What's new in 3.0
+* A fully compatible FFI for drop-in compatibiltiy with the https://github.com/google/brotli binaries
+  * custom allocators fully supported
+* Multithreaded compression so multiple threads can operate in unison on a single file
+* Concatenatability mode to add the feature requested in https://github.com/google/brotli/issues/628
+  * binary tool catbrotli can accomplish this if the first file was specified with -apendable and the second with -catable
+* validation mode where a file is double-checked to be able to be decompressed with the same settings; useful for benchmarking or fuzzing
+* Magic Number: where the brotli file can have a useful header with a few magic bytes, concatability info and a final output size for pre-allocating memory
+
 ## What's new in 2.5
-In 2.5 The callback also passes down an allocator to make new StaticCommands and PDFs and 256 bit floating point vectors.
-In 2.4 The callback with the compression intermediate representation now passes a full metablock at a time. Also these items are mutable
+* In 2.5 The callback also passes down an allocator to make new StaticCommands and PDFs and 256 bit floating point vectors.
+* In 2.4 The callback with the compression intermediate representation now passes a full metablock at a time. Also these items are mutable
 in case futher optimization is desired
 
 ## What's new in 2.3
 
-Flush now produces output instead of calling finish on the stream. This allows you to use the writer abstraction to
+* Flush now produces output instead of calling finish on the stream. This allows you to use the writer abstraction to
 get immediate output without having to resort to the CompressStream internal abstraction
 
 ## Project Requirements

c/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "brotli"
-version = "2.5.1"
+version = "3.0.0"
 authors = ["Daniel Reiter Horn <danielrh@dropbox.com>", "The Brotli Authors"]
 description = "A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe."
 license = "BSD-3-Clause/MIT"
@@ -19,16 +19,18 @@ crate-type=["cdylib", "rlib"]
 lto=true
 
 [dependencies]
-"alloc-no-stdlib" = "~1.3"
-"brotli-decompressor" = {version="~1.5"}
-
+"alloc-no-stdlib" = {version="2.0"}
+"brotli-decompressor" = {version="~2.0"}
+"alloc-stdlib" = {version="~0.2", optional=true}
 "packed_simd" = {version="0.3", optional=true}
+"sha2" = {version="~0.8", optional=true}
 
 [features]
-default=["no-stdlib-ffi-binding"]
+validation=["sha2"]
+default=["std", "no-stdlib-ffi-binding"]
 no-stdlib-ffi-binding = []
 seccomp = ["brotli-decompressor/seccomp"]
-no-stdlib = ["alloc-no-stdlib/no-stdlib", "brotli-decompressor/no-stdlib"]
+std = ["alloc-stdlib", "brotli-decompressor/std"]
 external-literal-probability = []
 disable-timer = ["brotli-decompressor/disable-timer"]
 benchmark = ["brotli-decompressor/benchmark"]

examples/compress.rs

@@ -1,9 +1,9 @@
 extern crate brotli;
-#[cfg(feature="no-stdlib")]
+#[cfg(not(feature="std"))]
 fn main() {
     panic!("For no-stdlib examples please see the tests")
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 fn main() {
     use std::io;
     use std::io::{Read, Write};

examples/decompress.rs

@@ -1,9 +1,9 @@
 extern crate brotli;
-#[cfg(feature="no-stdlib")]
+#[cfg(not(feature="std"))]
 fn main() {
     panic!("For no-stdlib examples please see the tests")
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 fn main() {
     use std::io;
     let stdin = &mut io::stdin();

src/bin/brotli.rs

@@ -17,7 +17,8 @@ extern crate sha2;
 extern crate alloc_no_stdlib;
 use brotli::enc::{BrotliEncoderParams, BrotliEncoderMaxCompressedSizeMulti, WorkerPool, compress_worker_pool, new_work_pool};
 use brotli::enc::threading::{SendAlloc,Owned, CompressionThreadResult, CompressMulti, BrotliEncoderThreadError};
-
+#[allow(unused_imports)]
+use brotli::HuffmanCode;
 use brotli::CustomRead;
 use core::ops;
 use brotli::enc::backward_references::BrotliEncoderMode;
@@ -236,9 +237,9 @@ declare_stack_allocator_struct!(CallocAllocatedFreelist, 8192, calloc);
 
 #[cfg(feature="seccomp")]
 pub fn decompress<InputType, OutputType>(r: &mut InputType,
-                                         mut w: &mut OutputType,
+                                         w: &mut OutputType,
                                          buffer_size: usize,
-                                         mut custom_dictionary:Rebox<u8>)
+                                         custom_dictionary:Rebox<u8>)
                                          -> Result<(), io::Error>
   where InputType: Read,
         OutputType: Write
@@ -249,10 +250,10 @@ pub fn decompress<InputType, OutputType>(r: &mut InputType,
 
   }
   core::mem::drop(custom_dictionary);
-  let mut u8_buffer =
+  let u8_buffer =
     unsafe { define_allocator_memory_pool!(4, u8, [0; 1024 * 1024 * 200], calloc) };
-  let mut u32_buffer = unsafe { define_allocator_memory_pool!(4, u32, [0; 16384], calloc) };
-  let mut hc_buffer =
+  let u32_buffer = unsafe { define_allocator_memory_pool!(4, u32, [0; 16384], calloc) };
+  let hc_buffer =
     unsafe { define_allocator_memory_pool!(4, HuffmanCode, [0; 1024 * 1024 * 16], calloc) };
   let mut alloc_u8 = CallocAllocatedFreelist::<u8>::new_allocator(u8_buffer.data, bzero);
   let alloc_u32 = CallocAllocatedFreelist::<u32>::new_allocator(u32_buffer.data, bzero);
@@ -417,7 +418,7 @@ pub fn compress<InputType, OutputType>(r: &mut InputType,
                                    Error::new(ErrorKind::UnexpectedEof, "Unexpected EOF"))
 }
 
-// This decompressor is defined unconditionally on whether no-stdlib is defined
+// This decompressor is defined unconditionally on whether std is defined
 // so we can exercise the code in any case
 pub struct BrotliDecompressor<R: Read>(brotli::DecompressorCustomIo<io::Error,
                                                                     IntoIoReader<R>,
@@ -480,11 +481,11 @@ fn read_custom_dictionary(filename :&str) -> Vec<u8> {
   ret
 }
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 fn has_stdlib() -> bool {
     true
 }
-#[cfg(feature="no-stdlib")]
+#[cfg(not(feature="std"))]
 fn has_stdlib() -> bool {
     false
 }

src/bin/integration_tests.rs

@@ -9,13 +9,13 @@ use super::alloc_no_stdlib::{Allocator, SliceWrapper, SliceWrapperMut};
 use super::brotli::BrotliResult;
 use super::brotli::BrotliState;
 use super::Rebox;
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 use super::brotli::{CompressorReader, CompressorWriter};
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 use super::brotli_decompressor::{Decompressor, DecompressorWriter};
 use core::cmp;
 use std::io;
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 use std::io::{Read, Write};
 use std::time::Duration;
 #[cfg(not(feature="disable-timer"))]
@@ -385,36 +385,36 @@ fn test_random_then_unicode_8() {
 fn test_random_then_unicode_9() {
     roundtrip_helper(RANDOM_THEN_UNICODE, 9, 22, false);
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 const random_then_unicode_compressed_size_9_5 : usize = 136542;
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 const random_then_unicode_compressed_size_9_5x : usize = 136045;
 
 
-#[cfg(feature="no-stdlib")]
+#[cfg(not(feature="std"))]
 const alice_compressed_size_10 : usize = 47490;
-#[cfg(feature="no-stdlib")]
+#[cfg(not(feature="std"))]
 const alice_compressed_size_11 : usize = 46496;
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[cfg(not(feature="float64"))]
 const alice_compressed_size_10 : usize = 47477;
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[cfg(not(feature="float64"))]
 const alice_compressed_size_11 : usize = 46487;
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[cfg(feature="float64")]
 const alice_compressed_size_10 : usize = 47515;
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[cfg(feature="float64")]
 const alice_compressed_size_11 : usize = 46510;
 
 
 
-#[cfg(feature="no-stdlib")] // approx log
+#[cfg(not(feature="std"))] // approx log
 const random_then_unicode_compressed_size_9_5 : usize = 136699;
-#[cfg(feature="no-stdlib")] // approx log
+#[cfg(not(feature="std"))] // approx log
 const random_then_unicode_compressed_size_9_5x : usize = 136095;
 
 #[test]
@@ -456,7 +456,7 @@ fn test_roundtrip_as_you_lik() {
   total_roundtrip_helper(include_bytes!("../../testdata/asyoulik.txt"));
 }
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 fn reader_helper(mut in_buf: &[u8], q: u32, lgwin: u32) {
   let original_buf = in_buf;
   let mut cmp = [0u8; 259];
@@ -504,7 +504,7 @@ fn reader_helper(mut in_buf: &[u8], q: u32, lgwin: u32) {
 
 /*
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 fn simple_reader_helper(mut in_buf: &[u8], q: u32, lgwin: u32) {
     let mut xinput = UnlimitedBuffer::new(&in_buf);
     let xenc;
@@ -551,41 +551,41 @@ fn simple_reader_helper(mut in_buf: &[u8], q: u32, lgwin: u32) {
   assert!(compressed_size < original_buf.len() * pct_ratio / 100);
 }
 */
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_reader_64x() {
   reader_helper(include_bytes!("../../testdata/64x"), 9, 20);
 }
 /*
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_simple_64x() {
     bogus_reader_helper(include_bytes!("../../testdata/64x"), 9, 20);
 }*/
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_reader_as_you_lik() {
   reader_helper(include_bytes!("../../testdata/asyoulik.txt"), 9, 20);
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_reader_quickfox_repeated() {
   reader_helper(include_bytes!("../../testdata/quickfox_repeated"), 9, 20);
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_reader_random_then_unicode() {
   reader_helper(include_bytes!("../../testdata/random_then_unicode"), 9, 20);
 }
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_reader_alice() {
   reader_helper(include_bytes!("../../testdata/alice29.txt"), 9, 22);
 }
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 fn writer_helper(mut in_buf: &[u8], buf_size: usize, q: u32, lgwin: u32, do_flush: bool) {
   let original_buf = in_buf;
   let mut output = UnlimitedBuffer::new(&[]);
@@ -634,28 +634,28 @@ fn writer_helper(mut in_buf: &[u8], buf_size: usize, q: u32, lgwin: u32, do_flus
   assert!(compressed.data.len() < original_buf.len() * pct_ratio / 100);
   }
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_writer_as_you_lik() {
   writer_helper(include_bytes!("../../testdata/asyoulik.txt"), 17, 9, 20, false);
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_writer_64x() {
   writer_helper(include_bytes!("../../testdata/64x"), 17, 9, 20, false);
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_writer_quickfox_repeated() {
   writer_helper(include_bytes!("../../testdata/quickfox_repeated"), 251, 9, 20, false);
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_writer_random_then_unicode() {
   writer_helper(include_bytes!("../../testdata/random_then_unicode"), 277, 9, 20, false);
 }
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 #[test]
 fn test_writer_alice() {
   writer_helper(include_bytes!("../../testdata/alice29.txt"), 299, 9, 22, true);

src/bin/test_custom_dict.rs

@@ -175,7 +175,7 @@ fn test_custom_dict_for_multithreading() {
     output.reset_read();
     super::decompress(&mut output, &mut rt, 4096, Rebox::default()).unwrap();
     assert_eq!(rt.data(), ALICE);
-    // no-stdlib mode has some approximations that make it 4 bytes bigger
+    // without setting std flag: approximation make it 4 bytes bigger
     if output.data().len() != 48568 {
        assert_eq!(output.data().len(), 48564); // as opposed to 46487 with standard settings
     }

src/bin/util.rs

@@ -177,7 +177,7 @@ use std::sync::RwLock;
 
 
 pub struct MTJoinable<T:Send+'static, U:Send+'static>(JoinHandle<T>, PhantomData<U>);
-#[cfg(feature="no-stdlib")]
+#[cfg(not(feature="std"))]
 impl<T:Send+'static, U:Send+'static+AnyBoxConstructor> Joinable<T, U> for MTJoinable<T, U> {
   fn join(self) -> Result<T, U> {
       match self.0.join() {
@@ -186,7 +186,7 @@ impl<T:Send+'static, U:Send+'static+AnyBoxConstructor> Joinable<T, U> for MTJoin
       }
   }
 }
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 impl<T:Send+'static, U:Send+'static+AnyBoxConstructor> Joinable<T, U> for MTJoinable<T, U> {
   fn join(self) -> Result<T, U> {
       match self.0.join() {

src/enc/brotli_bit_stream.rs

@@ -2,7 +2,7 @@
 #![allow(dead_code)]
 #![allow(unused_imports)]
 #![allow(unused_macros)]
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 use std::io::Write;
 use VERSION;
 use super::{v8, s16};
@@ -199,11 +199,11 @@ impl<'a, Alloc: BrotliAlloc> interface::CommandProcessor<'a> for CommandQueue<'a
 }
 
 
-#[cfg(not(feature="no-stdlib"))]
+#[cfg(feature="std")]
 fn warn_on_missing_free() {
     let _err = ::std::io::stderr().write(b"Need to free entropy_tally_scratch before dropping CommandQueue\n");
 }
-#[cfg(feature="no-stdlib")]
+#[cfg(not(feature="std"))]
 fn warn_on_missing_free() {
      // no way to warn in this case
 }