From 8215229100905bffa1b50d0cd70c14b04ce9da01 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 15 Nov 2015 18:15:18 -0800 Subject: [PATCH] Use a larger default block size for large isos. If the iso is > 2GB, the tools that don't support > 2048 block sizes probably won't support the file anyway. This is a smarter default. --- README.md | 4 ++-- cli/cli.cpp | 10 ++++----- src/compress.cpp | 55 ++++++++++++++++++++++++++++++++++++------------ src/compress.h | 8 ++++--- src/output.cpp | 10 ++++++--- src/output.h | 2 ++ src/sector.cpp | 4 ++-- src/sector.h | 6 ++++-- 8 files changed, 68 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index d141a938f..0d6b57208 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,8 @@ Multiple files may be specified. Inputs can be iso or cso files. --crc Log CRC32 checksums, ignore output files and methods --fast Use only basic zlib or lz4 for fastest result --decompress Write out to raw ISO, decompressing as needed - --block=N Specify a block size (default is 2048) - Most readers only support the 2048 size + --block=N Specify a block size (default depends on iso size) + Many readers only support the 2048 size --format=VER Specify cso version (options: cso1, cso2, zso) These are experimental, default is cso1 --use-zlib Enable trials with zlib for deflate compression diff --git a/cli/cli.cpp b/cli/cli.cpp index e03b0636b..47c80bb46 100644 --- a/cli/cli.cpp +++ b/cli/cli.cpp @@ -22,8 +22,8 @@ void show_help(const char *arg0) { fprintf(stderr, " --crc Log CRC32 checksums, ignore output files and methods\n"); fprintf(stderr, " --fast Use only basic zlib or lz4 for fastest result\n"); fprintf(stderr, " --decompress Write out to raw ISO, decompressing as needed\n"); - fprintf(stderr, " --block=N Specify a block size (default is 2048)\n"); - fprintf(stderr, " Most readers only support the 2048 size\n"); + fprintf(stderr, " --block=N Specify a block size (default depends on iso size)\n"); + fprintf(stderr, " Many readers only support the 2048 size\n"); fprintf(stderr, " --format=VER Specify cso version (options: cso1, cso2, zso)\n"); fprintf(stderr, " These are experimental, default is cso1\n"); // TODO: Bring this back once it's functional. @@ -112,7 +112,7 @@ struct Arguments { void default_args(Arguments &args) { args.threads = 0; - args.block_size = 2048; + args.block_size = maxcso::DEFAULT_BLOCK_SIZE; args.flags_fmt = 0; args.flags_use = 0; @@ -414,8 +414,8 @@ int main(int argc, char *argv[]) { task.error = error; task.block_size = args.block_size; task.flags = args.flags_final; - task.orig_max_cost = static_cast((args.orig_cost_percent * args.block_size) / 100); - task.lz4_max_cost = static_cast((args.lz4_cost_percent * args.block_size) / 100); + task.orig_max_cost_percent = args.orig_cost_percent; + task.lz4_max_cost_percent = args.lz4_cost_percent; tasks.push_back(std::move(task)); } diff --git a/src/compress.cpp b/src/compress.cpp index c643a56a9..8713a48a2 100644 --- a/src/compress.cpp +++ b/src/compress.cpp @@ -12,11 +12,18 @@ namespace maxcso { // Anything above this is insane. This value is even insane. static const uint32_t MAX_BLOCK_SIZE = 0x40000; +// These are the default block sizes. +static const uint32_t SMALL_BLOCK_SIZE = 2048; +static const uint32_t LARGE_BLOCK_SIZE = 16384; +// We use the LARGE_BLOCK_SIZE default for files larger than 2GB. +static const int64_t LARGE_BLOCK_SIZE_THRESH = 0x80000000; + // This actually handles decompression too. They're basically the same. class CompressionTask { public: CompressionTask(uv_loop_t *loop, const Task &t) - : task_(t), loop_(loop), input_(-1), inputHandler_(loop), outputHandler_(loop, t), output_(-1) { + : task_(t), loop_(loop), input_(-1), inputHandler_(loop), outputHandler_(loop, t), output_(-1), + blockSize_(0) { } ~CompressionTask() { Cleanup(); @@ -50,22 +57,30 @@ class CompressionTask { Input inputHandler_; Output outputHandler_; uv_file output_; + uint32_t blockSize_; }; void CompressionTask::Enqueue() { - if (task_.block_size > MAX_BLOCK_SIZE) { - Notify(TASK_INVALID_OPTION, "Block size too large"); - return; - } - if (task_.block_size < SECTOR_SIZE) { - Notify(TASK_INVALID_OPTION, "Block size too small, must be at least 2048"); - return; - } - if ((task_.block_size & (task_.block_size - 1)) != 0) { - Notify(TASK_INVALID_OPTION, "Block size must be a power of two"); - return; + if (task_.block_size == DEFAULT_BLOCK_SIZE) { + // Start with a small block size. + // We'll re-evaluate later. + blockSize_ = SMALL_BLOCK_SIZE; + } else { + if (task_.block_size > MAX_BLOCK_SIZE) { + Notify(TASK_INVALID_OPTION, "Block size too large"); + return; + } + if (task_.block_size < SECTOR_SIZE) { + Notify(TASK_INVALID_OPTION, "Block size too small, must be at least 2048"); + return; + } + if ((task_.block_size & (task_.block_size - 1)) != 0) { + Notify(TASK_INVALID_OPTION, "Block size must be a power of two"); + return; + } + blockSize_ = task_.block_size; } - if (!pool.SetBufferSize(task_.block_size * 2)) { + if (!pool.SetBufferSize(blockSize_ * 2)) { Notify(TASK_INVALID_OPTION, "Unable to update buffer size to match block size"); return; } @@ -134,7 +149,19 @@ void CompressionTask::BeginProcessing() { } else if (task_.flags & TASKFLAG_FMT_ZSO) { fmt = CSO_FMT_ZSO; } - outputHandler_.SetFile(output_, size, task_.block_size, fmt); + + // Now that we know the file size, check if we should resize the blockSize_. + if (task_.block_size == DEFAULT_BLOCK_SIZE && size >= LARGE_BLOCK_SIZE_THRESH) { + blockSize_ = LARGE_BLOCK_SIZE; + if (!pool.SetBufferSize(blockSize_ * 2)) { + // Abort reading. + inputHandler_.Pause(); + Notify(TASK_INVALID_OPTION, "Unable to update buffer size to match block size"); + return; + } + } + + outputHandler_.SetFile(output_, size, blockSize_, fmt); Notify(TASK_INPROGRESS, 0, size, 0); }); inputHandler_.Pipe(input_, [this](int64_t pos, uint8_t *sector) { diff --git a/src/compress.h b/src/compress.h index daee87af4..7cc672d67 100644 --- a/src/compress.h +++ b/src/compress.h @@ -7,7 +7,9 @@ namespace maxcso { -static const char *VERSION = "1.5.0"; +static const char *VERSION = "1.6.0"; + +static const uint32_t DEFAULT_BLOCK_SIZE = 0xFFFFFFFF; struct Task; @@ -57,8 +59,8 @@ struct Task { ErrorCallback error; uint32_t block_size; uint32_t flags; - uint32_t orig_max_cost; - uint32_t lz4_max_cost; + double orig_max_cost_percent; + double lz4_max_cost_percent; }; void Compress(const std::vector &tasks); diff --git a/src/output.cpp b/src/output.cpp index c56468e82..12afb56d7 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -9,9 +9,11 @@ namespace maxcso { static const size_t QUEUE_SIZE = 32; Output::Output(uv_loop_t *loop, const Task &task) - : loop_(loop), flags_(task.flags), state_(STATE_INIT), fmt_(CSO_FMT_CSO1), srcSize_(-1), index_(nullptr) { + : loop_(loop), flags_(task.flags), state_(STATE_INIT), fmt_(CSO_FMT_CSO1), + origMaxCostPercent_(task.orig_max_cost_percent), lz4MaxCostPercent_(task.lz4_max_cost_percent), + srcSize_(-1), index_(nullptr) { for (size_t i = 0; i < QUEUE_SIZE; ++i) { - freeSectors_.push_back(new Sector(flags_, task.orig_max_cost, task.lz4_max_cost)); + freeSectors_.push_back(new Sector(flags_)); } } @@ -78,8 +80,10 @@ void Output::SetFile(uv_file file, int64_t srcSize, uint32_t blockSize, CSOForma state_ |= STATE_HAS_FILE; + const uint32_t origMaxCost = static_cast((origMaxCostPercent_ * blockSize_) / 100); + const uint32_t lz4MaxCost = static_cast((lz4MaxCostPercent_ * blockSize_) / 100); for (Sector *sector : freeSectors_) { - sector->Setup(loop_, blockSize_, indexAlign_); + sector->Setup(loop_, blockSize_, indexAlign_, origMaxCost, lz4MaxCost); } } diff --git a/src/output.h b/src/output.h index f27eced37..8169075fc 100644 --- a/src/output.h +++ b/src/output.h @@ -46,6 +46,8 @@ class Output { uint32_t flags_; uint32_t state_; CSOFormat fmt_; + double origMaxCostPercent_; + double lz4MaxCostPercent_; uv_file file_; uv_fs_t req_; diff --git a/src/sector.cpp b/src/sector.cpp index 101fa3747..ca63ebfbf 100644 --- a/src/sector.cpp +++ b/src/sector.cpp @@ -22,8 +22,8 @@ static void EndZlib(z_stream *&z) { z = nullptr; } -Sector::Sector(uint32_t flags, uint32_t orig_max_cost, uint32_t lz4_max_cost) - : flags_(flags), origMaxCost_(orig_max_cost), lz4MaxCost_(lz4_max_cost), busy_(false), enqueued_(false), +Sector::Sector(uint32_t flags) + : flags_(flags), origMaxCost_(0), lz4MaxCost_(0), busy_(false), enqueued_(false), compress_(true), readySize_(0), buffer_(nullptr), best_(nullptr) { // Set up the zlib streams, which we will reuse each time we hit this sector. if (!(flags_ & TASKFLAG_NO_ZLIB_DEFAULT)) { diff --git a/src/sector.h b/src/sector.h index 911ce80f8..e9d0d3f43 100644 --- a/src/sector.h +++ b/src/sector.h @@ -23,13 +23,15 @@ enum SectorFormat { // Actually block. class Sector { public: - Sector(uint32_t flags, uint32_t orig_max_cost, uint32_t lz4_max_cost); + Sector(uint32_t flags); ~Sector(); - void Setup(uv_loop_t *loop, uint32_t blockSize, uint32_t align) { + void Setup(uv_loop_t *loop, uint32_t blockSize, uint32_t align, uint32_t origMaxCost, uint32_t lz4MaxCost) { loop_ = loop; blockSize_ = blockSize; align_ = align; + origMaxCost_ = origMaxCost; + lz4MaxCost_ = lz4MaxCost; } void Process(int64_t pos, uint8_t *buffer, SectorCallback ready);