Use a larger default block size for large isos.

If the iso is > 2GB, the tools that don't support > 2048 block sizes
probably won't support the file anyway.  This is a smarter default.

README.md

@@ -74,8 +74,8 @@ Multiple files may be specified.  Inputs can be iso or cso files.
    --crc           Log CRC32 checksums, ignore output files and methods
    --fast          Use only basic zlib or lz4 for fastest result
    --decompress    Write out to raw ISO, decompressing as needed
-   --block=N       Specify a block size (default is 2048)
-                   Most readers only support the 2048 size
+   --block=N       Specify a block size (default depends on iso size)
+                   Many readers only support the 2048 size
    --format=VER    Specify cso version (options: cso1, cso2, zso)
                    These are experimental, default is cso1
    --use-zlib      Enable trials with zlib for deflate compression

cli/cli.cpp

@@ -22,8 +22,8 @@ void show_help(const char *arg0) {
 	fprintf(stderr, "   --crc           Log CRC32 checksums, ignore output files and methods\n");
 	fprintf(stderr, "   --fast          Use only basic zlib or lz4 for fastest result\n");
 	fprintf(stderr, "   --decompress    Write out to raw ISO, decompressing as needed\n");
-	fprintf(stderr, "   --block=N       Specify a block size (default is 2048)\n");
-	fprintf(stderr, "                   Most readers only support the 2048 size\n");
+	fprintf(stderr, "   --block=N       Specify a block size (default depends on iso size)\n");
+	fprintf(stderr, "                   Many readers only support the 2048 size\n");
 	fprintf(stderr, "   --format=VER    Specify cso version (options: cso1, cso2, zso)\n");
 	fprintf(stderr, "                   These are experimental, default is cso1\n");
 	// TODO: Bring this back once it's functional.
@@ -112,7 +112,7 @@ struct Arguments {
 
 void default_args(Arguments &args) {
 	args.threads = 0;
-	args.block_size = 2048;
+	args.block_size = maxcso::DEFAULT_BLOCK_SIZE;
 
 	args.flags_fmt = 0;
 	args.flags_use = 0;
@@ -414,8 +414,8 @@ int main(int argc, char *argv[]) {
 		task.error = error;
 		task.block_size = args.block_size;
 		task.flags = args.flags_final;
-		task.orig_max_cost = static_cast<uint32_t>((args.orig_cost_percent * args.block_size) / 100);
-		task.lz4_max_cost = static_cast<uint32_t>((args.lz4_cost_percent * args.block_size) / 100);
+		task.orig_max_cost_percent = args.orig_cost_percent;
+		task.lz4_max_cost_percent = args.lz4_cost_percent;
 		tasks.push_back(std::move(task));
 	}
 

src/compress.cpp

@@ -12,11 +12,18 @@ namespace maxcso {
 // Anything above this is insane.  This value is even insane.
 static const uint32_t MAX_BLOCK_SIZE = 0x40000;
 
+// These are the default block sizes.
+static const uint32_t SMALL_BLOCK_SIZE = 2048;
+static const uint32_t LARGE_BLOCK_SIZE = 16384;
+// We use the LARGE_BLOCK_SIZE default for files larger than 2GB.
+static const int64_t LARGE_BLOCK_SIZE_THRESH = 0x80000000;
+
 // This actually handles decompression too.  They're basically the same.
 class CompressionTask {
 public:
 	CompressionTask(uv_loop_t *loop, const Task &t)
-		: task_(t), loop_(loop), input_(-1), inputHandler_(loop), outputHandler_(loop, t), output_(-1) {
+		: task_(t), loop_(loop), input_(-1), inputHandler_(loop), outputHandler_(loop, t), output_(-1),
+		blockSize_(0) {
 	}
 	~CompressionTask() {
 		Cleanup();
@@ -50,22 +57,30 @@ class CompressionTask {
 	Input inputHandler_;
 	Output outputHandler_;
 	uv_file output_;
+	uint32_t blockSize_;
 };
 
 void CompressionTask::Enqueue() {
-	if (task_.block_size > MAX_BLOCK_SIZE) {
-		Notify(TASK_INVALID_OPTION, "Block size too large");
-		return;
-	}
-	if (task_.block_size < SECTOR_SIZE) {
-		Notify(TASK_INVALID_OPTION, "Block size too small, must be at least 2048");
-		return;
-	}
-	if ((task_.block_size & (task_.block_size - 1)) != 0) {
-		Notify(TASK_INVALID_OPTION, "Block size must be a power of two");
-		return;
+	if (task_.block_size == DEFAULT_BLOCK_SIZE) {
+		// Start with a small block size.
+		// We'll re-evaluate later.
+		blockSize_ = SMALL_BLOCK_SIZE;
+	} else {
+		if (task_.block_size > MAX_BLOCK_SIZE) {
+			Notify(TASK_INVALID_OPTION, "Block size too large");
+			return;
+		}
+		if (task_.block_size < SECTOR_SIZE) {
+			Notify(TASK_INVALID_OPTION, "Block size too small, must be at least 2048");
+			return;
+		}
+		if ((task_.block_size & (task_.block_size - 1)) != 0) {
+			Notify(TASK_INVALID_OPTION, "Block size must be a power of two");
+			return;
+		}
+		blockSize_ = task_.block_size;
 	}
-	if (!pool.SetBufferSize(task_.block_size * 2)) {
+	if (!pool.SetBufferSize(blockSize_ * 2)) {
 		Notify(TASK_INVALID_OPTION, "Unable to update buffer size to match block size");
 		return;
 	}
@@ -134,7 +149,19 @@ void CompressionTask::BeginProcessing() {
 		} else if (task_.flags & TASKFLAG_FMT_ZSO) {
 			fmt = CSO_FMT_ZSO;
 		}
-		outputHandler_.SetFile(output_, size, task_.block_size, fmt);
+
+		// Now that we know the file size, check if we should resize the blockSize_.
+		if (task_.block_size == DEFAULT_BLOCK_SIZE && size >= LARGE_BLOCK_SIZE_THRESH) {
+			blockSize_ = LARGE_BLOCK_SIZE;
+			if (!pool.SetBufferSize(blockSize_ * 2)) {
+				// Abort reading.
+				inputHandler_.Pause();
+				Notify(TASK_INVALID_OPTION, "Unable to update buffer size to match block size");
+				return;
+			}
+		}
+
+		outputHandler_.SetFile(output_, size, blockSize_, fmt);
 		Notify(TASK_INPROGRESS, 0, size, 0);
 	});
 	inputHandler_.Pipe(input_, [this](int64_t pos, uint8_t *sector) {

src/compress.h

@@ -7,7 +7,9 @@
 
 namespace maxcso {
 
-static const char *VERSION = "1.5.0";
+static const char *VERSION = "1.6.0";
+
+static const uint32_t DEFAULT_BLOCK_SIZE = 0xFFFFFFFF;
 
 struct Task;
 
@@ -57,8 +59,8 @@ struct Task {
 	ErrorCallback error;
 	uint32_t block_size;
 	uint32_t flags;
-	uint32_t orig_max_cost;
-	uint32_t lz4_max_cost;
+	double orig_max_cost_percent;
+	double lz4_max_cost_percent;
 };
 
 void Compress(const std::vector<Task> &tasks);

src/output.cpp

@@ -9,9 +9,11 @@ namespace maxcso {
 static const size_t QUEUE_SIZE = 32;
 
 Output::Output(uv_loop_t *loop, const Task &task)
-	: loop_(loop), flags_(task.flags), state_(STATE_INIT), fmt_(CSO_FMT_CSO1), srcSize_(-1), index_(nullptr) {
+	: loop_(loop), flags_(task.flags), state_(STATE_INIT), fmt_(CSO_FMT_CSO1),
+	origMaxCostPercent_(task.orig_max_cost_percent), lz4MaxCostPercent_(task.lz4_max_cost_percent),
+	srcSize_(-1), index_(nullptr) {
 	for (size_t i = 0; i < QUEUE_SIZE; ++i) {
-		freeSectors_.push_back(new Sector(flags_, task.orig_max_cost, task.lz4_max_cost));
+		freeSectors_.push_back(new Sector(flags_));
 	}
 }
 
@@ -78,8 +80,10 @@ void Output::SetFile(uv_file file, int64_t srcSize, uint32_t blockSize, CSOForma
 
 	state_ |= STATE_HAS_FILE;
 
+	const uint32_t origMaxCost = static_cast<uint32_t>((origMaxCostPercent_ * blockSize_) / 100);
+	const uint32_t lz4MaxCost = static_cast<uint32_t>((lz4MaxCostPercent_ * blockSize_) / 100);
 	for (Sector *sector : freeSectors_) {
-		sector->Setup(loop_, blockSize_, indexAlign_);
+		sector->Setup(loop_, blockSize_, indexAlign_, origMaxCost, lz4MaxCost);
 	}
 }
 

src/output.h

@@ -46,6 +46,8 @@ class Output {
 	uint32_t flags_;
 	uint32_t state_;
 	CSOFormat fmt_;
+	double origMaxCostPercent_;
+	double lz4MaxCostPercent_;
 
 	uv_file file_;
 	uv_fs_t req_;

src/sector.cpp

@@ -22,8 +22,8 @@ static void EndZlib(z_stream *&z) {
 	z = nullptr;
 }
 
-Sector::Sector(uint32_t flags, uint32_t orig_max_cost, uint32_t lz4_max_cost)
-	: flags_(flags), origMaxCost_(orig_max_cost), lz4MaxCost_(lz4_max_cost), busy_(false), enqueued_(false),
+Sector::Sector(uint32_t flags)
+	: flags_(flags), origMaxCost_(0), lz4MaxCost_(0), busy_(false), enqueued_(false),
 	compress_(true), readySize_(0), buffer_(nullptr), best_(nullptr) {
 	// Set up the zlib streams, which we will reuse each time we hit this sector.
 	if (!(flags_ & TASKFLAG_NO_ZLIB_DEFAULT)) {

src/sector.h

@@ -23,13 +23,15 @@ enum SectorFormat {
 // Actually block.
 class Sector {
 public:
-	Sector(uint32_t flags, uint32_t orig_max_cost, uint32_t lz4_max_cost);
+	Sector(uint32_t flags);
 	~Sector();
 
-	void Setup(uv_loop_t *loop, uint32_t blockSize, uint32_t align) {
+	void Setup(uv_loop_t *loop, uint32_t blockSize, uint32_t align, uint32_t origMaxCost, uint32_t lz4MaxCost) {
 		loop_ = loop;
 		blockSize_ = blockSize;
 		align_ = align;
+		origMaxCost_ = origMaxCost;
+		lz4MaxCost_ = lz4MaxCost;
 	}
 
 	void Process(int64_t pos, uint8_t *buffer, SectorCallback ready);