From 395d115ca32a3fab0063c317cce53692cceac495 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sat, 25 Oct 2014 20:09:06 -0700 Subject: [PATCH] Add a feature to calculate a crc checksum. --- cli/cli.cpp | 52 +++++++++---- src/checksum.cpp | 158 ++++++++++++++++++++++++++++++++++++++++ src/checksum.h | 13 ++++ src/src.vcxproj | 2 + src/src.vcxproj.filters | 2 + 5 files changed, 211 insertions(+), 16 deletions(-) create mode 100644 src/checksum.cpp create mode 100644 src/checksum.h diff --git a/cli/cli.cpp b/cli/cli.cpp index 29c98e08b..a877c7760 100644 --- a/cli/cli.cpp +++ b/cli/cli.cpp @@ -1,8 +1,9 @@ -#include "../src/compress.h" #include #include #include #include +#include "../src/compress.h" +#include "../src/checksum.h" #include "../libuv/include/uv.h" void show_version() { @@ -17,6 +18,7 @@ void show_help(const char *arg0) { fprintf(stderr, "\n"); fprintf(stderr, " --threads=N Specify N threads for I/O and compression\n"); fprintf(stderr, " --quiet Suppress status output\n"); + fprintf(stderr, " --crc Log CRC32 checksums, ignore output files and methods\n"); fprintf(stderr, " --fast Use only basic zlib for fastest result\n"); // TODO: Bring this back once it's functional. //fprintf(stderr, " --smallest Force compression of all sectors for smallest result\n"); @@ -53,12 +55,14 @@ struct Arguments { int threads; uint32_t flags; bool quiet; + bool crc; }; void default_args(Arguments &args) { args.threads = 0; args.flags = maxcso::TASKFLAG_NO_ZOPFLI; args.quiet = false; + args.crc = false; } int parse_args(Arguments &args, int argc, char *argv[]) { @@ -74,6 +78,8 @@ int parse_args(Arguments &args, int argc, char *argv[]) { return 1; } else if (has_arg_value(i, argv, "--threads", val)) { args.threads = atoi(val); + } else if (has_arg(i, argv, "--crc")) { + args.crc = true; } else if (has_arg(i, argv, "--quiet")) { args.quiet = true; } else if (has_arg(i, argv, "--fast")) { @@ -127,22 +133,30 @@ int validate_args(const char *arg0, Arguments &args) { return 1; } - // Automatically write to .cso files if not specified. - for (size_t i = args.outputs.size(); i < args.inputs.size(); ++i) { - if (args.inputs[i].size() <= 4) { - continue; + if (args.crc) { + if (args.outputs.size()) { + show_help(arg0); + fprintf(stderr, "\nERROR: Output files not used with --crc.\n"); + return 1; } + } else { + // Automatically write to .cso files if not specified. + for (size_t i = args.outputs.size(); i < args.inputs.size(); ++i) { + if (args.inputs[i].size() <= 4) { + continue; + } - const std::string ext = args.inputs[i].substr(args.inputs[i].size() - 4); - if (ext == ".iso" || ext == ".ISO") { - args.outputs.push_back(args.inputs[i].substr(0, args.inputs[i].size() - 4) + ".cso"); + const std::string ext = args.inputs[i].substr(args.inputs[i].size() - 4); + if (ext == ".iso" || ext == ".ISO") { + args.outputs.push_back(args.inputs[i].substr(0, args.inputs[i].size() - 4) + ".cso"); + } } - } - if (args.inputs.size() != args.outputs.size()) { - show_help(arg0); - fprintf(stderr, "\nERROR: Too few output files.\n"); - return 1; + if (args.inputs.size() != args.outputs.size()) { + show_help(arg0); + fprintf(stderr, "\nERROR: Too few output files.\n"); + return 1; + } } if (args.inputs.empty()) { @@ -268,14 +282,20 @@ int main(int argc, char *argv[]) { for (size_t i = 0; i < args.inputs.size(); ++i) { maxcso::Task task; task.input = args.inputs[i]; - task.output = args.outputs[i]; + if (!args.crc) { + task.output = args.outputs[i]; + } task.progress = progress; task.error = error; task.flags = args.flags; tasks.push_back(std::move(task)); } - - maxcso::Compress(tasks); + + if (args.crc) { + maxcso::Checksum(tasks); + } else { + maxcso::Compress(tasks); + } uv_tty_reset_mode(); uv_loop_close(&loop); diff --git a/src/checksum.cpp b/src/checksum.cpp new file mode 100644 index 000000000..dc0be3777 --- /dev/null +++ b/src/checksum.cpp @@ -0,0 +1,158 @@ +#include +#include +#include +#include "checksum.h" +#include "uv_helper.h" +#include "input.h" +#include "buffer_pool.h" +#include "cso.h" +#define ZLIB_CONST +#include "zlib.h" + +namespace maxcso { + +class ChecksumTask { +public: + ChecksumTask(uv_loop_t *loop, const Task &t) + : task_(t), loop_(loop), input_(-1), inputHandler_(loop) { + } + ~ChecksumTask() { + Cleanup(); + } + + void Enqueue(); + void Cleanup(); + +private: + void HandleBuffer(uint8_t *buffer); + + void Notify(TaskStatus status, int64_t pos = -1, int64_t total = -1, int64_t written = -1) { + if (status == TASK_INPROGRESS || status == TASK_SUCCESS) { + task_.progress(&task_, status, pos, total, written); + } else { + task_.error(&task_, status, nullptr); + } + } + void Notify(TaskStatus status, const char *reason) { + task_.error(&task_, status, reason); + } + + void BeginProcessing(); + + UVHelper uv_; + const Task &task_; + uv_loop_t *loop_; + + uv_fs_t read_; + + uv_file input_; + Input inputHandler_; + + int64_t pos_; + int64_t size_; + uint32_t crc_; + std::map pendingBuffers_; +}; + +void ChecksumTask::Enqueue() { + // We open input and output in order in case there are errors. + uv_.fs_open(loop_, &read_, task_.input.c_str(), O_RDONLY, 0444, [this](uv_fs_t *req) { + if (req->result < 0) { + Notify(TASK_BAD_INPUT, "Could not open input file"); + } else { + input_ = static_cast(req->result); + + BeginProcessing(); + } + uv_fs_req_cleanup(req); + }); +} + +void ChecksumTask::Cleanup() { + if (input_ >= 0) { + uv_fs_close(loop_, &read_, input_, nullptr); + uv_fs_req_cleanup(&read_); + input_ = -1; + } + for (auto pair : pendingBuffers_) { + pool.Release(pair.second); + } + pendingBuffers_.clear(); +} + +void ChecksumTask::BeginProcessing() { + pos_ = 0; + + inputHandler_.OnFinish([this](bool success, const char *reason) { + if (!success) { + Notify(TASK_INVALID_DATA, reason); + } + }); + + inputHandler_.OnBegin([this](int64_t size) { + crc_ = crc32(0L, Z_NULL, 0); + size_ = size; + Notify(TASK_INPROGRESS, 0, size, 0); + }); + inputHandler_.Pipe(input_, [this](int64_t pos, uint8_t *buffer) { + // In case we allow the buffers to come out of order, let's use a queue. + if (pos_ == pos) { + HandleBuffer(buffer); + } else { + pendingBuffers_[pos] = buffer; + } + }); +} + +void ChecksumTask::HandleBuffer(uint8_t *buffer) { + // We're doing this on the main thread, but it should be okay. + crc_ = crc32(crc_, buffer, SECTOR_SIZE); + pool.Release(buffer); + pos_ += SECTOR_SIZE; + + // Flush any in the queue that we can now use. + if (!pendingBuffers_.empty()) { + auto it = pendingBuffers_.begin(); + auto begin = it; + for (; it != pendingBuffers_.end(); ++it) { + if (it->first != pos_) { + break; + } + + crc_ = crc32(crc_, it->second, SECTOR_SIZE); + pool.Release(it->second); + pos_ += SECTOR_SIZE; + } + + // If we used any, erase them. + if (begin != it) { + pendingBuffers_.erase(begin, it); + } + } + + Notify(TASK_INPROGRESS, pos_, size_, 0); + + if (pos_ == size_) { + char temp[128]; + sprintf(temp, "CRC32: %08x", crc_); + Notify(TASK_SUCCESS, temp); + } +} + +void Checksum(const std::vector &tasks) { + uv_loop_t loop; + uv_loop_init(&loop); + + for (const Task t : tasks) { + ChecksumTask task(&loop, t); + task.Enqueue(); + uv_run(&loop, UV_RUN_DEFAULT); + } + + // Run any remaining events from destructors. + uv_run(&loop, UV_RUN_DEFAULT); + + uv_loop_close(&loop); +} + +}; diff --git a/src/checksum.h b/src/checksum.h new file mode 100644 index 000000000..477ebc9c0 --- /dev/null +++ b/src/checksum.h @@ -0,0 +1,13 @@ +#pragma once + +#include +#include +#include +#include +#include "compress.h" + +namespace maxcso { + +void Checksum(const std::vector &tasks); + +}; diff --git a/src/src.vcxproj b/src/src.vcxproj index 407ac7288..b22b5dfa4 100644 --- a/src/src.vcxproj +++ b/src/src.vcxproj @@ -20,6 +20,7 @@ + @@ -28,6 +29,7 @@ + diff --git a/src/src.vcxproj.filters b/src/src.vcxproj.filters index 3a3c9dd74..2f51adc5c 100644 --- a/src/src.vcxproj.filters +++ b/src/src.vcxproj.filters @@ -7,6 +7,7 @@ + @@ -17,5 +18,6 @@ + \ No newline at end of file