Skip to content

Commit

Permalink
Add support for reading DAX files.
Browse files Browse the repository at this point in the history 
Not super efficient, but functional at least.
  • Loading branch information
@unknownbrackets
unknownbrackets committed Oct 25, 2014
1 parent b7da875 commit 2f08cf6
Show file tree
Hide file tree
Showing 6 changed files with 143 additions and 31 deletions.
5 changes: 3 additions & 2 deletions src/buffer_pool.h
View file
Expand Up @@ -6,11 +6,12 @@

namespace maxcso {

// All buffers are 4 KB. Might experiment with different later.
// All buffers are 16 KB. Might experiment with different later.
// We compress 2 KB sectors, so this is really too much. zlib has a 5 byte overhead.
// However, we also decompress DAX 8KB buffers.
class BufferPool {
public:
static const int BUFFER_SIZE = 4096;
static const int BUFFER_SIZE = 16384;

BufferPool();
~BufferPool();
Expand Down
26 changes: 26 additions & 0 deletions src/dax.h
View file
@@ -0,0 +1,26 @@
#pragma once

#include <cstdint>

namespace maxcso {

static const char *DAX_MAGIC = "DAX\0";
static const uint32_t DAX_FRAME_SIZE = 0x2000;
static const uint32_t DAX_FRAME_MASK = 0x1FFF;
static const uint8_t DAX_FRAME_SHIFT = 13;

// TODO: Endian-ify?
struct DAXHeader {
char magic[4];
uint32_t uncompressed_size;
uint32_t version;
uint32_t nc_areas;
uint32_t unused[4];
};

struct DAXNCArea {
uint32_t start;
uint32_t count;
};

};
126 changes: 101 additions & 25 deletions src/input.cpp
View file
@@ -1,20 +1,26 @@
#include "input.h"
#include "buffer_pool.h"
#include "cso.h"
#include "dax.h"
#define ZLIB_CONST
#include "zlib.h"

namespace maxcso {

Input::Input(uv_loop_t *loop)
: loop_(loop), type_(UNKNOWN), paused_(false), resumeShouldRead_(false), size_(-1), cache_(nullptr), csoIndex_(nullptr) {
: loop_(loop), type_(UNKNOWN), paused_(false), resumeShouldRead_(false), size_(-1), cache_(nullptr),
csoIndex_(nullptr), daxSize_(nullptr), daxIsNC_(nullptr) {
}

Input::~Input() {
delete [] cache_;
cache_ = nullptr;
delete [] csoIndex_;
csoIndex_ = nullptr;
delete [] daxSize_;
daxSize_ = nullptr;
delete [] daxIsNC_;
daxIsNC_ = nullptr;
}

void Input::OnFinish(InputFinishCallback finish) {
Expand Down Expand Up @@ -67,7 +73,7 @@ void Input::DetectFormat() {
const uv_buf_t buf = uv_buf_init(reinterpret_cast<char *>(csoIndex_), bytes);
SetupCache(header->sector_size);

uv_.fs_read(loop_, &req_, file_, &buf, 1, 24, [this, bytes](uv_fs_t *req) {
uv_.fs_read(loop_, &req_, file_, &buf, 1, sizeof(CSOHeader), [this, bytes](uv_fs_t *req) {
if (req->result != bytes) {
// Index wasn't all there, this file is corrupt.
finish_(false, "Unable to read entire index");
Expand All @@ -76,6 +82,57 @@ void Input::DetectFormat() {
}
uv_fs_req_cleanup(req);

begin_(size_);
ReadSector();
});
}
} else if (!memcmp(headerBuf, DAX_MAGIC, 4)) {
type_ = DAX;
const DAXHeader *const header = reinterpret_cast<DAXHeader *>(headerBuf);
if (header->version > 1) {
finish_(false, "DAX header indicates unsupported version");
} else if ((header->uncompressed_size & SECTOR_MASK) != 0) {
finish_(false, "DAX uncompressed size not aligned to sector size");
} else {
size_ = header->uncompressed_size;

const uint32_t frames = static_cast<uint32_t>((size_ + DAX_FRAME_SIZE - 1) >> DAX_FRAME_SHIFT);
daxIndex_ = new uint32_t[frames];
daxSize_ = new uint16_t[frames];
daxIsNC_ = new bool[frames];
memset(daxIsNC_, 0, sizeof(bool) * frames);

uv_buf_t bufs[3];
int nbufs = 2;
uint32_t bytes = frames * (sizeof(uint32_t) + sizeof(uint16_t));
bufs[0] = uv_buf_init(reinterpret_cast<char *>(daxIndex_), frames * sizeof(uint32_t));
bufs[1] = uv_buf_init(reinterpret_cast<char *>(daxSize_), frames * sizeof(uint16_t));

int nareas = header->version >= 1 ? header->nc_areas : 0;
DAXNCArea *areas = nareas > 0 ? new DAXNCArea[nareas] : nullptr;
if (areas != nullptr) {
bufs[nbufs++] = uv_buf_init(reinterpret_cast<char *>(areas), nareas * sizeof(DAXNCArea));
bytes += nareas * sizeof(DAXNCArea);
}
SetupCache(DAX_FRAME_SIZE);

uv_.fs_read(loop_, &req_, file_, bufs, nbufs, sizeof(DAXHeader), [this, bytes, areas, nareas](uv_fs_t *req) {
if (req->result != bytes) {
// Index wasn't all there, this file is corrupt.
finish_(false, "Unable to read entire index");
uv_fs_req_cleanup(req);
return;
}
uv_fs_req_cleanup(req);

// Map the areas to an index and free.
for (int i = 0; i < nareas; ++i) {
for (uint32_t frame = 0; frame < areas[i].count; ++frame) {
daxIsNC_[areas[i].start + frame] = true;
}
}
delete [] areas;

begin_(size_);
ReadSector();
});
Expand Down Expand Up @@ -126,7 +183,10 @@ void Input::ReadSector() {
return;
}

// Position of data in file.
int64_t pos = pos_;
// Offset into position where our data is.
uint32_t offset = 0;
unsigned int len = SECTOR_SIZE;
bool compressed = false;
switch (type_) {
Expand All @@ -149,17 +209,29 @@ void Input::ReadSector() {
}
}
break;
case DAX:
{
const uint32_t frame = static_cast<uint32_t>(pos_ >> DAX_FRAME_SHIFT);
pos = daxIndex_[frame];
len = daxSize_[frame];
compressed = !daxIsNC_[frame];
offset = pos_ & DAX_FRAME_MASK;

if (!compressed && offset != 0) {
pos += offset;
offset = 0;
}
}
}

// This ends up being owned by the compressor.
uint8_t *const readBuf = pool.Alloc();

if (pos >= cachePos_ && pos + len <= cachePos_ + cacheSize_) {
// Already read in, let's just reuse.
memcpy(readBuf, cache_ + pos - cachePos_, len);
if (compressed) {
EnqueueDecompressSector(readBuf, len);
EnqueueDecompressSector(cache_ + pos - cachePos_, len, offset);
} else {
memcpy(readBuf, cache_ + pos - cachePos_, len);
callback_(pos_, readBuf);

pos_ += SECTOR_SIZE;
Expand All @@ -168,18 +240,18 @@ void Input::ReadSector() {
} else {
const uv_buf_t buf = uv_buf_init(reinterpret_cast<char *>(cache_), cacheSize_);
cachePos_ = pos;
uv_.fs_read(loop_, &req_, file_, &buf, 1, pos, [this, readBuf, len, compressed](uv_fs_t *req) {
uv_.fs_read(loop_, &req_, file_, &buf, 1, pos, [this, readBuf, len, offset, compressed](uv_fs_t *req) {
if (req->result < len) {
finish_(false, "Unable to read entire sector");
uv_fs_req_cleanup(req);
return;
}
uv_fs_req_cleanup(req);

memcpy(readBuf, cache_, len);
if (compressed) {
EnqueueDecompressSector(readBuf, len);
EnqueueDecompressSector(cache_, len, offset);
} else {
memcpy(readBuf, cache_, len);
callback_(pos_, readBuf);

pos_ += SECTOR_SIZE;
Expand All @@ -189,21 +261,23 @@ void Input::ReadSector() {
}
}

void Input::EnqueueDecompressSector(uint8_t *buffer, uint32_t len) {
void Input::EnqueueDecompressSector(uint8_t *src, uint32_t len, uint32_t offset) {
// We swap this with the compressed buf, and free the readBuf.
uint8_t *const actualBuf = pool.Alloc();
csoError_.clear();
uv_.queue_work(loop_, &work_, [this, actualBuf, buffer, len](uv_work_t *req) {
if (!DecompressSector(actualBuf, buffer, len, csoError_)) {
if (csoError_.empty()) {
csoError_ = "Unknown error";
decompressError_.clear();
uv_.queue_work(loop_, &work_, [this, actualBuf, src, len](uv_work_t *req) {
if (!DecompressSector(actualBuf, src, len, type_, decompressError_)) {
if (decompressError_.empty()) {
decompressError_ = "Unknown error";
}
}
}, [this, actualBuf, buffer](uv_work_t *req, int status) {
pool.Release(buffer);
}, [this, actualBuf, offset](uv_work_t *req, int status) {
if (offset != 0) {
memmove(actualBuf, actualBuf + offset, SECTOR_SIZE);
}

if (!csoError_.empty()) {
finish_(false, csoError_.c_str());
if (!decompressError_.empty()) {
finish_(false, decompressError_.c_str());
pool.Release(actualBuf);
} else if (status == -1) {
finish_(false, "Decompression work failed");
Expand All @@ -229,11 +303,11 @@ void Input::Resume() {
}
}

bool Input::DecompressSector(uint8_t *dst, const uint8_t *src, unsigned int len, std::string &err) {
bool Input::DecompressSector(uint8_t *dst, const uint8_t *src, unsigned int len, FileType type, std::string &err) {
z_stream z;
memset(&z, 0, sizeof(z));
// TODO: inflateReset2?
if (inflateInit2(&z, -15) != Z_OK) {
if (inflateInit2(&z, type == CISO ? -15 : 15) != Z_OK) {
err = z.msg ? z.msg : "Unable to initialize inflate";
return false;
}
Expand All @@ -243,17 +317,19 @@ bool Input::DecompressSector(uint8_t *dst, const uint8_t *src, unsigned int len,
z.avail_out = BufferPool::BUFFER_SIZE;
z.next_in = src;

const int status = inflate(&z, Z_FULL_FLUSH);
const int status = inflate(&z, Z_FINISH);
if (status != Z_STREAM_END) {
err = z.msg ? z.msg : "Inflate failed";
inflateEnd(&z);
return false;
}

if (z.avail_out != BufferPool::BUFFER_SIZE - SECTOR_SIZE) {
err = "Expected to decompress into a full sector";
inflateEnd(&z);
return false;
if (type == CISO) {
if (z.avail_out != BufferPool::BUFFER_SIZE - SECTOR_SIZE) {
err = "Expected to decompress into a full sector";
inflateEnd(&z);
return false;
}
}

inflateEnd(&z);
Expand Down
15 changes: 11 additions & 4 deletions src/input.h
View file
Expand Up @@ -23,15 +23,17 @@ class Input {
void DetectFormat();
void SetupCache(uint32_t minSize);
void ReadSector();
void EnqueueDecompressSector(uint8_t *buffer, uint32_t len);
bool DecompressSector(uint8_t *dst, const uint8_t *src, unsigned int len, std::string &err);
void EnqueueDecompressSector(uint8_t *src, uint32_t len, uint32_t offset);

enum FileType {
UNKNOWN,
ISO,
CISO,
DAX,
};

bool DecompressSector(uint8_t *dst, const uint8_t *src, unsigned int len, FileType type, std::string &err);

UVHelper uv_;
uv_loop_t *loop_;

Expand All @@ -51,10 +53,15 @@ class Input {
int64_t cachePos_;
int32_t cacheSize_;

std::string decompressError_;
uint8_t csoShift_;
// TODO: Endian?
uint32_t *csoIndex_;
std::string csoError_;
union {
uint32_t *csoIndex_;
uint32_t *daxIndex_;
};
uint16_t *daxSize_;
bool *daxIsNC_;
};

};
1 change: 1 addition & 0 deletions src/src.vcxproj
View file
Expand Up @@ -30,6 +30,7 @@
<ClInclude Include="buffer_pool.h" />
<ClInclude Include="compress.h" />
<ClInclude Include="cso.h" />
<ClInclude Include="dax.h" />
<ClInclude Include="input.h" />
<ClInclude Include="output.h" />
<ClInclude Include="sector.h" />
Expand Down
1 change: 1 addition & 0 deletions src/src.vcxproj.filters
View file
Expand Up @@ -16,5 +16,6 @@
<ClInclude Include="output.h" />
<ClInclude Include="sector.h" />
<ClInclude Include="cso.h" />
<ClInclude Include="dax.h" />
</ItemGroup>
</Project>

0 comments on commit 2f08cf6

Please sign in to comment.