forked from cculianu/Fulcrum
/
RecordFile.h
130 lines (109 loc) · 7.1 KB
/
RecordFile.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
//
// Fulcrum - A fast & nimble SPV Server for Bitcoin Cash
// Copyright (C) 2019-2020 Calin A. Culianu <calin.culianu@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program (see LICENSE.txt). If not, see
// <https://www.gnu.org/licenses/>.
//
#pragma once
#include "Common.h"
#include <QByteArray>
#include <QFile>
#include <QString>
#include <atomic>
#include <cstdint>
#include <optional>
#include <shared_mutex>
#include <memory>
/// A low-level class for reading/writing fixed-sized records indexed by an index number. Basically, this is a
/// file-backed array. We do it this way to save some space in the DB when the key is just a sequential index
/// and there is no sense in storing it. Used by Storage for the TxNum -> TxHash map, for instance.
class RecordFile
{
public:
struct FileError : public Exception { using Exception::Exception; ~FileError() override; };
struct FileFormatError : public FileError { using FileError::FileError; ~FileFormatError() override; };
struct FileOpenError : public FileError { using FileError::FileError; ~FileOpenError() override; };
/// Throws Exception (typically one of the above Exceptions) if it cannot open fileName, or if filename was opened
/// but doesn't seem cromulent (bad magic, bad size, etc).
/// Note 'fileName' will be created if it does not already exist and initialized with the magicBytes and header.
RecordFile(const QString &fileName, size_t recordSize, uint32_t magicBytes = 0x002367f0) noexcept(false);
~RecordFile();
size_t recordSize() const { return recsz; }
uint32_t magicBytes() const { return magic; }
QString fileName() const { return file.fileName(); /* nb: assumption is file.fileName() is thread-safe */ }
uint64_t numRecords() const { return nrecs; }
/// Thread-safe. Implicitly opens a private copy of the file and reads record number recNum from the file. The
/// first record is recNum = 0, the second is recNum = 1. Each record is separated by recordSize() bytes in the
/// file.
/// Returns a QByteArray of size recsz or an empty QByteArray on error.
QByteArray readRecord(uint64_t recNum, QString *errStr = nullptr) const;
/// Thread-safe. Like the above but does a batch read of count records sequantially starting at recNumStart.
/// If the returned vector is not 'count' sized, either not enough records exist in the file or an error occurred
/// (and *errStr will contain the error message).
std::vector<QByteArray> readRecords(uint64_t recNumStart, size_t count, QString *errStr = nullptr) const;
/// Thread-safe. Implicitly opens a private copy of the file and reads recNums from the file. Under non-error
/// circumstances, the returned array will be of the same size as the recNums array, with corresponding indices
/// containing the data obtained per recNum. On error the returned array will be shorter than anticipated
/// and *errStr (if specified) will be set appropriately. Note that the recNums array is not "de-duplicated".
std::vector<QByteArray> readRandomRecords(const std::vector<uint64_t> & recNums, QString *errStr = nullptr) const;
/// Thread-safe, but it does take an exclusive lock. Appends data to the file. The new record number is returned.
/// Note that an error leads to an optional with no value being returned. Data *must* be recordSize() bytes.
/// Note: updateHeader is a performance optimization. If it's false, we don't write the new number of records
/// to the header this call. Use this in a loop and specify updateHeader = true for the last iteration as a
/// performance saving measaure. For even better performance, consider using the beginBatchAppend() method
/// which cuts down further on redundant checks (deferring them until the very end when the batch context ends).
std::optional<uint64_t> appendRecord(const QByteArray &data, bool updateHeader = true, QString *errStr = nullptr);
/// Deletes every record from the file starting with newNumRecords until the end of the file. Updates the header
/// and internal counter to reflect the new count. Returns the new numRecords() of the file (under non-error
/// circumstances this should be identical to the supplied argument, newNumRecords).
uint64_t truncate(uint64_t newNumRecords, QString *errStr = nullptr);
class BatchAppendContext {
RecordFile & rf;
std::unique_lock<std::shared_mutex> lock;
// internal use. Used by RecordFile::beginBatchAppend
BatchAppendContext(RecordFile &);
friend class ::RecordFile;
public:
/// updates the header with the new count, does some checks (may quit app with Fatal() if checks fail), releases lock
~BatchAppendContext();
/// Append a record to the end of the file. Does not write to the file header until d'tor is called (at which
/// point the file's record count is updated in the header).
/// Note that it is imperative that the passed-in data be sized recordSize(). No checks are done as a
/// performance shortcut! If you pass in data that is not recsz bytes, the file may be corrupted or the app
/// may quit with a fatal error.
bool append(const QByteArray &data, QString *errStr = nullptr);
};
/// May throw Exception if the file is in an inconsistent state or if cannot seek (low-level IO error).
/// Otherwise returnes a locking context. Use context.append() to write batches of records to the end of the file.
/// The locked context is released on BatchAppendContext destruction (at which time the file's header is also updated
/// to reflect the new counts).
BatchAppendContext beginBatchAppend();
/// Flushes pending writes. Thread-safe. Returns true if the flush succeeds, false otherwise.
bool flush();
private:
mutable std::shared_mutex rwlock;
friend class RecordFile::BatchAppendContext;
const size_t recsz;
const uint32_t magic;
QFile file; ///< this is kept open throughout the lifetime of this instance; and is the instance used to write to the file. readers open up a new QFile each time.
std::atomic<uint64_t> nrecs = 0;
std::atomic_bool ok = false;
static constexpr size_t hdrsz = sizeof(magic) + sizeof(uint64_t);
static constexpr qint64 offset0() { return hdrsz; }
static constexpr qint64 offsetOfNRecs() { return sizeof(magic); }
qint64 offsetOfRec(uint64_t recNum) const { return qint64(offset0() + recNum*recsz); }
QByteArray readRandomCommon(QFile & f, uint64_t recNum, QString *errStr = nullptr) const;
bool writeNewSizeToHeader(QString *errStr = nullptr, bool flush = false);
};