Merge pull request #202 from mongodb-partners/use_official_rocksdb

Use official rocksdb

SConscript

@@ -9,10 +9,13 @@ conf = Configure(env)
 if conf.CheckLibWithHeader("lz4", ["lz4.h","lz4hc.h"], "C", "LZ4_versionNumber();", autoadd=False ):
     dynamic_syslibdeps.append("lz4")
 
-conf.Finish()
 
 env.InjectMongoIncludePaths()
 
+env.InjectThirdParty(libraries=['s2',]) # for Encoder and Decoder
+
+conf.Finish()
+
 env.Library(
     target= 'storage_rocks_base',
     source= [
@@ -29,6 +32,10 @@ env.Library(
         'src/rocks_oplog_manager.cpp',
         'src/rocks_begin_transaction_block.cpp',
         'src/rocks_prepare_conflict.cpp',
+# TODO(wolfkdy): move totdb files into a seperate compile-unit
+        'src/totdb/totransaction_impl.cpp',
+        'src/totdb/totransaction_db_impl.cpp',
+        'src/totdb/totransaction_prepare_iterator.cpp',
         env.Idlc('src/rocks_parameters.idl')[0],
         env.Idlc('src/rocks_global_options.idl')[0],
         'src/rocks_parameters.cpp',
@@ -53,6 +60,7 @@ env.Library(
         '$BUILD_DIR/mongo/util/concurrency/ticketholder',
         '$BUILD_DIR/mongo/util/processinfo',
         '$BUILD_DIR/third_party/shim_snappy',
+        '$BUILD_DIR/third_party/s2/util/coding/coding',
     ],
     LIBDEPS_PRIVATE= [
         '$BUILD_DIR/mongo/db/snapshot_window_options',
@@ -132,3 +140,13 @@ env.CppUnitTest(
         '$BUILD_DIR/mongo/db/repl/replmocks',
     ],
 )
+
+env.CppUnitTest(
+    target='totdb_test',
+    source=[
+        'src/totdb/totransaction_test.cpp',
+    ],
+    LIBDEPS=[
+        'storage_rocks_mock',
+    ],
+)

src/rocks_begin_transaction_block.h

@@ -29,8 +29,8 @@
 
 #pragma once
 
-#include <rocksdb/utilities/totransaction.h>
-#include <rocksdb/utilities/totransaction_db.h>
+#include "mongo/db/modules/rocks/src/totdb/totransaction.h"
+#include "mongo/db/modules/rocks/src/totdb/totransaction_db.h"
 #include "mongo/base/status.h"
 #include "mongo/bson/timestamp.h"
 #include "mongo/db/storage/recovery_unit.h"

src/rocks_compaction_scheduler.h

@@ -70,6 +70,7 @@ namespace mongo {
         ~RocksCompactionScheduler();
 
         void start(rocksdb::TOTransactionDB* db, rocksdb::ColumnFamilyHandle* cf);
+        void stop();
 
         static int getSkippedDeletionsThreshold() { return kSkippedDeletionsThreshold; }
 
@@ -78,7 +79,6 @@ namespace mongo {
         // schedule compact range operation for execution in _compactionThread
         void compactAll();
         Status compactOplog(rocksdb::ColumnFamilyHandle* cf, const std::string& begin, const std::string& end);
-        Status rollbackToStable(rocksdb::ColumnFamilyHandle* cf);
 
         rocksdb::CompactionFilterFactory* createCompactionFilterFactory();
         std::unordered_map<uint32_t, BSONObj> getDroppedPrefixes() const;
@@ -111,7 +111,6 @@ namespace mongo {
         void compactDroppedPrefix(rocksdb::ColumnFamilyHandle* cf, const std::string& prefix);
         void compact(rocksdb::ColumnFamilyHandle* cf, const std::string& begin,
                      const std::string& end, bool rangeDropped, uint32_t order,
-                     const bool trimHistory,
                      boost::optional<std::shared_ptr<Notification<Status>>>);
         void droppedPrefixCompacted(const std::string& prefix, bool opSucceeded);
 

src/rocks_counter_manager.cpp

@@ -57,12 +57,15 @@ namespace mongo {
             }
         }
         std::string value;
-        auto s = _db->Get(rocksdb::ReadOptions(), _cf, counterKey, &value);
-        if (s.IsNotFound()) {
-            return 0;
+        {
+            auto txn = _db->makeTxn();
+            auto readopts = rocksdb::ReadOptions();
+            auto s = txn->Get(readopts, _cf, counterKey, &value);
+            if (s.IsNotFound()) {
+                return 0;
+            }
+            invariantRocksOK(s);
         }
-        invariantRocksOK(s);
-
         int64_t ret;
         invariant(sizeof(ret) == value.size());
         memcpy(&ret, value.data(), sizeof(ret));
@@ -73,7 +76,7 @@ namespace mongo {
     void RocksCounterManager::updateCounter(const std::string& counterKey, long long count) {
         if (_crashSafe) {
             int64_t storage;
-            auto txn = _makeTxn();
+            auto txn = _db->makeTxn();
             invariantRocksOK(txn->Put(_cf, counterKey, _encodeCounter(count, &storage)));
             invariantRocksOK(txn->Commit());
         } else {
@@ -83,7 +86,7 @@ namespace mongo {
             if (_syncCounter >= kSyncEvery) {
                 // let's sync this now. piggyback on writeBatch
                 int64_t storage;
-                auto txn = _makeTxn();
+                auto txn = _db->makeTxn();
                 for (const auto& counter : _counters) {
                     invariantRocksOK(
                         txn->Put(_cf, counter.first, _encodeCounter(counter.second, &storage)));
@@ -100,7 +103,7 @@ namespace mongo {
         if (_counters.size() == 0) {
             return;
         }
-        auto txn = _makeTxn();
+        auto txn = _db->makeTxn();
         int64_t storage;
         for (const auto& counter : _counters) {
             invariantRocksOK(txn->Put(_cf, counter.first, _encodeCounter(counter.second, &storage)));
@@ -115,9 +118,4 @@ namespace mongo {
         return rocksdb::Slice(reinterpret_cast<const char*>(storage), sizeof(*storage));
     }
 
-    std::unique_ptr<rocksdb::TOTransaction> RocksCounterManager::_makeTxn() {
-        rocksdb::WriteOptions options;
-        rocksdb::TOTransactionOptions txnOptions;
-        return std::unique_ptr<rocksdb::TOTransaction>(_db->BeginTransaction(options, txnOptions));
-    }
 }  // namespace mongo

src/rocks_counter_manager.h

@@ -37,8 +37,8 @@
 
 #include <rocksdb/db.h>
 #include <rocksdb/slice.h>
-#include <rocksdb/utilities/totransaction.h>
-#include <rocksdb/utilities/totransaction_db.h>
+#include "mongo/db/modules/rocks/src/totdb/totransaction.h"
+#include "mongo/db/modules/rocks/src/totdb/totransaction_db.h"
 
 #include "mongo/base/string_data.h"
 #include "mongo/platform/mutex.h"
@@ -61,8 +61,6 @@ namespace mongo {
     private:
         static rocksdb::Slice _encodeCounter(long long counter, int64_t* storage);
 
-        std::unique_ptr<rocksdb::TOTransaction> _makeTxn();
-
         rocksdb::TOTransactionDB* _db;  // not owned
 
         rocksdb::ColumnFamilyHandle* _cf;  // not owned