Db namespaces changes.

nimbus/beacon/merge_tracker.nim

@@ -37,10 +37,12 @@ type
 # ------------------------------------------------------------------------------
 
 proc writeStatus(db: CoreDbRef, status: TransitionStatus) =
-  db.kvt.put(transitionStatusKey().toOpenArray(), rlp.encode(status))
+  let key = transitionStatusKey()
+  db.kvt(key.namespace).put(key.toOpenArray(), rlp.encode(status))
 
 proc readStatus(db: CoreDbRef): TransitionStatus =
-  var bytes = db.kvt.get(transitionStatusKey().toOpenArray())
+  let key = transitionStatusKey()
+  var bytes = db.kvt(key.namespace).get(key.toOpenArray())
   if bytes.len > 0:
     try:
       result = rlp.decode(bytes, typeof result)

nimbus/common/common.nim

@@ -406,14 +406,16 @@ proc consensus*(com: CommonRef, header: BlockHeader): ConsensusType
 
 proc initializeEmptyDb*(com: CommonRef)
     {.gcsafe, raises: [CatchableError].} =
-  let kvt = com.db.kvt()
-  if canonicalHeadHashKey().toOpenArray notin kvt:
+  let
+    key = canonicalHeadHashKey()
+    kvt = com.db.kvt(key.namespace)
+  if key.toOpenArray notin kvt:
     trace "Writing genesis to DB"
     doAssert(com.genesisHeader.blockNumber.isZero,
       "can't commit genesis block with number > 0")
     discard com.db.persistHeaderToDb(com.genesisHeader,
       com.consensusType == ConsensusType.POS)
-    doAssert(canonicalHeadHashKey().toOpenArray in kvt)
+    doAssert(key.toOpenArray in kvt)
 
 proc syncReqNewHead*(com: CommonRef; header: BlockHeader)
     {.gcsafe, raises: [].} =

nimbus/core/clique/snapshot/snapshot_desc.nim

@@ -147,7 +147,8 @@ proc loadSnapshot*(cfg: CliqueCfg; hash: Hash256):
   ## Load an existing snapshot from the database.
   var s = Snapshot(cfg: cfg)
   try:
-    let rc = s.cfg.db.newKvt(Shared).get(hash.cliqueSnapshotKey.toOpenArray)
+    let key = cliqueSnapshotKey(hash)
+    let rc = s.cfg.db.newKvt(key.namespace, Shared).get(key.toOpenArray)
     if rc.isOk:
       s.data = rc.value.decode(SnapshotData)
     else:
@@ -163,12 +164,12 @@ proc storeSnapshot*(cfg: CliqueCfg; s: Snapshot): CliqueOkResult =
   ## Insert the snapshot into the database.
   try:
     let
-      key = s.data.blockHash.cliqueSnapshotKey
+      key = cliqueSnapshotKey(s.data.blockHash)
       val = rlp.encode(s.data)
-      db  = s.cfg.db.newKvt(Companion)
-    db.put(key.toOpenArray, val).isOkOr:
+      kvt = s.cfg.db.newKvt(key.namespace, Companion)
+    kvt.put(key.toOpenArray, val).isOkOr:
       error logTxt "put() failed", `error`=($$error)
-    db.persistent()
+    kvt.persistent()
 
     cfg.nSnaps.inc
     cfg.snapsData += val.len.uint

nimbus/db/aristo/aristo_desc/desc_error.nim

@@ -236,7 +236,6 @@ type
 
     # RocksDB backend
     RdbBeCantCreateDataDir
-    RdbBeCantCreateBackupDir
     RdbBeCantCreateTmpDir
     RdbBeDriverInitError
     RdbBeDriverGetError
@@ -268,7 +267,7 @@ type
     AccVtxUnsupported
     AccNodeUnsupported
     PayloadTypeUnsupported
-    
+
     # Miscelaneous handy helpers
     AccRootUnacceptable
     MptContextMissing

nimbus/db/aristo/aristo_init/rocks_db/rdb_desc.nim

@@ -41,7 +41,6 @@ type
 const
   BaseFolder* = "nimbus"           # Same as for Legacy DB
   DataFolder* = "aristo"           # Legacy DB has "data"
-  BackupFolder* = "aristo-history" # Legacy DB has "backups"
   SstCache* = "bulkput"            # Rocks DB bulk load file name in temp folder
   TempFolder* = "tmp"              # No `tmp` directory used with legacy DB
 
@@ -55,9 +54,6 @@ func baseDir*(rdb: RdbInst): string =
 func dataDir*(rdb: RdbInst): string =
   rdb.baseDir / DataFolder
 
-func backupsDir*(rdb: RdbInst): string =
-  rdb.basePath / BaseFolder / BackupFolder
-
 func cacheDir*(rdb: RdbInst): string =
   rdb.dataDir / TempFolder
 

nimbus/db/aristo/aristo_init/rocks_db/rdb_init.nim

@@ -47,16 +47,11 @@ proc init*(
 
   let
     dataDir = rdb.dataDir
-    backupsDir = rdb.backupsDir
 
   try:
     dataDir.createDir
   except OSError, IOError:
     return err((RdbBeCantCreateDataDir, ""))
-  try:
-    backupsDir.createDir
-  except OSError, IOError:
-    return err((RdbBeCantCreateBackupDir, ""))
   try:
     rdb.cacheDir.createDir
   except OSError, IOError:
@@ -68,7 +63,7 @@ proc init*(
   let rc = openRocksDb(dataDir, dbOpts)
   if rc.isErr:
     let error = RdbBeDriverInitError
-    debug logTxt "driver failed", dataDir, backupsDir, openMax,
+    debug logTxt "driver failed", dataDir, openMax,
       error, info=rc.error
     return err((RdbBeDriverInitError, rc.error))
 

nimbus/db/core_db/backend/aristo_db.nim

@@ -20,7 +20,9 @@ import
   ../../kvt,
   ../../kvt/[kvt_desc, kvt_init, kvt_tx, kvt_walk],
   ".."/[base, base/base_desc],
-  ./aristo_db/[common_desc, handlers_aristo, handlers_kvt]
+  ./aristo_db/[common_desc, handlers_aristo, handlers_kvt],
+  ../../storage_types
+
 
 import
   ../../aristo/aristo_init/memory_only as aristo_memory_only
@@ -135,7 +137,8 @@ proc baseMethods(
           ): CoreDbRc[CoreDbTrieRef] =
       db.adbBase.getTrie(kind, root, address, "getTrieFn()"),
 
-    newKvtFn: proc(saveMode: CoreDbSaveFlags): CoreDbRc[CoreDxKvtRef] =
+    newKvtFn: proc(namespace: DbNamespace, saveMode: CoreDbSaveFlags): CoreDbRc[CoreDxKvtRef] =
+      # TODO: use namespace
       db.kdbBase.gc()
       db.kdbBase.newKvtHandler(saveMode, "newKvtFn()"),
 

nimbus/db/core_db/backend/legacy_db.nim

@@ -14,6 +14,7 @@ import
   eth/[common, rlp, trie/db, trie/hexary],
   stew/byteutils,
   results,
+  ../../storage_types,
   ../../../errors,
   ".."/[base, base/base_desc]
 
@@ -27,6 +28,7 @@ type
     kvt: CoreDxKvtRef       ## Cache, no need to rebuild methods descriptor
     tdb: TrieDatabaseRef    ## Descriptor reference copy captured with closures
     top: LegacyCoreDxTxRef  ## Top transaction (if any)
+    nsDbs: Table[DbNamespace, TrieDatabaseRef]
 
   LegacyDbClose* = proc() {.gcsafe, raises: [].}
     ## Custom destructor
@@ -79,6 +81,7 @@ proc init*(
     db: LegacyDbRef;
     dbType: CoreDbType;
     tdb: TrieDatabaseRef;
+    nsDbs: Table[DbNamespace, TrieDatabaseRef] = initTable[DbNamespace, TrieDatabaseRef]();
     closeDb = LegacyDbClose(nil);
       ): CoreDbRef
       {.gcsafe.}
@@ -223,9 +226,15 @@ proc newRecorderRef(
 # Private database method function tables
 # ------------------------------------------------------------------------------
 
-proc kvtMethods(db: LegacyDbRef): CoreDbKvtFns =
+proc kvtMethods(db: LegacyDbRef, namespace: DbNamespace): CoreDbKvtFns {.gcsafe.} =
   ## Key-value database table handlers
-  let tdb = db.tdb
+
+  let tdb = if db.dbType == LegacyDbPersistent and namespace != DbNamespace.default:
+    doAssert db.nsDbs.hasKey(namespace)
+    db.nsDbs.getOrDefault(namespace)
+  else:
+    db.tdb
+
   CoreDbKvtFns(
     backendFn: proc(): CoreDbKvtBackendRef =
       db.bless(LegacyCoreDbKvtBE(tdb: tdb)),
@@ -255,7 +264,10 @@ proc kvtMethods(db: LegacyDbRef): CoreDbKvtFns =
       ok(),
 
     forgetFn: proc(): CoreDbRc[void] =
-      ok())
+      ok(),
+
+    namespaceFn: proc(namespace: DbNamespace): CoreDxKvtRef {.gcsafe.} =
+      CoreDxKvtRef(methods: db.kvtMethods(namespace)))
 
 proc mptMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbMptFns =
   ## Hexary trie database handlers
@@ -469,8 +481,8 @@ proc baseMethods(
           trie.accPath = @(address.unsafeGet.keccakHash.data)
       ok(db.bless trie),
 
-    newKvtFn: proc(saveMode: CoreDbSaveFlags): CoreDbRc[CoreDxKvtRef] =
-      ok(db.kvt),
+    newKvtFn: proc(namespace: DbNamespace, saveMode: CoreDbSaveFlags): CoreDbRc[CoreDxKvtRef] =
+      ok(db.kvt.namespace(namespace)),
 
     newMptFn: proc(
         trie: CoreDbTrieRef,
@@ -525,13 +537,15 @@ proc init*(
     db: LegacyDbRef;
     dbType: CoreDbType;
     tdb: TrieDatabaseRef;
+    nsDbs: Table[DbNamespace, TrieDatabaseRef] = initTable[DbNamespace, TrieDatabaseRef]();
     closeDb = LegacyDbClose(nil);
      ): CoreDbRef =
   ## Constructor helper
 
   # Local extensions
   db.tdb = tdb
-  db.kvt = db.bless CoreDxKvtRef(methods: db.kvtMethods())
+  db.nsDbs = nsDbs
+  db.kvt = db.bless CoreDxKvtRef(methods: db.kvtMethods(DbNamespace.default))
 
   # Base descriptor
   db.dbType = dbType

nimbus/db/core_db/backend/legacy_rocksdb.nim

@@ -11,11 +11,13 @@
 {.push raises: [].}
 
 import
+  std/[tables, sequtils],
   eth/trie/db,
   eth/db/kvstore,
   rocksdb,
   ../base,
   ./legacy_db,
+  ../../storage_types,
   ../../kvstore_rocksdb
 
 type
@@ -26,6 +28,20 @@ type
     kv: KvStoreRef
     rdb: RocksStoreRef
 
+proc `$`*(ns: DbNamespace): string =
+  if ns == DbNamespace.default:
+    return "default"
+  $ord(ns)
+
+proc getNamespaces(): seq[DbNamespace] =
+  var namespaces = newSeq[DbNamespace]()
+  for ns in DbNamespace.items():
+    # Don't include the default namespace in the list because the main
+    # RocksStoreRef uses the default namespace already
+    if ns != DbNamespace.default:
+      namespaces.add(ns)
+  namespaces
+
 # TODO KvStore is a virtual interface and TrieDB is a virtual interface - one
 #      will be enough eventually - unless the TrieDB interface gains operations
 #      that are not typical to KvStores
@@ -45,10 +61,17 @@ proc del(db: ChainDB, key: openArray[byte]): bool =
   db.kv.del(key).expect("working database")
 
 proc newChainDB(path: string): KvResult[ChainDB] =
-  let rdb = RocksStoreRef.init(path, "nimbus").valueOr:
+  let rdb = RocksStoreRef.init(
+      path,
+      "nimbus",
+      namespaces = getNamespaces().mapIt($it)).valueOr:
     return err(error)
   ok(ChainDB(kv: kvStore rdb, rdb: rdb))
 
+proc withNamespace(db: ChainDB, ns: string): KvResult[ChainDB] =
+  let nsDb = ? db.rdb.openNamespace(ns)
+  ok(ChainDB(kv: kvStore nsDb, rdb: db.rdb))
+
 # ------------------------------------------------------------------------------
 # Public constructor and low level data retrieval, storage & transation frame
 # ------------------------------------------------------------------------------
@@ -59,10 +82,18 @@ proc newLegacyPersistentCoreDbRef*(path: string): CoreDbRef =
     let msg = "DB initialisation : " & error
     raise (ref ResultDefect)(msg: msg)
 
+  var nsMap = initTable[DbNamespace, TrieDatabaseRef]()
+
+  for ns in getNamespaces():
+    let namespace = backend.withNamespace($ns).valueOr:
+      let msg = "DB initialisation : " & error
+      raise (ref ResultDefect)(msg: msg)
+    nsMap[ns] = trieDB(namespace)
+
   proc done() =
     backend.rdb.close()
 
-  LegaPersDbRef(rdb: backend.rdb).init(LegacyDbPersistent, backend.trieDB, done)
+  LegaPersDbRef(rdb: backend.rdb).init(LegacyDbPersistent, trieDB(backend), nsMap, done)
 
 # ------------------------------------------------------------------------------
 # Public helper for direct backend access

nimbus/db/core_db/base.nim

@@ -15,6 +15,7 @@ import
   chronicles,
   eth/common,
   results,
+  ../storage_types,
   "../.."/[constants, errors],
   ./base/[api_new_desc, api_tracking, base_desc]
 
@@ -437,7 +438,7 @@ proc getTrie*(
 # Public key-value table methods
 # ------------------------------------------------------------------------------
 
-proc newKvt*(db: CoreDbRef; saveMode = AutoSave): CoreDxKvtRef =
+proc newKvt*(db: CoreDbRef; namespace: DbNamespace, saveMode = AutoSave): CoreDxKvtRef =
   ## Constructor, will defect on failure.
   ##
   ## Depending on the argument `saveMode`, the contructed object will have
@@ -476,10 +477,13 @@ proc newKvt*(db: CoreDbRef; saveMode = AutoSave): CoreDxKvtRef =
   ## function argument.
   ##
   db.setTrackNewApi BaseNewKvtFn
-  result = db.methods.newKvtFn(saveMode).valueOr:
+  result = db.methods.newKvtFn(namespace, saveMode).valueOr:
     raiseAssert error.prettyText()
   db.ifTrackNewApi: debug newApiTxt, ctx, elapsed, saveMode
 
+proc newDefaultKvt*(db: CoreDbRef; saveMode = AutoSave): CoreDxKvtRef =
+  newKvt(db, DbNamespace.default, saveMode)
+
 proc get*(kvt: CoreDxKvtRef; key: openArray[byte]): CoreDbRc[Blob] =
   ## This function always returns a non-empty `Blob` or an error code.
   kvt.setTrackNewApi KvtGetFn
@@ -555,6 +559,10 @@ proc forget*(dsc: CoreDxKvtRef): CoreDbRc[void] {.discardable.} =
   result = dsc.methods.forgetFn()
   dsc.ifTrackNewApi: debug newApiTxt, ctx, elapsed, result
 
+proc namespace*(dsc: CoreDxKvtRef, namespace: DbNamespace): CoreDxKvtRef =
+  ## TODO:
+  dsc.methods.namespaceFn(namespace)
+
 # ------------------------------------------------------------------------------
 # Public Merkle Patricia Tree, hexary trie constructors
 # ------------------------------------------------------------------------------
@@ -1029,12 +1037,18 @@ when ProvideLegacyAPI:
 
   # ----------------
 
-  proc kvt*(db: CoreDbRef): CoreDbKvtRef =
+  proc kvt*(db: CoreDbRef, namespace: DbNamespace): CoreDbKvtRef =
     ## Legacy pseudo constructor, see `toKvt()` for production constructor
     db.setTrackLegaApi LegaNewKvtFn
-    result = db.newKvt().CoreDbKvtRef
     db.ifTrackLegaApi: debug legaApiTxt, ctx, elapsed, result
 
+    result = db.newKvt(namespace = namespace).CoreDbKvtRef
+
+    db.ifTrackLegaApi: debug legaApiTxt, ctx, elapsed, result
+
+  proc defaultKvt*(db: CoreDbRef): CoreDbKvtRef =
+    kvt(db, DbNamespace.default)
+
   proc get*(kvt: CoreDbKvtRef; key: openArray[byte]): Blob =
     kvt.setTrackLegaApi LegaKvtGetFn
     result = kvt.distinctBase.getOrEmpty(key).expect $ctx
@@ -1047,7 +1061,7 @@ when ProvideLegacyAPI:
 
   proc put*(kvt: CoreDbKvtRef; key: openArray[byte]; val: openArray[byte]) =
     kvt.setTrackLegaApi LegaKvtPutFn
-    kvt.distinctBase.parent.newKvt().put(key, val).expect $ctx
+    kvt.distinctBase.put(key, val).expect $ctx
     kvt.ifTrackLegaApi:
       debug legaApiTxt, ctx, elapsed, key=key.toStr, val=val.toLenStr