Stateless mode (not yet working!!!)

Makefile

@@ -138,6 +138,10 @@ ifneq ($(ENABLE_EVMC), 0)
   T8N_PARAMS := -d:chronicles_enabled=off
 endif
 
+ifneq ($(ENABLE_SPECULATIVE_EXECUTION), 0)
+  NIM_PARAMS += -d:evm_speculative_execution
+endif
+
 # disabled by default, enable with ENABLE_VMLOWMEM=1
 ifneq ($(if $(ENABLE_VMLOWMEM),$(ENABLE_VMLOWMEM),0),0)
   NIM_PARAMS += -d:lowmem:1

nimbus/core/executor/process_block.nim

@@ -117,7 +117,7 @@ proc procBlkPreamble(vmState: BaseVMState;
 
 proc procBlkEpilogue(vmState: BaseVMState;
                      header: BlockHeader; body: BlockBody): bool
-    {.gcsafe, raises: [RlpError].} =
+    {.gcsafe, raises: [RlpError, CatchableError].} =
   # Reward beneficiary
   vmState.mutateStateDB:
     if vmState.generateWitness:

nimbus/core/executor/process_transaction.nim

@@ -59,6 +59,29 @@ proc commitOrRollbackDependingOnGasUsed(vmState: BaseVMState, accTx: SavePoint,
     vmState.gasPool += tx.gasLimit - gasBurned
     return ok(gasBurned)
 
+# For stateless mode with on-demand fetching, we need to do
+# this, because it's possible for deletion to result in a
+# branch node with only one child, which then needs to be
+# transformed into an extension node or leaf node (or
+# grafted onto one), but we don't actually have that node
+# yet so we have to fetch it and then retry.
+proc repeatedlyTryToPersist(vmState: BaseVMState, fork: EVMFork): Future[void] {.async.} =
+  #info("repeatedlyTryToPersist about to get started")
+  var i = 0
+  while i < 100:
+    #info("repeatedlyTryToPersist making an attempt to persist", i)
+    try:
+      await vmState.stateDB.asyncPersist(
+        clearEmptyAccount = fork >= FkSpurious,
+        clearCache = false)
+      return
+    except MissingNodesError as e:
+      #warn("repeatedlyTryToPersist found missing paths", missingPaths=e.paths, missingNodeHashes=e.nodeHashes)
+      await fetchAndPopulateNodes(vmState, e.paths, e.nodeHashes)
+      i += 1
+  error("repeatedlyTryToPersist failed after 100 tries")
+  raise newException(CatchableError, "repeatedlyTryToPersist failed after 100 tries")
+
 proc asyncProcessTransactionImpl(
     vmState: BaseVMState; ## Parent accounts environment for transaction
     tx:      Transaction; ## Transaction to validate
@@ -112,9 +135,7 @@ proc asyncProcessTransactionImpl(
 
   if vmState.generateWitness:
     vmState.stateDB.collectWitnessData()
-  vmState.stateDB.persist(
-    clearEmptyAccount = fork >= FkSpurious,
-    clearCache = false)
+  await repeatedlyTryToPersist(vmState, fork)
 
   return res
 

nimbus/core/tx_pool/tx_tasks/tx_packer.nim

@@ -68,7 +68,7 @@ template safeExecutor(info: string; code: untyped) =
     raise newException(TxPackerError, info & "(): " & $e.name & " -- " & e.msg)
 
 proc persist(pst: TxPackerStateRef)
-    {.gcsafe,raises: [RlpError].} =
+    {.gcsafe,raises: [RlpError, CatchableError].} =
   ## Smart wrapper
   if not pst.cleanState:
     let fork = pst.xp.chain.nextFork

nimbus/db/accounts_cache.nim

@@ -1,8 +1,11 @@
 import
   std/[tables, hashes, sets],
-  eth/[common, rlp], eth/trie/[hexary, db, trie_defs],
+  chronos,
+  eth/[common, rlp], eth/trie/[hexary, db, trie_defs, nibbles],
+  ../utils/functors/possible_futures,
   ../constants, ../utils/utils, storage_types,
   ../../stateless/multi_keys,
+  ../evm/async/speculex,
   ./distinct_tries,
   ./access_list as ac_access_list
 
@@ -21,12 +24,14 @@ type
 
   AccountFlags = set[AccountFlag]
 
+  StorageCell* = SpeculativeExecutionCell[UInt256]
+
   RefAccount = ref object
     account: Account
     flags: AccountFlags
     code: seq[byte]
     originalStorage: TableRef[UInt256, UInt256]
-    overlayStorage: Table[UInt256, UInt256]
+    overlayStorage: Table[UInt256, StorageCell]
 
   WitnessData* = object
     storageKeys*: HashSet[UInt256]
@@ -261,11 +266,15 @@ proc originalStorageValue(acc: RefAccount, slot: UInt256, db: TrieDatabaseRef):
 
   acc.originalStorage[slot] = result
 
-proc storageValue(acc: RefAccount, slot: UInt256, db: TrieDatabaseRef): UInt256 =
-  acc.overlayStorage.withValue(slot, val) do:
-    return val[]
+proc storageCell(acc: RefAccount, slot: UInt256, db: TrieDatabaseRef): StorageCell =
+  acc.overlayStorage.withValue(slot, cell) do:
+    return cell[]
   do:
-    result = acc.originalStorageValue(slot, db)
+    return pureCell(acc.originalStorageValue(slot, db))
+
+# FIXME-removeSynchronousInterface: we use this down below, but I don't think that's okay anymore.
+proc storageValue(acc: RefAccount, slot: UInt256, db: TrieDatabaseRef): UInt256 =
+  waitForValueOf(storageCell(acc, slot, db))
 
 proc kill(acc: RefAccount) =
   acc.flags.excl Alive
@@ -296,7 +305,7 @@ proc persistCode(acc: RefAccount, db: TrieDatabaseRef) =
     else:
       db.put(contractHashKey(acc.account.codeHash).toOpenArray, acc.code)
 
-proc persistStorage(acc: RefAccount, db: TrieDatabaseRef, clearCache: bool) =
+proc asyncPersistStorage(acc: RefAccount, db: TrieDatabaseRef, clearCache: bool): Future[void] {.async.} =
   if acc.overlayStorage.len == 0:
     # TODO: remove the storage too if we figure out
     # how to create 'virtual' storage room for each account
@@ -307,9 +316,10 @@ proc persistStorage(acc: RefAccount, db: TrieDatabaseRef, clearCache: bool) =
 
   var storageTrie = getStorageTrie(db, acc)
 
-  for slot, value in acc.overlayStorage:
+  for slot, valueCell in acc.overlayStorage:
     let slotAsKey = createTrieKeyFromSlot slot
 
+    let value = await valueCell.toFuture
     if value > 0:
       let encodedValue = rlp.encode(value)
       storageTrie.putSlotBytes(slotAsKey, encodedValue)
@@ -327,7 +337,8 @@ proc persistStorage(acc: RefAccount, db: TrieDatabaseRef, clearCache: bool) =
   if not clearCache:
     # if we preserve cache, move the overlayStorage
     # to originalStorage, related to EIP2200, EIP1283
-    for slot, value in acc.overlayStorage:
+    for slot, valueCell in acc.overlayStorage:
+      let value = unsafeGetAlreadyAvailableValue(valueCell)
       if value > 0:
         acc.originalStorage[slot] = value
       else:
@@ -390,11 +401,15 @@ proc getCommittedStorage*(ac: AccountsCache, address: EthAddress, slot: UInt256)
     return
   acc.originalStorageValue(slot, ac.db)
 
-proc getStorage*(ac: AccountsCache, address: EthAddress, slot: UInt256): UInt256 {.inline.} =
+proc getStorageCell*(ac: AccountsCache, address: EthAddress, slot: UInt256): StorageCell =
   let acc = ac.getAccount(address, false)
   if acc.isNil:
-    return
-  acc.storageValue(slot, ac.db)
+    return pureCell(UInt256.zero)
+  return acc.storageCell(slot, ac.db)
+
+# FIXME-removeSynchronousInterface
+proc getStorage*(ac: AccountsCache, address: EthAddress, slot: UInt256): UInt256 =
+  waitForValueOf(getStorageCell(ac, address, slot))
 
 proc hasCodeOrNonce*(ac: AccountsCache, address: EthAddress): bool {.inline.} =
   let acc = ac.getAccount(address, false)
@@ -460,15 +475,21 @@ proc setCode*(ac: AccountsCache, address: EthAddress, code: seq[byte]) =
     acc.code = code
     acc.flags.incl CodeChanged
 
-proc setStorage*(ac: AccountsCache, address: EthAddress, slot, value: UInt256) =
+proc setStorageCell*(ac: AccountsCache, address: EthAddress, slot: UInt256, cell: StorageCell) =
   let acc = ac.getAccount(address)
   acc.flags.incl {Alive}
+  # FIXME-removeSynchronousInterface: ugh, this seems like a problem (that we need the values to be able to check whether they're equal)
   let oldValue = acc.storageValue(slot, ac.db)
+  let value = waitForValueOf(cell)
   if oldValue != value:
     var acc = ac.makeDirty(address)
-    acc.overlayStorage[slot] = value
+    acc.overlayStorage[slot] = cell
     acc.flags.incl StorageChanged
 
+# FIXME-removeSynchronousInterface
+proc setStorage*(ac: AccountsCache, address: EthAddress, slot: UInt256, value: UInt256) =
+  setStorageCell(ac, address, slot, pureCell(value))
+
 proc clearStorage*(ac: AccountsCache, address: EthAddress) =
   let acc = ac.getAccount(address)
   acc.flags.incl {Alive}
@@ -528,9 +549,52 @@ proc clearEmptyAccounts(ac: AccountsCache) =
     ac.deleteEmptyAccount(ripemdAddr)
     ac.ripemdSpecial = false
 
-proc persist*(ac: AccountsCache,
-              clearEmptyAccount: bool = false,
-              clearCache: bool = true) =
+
+type MissingNodesError* = ref object of Defect
+  paths*: seq[seq[seq[byte]]]
+  nodeHashes*: seq[Hash256]
+
+# FIXME-Adam: Move this elsewhere.
+# Also, I imagine there's a more efficient way to do this.
+proc padRight[V](s: seq[V], n: int, v: V): seq[V] =
+  for sv in s:
+    result.add(sv)
+  while result.len < n:
+    result.add(v)
+
+proc padRightWithZeroes(s: NibblesSeq, n: int): NibblesSeq =
+  initNibbleRange(padRight(s.getBytes, (n + 1) div 2, byte(0)))
+
+# FIXME-Adam: Why can I never find the conversion function I need?
+func toHash*(value: seq[byte]): Hash256 =
+  doAssert(value.len == 32)
+  var byteArray: array[32, byte]
+  for i, b in value:
+    byteArray[i] = b
+  result.data = byteArray
+
+func encodePath(path: NibblesSeq): seq[byte] =
+  if path.len == 64:
+    path.getBytes
+  else:
+    hexPrefixEncode(path)
+
+proc createMissingNodesErrorForAccount(missingAccountPath: NibblesSeq, nodeHash: Hash256): MissingNodesError =
+  MissingNodesError(
+    paths: @[@[encodePath(padRightWithZeroes(missingAccountPath, 64))]],
+    nodeHashes: @[nodeHash]
+  )
+
+proc createMissingNodesErrorForSlot(address: EthAddress, missingSlotPath: NibblesSeq, nodeHash: Hash256): MissingNodesError =
+  MissingNodesError(
+    paths: @[@[@(address.keccakHash.data), encodePath(padRightWithZeroes(missingSlotPath, 64))]],
+    nodeHashes: @[nodeHash]
+  )
+
+
+proc asyncPersist*(ac: AccountsCache,
+                   clearEmptyAccount: bool = false,
+                   clearCache: bool = true): Future[void] {.async.} =
   # make sure all savepoint already committed
   doAssert(ac.savePoint.parentSavepoint.isNil)
   var cleanAccounts = initHashSet[EthAddress]()
@@ -549,10 +613,19 @@ proc persist*(ac: AccountsCache,
       if StorageChanged in acc.flags:
         # storageRoot must be updated first
         # before persisting account into merkle trie
-        acc.persistStorage(ac.db, clearCache)
+        #
+        # Also, see the comment on repeatedlyTryToPersist in
+        # process_transaction.nim.
+        try:
+          await acc.asyncPersistStorage(ac.db, clearCache)
+        except MissingNodeError as e:
+          raise createMissingNodesErrorForSlot(address, e.path, toHash(e.nodeHashBytes))
       ac.trie.putAccountBytes address, rlp.encode(acc.account)
     of Remove:
-      ac.trie.delAccountBytes address
+      try:
+        ac.trie.delAccountBytes address
+      except MissingNodeError as e:
+        raise createMissingNodesErrorForAccount(e.path, toHash(e.nodeHashBytes))
       if not clearCache:
         cleanAccounts.incl address
     of DoNothing:
@@ -576,6 +649,12 @@ proc persist*(ac: AccountsCache,
 
   ac.isDirty = false
 
+# FIXME-removeSynchronousInterface
+proc persist*(ac: AccountsCache,
+              clearEmptyAccount: bool = false,
+              clearCache: bool = true) =
+  waitFor(asyncPersist(ac, clearEmptyAccount, clearCache))
+
 iterator addresses*(ac: AccountsCache): EthAddress =
   # make sure all savepoint already committed
   doAssert(ac.savePoint.parentSavepoint.isNil)
@@ -630,7 +709,8 @@ func update(wd: var WitnessData, acc: RefAccount) =
       if v.isZero: continue
       wd.storageKeys.incl k
 
-  for k, v in acc.overlayStorage:
+  for k, cell in acc.overlayStorage:
+    let v = unsafeGetAlreadyAvailableValue(cell)  # FIXME-Adam: should be resolved by now, I think? wait, maybe not?
     if v.isZero and k notin wd.storageKeys:
       continue
     if v.isZero and k in wd.storageKeys:

nimbus/db/distinct_tries.nim

@@ -27,10 +27,10 @@ proc isPruning*  (trie: AccountsTrie | StorageTrie): bool            = distinctB
 
 
 template initAccountsTrie*(db: DB, rootHash: KeccakHash, isPruning = true): AccountsTrie =
-  AccountsTrie(initSecureHexaryTrie(db, rootHash, isPruning))
+  AccountsTrie(initSecureHexaryTrie(db, rootHash, isPruning, true))
 
 template initAccountsTrie*(db: DB, isPruning = true): AccountsTrie =
-  AccountsTrie(initSecureHexaryTrie(db, isPruning))
+  AccountsTrie(initSecureHexaryTrie(db, isPruning, true))
 
 proc getAccountBytes*(trie: AccountsTrie, address: EthAddress): seq[byte] =
   SecureHexaryTrie(trie).get(address)
@@ -47,10 +47,10 @@ proc delAccountBytes*(trie: var AccountsTrie, address: EthAddress) =
 
 
 template initStorageTrie*(db: DB, rootHash: KeccakHash, isPruning = true): StorageTrie =
-  StorageTrie(initSecureHexaryTrie(db, rootHash, isPruning))
+  StorageTrie(initSecureHexaryTrie(db, rootHash, isPruning, true))
 
 template initStorageTrie*(db: DB, isPruning = true): StorageTrie =
-  StorageTrie(initSecureHexaryTrie(db, isPruning))
+  StorageTrie(initSecureHexaryTrie(db, isPruning, true))
 
 template createTrieKeyFromSlot*(slot: UInt256): auto =
   # XXX: This is too expensive. Similar to `createRangeFromAddress`

nimbus/db/incomplete_db.nim

@@ -30,13 +30,11 @@ proc ifNodesExistGetStorageBytesWithinAccount*(storageTrie: StorageTrie, slotAsK
 
 
 proc populateDbWithNodes*(db: TrieDatabaseRef, nodes: seq[seq[byte]]) =
-  error("GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG AARDVARK: populateDbWithNodes received nodes, about to populate", nodes)   # AARDVARK not an error, I just want it to stand out
   for nodeBytes in nodes:
     let nodeHash = keccakHash(nodeBytes)
-    info("AARDVARK: populateDbWithNodes about to add node", nodeHash, nodeBytes)
     db.put(nodeHash.data, nodeBytes)
 
-# AARDVARK: just make the callers call populateDbWithNodes directly?
+# FIXME-Adam: just make the callers call populateDbWithNodes directly?
 proc populateDbWithBranch*(db: TrieDatabaseRef, branch: seq[seq[byte]]) =
   for nodeBytes in branch:
     let nodeHash = keccakHash(nodeBytes)

nimbus/evm/async/data_sources.nim

@@ -4,6 +4,7 @@ import
   stint,
   eth/common,
   eth/trie/db,
+  ../../sync/protocol,
   ../../db/db_chain
 
 type
@@ -12,8 +13,7 @@ type
     ifNecessaryGetCode*:        proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.gcsafe.}
     ifNecessaryGetAccount*:     proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.gcsafe.}
     ifNecessaryGetBlockHeaderByNumber*: proc(chainDB: ChainDBRef, blockNumber: BlockNumber): Future[void] {.gcsafe.}
-    # FIXME-Adam: Later.
-    #fetchNodes*: proc(stateRoot: Hash256, paths: seq[seq[seq[byte]]], nodeHashes: seq[Hash256]): Future[seq[seq[byte]]] {.gcsafe.}
+    fetchNodes*: proc(stateRoot: Hash256, paths: seq[SnapTriePaths], nodeHashes: seq[Hash256]): Future[seq[seq[byte]]] {.gcsafe.}
     fetchBlockHeaderWithHash*: proc(h: Hash256): Future[BlockHeader] {.gcsafe.}
     fetchBlockHeaderWithNumber*: proc(n: BlockNumber): Future[BlockHeader] {.gcsafe.}
     fetchBlockHeaderAndBodyWithHash*: proc(h: Hash256): Future[(BlockHeader, BlockBody)] {.gcsafe.}