Snap sync moving root/healing implementation

packages/client/src/sync/fetcher/accountfetcher.ts

@@ -1,3 +1,4 @@
+import type { BlockHeader } from '@ethereumjs/block'
 import { DefaultStateManager } from '@ethereumjs/statemanager'
 import { Trie } from '@ethereumjs/trie'
 import {
@@ -100,6 +101,7 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
     this.accountTrie = this.stateManager['_getAccountTrie']()
 
     this.debug = createDebugLogger('client:AccountFetcher')
+    this.debug('dbg101')
 
     this.storageFetcher = new StorageFetcher({
       config: this.config,
@@ -585,6 +587,11 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
 
   updateStateRoot(stateRoot: Uint8Array) {
     this.root = stateRoot
+
+    this.storageFetcher.updateStateRoot(stateRoot)
+
+    // TODO trieNodeFetcher needs to be constantly healing while other fetchers work
+    this.trieNodeFetcher.updateStateRoot(stateRoot)
   }
 
   nextTasks(): void {

packages/client/src/sync/fetcher/storagefetcher.ts

@@ -617,6 +617,10 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
     return tasks
   }
 
+  updateStateRoot(stateRoot: Uint8Array) {
+    this.root = stateRoot
+  }
+
   nextTasks(): void {
     this.debug(
       `Entering nextTasks with primary queue length of ${this.storageRequests.length} and secondary queue length of ${this.fragmentedRequests.length}`

packages/client/src/sync/fetcher/trienodefetcher.ts

@@ -61,6 +61,7 @@ type FetchedNodeData = {
 }
 
 type NodeRequestData = {
+  requested: boolean
   nodeHash: string
   nodeParentHash: string
   parentAccountHash?: string // for leaf account nodes that contain a storage component
@@ -117,6 +118,7 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
 
     // will always start with root node as first set of node requests
     this.pathToNodeRequestData.setElement('', {
+      requested: false,
       nodeHash: bytesToHex(this.root),
       nodeParentHash: '', // root node does not have a parent
     } as NodeRequestData)
@@ -263,6 +265,7 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
                 storagePath,
               ].join('/')
               this.pathToNodeRequestData.setElement(syncPath, {
+                requested: false,
                 nodeHash: bytesToHex(storageRoot),
                 nodeParentHash: nodeHash,
                 parentAccountHash: nodeHash,
@@ -299,6 +302,7 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
               pathString
             ) as NodeRequestData
             this.pathToNodeRequestData.setElement(childNode.path, {
+              requested: false,
               nodeHash: bytesToHex(childNode.nodeHash as Uint8Array),
               nodeParentHash: nodeHash, // TODO root node does not have a parent, so handle that in the leaf callback when checking if dependencies are met recursively
               parentAccountHash,
@@ -411,17 +415,25 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
       if (this.pathToNodeRequestData.size() > 0) {
         let { pathStrings } = this.getSortedPathStrings() // TODO pass in number of paths to return
         while (tasks.length < maxTasks && pathStrings.length > 0) {
-          const requestedPathStrings = pathStrings.slice(0, max)
+          const pendingPathStrings = pathStrings.slice(0, max)
           pathStrings = pathStrings.slice(max + 1)
-          for (const pathString of requestedPathStrings) {
-            const nodeHash = this.pathToNodeRequestData.getElementByKey(pathString)?.nodeHash // TODO return node set too from sorted function and avoid lookups here
-            if (nodeHash === undefined) throw Error('Path should exist')
-            this.requestedNodeToPath.set(nodeHash as unknown as string, pathString)
+          const neededPathStrings = []
+          for (const pathString of pendingPathStrings) {
+            const requestData = this.pathToNodeRequestData.getElementByKey(pathString) // TODO return node set too from sorted function and avoid lookups here
+            if (requestData === undefined) throw Error('Path should exist')
+            if (requestData.requested === false) {
+              this.requestedNodeToPath.set(requestData.nodeHash as unknown as string, pathString)
+              this.pathToNodeRequestData.setElement(pathString, {
+                ...requestData,
+                requested: true,
+              })
+              neededPathStrings.push(pathString)
+            }
           }
           this.debug('At start of mergeAndFormatPaths')
-          const paths = mergeAndFormatKeyPaths(requestedPathStrings) as unknown as Uint8Array[][]
+          const paths = mergeAndFormatKeyPaths(neededPathStrings) as unknown as Uint8Array[][]
           tasks.push({
-            pathStrings: requestedPathStrings,
+            pathStrings: neededPathStrings,
             paths,
           })
           this.debug(`Created new tasks num=${tasks.length}`)
@@ -435,6 +447,10 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
     return tasks
   }
 
+  updateStateRoot(stateRoot: Uint8Array) {
+    this.root = stateRoot
+  }
+
   nextTasks(): void {
     try {
       if (this.in.length === 0) {

packages/client/test/sync/fetcher/trienodefetcher.spec.ts

@@ -214,6 +214,7 @@ describe('[TrieNodeFetcher]', async () => {
 
     fetcher.pathToNodeRequestData = new OrderedMap<string, any>()
     fetcher.pathToNodeRequestData.setElement('', {
+      requested: false,
       nodeHash: bytesToHex(new Uint8Array(0)),
       nodeParentHash: '',
     })