feat: add historical state regen

packages/api/src/beacon/routes/debug.ts

@@ -92,7 +92,7 @@ export type Api = {
     format?: ResponseFormat
   ): Promise<
     ApiClientResponse<{
-      [HttpStatusCode.OK]: Uint8Array | {data: allForks.BeaconState; executionOptimistic: ExecutionOptimistic};
+      [HttpStatusCode.OK]: Uint8Array | {data: allForks.BeaconState};
     }>
   >;
 

packages/api/test/unit/beacon/testData/debug.ts

@@ -46,7 +46,7 @@ export const testData: GenericServerTestCases<Api> = {
   },
   getState: {
     args: ["head", "json"],
-    res: {executionOptimistic: true, data: ssz.phase0.BeaconState.defaultValue()},
+    res: {data: ssz.phase0.BeaconState.defaultValue()},
   },
   getStateV2: {
     args: ["head", "json"],

packages/beacon-node/src/api/impl/beacon/state/index.ts

@@ -14,7 +14,7 @@ import {
   filterStateValidatorsByStatus,
   getStateValidatorIndex,
   getValidatorStatus,
-  resolveStateId,
+  getStateResponse,
   toValidatorResponse,
 } from "./utils.js";
 
@@ -25,7 +25,7 @@ export function getBeaconStateApi({
   async function getState(
     stateId: routes.beacon.StateId
   ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean}> {
-    return resolveStateId(chain, stateId);
+    return getStateResponse(chain, stateId);
   }
 
   return {
@@ -77,7 +77,7 @@ export function getBeaconStateApi({
     },
 
     async getStateValidators(stateId, filters) {
-      const {state, executionOptimistic} = await resolveStateId(chain, stateId);
+      const {state, executionOptimistic} = await getStateResponse(chain, stateId);
       const currentEpoch = getCurrentEpoch(state);
       const {validators, balances} = state; // Get the validators sub tree once for all the loop
       const {pubkey2index} = chain.getHeadState().epochCtx;
@@ -128,7 +128,7 @@ export function getBeaconStateApi({
     },
 
     async getStateValidator(stateId, validatorId) {
-      const {state, executionOptimistic} = await resolveStateId(chain, stateId);
+      const {state, executionOptimistic} = await getStateResponse(chain, stateId);
       const {pubkey2index} = chain.getHeadState().epochCtx;
 
       const resp = getStateValidatorIndex(validatorId, state, pubkey2index);
@@ -149,7 +149,7 @@ export function getBeaconStateApi({
     },
 
     async getStateValidatorBalances(stateId, indices) {
-      const {state, executionOptimistic} = await resolveStateId(chain, stateId);
+      const {state, executionOptimistic} = await getStateResponse(chain, stateId);
 
       if (indices) {
         const headState = chain.getHeadState();
@@ -186,7 +186,7 @@ export function getBeaconStateApi({
     },
 
     async getEpochCommittees(stateId, filters) {
-      const {state, executionOptimistic} = await resolveStateId(chain, stateId);
+      const {state, executionOptimistic} = await getStateResponse(chain, stateId);
 
       const stateCached = state as CachedBeaconStateAltair;
       if (stateCached.epochCtx === undefined) {
@@ -228,7 +228,7 @@ export function getBeaconStateApi({
      */
     async getEpochSyncCommittees(stateId, epoch) {
       // TODO: Should pick a state with the provided epoch too
-      const {state, executionOptimistic} = await resolveStateId(chain, stateId);
+      const {state, executionOptimistic} = await getStateResponse(chain, stateId);
 
       // TODO: If possible compute the syncCommittees in advance of the fork and expose them here.
       // So the validators can prepare and potentially attest the first block. Not critical tho, it's very unlikely

packages/beacon-node/src/api/impl/beacon/state/utils.ts

@@ -1,56 +1,41 @@
 import {fromHexString} from "@chainsafe/ssz";
+import {ChainForkConfig} from "@lodestar/config";
 import {routes} from "@lodestar/api";
 import {FAR_FUTURE_EPOCH, GENESIS_SLOT} from "@lodestar/params";
 import {BeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition";
-import {BLSPubkey, phase0} from "@lodestar/types";
+import {BLSPubkey, allForks, phase0} from "@lodestar/types";
 import {Epoch, ValidatorIndex} from "@lodestar/types";
-import {IBeaconChain, StateGetOpts} from "../../../../chain/index.js";
+import {IBeaconChain} from "../../../../chain/index.js";
 import {ApiError, ValidationError} from "../../errors.js";
-import {isOptimisticBlock} from "../../../../util/forkChoice.js";
+import {getSlotFromBeaconStateSerialized} from "../../../../util/sszBytes.js";
 
-export async function resolveStateId(
-  chain: IBeaconChain,
-  stateId: routes.beacon.StateId,
-  opts?: StateGetOpts
-): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean}> {
-  const stateRes = await resolveStateIdOrNull(chain, stateId, opts);
-  if (!stateRes) {
-    throw new ApiError(404, `No state found for id '${stateId}'`);
-  }
-
-  return stateRes;
+export function deserializeBeaconStateSerialized(config: ChainForkConfig, data: Uint8Array): allForks.BeaconState {
+  const slot = getSlotFromBeaconStateSerialized(data);
+  return config.getForkTypes(slot).BeaconState.deserialize(data);
 }
 
-async function resolveStateIdOrNull(
-  chain: IBeaconChain,
-  stateId: routes.beacon.StateId,
-  opts?: StateGetOpts
-): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean} | null> {
+export function resolveStateId(chain: IBeaconChain, stateId: routes.beacon.StateId): string | number {
   if (stateId === "head") {
-    // TODO: This is not OK, head and headState must be fetched atomically
     const head = chain.forkChoice.getHead();
-    const headState = chain.getHeadState();
-    return {state: headState, executionOptimistic: isOptimisticBlock(head)};
+    return head.stateRoot;
   }
 
   if (stateId === "genesis") {
-    return chain.getStateBySlot(GENESIS_SLOT, opts);
+    return GENESIS_SLOT;
   }
 
   if (stateId === "finalized") {
     const block = chain.forkChoice.getFinalizedBlock();
-    const state = await chain.getStateByStateRoot(block.stateRoot, opts);
-    return state && {state: state.state, executionOptimistic: isOptimisticBlock(block)};
+    return block.stateRoot;
   }
 
   if (stateId === "justified") {
     const block = chain.forkChoice.getJustifiedBlock();
-    const state = await chain.getStateByStateRoot(block.stateRoot, opts);
-    return state && {state: state.state, executionOptimistic: isOptimisticBlock(block)};
+    return block.stateRoot;
   }
 
   if (typeof stateId === "string" && stateId.startsWith("0x")) {
-    return chain.getStateByStateRoot(stateId, opts);
+    return stateId as string;
   }
 
   // id must be slot
@@ -59,7 +44,49 @@ async function resolveStateIdOrNull(
     throw new ValidationError(`Invalid block id '${stateId}'`, "blockId");
   }
 
-  return chain.getStateBySlot(blockSlot, opts);
+  return blockSlot;
+}
+
+export async function getStateResponse(
+  chain: IBeaconChain,
+  stateId: routes.beacon.StateId
+): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean}> {
+  const rootOrSlot = resolveStateId(chain, stateId);
+
+  let state: {state: BeaconStateAllForks; executionOptimistic: boolean} | null = null;
+  if (typeof rootOrSlot === "string") {
+    state = await chain.getStateByStateRoot(rootOrSlot);
+  } else if (typeof rootOrSlot === "number") {
+    state = await chain.getStateBySlot(rootOrSlot);
+  }
+
+  if (state == null) {
+    throw new ApiError(404, `No state found for id '${stateId}'`);
+  }
+  return state;
+}
+
+export async function getStateResponseWithRegen(
+  chain: IBeaconChain,
+  stateId: routes.beacon.StateId
+): Promise<{state: BeaconStateAllForks | Uint8Array; executionOptimistic: boolean}> {
+  const rootOrSlot = resolveStateId(chain, stateId);
+
+  let state: {state: BeaconStateAllForks | Uint8Array; executionOptimistic: boolean} | null = null;
+  if (typeof rootOrSlot === "string") {
+    state = await chain.getStateByStateRoot(rootOrSlot, {allowRegen: true});
+  } else if (typeof rootOrSlot === "number") {
+    if (rootOrSlot >= chain.forkChoice.getFinalizedBlock().slot) {
+      state = await chain.getStateBySlot(rootOrSlot, {allowRegen: true});
+    } else {
+      state = await chain.getHistoricalStateBySlot(rootOrSlot);
+    }
+  }
+
+  if (state == null) {
+    throw new ApiError(404, `No state found for id '${stateId}'`);
+  }
+  return state;
 }
 
 /**

packages/beacon-node/src/api/impl/debug/index.ts

@@ -1,5 +1,5 @@
 import {routes, ServerApi, ResponseFormat} from "@lodestar/api";
-import {resolveStateId} from "../beacon/state/utils.js";
+import {deserializeBeaconStateSerialized, getStateResponseWithRegen} from "../beacon/state/utils.js";
 import {ApiModules} from "../types.js";
 import {isOptimisticBlock} from "../../../util/forkChoice.js";
 
@@ -37,24 +37,41 @@ export function getDebugApi({chain, config}: Pick<ApiModules, "chain" | "config"
     },
 
     async getState(stateId: string | number, format?: ResponseFormat) {
-      const {state} = await resolveStateId(chain, stateId, {allowRegen: true});
+      const {state} = await getStateResponseWithRegen(chain, stateId);
       if (format === "ssz") {
-        // Casting to any otherwise Typescript doesn't like the multi-type return
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-explicit-any
-        return state.serialize() as any;
+        if (state instanceof Uint8Array) {
+          return state;
+        }
+        return state.serialize();
       } else {
+        if (state instanceof Uint8Array) {
+          return {data: deserializeBeaconStateSerialized(config, state)};
+        }
         return {data: state.toValue()};
       }
     },
 
     async getStateV2(stateId: string | number, format?: ResponseFormat) {
-      const {state} = await resolveStateId(chain, stateId, {allowRegen: true});
+      const {state, executionOptimistic} = await getStateResponseWithRegen(chain, stateId);
       if (format === "ssz") {
-        // Casting to any otherwise Typescript doesn't like the multi-type return
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-explicit-any
-        return state.serialize() as any;
+        if (state instanceof Uint8Array) {
+          return state;
+        }
+        return state.serialize();
       } else {
-        return {data: state.toValue(), version: config.getForkName(state.slot)};
+        if (state instanceof Uint8Array) {
+          const data = deserializeBeaconStateSerialized(config, state);
+          return {
+            data,
+            version: config.getForkName(data.slot),
+            executionOptimistic,
+          };
+        }
+        return {
+          data: state.toValue(),
+          version: config.getForkName(state.slot),
+          executionOptimistic,
+        };
       }
     },
   };

packages/beacon-node/src/api/impl/proof/index.ts

@@ -1,7 +1,7 @@
 import {createProof, ProofType} from "@chainsafe/persistent-merkle-tree";
 import {routes, ServerApi} from "@lodestar/api";
 import {ApiModules} from "../types.js";
-import {resolveStateId} from "../beacon/state/utils.js";
+import {getStateResponse} from "../beacon/state/utils.js";
 import {resolveBlockId} from "../beacon/blocks/utils.js";
 import {ApiOptions} from "../../options.js";
 
@@ -20,7 +20,7 @@ export function getProofApi(
         throw new Error("Requested proof is too large.");
       }
 
-      const {state} = await resolveStateId(chain, stateId);
+      const {state} = await getStateResponse(chain, stateId);
 
       // Commit any changes before computing the state root. In normal cases the state should have no changes here
       state.commit();

packages/beacon-node/src/chain/chain.ts

@@ -76,6 +76,7 @@ import {BlockAttributes, produceBlockBody, produceCommonBlockBody} from "./produ
 import {computeNewStateRoot} from "./produceBlock/computeNewStateRoot.js";
 import {BlockInput} from "./blocks/types.js";
 import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js";
+import {HistoricalStateRegen} from "./historicalState/index.js";
 import {ShufflingCache} from "./shufflingCache.js";
 import {StateContextCache} from "./stateCache/stateContextCache.js";
 import {SeenGossipBlockInput} from "./seenCache/index.js";
@@ -108,6 +109,7 @@ export class BeaconChain implements IBeaconChain {
   readonly regen: QueuedStateRegenerator;
   readonly lightClientServer: LightClientServer;
   readonly reprocessController: ReprocessController;
+  readonly historicalStateRegen?: HistoricalStateRegen;
 
   // Ops pool
   readonly attestationPool: AttestationPool;
@@ -165,6 +167,7 @@ export class BeaconChain implements IBeaconChain {
       eth1,
       executionEngine,
       executionBuilder,
+      historicalStateRegen,
     }: {
       config: BeaconConfig;
       db: IBeaconDb;
@@ -177,6 +180,7 @@ export class BeaconChain implements IBeaconChain {
       eth1: IEth1ForBlockProduction;
       executionEngine: IExecutionEngine;
       executionBuilder?: IExecutionBuilder;
+      historicalStateRegen?: HistoricalStateRegen;
     }
   ) {
     this.opts = opts;
@@ -191,6 +195,7 @@ export class BeaconChain implements IBeaconChain {
     this.eth1 = eth1;
     this.executionEngine = executionEngine;
     this.executionBuilder = executionBuilder;
+    this.historicalStateRegen = historicalStateRegen;
     const signal = this.abortController.signal;
     const emitter = new ChainEventEmitter();
     // by default, verify signatures on both main threads and worker threads
@@ -401,12 +406,24 @@ export class BeaconChain implements IBeaconChain {
         return state && {state, executionOptimistic: isOptimisticBlock(block)};
       }
     } else {
-      // request for finalized state
+      return null;
+    }
+  }
+
+  async getHistoricalStateBySlot(slot: number): Promise<{state: Uint8Array; executionOptimistic: boolean} | null> {
+    const finalizedBlock = this.forkChoice.getFinalizedBlock();
 
-      // do not attempt regen, just check if state is already in DB
-      const state = await this.db.stateArchive.get(slot);
-      return state && {state, executionOptimistic: false};
+    if (slot >= finalizedBlock.slot) {
+      return null;
+    }
+
+    // request for finalized state using historical state regen
+    const stateSerialized = await this.historicalStateRegen?.getHistoricalState(slot);
+    if (!stateSerialized) {
+      return null;
     }
+
+    return {state: stateSerialized, executionOptimistic: false};
   }
 
   async getStateByStateRoot(