feat: generalize blobs to data for extension ready for ils and/or data columns

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -4,7 +4,14 @@ import {computeEpochAtSlot, computeTimeAtSlot, reconstructFullBlockOrContents} f
 import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {sleep, toHex} from "@lodestar/utils";
 import {allForks, deneb, isSignedBlockContents, ProducedBlockSource} from "@lodestar/types";
-import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput, BlobsSource} from "../../../../chain/blocks/types.js";
+import {
+  BlockSource,
+  getBlockInput,
+  ImportBlockOpts,
+  BlockInput,
+  BlobsSource,
+  BlockInputDataBlobs,
+} from "../../../../chain/blocks/types.js";
 import {promiseAllMaybeAsync} from "../../../../util/promises.js";
 import {isOptimisticBlock} from "../../../../util/forkChoice.js";
 import {computeBlobSidecars} from "../../../../util/blobs.js";
@@ -47,21 +54,25 @@ export function getBeaconBlockApi({
     if (isSignedBlockContents(signedBlockOrContents)) {
       ({signedBlock} = signedBlockOrContents);
       blobSidecars = computeBlobSidecars(config, signedBlock, signedBlockOrContents);
-      blockForImport = getBlockInput.postDeneb(
+      const blockData = {
+        fork: config.getForkName(signedBlock.message.slot),
+        blobs: blobSidecars,
+        blobsSource: BlobsSource.api,
+        blobsBytes: blobSidecars.map(() => null),
+      } as BlockInputDataBlobs;
+      blockForImport = getBlockInput.availableData(
         config,
         signedBlock,
         BlockSource.api,
-        blobSidecars,
-        BlobsSource.api,
         // don't bundle any bytes for block and blobs
         null,
-        blobSidecars.map(() => null)
+        blockData
       );
     } else {
       signedBlock = signedBlockOrContents;
       blobSidecars = [];
       // TODO: Once API supports submitting data as SSZ, replace null with blockBytes
-      blockForImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.api, null);
+      blockForImport = getBlockInput.preData(config, signedBlock, BlockSource.api, null);
     }
 
     // check what validations have been requested before broadcasting and publishing the block

packages/beacon-node/src/api/impl/validator/index.ts

@@ -34,7 +34,7 @@ import {
   isBlockContents,
   phase0,
 } from "@lodestar/types";
-import {ExecutionStatus} from "@lodestar/fork-choice";
+import {ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
 import {toHex, resolveOrRacePromises, prettyWeiToEth} from "@lodestar/utils";
 import {
   AttestationError,
@@ -315,6 +315,16 @@ export function getValidatorApi({
       );
   }
 
+  function notOnOutOfRangeData(beaconBlockRoot: Root): void {
+    const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot);
+    if (!protoBeaconBlock) {
+      throw new ApiError(400, "Block not in forkChoice");
+    }
+
+    if (protoBeaconBlock.dataAvailabilityStatus === DataAvailabilityStatus.OutOfRange)
+      throw new NodeIsSyncing("Block's data is out of range and not validated");
+  }
+
   const produceBuilderBlindedBlock = async function produceBuilderBlindedBlock(
     slot: Slot,
     randaoReveal: BLSSignature,
@@ -367,6 +377,7 @@ export function getValidatorApi({
     } else {
       parentBlockRoot = inParentBlockRoot;
     }
+    notOnOutOfRangeData(parentBlockRoot);
 
     let timer;
     try {
@@ -434,6 +445,7 @@ export function getValidatorApi({
     } else {
       parentBlockRoot = inParentBlockRoot;
     }
+    notOnOutOfRangeData(parentBlockRoot);
 
     let timer;
     try {
@@ -509,6 +521,7 @@ export function getValidatorApi({
       // handler, in which case this should just return.
       chain.forkChoice.updateTime(slot);
       const parentBlockRoot = fromHexString(chain.recomputeForkChoiceHead().blockRoot);
+      notOnOutOfRangeData(parentBlockRoot);
 
       const fork = config.getForkName(slot);
       // set some sensible opts
@@ -838,11 +851,15 @@ export function getValidatorApi({
       // Check the execution status as validator shouldn't vote on an optimistic head
       // Check on target is sufficient as a valid target would imply a valid source
       notOnOptimisticBlockRoot(targetRoot);
+      notOnOutOfRangeData(targetRoot);
 
       // To get the correct source we must get a state in the same epoch as the attestation's epoch.
       // An epoch transition may change state.currentJustifiedCheckpoint
       const attEpochState = await chain.getHeadStateAtEpoch(attEpoch, RegenCaller.produceAttestationData);
 
+      // TODO confirm if the below is correct assertion
+      // notOnOutOfRangeData(attEpochState.currentJustifiedCheckpoint.root);
+
       return {
         data: {
           slot,
@@ -878,6 +895,7 @@ export function getValidatorApi({
 
       // Check the execution status as validator shouldn't contribute on an optimistic head
       notOnOptimisticBlockRoot(beaconBlockRoot);
+      notOnOutOfRangeData(beaconBlockRoot);
 
       const contribution = chain.syncCommitteeMessagePool.getContribution(subcommitteeIndex, slot, beaconBlockRoot);
       if (!contribution) throw new ApiError(500, "No contribution available");

packages/beacon-node/src/chain/blocks/importBlock.ts

@@ -58,7 +58,7 @@ export async function importBlock(
   fullyVerifiedBlock: FullyVerifiedBlock,
   opts: ImportBlockOpts
 ): Promise<void> {
-  const {blockInput, postState, parentBlockSlot, executionStatus} = fullyVerifiedBlock;
+  const {blockInput, postState, parentBlockSlot, executionStatus, dataAvailabilityStatus} = fullyVerifiedBlock;
   const {block, source} = blockInput;
   const {slot: blockSlot} = block.message;
   const blockRoot = this.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message);
@@ -70,8 +70,8 @@ export async function importBlock(
   const blockDelaySec = (fullyVerifiedBlock.seenTimestampSec - postState.genesisTime) % this.config.SECONDS_PER_SLOT;
   const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
 
-  // this is just a type assertion since blockinput with blobsPromise type will not end up here
-  if (blockInput.type === BlockInputType.blobsPromise) {
+  // this is just a type assertion since blockinput with dataPromise type will not end up here
+  if (blockInput.type === BlockInputType.dataPromise) {
     throw Error("Unavailable block can not be imported in forkchoice");
   }
 
@@ -90,7 +90,8 @@ export async function importBlock(
     postState,
     blockDelaySec,
     this.clock.currentSlot,
-    executionStatus
+    executionStatus,
+    dataAvailabilityStatus
   );
 
   // This adds the state necessary to process the next block
@@ -108,11 +109,11 @@ export async function importBlock(
       executionOptimistic: blockSummary != null && isOptimisticBlock(blockSummary),
     });
 
-    // blobsPromise will not end up here, but preDeneb could. In future we might also allow syncing
+    // dataPromise will not end up here, but preDeneb could. In future we might also allow syncing
     // out of data range blocks and import then in forkchoice although one would not be able to
     // attest and propose with such head similar to optimistic sync
-    if (blockInput.type === BlockInputType.postDeneb) {
-      const {blobsSource, blobs} = blockInput;
+    if (blockInput.type === BlockInputType.availableData) {
+      const {blobsSource, blobs} = blockInput.blockData;
 
       this.metrics?.importBlock.blobsBySource.inc({blobsSource});
       for (const blobSidecar of blobs) {

packages/beacon-node/src/chain/blocks/index.ts

@@ -87,9 +87,8 @@ export async function processBlocks(
         postState: postStates[i],
         parentBlockSlot: parentSlots[i],
         executionStatus: executionStatuses[i],
-        // Currently dataAvailableStatus is not used upstream but that can change if we
         // start supporting optimistic syncing/processing
-        dataAvailableStatus: dataAvailabilityStatuses[i],
+        dataAvailabilityStatus: dataAvailabilityStatuses[i],
         proposerBalanceDelta: proposerBalanceDeltas[i],
         // TODO: Make this param mandatory and capture in gossip
         seenTimestampSec: opts.seenTimestampSec ?? Math.floor(Date.now() / 1000),

packages/beacon-node/src/chain/blocks/types.ts

@@ -1,13 +1,15 @@
-import {CachedBeaconStateAllForks, computeEpochAtSlot, DataAvailableStatus} from "@lodestar/state-transition";
-import {MaybeValidExecutionStatus} from "@lodestar/fork-choice";
+import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition";
+import {MaybeValidExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
 import {allForks, deneb, Slot, RootHex} from "@lodestar/types";
-import {ForkSeq} from "@lodestar/params";
+import {ForkSeq, ForkName} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
 
 export enum BlockInputType {
-  preDeneb = "preDeneb",
-  postDeneb = "postDeneb",
-  blobsPromise = "blobsPromise",
+  preData = "preData",
+  // data is out of available window, can be used to sync forward and keep adding to forkchoice
+  outOfRangeData = "outOfRangeData",
+  availableData = "availableData",
+  dataPromise = "dataPromise",
 }
 
 /** Enum to represent where blocks come from */
@@ -32,20 +34,30 @@ export enum GossipedInputType {
 }
 
 export type BlobsCache = Map<number, {blobSidecar: deneb.BlobSidecar; blobBytes: Uint8Array | null}>;
+
+type ForkBlobsInfo = {fork: ForkName.deneb};
+type BlobsData = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]; blobsSource: BlobsSource};
+export type BlockInputDataBlobs = ForkBlobsInfo & BlobsData;
+export type BlockInputData = BlockInputDataBlobs;
+
+type BlobsInputCache = {blobsCache: BlobsCache};
+export type BlockInputCacheBlobs = ForkBlobsInfo & BlobsInputCache;
+
 export type BlockInputBlobs = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]; blobsSource: BlobsSource};
-type CachedBlobs = {
-  blobsCache: BlobsCache;
-  availabilityPromise: Promise<BlockInputBlobs>;
-  resolveAvailability: (blobs: BlockInputBlobs) => void;
-};
+type Availability<T> = {availabilityPromise: Promise<T>; resolveAvailability: (data: T) => void};
+
+type CachedBlobs = BlobsInputCache & Availability<BlockInputDataBlobs>;
+export type CachedData = ForkBlobsInfo & CachedBlobs;
 
 export type BlockInput = {block: allForks.SignedBeaconBlock; source: BlockSource; blockBytes: Uint8Array | null} & (
-  | {type: BlockInputType.preDeneb}
-  | ({type: BlockInputType.postDeneb} & BlockInputBlobs)
+  | {type: BlockInputType.preData | BlockInputType.outOfRangeData}
+  | ({type: BlockInputType.availableData} & {blockData: BlockInputData})
   // the blobsSource here is added to BlockInputBlobs when availability is resolved
-  | ({type: BlockInputType.blobsPromise} & CachedBlobs)
+  | ({type: BlockInputType.dataPromise} & {cachedData: CachedData})
 );
-export type NullBlockInput = {block: null; blockRootHex: RootHex; blockInputPromise: Promise<BlockInput>} & CachedBlobs;
+export type NullBlockInput = {block: null; blockRootHex: RootHex; blockInputPromise: Promise<BlockInput>} & {
+  cachedData: CachedData;
+};
 
 export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean {
   return (
@@ -56,7 +68,7 @@ export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clo
 }
 
 export const getBlockInput = {
-  preDeneb(
+  preData(
     config: ChainForkConfig,
     block: allForks.SignedBeaconBlock,
     source: BlockSource,
@@ -66,61 +78,70 @@ export const getBlockInput = {
       throw Error(`Post Deneb block slot ${block.message.slot}`);
     }
     return {
-      type: BlockInputType.preDeneb,
+      type: BlockInputType.preData,
+      block,
+      source,
+      blockBytes,
+    };
+  },
+
+  outOfRangeData(
+    config: ChainForkConfig,
+    block: allForks.SignedBeaconBlock,
+    source: BlockSource,
+    blockBytes: Uint8Array | null
+  ): BlockInput {
+    if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
+      throw Error(`Pre Deneb block slot ${block.message.slot}`);
+    }
+    return {
+      type: BlockInputType.outOfRangeData,
       block,
       source,
       blockBytes,
     };
   },
 
-  postDeneb(
+  availableData(
     config: ChainForkConfig,
     block: allForks.SignedBeaconBlock,
     source: BlockSource,
-    blobs: deneb.BlobSidecars,
-    blobsSource: BlobsSource,
     blockBytes: Uint8Array | null,
-    blobsBytes: (Uint8Array | null)[]
+    blockData: BlockInputData
   ): BlockInput {
     if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
       throw Error(`Pre Deneb block slot ${block.message.slot}`);
     }
     return {
-      type: BlockInputType.postDeneb,
+      type: BlockInputType.availableData,
       block,
       source,
-      blobs,
-      blobsSource,
       blockBytes,
-      blobsBytes,
+      blockData,
     };
   },
 
-  blobsPromise(
+  dataPromise(
     config: ChainForkConfig,
     block: allForks.SignedBeaconBlock,
     source: BlockSource,
-    blobsCache: BlobsCache,
     blockBytes: Uint8Array | null,
-    availabilityPromise: Promise<BlockInputBlobs>,
-    resolveAvailability: (blobs: BlockInputBlobs) => void
+    cachedData: CachedData
   ): BlockInput {
     if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
       throw Error(`Pre Deneb block slot ${block.message.slot}`);
     }
     return {
-      type: BlockInputType.blobsPromise,
+      type: BlockInputType.dataPromise,
       block,
       source,
-      blobsCache,
       blockBytes,
-      availabilityPromise,
-      resolveAvailability,
+      cachedData,
     };
   },
 };
 
-export function getBlockInputBlobs(blobsCache: BlobsCache): Omit<BlockInputBlobs, "blobsSource"> {
+export function getBlockInputBlobs(blobsCache: BlobsCache): Omit<BlobsData, "blobsSource"> {
   const blobs = [];
   const blobsBytes = [];
 
@@ -206,7 +227,7 @@ export type FullyVerifiedBlock = {
    * used in optimistic sync or for merge block
    */
   executionStatus: MaybeValidExecutionStatus;
-  dataAvailableStatus: DataAvailableStatus;
+  dataAvailabilityStatus: DataAvailabilityStatus;
   /** Seen timestamp seconds */
   seenTimestampSec: number;
 };

packages/beacon-node/src/chain/blocks/verifyBlock.ts

@@ -7,7 +7,7 @@ import {
 } from "@lodestar/state-transition";
 import {bellatrix, deneb} from "@lodestar/types";
 import {ForkName} from "@lodestar/params";
-import {ProtoBlock, ExecutionStatus} from "@lodestar/fork-choice";
+import {ProtoBlock, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
 import {ChainForkConfig} from "@lodestar/config";
 import {Logger} from "@lodestar/utils";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
@@ -44,7 +44,7 @@ export async function verifyBlocksInEpoch(
   postStates: CachedBeaconStateAllForks[];
   proposerBalanceDeltas: number[];
   segmentExecStatus: SegmentExecStatus;
-  dataAvailabilityStatuses: DataAvailableStatus[];
+  dataAvailabilityStatuses: DataAvailabilityStatus[];
   availableBlockInputs: BlockInput[];
 }> {
   const blocks = blocksInput.map(({block}) => block);
@@ -169,7 +169,7 @@ export async function verifyBlocksInEpoch(
         blocksInput.length === 1 &&
         // gossip blocks have seenTimestampSec
         opts.seenTimestampSec !== undefined &&
-        blocksInput[0].type !== BlockInputType.preDeneb &&
+        blocksInput[0].type !== BlockInputType.preData &&
         executionStatuses[0] === ExecutionStatus.Valid
       ) {
         // Find the max time when the block was actually verified