From 7be14003912a593bffc394cbe08310602d3b79c6 Mon Sep 17 00:00:00 2001 From: raychu86 <14917648+raychu86@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:15:55 -0700 Subject: [PATCH 1/3] Parallelize deserialization of prev certificate ids --- Cargo.lock | 1 + ledger/narwhal/batch-header/Cargo.toml | 6 +++++- ledger/narwhal/batch-header/src/bytes.rs | 13 ++++++++----- ledger/narwhal/batch-header/src/lib.rs | 6 +++++- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f91e9879a7..e95411f9e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3196,6 +3196,7 @@ version = "0.16.19" dependencies = [ "bincode", "indexmap 2.1.0", + "rayon", "serde_json", "snarkvm-console", "snarkvm-ledger-narwhal-batch-header", diff --git a/ledger/narwhal/batch-header/Cargo.toml b/ledger/narwhal/batch-header/Cargo.toml index 8ec3fd22cd..56037b9b45 100644 --- a/ledger/narwhal/batch-header/Cargo.toml +++ b/ledger/narwhal/batch-header/Cargo.toml @@ -24,7 +24,7 @@ license = "Apache-2.0" edition = "2021" [features] -default = [ ] +default = [ "rayon" ] serial = [ "console/serial" ] wasm = [ "console/wasm" ] test-helpers = [ "narwhal-transmission-id/test-helpers", "time" ] @@ -43,6 +43,10 @@ version = "=0.16.19" version = "2.0" features = [ "serde" ] +[dependencies.rayon] +version = "1" +optional = true + [dependencies.serde_json] version = "1.0" features = [ "preserve_order" ] diff --git a/ledger/narwhal/batch-header/src/bytes.rs b/ledger/narwhal/batch-header/src/bytes.rs index ceb455192e..7d01157e42 100644 --- a/ledger/narwhal/batch-header/src/bytes.rs +++ b/ledger/narwhal/batch-header/src/bytes.rs @@ -60,12 +60,15 @@ impl FromBytes for BatchHeader { Self::MAX_CERTIFICATES ))); } + + // Read the previous certificate ID bytes. + let mut previous_certificate_id_bytes = + vec![0u8; num_previous_certificate_ids as usize * Field::::size_in_bytes()]; + reader.read_exact(&mut previous_certificate_id_bytes)?; // Read the previous certificate IDs. - let mut previous_certificate_ids = IndexSet::new(); - for _ in 0..num_previous_certificate_ids { - // Read the certificate ID. - previous_certificate_ids.insert(Field::read_le(&mut reader)?); - } + let previous_certificate_ids = cfg_chunks!(previous_certificate_id_bytes, Field::::size_in_bytes()) + .map(Field::read_le) + .collect::, _>>()?; // Read the signature. let signature = Signature::read_le(&mut reader)?; diff --git a/ledger/narwhal/batch-header/src/lib.rs b/ledger/narwhal/batch-header/src/lib.rs index ce79fc6c6a..72e25d1f73 100644 --- a/ledger/narwhal/batch-header/src/lib.rs +++ b/ledger/narwhal/batch-header/src/lib.rs @@ -26,9 +26,13 @@ use console::{ prelude::*, types::Field, }; -use indexmap::IndexSet; use narwhal_transmission_id::TransmissionID; +use indexmap::IndexSet; + +#[cfg(not(feature = "serial"))] +use rayon::prelude::*; + #[derive(Clone, PartialEq, Eq)] pub struct BatchHeader { /// The batch ID, defined as the hash of the author, round number, timestamp, transmission IDs, From 4bac147d50435ddde130d090a9059be542e7c8bf Mon Sep 17 00:00:00 2001 From: raychu86 <14917648+raychu86@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:41:19 -0700 Subject: [PATCH 2/3] Parallelize deserialization of committee members --- ledger/committee/Cargo.toml | 6 +++++- ledger/committee/src/bytes.rs | 30 +++++++++++++++++++----------- ledger/committee/src/lib.rs | 5 ++++- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/ledger/committee/Cargo.toml b/ledger/committee/Cargo.toml index a9cb6dad8c..8c784ee899 100644 --- a/ledger/committee/Cargo.toml +++ b/ledger/committee/Cargo.toml @@ -24,7 +24,7 @@ license = "Apache-2.0" edition = "2021" [features] -default = [ ] +default = [ "rayon" ] serial = [ "console/serial" ] wasm = [ "console/wasm" ] metrics = [ "dep:metrics" ] @@ -74,6 +74,10 @@ optional = true version = "0.4" optional = true +[dependencies.rayon] +version = "1" +optional = true + [dependencies.test-strategy] version = "0.3.1" optional = true diff --git a/ledger/committee/src/bytes.rs b/ledger/committee/src/bytes.rs index 52910b99e9..b700ecbd7e 100644 --- a/ledger/committee/src/bytes.rs +++ b/ledger/committee/src/bytes.rs @@ -37,18 +37,26 @@ impl FromBytes for Committee { Self::MAX_COMMITTEE_SIZE, ))); } + + // Calculate the number of bytes per member. Each member is a (address, stake, is_open) tuple. + let member_byte_size = Address::::size_in_bytes() + 8 + 1; + // Read the member bytes. + let mut member_bytes = vec![0u8; num_members as usize * member_byte_size]; + reader.read_exact(&mut member_bytes)?; // Read the members. - let mut members = IndexMap::with_capacity(num_members as usize); - for _ in 0..num_members { - // Read the address. - let member = Address::read_le(&mut reader)?; - // Read the stake. - let stake = u64::read_le(&mut reader)?; - // Read the is_open flag. - let is_open = bool::read_le(&mut reader)?; - // Insert the member and (stake, is_open). - members.insert(member, (stake, is_open)); - } + let members = cfg_chunks!(member_bytes, member_byte_size) + .map(|mut bytes| { + // Read the address. + let member = Address::::read_le(&mut bytes)?; + // Read the stake. + let stake = u64::read_le(&mut bytes)?; + // Read the is_open flag. + let is_open = bool::read_le(&mut bytes)?; + // Insert the member and (stake, is_open). + Ok((member, (stake, is_open))) + }) + .collect::, std::io::Error>>()?; + // Read the total stake. let total_stake = u64::read_le(&mut reader)?; // Construct the committee. diff --git a/ledger/committee/src/lib.rs b/ledger/committee/src/lib.rs index 75b4a988a8..56cb5d1e36 100644 --- a/ledger/committee/src/lib.rs +++ b/ledger/committee/src/lib.rs @@ -28,11 +28,14 @@ use console::{ program::{Literal, LiteralType}, types::{Address, Field}, }; +use ledger_narwhal_batch_header::BatchHeader; use indexmap::IndexMap; -use ledger_narwhal_batch_header::BatchHeader; use std::collections::HashSet; +#[cfg(not(feature = "serial"))] +use rayon::prelude::*; + /// The minimum amount of stake required for a validator to bond. pub const MIN_VALIDATOR_STAKE: u64 = 1_000_000_000_000u64; // microcredits /// The minimum amount of stake required for a delegator to bond. From 230d42bd9c8d29a0f6e14d679139d38064cca14c Mon Sep 17 00:00:00 2001 From: raychu86 <14917648+raychu86@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:28:15 -0700 Subject: [PATCH 3/3] clippy --- ledger/committee/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ledger/committee/src/lib.rs b/ledger/committee/src/lib.rs index 56cb5d1e36..7c826e4bc4 100644 --- a/ledger/committee/src/lib.rs +++ b/ledger/committee/src/lib.rs @@ -352,7 +352,6 @@ mod tests { use console::prelude::TestRng; use parking_lot::RwLock; - use rayon::prelude::*; use std::sync::Arc; type CurrentNetwork = console::network::MainnetV0; @@ -362,7 +361,7 @@ mod tests { // Initialize a tracker for the leaders. let leaders = Arc::new(RwLock::new(IndexMap::, i64>::new())); // Iterate through the rounds. - (1..=num_rounds).into_par_iter().for_each(|round| { + cfg_into_iter!(1..=num_rounds).for_each(|round| { // Compute the leader. let leader = committee.get_leader(round).unwrap(); // Increment the leader count for the current leader.