forked from bitcoin/bitcoin
-
Notifications
You must be signed in to change notification settings - Fork 73
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
43 changed files
with
1,600 additions
and
290 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,200 @@ | ||
| #!/usr/bin/env bash | ||
| # Demonstrate the creation and usage of UTXO snapshots. | ||
| # | ||
| # A server node starts up, IBDs up to a certain height, then generates a UTXO | ||
| # snapshot at that point. | ||
| # | ||
| # The server then downloads more blocks (to create a diff from the snapshot). | ||
| # | ||
| # We bring a client up, load the UTXO snapshot, and we show the client sync to | ||
| # the "network tip" and then start a background validation of the snapshot it | ||
| # loaded. We see the background validation chainstate removed after validation | ||
| # completes. | ||
| # | ||
|
|
||
| export LC_ALL=C | ||
| set -e | ||
|
|
||
| BASE_HEIGHT=${1:-30000} | ||
| INCREMENTAL_HEIGHT=20000 | ||
| FINAL_HEIGHT=$(($BASE_HEIGHT + $INCREMENTAL_HEIGHT)) | ||
|
|
||
| SERVER_DATADIR="$(pwd)/utxodemo-data-server-$BASE_HEIGHT" | ||
| CLIENT_DATADIR="$(pwd)/utxodemo-data-client-$BASE_HEIGHT" | ||
| UTXO_DAT_FILE="$(pwd)/utxo.$BASE_HEIGHT.dat" | ||
|
|
||
| # Chosen to try to not interfere with any running bitcoind processes. | ||
| SERVER_PORT=8633 | ||
| SERVER_RPC_PORT=8632 | ||
|
|
||
| CLIENT_PORT=8733 | ||
| CLIENT_RPC_PORT=8732 | ||
|
|
||
| SERVER_PORTS="-port=${SERVER_PORT} -rpcport=${SERVER_RPC_PORT}" | ||
| CLIENT_PORTS="-port=${CLIENT_PORT} -rpcport=${CLIENT_RPC_PORT}" | ||
|
|
||
| # Ensure the client exercises all indexes to test that snapshot use works | ||
| # properly with indexes. | ||
| ALL_INDEXES="-txindex -coinstatsindex -blockfilterindex=1" | ||
|
|
||
| if ! command -v jq >/dev/null ; then | ||
| echo "This script requires jq to parse JSON RPC output. Please install it." | ||
| echo "(e.g. sudo apt install jq)" | ||
| exit 1 | ||
| fi | ||
|
|
||
| DUMP_OUTPUT="dumptxoutset-output-$BASE_HEIGHT.json" | ||
|
|
||
| finish() { | ||
| echo | ||
| echo "Killing server and client PIDs ($SERVER_PID, $CLIENT_PID) and cleaning up datadirs" | ||
| echo | ||
| rm -f "$UTXO_DAT_FILE" "$DUMP_OUTPUT" | ||
| rm -rf "$SERVER_DATADIR" "$CLIENT_DATADIR" | ||
| kill -9 "$SERVER_PID" "$CLIENT_PID" | ||
| } | ||
|
|
||
| trap finish EXIT | ||
|
|
||
| # Need to specify these to trick client into accepting server as a peer | ||
| # it can IBD from, otherwise the default values prevent IBD from the server node. | ||
| EARLY_IBD_FLAGS="-maxtipage=9223372036854775207 -minimumchainwork=0x00" | ||
|
|
||
| server_rpc() { | ||
| ./src/bitcoin-cli -rpcport=$SERVER_RPC_PORT -datadir="$SERVER_DATADIR" "$@" | ||
| } | ||
| client_rpc() { | ||
| ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir="$CLIENT_DATADIR" "$@" | ||
| } | ||
| server_sleep_til_boot() { | ||
| while ! server_rpc ping >/dev/null 2>&1; do sleep 0.1; done | ||
| } | ||
| client_sleep_til_boot() { | ||
| while ! client_rpc ping >/dev/null 2>&1; do sleep 0.1; done | ||
| } | ||
|
|
||
| mkdir -p "$SERVER_DATADIR" "$CLIENT_DATADIR" | ||
|
|
||
| echo "Hi, welcome to the assumeutxo demo/test" | ||
| echo | ||
| echo "We're going to" | ||
| echo | ||
| echo " - start up a 'server' node, sync it via mainnet IBD to height ${BASE_HEIGHT}" | ||
| echo " - create a UTXO snapshot at that height" | ||
| echo " - IBD ${INCREMENTAL_HEIGHT} more blocks on top of that" | ||
| echo | ||
| echo "then we'll demonstrate assumeutxo by " | ||
| echo | ||
| echo " - starting another node (the 'client') and loading the snapshot in" | ||
| echo " * first you'll have to modify the code slightly (chainparams) and recompile" | ||
| echo " * don't worry, we'll make it easy" | ||
| echo " - observing the client sync ${INCREMENTAL_HEIGHT} blocks on top of the snapshot from the server" | ||
| echo " - observing the client validate the snapshot chain via background IBD" | ||
| echo | ||
| read -p "Press [enter] to continue" _ | ||
|
|
||
| echo | ||
| echo "-- Starting the demo. You might want to run the two following commands in" | ||
| echo " separate terminal windows:" | ||
| echo | ||
| echo " watch -n0.1 tail -n 30 $SERVER_DATADIR/debug.log" | ||
| echo " watch -n0.1 tail -n 30 $CLIENT_DATADIR/debug.log" | ||
| echo | ||
| read -p "Press [enter] to continue" _ | ||
|
|
||
| echo | ||
| echo "-- IBDing the blocks (height=$BASE_HEIGHT) required to the server node..." | ||
| ./src/bitcoind -logthreadnames=1 $SERVER_PORTS \ | ||
| -datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -stopatheight="$BASE_HEIGHT" >/dev/null | ||
|
|
||
| echo | ||
| echo "-- Creating snapshot at ~ height $BASE_HEIGHT ($UTXO_DAT_FILE)..." | ||
| sleep 2 | ||
| ./src/bitcoind -logthreadnames=1 $SERVER_PORTS \ | ||
| -datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -connect=0 -listen=0 >/dev/null & | ||
| SERVER_PID="$!" | ||
|
|
||
| server_sleep_til_boot | ||
| server_rpc dumptxoutset "$UTXO_DAT_FILE" > "$DUMP_OUTPUT" | ||
| cat "$DUMP_OUTPUT" | ||
| kill -9 "$SERVER_PID" | ||
|
|
||
| RPC_BASE_HEIGHT=$(jq -r .base_height < "$DUMP_OUTPUT") | ||
| RPC_AU=$(jq -r .txoutset_hash < "$DUMP_OUTPUT") | ||
| RPC_NCHAINTX=$(jq -r .nchaintx < "$DUMP_OUTPUT") | ||
| RPC_BLOCKHASH=$(jq -r .base_hash < "$DUMP_OUTPUT") | ||
|
|
||
| # Wait for server to shutdown... | ||
| while server_rpc ping >/dev/null 2>&1; do sleep 0.1; done | ||
|
|
||
| echo | ||
| echo "-- Now: add the following to CMainParams::m_assumeutxo_data" | ||
| echo " in src/kernel/chainparams.cpp, and recompile:" | ||
| echo | ||
| echo " {${RPC_BASE_HEIGHT}, AssumeutxoHash{uint256S(\"0x${RPC_AU}\")}, ${RPC_NCHAINTX}, uint256S(\"0x${RPC_BLOCKHASH}\")}," | ||
| echo | ||
| echo | ||
| echo "-- IBDing more blocks to the server node (height=$FINAL_HEIGHT) so there is a diff between snapshot and tip..." | ||
| ./src/bitcoind $SERVER_PORTS -logthreadnames=1 -datadir="$SERVER_DATADIR" \ | ||
| $EARLY_IBD_FLAGS -stopatheight="$FINAL_HEIGHT" >/dev/null | ||
|
|
||
| echo | ||
| echo "-- Starting the server node to provide blocks to the client node..." | ||
| ./src/bitcoind $SERVER_PORTS -logthreadnames=1 -debug=net -datadir="$SERVER_DATADIR" \ | ||
| $EARLY_IBD_FLAGS -connect=0 -listen=1 >/dev/null & | ||
| SERVER_PID="$!" | ||
| server_sleep_til_boot | ||
|
|
||
| echo | ||
| echo "-- Okay, what you're about to see is the client starting up and activating the snapshot." | ||
| echo " I'm going to display the top 14 log lines from the client on top of an RPC called" | ||
| echo " getchainstates, which is like getblockchaininfo but for both the snapshot and " | ||
| echo " background validation chainstates." | ||
| echo | ||
| echo " You're going to first see the snapshot chainstate sync to the server's tip, then" | ||
| echo " the background IBD chain kicks in to validate up to the base of the snapshot." | ||
| echo | ||
| echo " Once validation of the snapshot is done, you should see log lines indicating" | ||
| echo " that we've deleted the background validation chainstate." | ||
| echo | ||
| echo " Once everything completes, exit the watch command with CTRL+C." | ||
| echo | ||
| read -p "When you're ready for all this, hit [enter]" _ | ||
|
|
||
| echo | ||
| echo "-- Starting the client node to get headers from the server, then load the snapshot..." | ||
| ./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" \ | ||
| -connect=0 -addnode=127.0.0.1:$SERVER_PORT -debug=net $EARLY_IBD_FLAGS >/dev/null & | ||
| CLIENT_PID="$!" | ||
| client_sleep_til_boot | ||
|
|
||
| echo | ||
| echo "-- Initial state of the client:" | ||
| client_rpc getchainstates | ||
|
|
||
| echo | ||
| echo "-- Loading UTXO snapshot into client..." | ||
| client_rpc loadtxoutset "$UTXO_DAT_FILE" | ||
|
|
||
| watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat" | ||
|
|
||
| echo | ||
| echo "-- Okay, now I'm going to restart the client to make sure that the snapshot chain reloads " | ||
| echo " as the main chain properly..." | ||
| echo | ||
| echo " Press CTRL+C after you're satisfied to exit the demo" | ||
| echo | ||
| read -p "Press [enter] to continue" | ||
|
|
||
| while kill -0 "$CLIENT_PID"; do | ||
| sleep 1 | ||
| done | ||
| ./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" -connect=0 \ | ||
| -addnode=127.0.0.1:$SERVER_PORT "$EARLY_IBD_FLAGS" >/dev/null & | ||
| CLIENT_PID="$!" | ||
| client_sleep_til_boot | ||
|
|
||
| watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat" | ||
|
|
||
| echo | ||
| echo "-- Done!" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| Pruning | ||
| ------- | ||
|
|
||
| When using assumeutxo with `-prune`, the prune budget may be exceeded if it is set | ||
| lower than 1100MB (i.e. `MIN_DISK_SPACE_FOR_BLOCK_FILES * 2`). Prune budget is normally | ||
| split evenly across each chainstate, unless the resulting prune budget per chainstate | ||
| is beneath `MIN_DISK_SPACE_FOR_BLOCK_FILES` in which case that value will be used. | ||
|
|
||
| RPC | ||
| --- | ||
|
|
||
| `loadtxoutset` has been added, which allows loading a UTXO snapshot of the format | ||
| generated by `dumptxoutset`. Once this snapshot is loaded, its contents will be | ||
| deserialized into a second chainstate data structure, which is then used to sync to | ||
| the network's tip under a security model very much like `assumevalid`. | ||
|
|
||
| Meanwhile, the original chainstate will complete the initial block download process in | ||
| the background, eventually validating up to the block that the snapshot is based upon. | ||
|
|
||
| The result is a usable bitcoind instance that is current with the network tip in a | ||
| matter of minutes rather than hours. UTXO snapshot are typically obtained via | ||
| third-party sources (HTTP, torrent, etc.) which is reasonable since their contents | ||
| are always checked by hash. | ||
|
|
||
| You can find more information on this process in the `assumeutxo` design | ||
| document (<https://github.com/bitcoin/bitcoin/blob/master/doc/design/assumeutxo.md>). | ||
|
|
||
| `getchainstates` has been added to aid in monitoring the assumeutxo sync process. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.