Use Rust version of MerkleSet (#17908)

* drop in merkle set from rust

* add test of non-inclusion

* migrate sync_utils to use chia_rs

* remove unused import

* flake8 fixes

* mypy fix

* more lint fixes

* isort

* remove old merkle_set.py

* make test proof of exlcusion more explicit

* covnert test_merkle_set to use chia_rs MerkleSet

* isort again

chia/_tests/blockchain/test_blockchain.py

@@ -8,7 +8,7 @@
 from typing import List, Optional
 
 import pytest
-from chia_rs import AugSchemeMPL, G2Element
+from chia_rs import AugSchemeMPL, G2Element, MerkleSet
 from clvm.casts import int_to_bytes
 
 from chia._tests.blockchain.blockchain_test_utils import (
@@ -53,7 +53,6 @@
 from chia.util.generator_tools import get_block_header
 from chia.util.hash import std_hash
 from chia.util.ints import uint8, uint32, uint64
-from chia.util.merkle_set import MerkleSet
 from chia.util.misc import available_logical_cores
 from chia.util.recursive_replace import recursive_replace
 from chia.util.vdf_prover import get_vdf_info_and_proof

chia/_tests/core/test_merkle_set.py

@@ -8,13 +8,12 @@
 from typing import List, Optional, Tuple
 
 import pytest
-from chia_rs import Coin, compute_merkle_set_root
+from chia_rs import Coin, MerkleSet, compute_merkle_set_root, confirm_included_already_hashed
 
 from chia.simulator.block_tools import BlockTools
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.util.hash import std_hash
 from chia.util.ints import uint64
-from chia.util.merkle_set import MerkleSet, confirm_included_already_hashed
 from chia.util.misc import to_batches
 from chia.wallet.util.wallet_sync_utils import validate_additions, validate_removals
 
@@ -53,21 +52,21 @@ def hashdown(buf: bytes) -> bytes32:
 @pytest.mark.anyio
 async def test_merkle_set_invalid_hash_size() -> None:
     # this is too large
-    with pytest.raises(AssertionError):
+    with pytest.raises(ValueError):
         MerkleSet([bytes([0x80] + [0] * 32)])  # type: ignore[list-item]
 
     with pytest.raises(ValueError, match="could not convert slice to array"):
         compute_merkle_set_root([bytes([0x80] + [0] * 32)])
 
     # this is too small
-    with pytest.raises(AssertionError):
+    with pytest.raises(ValueError):
         MerkleSet([bytes([0x80] + [0] * 30)])  # type: ignore[list-item]
 
     with pytest.raises(ValueError, match="could not convert slice to array"):
         compute_merkle_set_root([bytes([0x80] + [0] * 30)])
 
     # empty
-    with pytest.raises(AssertionError):
+    with pytest.raises(ValueError):
         MerkleSet([b""])  # type: ignore[list-item]
 
     with pytest.raises(ValueError, match="could not convert slice to array"):

chia/_tests/wallet/sync/test_wallet_sync.py

@@ -9,6 +9,7 @@
 
 import pytest
 from aiosqlite import Error as AIOSqliteError
+from chia_rs import confirm_not_included_already_hashed
 from colorlog import getLogger
 
 from chia._tests.connection_utils import disconnect_all, disconnect_all_and_reconnect
@@ -526,8 +527,10 @@ async def test_request_additions_errors(simulator_and_wallet: OldSimulatorsAndWa
         await full_node_api.request_additions(RequestAdditions(last_block.height, std_hash(b""), [ph]))
 
     # No results
+    fake_coin = std_hash(b"")
+    assert ph != fake_coin
     res1 = await full_node_api.request_additions(
-        RequestAdditions(last_block.height, last_block.header_hash, [std_hash(b"")])
+        RequestAdditions(last_block.height, last_block.header_hash, [fake_coin])
     )
     assert res1 is not None
     response = RespondAdditions.from_bytes(res1.data)
@@ -536,6 +539,16 @@ async def test_request_additions_errors(simulator_and_wallet: OldSimulatorsAndWa
     assert response.proofs is not None
     assert len(response.proofs) == 1
     assert len(response.coins) == 1
+    full_block = await full_node_api.full_node.block_store.get_full_block(last_block.header_hash)
+    assert full_block is not None
+    assert full_block.foliage_transaction_block is not None
+    root = full_block.foliage_transaction_block.additions_root
+    assert confirm_not_included_already_hashed(root, response.proofs[0][0], response.proofs[0][1])
+    # proofs is a tuple of (puzzlehash, proof, proof_2)
+    # proof is a proof of inclusion (or exclusion) of that puzzlehash
+    # proof_2 is a proof of all the coins with that puzzlehash
+    # all coin names are concatenated and hashed into one entry in the merkle set for proof_2
+    # the response contains the list of coins so you can check the proof_2
 
     assert response.proofs[0][0] == std_hash(b"")
     assert response.proofs[0][1] is not None

chia/full_node/full_node_api.py

@@ -10,7 +10,7 @@
 from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, cast
 
 import anyio
-from chia_rs import AugSchemeMPL, G1Element, G2Element
+from chia_rs import AugSchemeMPL, G1Element, G2Element, MerkleSet
 from chiabip158 import PyBIP158
 
 from chia.consensus.block_creation import create_unfinished_block
@@ -66,7 +66,6 @@
 from chia.util.hash import std_hash
 from chia.util.ints import uint8, uint32, uint64, uint128
 from chia.util.limited_semaphore import LimitedSemaphoreFullError
-from chia.util.merkle_set import MerkleSet
 
 if TYPE_CHECKING:
     from chia.full_node.full_node import FullNode