Add withdrawals (EIP-4895) (#434)

* Update IPFS gateway

* Add format parameter to test_export_blocks_job

* Add withdrawals field to block model

* Bump package version

blockchainetl/exporters.py

@@ -119,9 +119,16 @@ def serialize_field(self, field, name, value):
         return serializer(value)
 
     def _join_if_needed(self, value):
+        def to_string(x):
+            if isinstance(x, dict):
+                # Separators without whitespace for compact format.
+                return JSONEncoder(separators=(',', ':')).encode(x)
+            else:
+                return str(x)
+
         if isinstance(value, (list, tuple)):
             try:
-                return self._join_multivalued.join(str(x) for x in value)
+                return self._join_multivalued.join(to_string(x) for x in value)
             except TypeError:  # list in value may not contain strings
                 pass
         return value

ethereumetl/domain/block.py

@@ -40,7 +40,9 @@ def __init__(self):
         self.gas_limit = None
         self.gas_used = None
         self.timestamp = None
+        self.withdrawals_root = None
 
         self.transactions = []
         self.transaction_count = 0
         self.base_fee_per_gas = 0
+        self.withdrawals = []

ethereumetl/ipfs/origin.py

@@ -6,7 +6,7 @@
 
 logger = logging.getLogger('origin')
 
-IPFS_PRIMARY_GATEWAY_URL = 'https://ipfs-prod.ogn.app/ipfs'
+IPFS_PRIMARY_GATEWAY_URL = 'https://cf-ipfs.com/ipfs'
 IPFS_SECONDARY_GATEWAY_URL = 'https://gateway.ipfs.io/ipfs'
 
 # Returns an IPFS client that can be used to fetch Origin Protocol's data.

ethereumetl/jobs/exporters/blocks_and_transactions_item_exporter.py

@@ -42,7 +42,9 @@
     'gas_used',
     'timestamp',
     'transaction_count',
-    'base_fee_per_gas'
+    'base_fee_per_gas',
+    'withdrawals_root',
+    'withdrawals'
 ]
 
 TRANSACTION_FIELDS_TO_EXPORT = [

ethereumetl/mappers/block_mapper.py

@@ -53,6 +53,7 @@ def json_dict_to_block(self, json_dict):
         block.gas_used = hex_to_dec(json_dict.get('gasUsed'))
         block.timestamp = hex_to_dec(json_dict.get('timestamp'))
         block.base_fee_per_gas = hex_to_dec(json_dict.get('baseFeePerGas'))
+        block.withdrawals_root = json_dict.get('withdrawalsRoot')
 
         if 'transactions' in json_dict:
             block.transactions = [
@@ -63,8 +64,22 @@ def json_dict_to_block(self, json_dict):
 
             block.transaction_count = len(json_dict['transactions'])
 
+        if 'withdrawals' in json_dict:
+            block.withdrawals = self.parse_withdrawals(json_dict['withdrawals'])
+
         return block
 
+    def parse_withdrawals(self, withdrawals):
+        return [
+            {
+                "index": hex_to_dec(withdrawal["index"]),
+                "validator_index": hex_to_dec(withdrawal["validatorIndex"]),
+                "address": withdrawal["address"],
+                "amount": hex_to_dec(withdrawal["amount"]),
+            }
+            for withdrawal in withdrawals
+        ]
+
     def block_to_dict(self, block):
         return {
             'type': 'block',
@@ -86,5 +101,7 @@ def block_to_dict(self, block):
             'gas_used': block.gas_used,
             'timestamp': block.timestamp,
             'transaction_count': block.transaction_count,
-            'base_fee_per_gas': block.base_fee_per_gas
+            'base_fee_per_gas': block.base_fee_per_gas,
+            'withdrawals_root': block.withdrawals_root,
+            'withdrawals': block.withdrawals,
         }

setup.py

@@ -11,7 +11,7 @@ def read(fname):
 
 setup(
     name='ethereum-etl',
-    version='2.1.2',
+    version='2.2.0',
     author='Evgeny Medvedev',
     author_email='evge.medvedev@gmail.com',
     description='Tools for exporting Ethereum blockchain data to CSV or JSON',

tests/ethereumetl/job/helpers.py

@@ -21,6 +21,12 @@ def get_web3_provider(provider_type, read_resource_lambda=None, batch=False):
             provider = BatchHTTPProvider(provider_url)
         else:
             provider = HTTPProvider(provider_url)
+    elif provider_type == 'goerli':
+        provider_url = os.environ.get('GOERLI_PROVIDER_URL', 'https://goerli.infura.io/v3/7aef3f0cd1f64408b163814b22cc643c')
+        if batch:
+            provider = BatchHTTPProvider(provider_url)
+        else:
+            provider = HTTPProvider(provider_url)
     else:
         raise ValueError('Provider type {} is unexpected'.format(provider_type))
     return provider

tests/ethereumetl/job/test_export_blocks_job.py

@@ -36,18 +36,21 @@ def read_resource(resource_group, file_name):
     return tests.resources.read_resource([RESOURCE_GROUP, resource_group], file_name)
 
 
-@pytest.mark.parametrize("start_block,end_block,batch_size,resource_group,web3_provider_type", [
-    (0, 0, 1, 'block_without_transactions', 'mock'),
-    (483920, 483920, 1, 'block_with_logs', 'mock'),
-    (47218, 47219, 1, 'blocks_with_transactions', 'mock'),
-    (47218, 47219, 2, 'blocks_with_transactions', 'mock'),
-    skip_if_slow_tests_disabled((0, 0, 1, 'block_without_transactions', 'infura')),
-    skip_if_slow_tests_disabled((483920, 483920, 1, 'block_with_logs', 'infura')),
-    skip_if_slow_tests_disabled((47218, 47219, 2, 'blocks_with_transactions', 'infura')),
+@pytest.mark.parametrize("start_block,end_block,batch_size,resource_group,web3_provider_type,format", [
+    (0, 0, 1, 'block_without_transactions', 'mock', 'csv'),
+    (483920, 483920, 1, 'block_with_logs', 'mock', 'csv'),
+    (47218, 47219, 1, 'blocks_with_transactions', 'mock', 'csv'),
+    (47218, 47219, 2, 'blocks_with_transactions', 'mock', 'csv'),
+    skip_if_slow_tests_disabled((0, 0, 1, 'block_without_transactions', 'infura', 'csv')),
+    skip_if_slow_tests_disabled((483920, 483920, 1, 'block_with_logs', 'infura', 'csv')),
+    skip_if_slow_tests_disabled((47218, 47219, 2, 'blocks_with_transactions', 'infura', 'csv')),
+    # TODO: Update these tests after Shanghai:
+    skip_if_slow_tests_disabled((8656134, 8656135, 2, 'blocks_with_transactions_goerli', 'goerli', 'csv')),
+    skip_if_slow_tests_disabled((8656134, 8656135, 2, 'blocks_with_transactions_goerli', 'goerli', 'json')),
 ])
-def test_export_blocks_job(tmpdir, start_block, end_block, batch_size, resource_group, web3_provider_type):
-    blocks_output_file = str(tmpdir.join('actual_blocks.csv'))
-    transactions_output_file = str(tmpdir.join('actual_transactions.csv'))
+def test_export_blocks_job(tmpdir, start_block, end_block, batch_size, resource_group, web3_provider_type, format):
+    blocks_output_file = str(tmpdir.join(f'actual_blocks.{format}'))
+    transactions_output_file = str(tmpdir.join(f'actual_transactions.{format}'))
 
     job = ExportBlocksJob(
         start_block=start_block, end_block=end_block, batch_size=batch_size,
@@ -62,9 +65,9 @@ def test_export_blocks_job(tmpdir, start_block, end_block, batch_size, resource_
     job.run()
 
     compare_lines_ignore_order(
-        read_resource(resource_group, 'expected_blocks.csv'), read_file(blocks_output_file)
+        read_resource(resource_group, f'expected_blocks.{format}'), read_file(blocks_output_file)
     )
 
     compare_lines_ignore_order(
-        read_resource(resource_group, 'expected_transactions.csv'), read_file(transactions_output_file)
+        read_resource(resource_group, f'expected_transactions.{format}'), read_file(transactions_output_file)
     )

tests/ethereumetl/streaming/test_stream.py

@@ -45,6 +45,8 @@ def read_resource(resource_group, file_name):
     skip_if_slow_tests_disabled([1755634, 1755635, 1, 'blocks_1755634_1755635', EntityType.ALL_FOR_INFURA, 'infura']),
     (508110, 508110, 1, 'blocks_508110_508110', ['trace', 'contract', 'token'], 'mock'),
     (2112234, 2112234, 1, 'blocks_2112234_2112234', ['trace', 'contract', 'token'], 'mock'),
+    # TODO: Update these tests after Shanghai:
+    skip_if_slow_tests_disabled([8656134, 8656135, 1, 'blocks_8656134_8656135_goerli', EntityType.ALL_FOR_INFURA, 'goerli']),
 ])
 def test_stream(tmpdir, start_block, end_block, batch_size, resource_group, entity_types, provider_type):
     try:

tests/resources/test_export_blocks_job/block_with_logs/expected_blocks.csv

@@ -1,2 +1,2 @@
-number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,receipts_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count,base_fee_per_gas
-483920,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,0x2610dc6eb941f4bcbddfd2362b999087ccd956e978f0ece4f8da96851283a2ba,0x57a633e01197dc86,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000800000000000000000000000000000800000000000000000000000000000008000000000000000000000000000000000000021000000080000000004000008000000000000000000000400000000000000000000000000000000400000000000000000000000000000000000000010000000000000000000000000000000000000000400000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000010000000000000000000000000000000000000000000000004000000000000000000000000000000000000040080000,0x2744d46ab0647ed91a9bbd08e19d3bb67491067e8cbe04a276ad2afde5ecd65e,0x48b17dd0031aa97d748a886c912539de22997e861d631fd1eb6509fbabef9651,0xada95dd1e1590fe095e67c58f41d633193b238e0e0c588de46682db595738f0b,0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5,7298514125186,2571481026230204460,1113,0xd783010203844765746887676f312e342e32856c696e7578,3141592,143706,1446561880,4,
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,receipts_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count,base_fee_per_gas,withdrawals_root,withdrawals
+483920,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,0x2610dc6eb941f4bcbddfd2362b999087ccd956e978f0ece4f8da96851283a2ba,0x57a633e01197dc86,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000800000000000000000000000000000800000000000000000000000000000008000000000000000000000000000000000000021000000080000000004000008000000000000000000000400000000000000000000000000000000400000000000000000000000000000000000000010000000000000000000000000000000000000000400000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000010000000000000000000000000000000000000000000000004000000000000000000000000000000000000040080000,0x2744d46ab0647ed91a9bbd08e19d3bb67491067e8cbe04a276ad2afde5ecd65e,0x48b17dd0031aa97d748a886c912539de22997e861d631fd1eb6509fbabef9651,0xada95dd1e1590fe095e67c58f41d633193b238e0e0c588de46682db595738f0b,0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5,7298514125186,2571481026230204460,1113,0xd783010203844765746887676f312e342e32856c696e7578,3141592,143706,1446561880,4,,,

tests/resources/test_export_blocks_job/block_without_transactions/expected_blocks.csv

@@ -1,2 +1,2 @@
-number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,receipts_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count,base_fee_per_gas
-0,0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3,0x0000000000000000000000000000000000000000000000000000000000000000,0x0000000000000042,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421,0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544,0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421,0x0000000000000000000000000000000000000000,17179869184,17179869184,540,0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa,5000,0,0,0,
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,receipts_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count,base_fee_per_gas,withdrawals_root,withdrawals
+0,0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3,0x0000000000000000000000000000000000000000000000000000000000000000,0x0000000000000042,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421,0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544,0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421,0x0000000000000000000000000000000000000000,17179869184,17179869184,540,0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa,5000,0,0,0,,,

tests/resources/test_export_blocks_job/blocks_with_transactions/expected_blocks.csv

@@ -1,3 +1,3 @@
-number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,receipts_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count,base_fee_per_gas
-47218,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0xfc1dd3249585b593ad18822a95873c75779ccbe8a420b457be380b977fc38e87,0xfaa1e51d379b21f7,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xdef39a3afcfb55fabd16d7d5a19d8b078cb95a117ea9596af289525eb8592982,0xab6d1b109c04c1f43f86da33ece31689d9ba1c44ea809d0979c05057a0703c28,0xf0afaf08454f89092907f820c40db77afbd0270f179166907dc110f9723972d7,0x9746c7e1ef2bd21ff3997fa467593a89cb852bd0,1460233976906,44246932724217368,766,0x476574682f76312e302e312f77696e646f77732f676f312e342e32,42085,42000,1438936285,2,
-47219,0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0x52cf720359834975,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xd757552351d6a714feda148f8f6f1283e000e05ea407352d45bc9dfc4c16ffe9,0x05a16e52dbacec805dc881439ef54338e8324ee133a4dbbb8ab17f8c73290054,0x9c8327de15d9d1668feb6e3583ddc337fb857620a5a3ff66d0d66148864b9d55,0xf927a40c8b7f6e07c5af7fa2155b4864a4112b13,1459520972035,44248392245189403,763,0x476574682f76312e302e312f6c696e75782f676f312e342e32,42125,42000,1438936326,2,
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,receipts_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count,base_fee_per_gas,withdrawals_root,withdrawals
+47218,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0xfc1dd3249585b593ad18822a95873c75779ccbe8a420b457be380b977fc38e87,0xfaa1e51d379b21f7,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xdef39a3afcfb55fabd16d7d5a19d8b078cb95a117ea9596af289525eb8592982,0xab6d1b109c04c1f43f86da33ece31689d9ba1c44ea809d0979c05057a0703c28,0xf0afaf08454f89092907f820c40db77afbd0270f179166907dc110f9723972d7,0x9746c7e1ef2bd21ff3997fa467593a89cb852bd0,1460233976906,44246932724217368,766,0x476574682f76312e302e312f77696e646f77732f676f312e342e32,42085,42000,1438936285,2,,,
+47219,0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0x52cf720359834975,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xd757552351d6a714feda148f8f6f1283e000e05ea407352d45bc9dfc4c16ffe9,0x05a16e52dbacec805dc881439ef54338e8324ee133a4dbbb8ab17f8c73290054,0x9c8327de15d9d1668feb6e3583ddc337fb857620a5a3ff66d0d66148864b9d55,0xf927a40c8b7f6e07c5af7fa2155b4864a4112b13,1459520972035,44248392245189403,763,0x476574682f76312e302e312f6c696e75782f676f312e342e32,42125,42000,1438936326,2,,,