Merge remote-tracking branch 'origin/main' into 7765-token-metadata-a…

…cceptance-tests Signed-off-by: Jeff Schmidt <jeffrey.schmidt@swirldslabs.com>
hashgraph · May 8, 2024 · f9bc648 · f9bc648
2 parents 6d32381 + 13b6840
commit f9bc648
Show file tree

Hide file tree

Showing 57 changed files with 857 additions and 195 deletions.
diff --git a/.github/workflows/charts.yml b/.github/workflows/charts.yml
@@ -30,7 +30,7 @@ jobs:
           python-version: "3.10"
 
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Install Helm
         uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
@@ -55,7 +55,7 @@ jobs:
           python-version: "3.10"
 
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Install Wget
         run: |

diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -65,7 +65,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Setup Node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2

diff --git a/.github/workflows/release-automation.yml b/.github/workflows/release-automation.yml
@@ -60,7 +60,7 @@ jobs:
           EOF
 
       - name: Checkout Repository
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           fetch-depth: 0
           ref: main
@@ -156,7 +156,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Repository
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           ref: main
 

diff --git a/.github/workflows/release-integration.yml b/.github/workflows/release-integration.yml
@@ -41,7 +41,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Get version
         run: echo "VERSION=$(grep -oP 'version=\K.+' gradle.properties)" >> $GITHUB_ENV
@@ -102,7 +102,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           ref: deploy
 

diff --git a/.github/workflows/release-production.yml b/.github/workflows/release-production.yml
@@ -42,7 +42,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Get tag
         run: echo "TAG=${GITHUB_REF#refs/*/v}" >> $GITHUB_ENV
@@ -106,7 +106,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Publish helm chart
         uses: stefanprodan/helm-gh-pages@0ad2bb377311d61ac04ad9eb6f252fb68e207260 # v1.7.0
@@ -132,7 +132,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Authenticate to Google Cloud
         id: google-auth

diff --git a/.github/workflows/rosetta.yml b/.github/workflows/rosetta.yml
@@ -33,7 +33,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Setup Node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2

diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
@@ -33,7 +33,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Setup Node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@@ -81,7 +81,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           path: tmp
 
@@ -102,7 +102,7 @@ jobs:
           go-version: 1.22
 
       - name: Run Gosec Security Scanner
-        uses: securego/gosec@cf29d543e241f74432305ac938071eac9e36da7f # master
+        uses: securego/gosec@d3b2359ae29fe344f4df5bf3a8c6491fb7c46c02 # master
         with:
           args: ./...
 
@@ -117,7 +117,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
 
@@ -159,7 +159,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Setup Node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@@ -206,7 +206,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Setup Node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2

diff --git a/.github/workflows/snyk-monitor.yml b/.github/workflows/snyk-monitor.yml
@@ -28,7 +28,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout Code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
 
       - name: Setup Node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -57,7 +57,7 @@ dependencies {
         api("com.github.vertical-blank:sql-formatter:2.0.5")
         api("org.bouncycastle:bcprov-jdk18on:1.78.1")
         api("com.bucket4j:bucket4j-core:8.10.1")
-        api("com.google.cloud:spring-cloud-gcp-dependencies:5.2.0")
+        api("com.google.cloud:spring-cloud-gcp-dependencies:5.2.1")
         api("com.google.guava:guava:33.2.0-jre")
         api("com.google.protobuf:protobuf-java:$protobufVersion")
         api("com.graphql-java-generator:graphql-java-client-runtime:2.4")

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -697,6 +697,7 @@ value, it is recommended to only populate overridden properties in the custom `a
 | `hedera.mirror.web3.evm.minAutoRenewDuration`                 | 2592000                                           | Minimum duration for auto-renew account                                                                                                                                                       |
 | `hedera.mirror.web3.evm.network`                              | TESTNET                                           | Which Hedera network to use. Can be either `MAINNET`, `PREVIEWNET`, `TESTNET` or `OTHER`                                                                                                      |
 | `hedera.mirror.web3.evm.rateLimit`                            | 500                                               | Maximum RPS limit                                                                                                                                                                             |
+| `hedera.mirror.web3.evm.feesTokenTransferUsageMultiplier`     | 380                                               | Used to calculate token transfer fees                                                                                                                                                         |
 | `hedera.mirror.web3.evm.trace.enabled`                        | false                                             | Flag enabling tracer                                                                                                                                                                          |
 | `hedera.mirror.web3.evm.trace.contract`                       | []                                                | A set with contract addresses to filter. By default it is empty to indicate it will trace all contract addresses.                                                                             |
 | `hedera.mirror.web3.evm.trace.status`                         | []                                                | A set with frame statuses to filter. By default it is empty to indicate it will trace all frames regardless of status.                                                                        |
diff --git a/docs/database/README.md b/docs/database/README.md
@@ -264,18 +264,20 @@ since open access until April 2023.
 | Coordinator | 2     | 7    | 24 GB  | 256 GB       |
 | Worker      | 3     | 7    | 24 GB  | 3 TB         |
 
-Following are the prerequisites and steps for migrating V1 data to V2.
+Following are the prerequisites and steps for migrating V1 data to V2. As of mirror node v0.103.0, the migration script is expected to migrate full mainnet data in 7 days.
 
 1. Create a Citus cluster with enough resources (Disk, CPU and memory). For GKE, an e2-custom-6-32768 can be used.
-2. Ensure the source and target schemas are compatible by deploying the same version to both.
-3. Populate correct values for the source and target configuration in the
-   [migration.config](/hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.config).
-4. Get the correct version of [flyway](https://flywaydb.org/documentation/usage/commandline/) based on your OS and
+2. Ensure the source and target schemas are compatible by deploying the same version to both. Be sure to leave importer disabled for the target deployment.
+3. Get the correct version of [flyway](https://flywaydb.org/documentation/usage/commandline/) based on your OS and
    update it in the `FLYWAY_URL` field in the `migration.config` file. The default is set to the linux version.
-5. Stop the [Importer](/docs/importer/README.md) process.
-6. Run the [migration.sh](/hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.sh) script. Due to the time it will take to complete the migration,
+4. Stop the [Importer](/docs/importer/README.md) process on the source.
+5. Create a clone of the source database to use as the source for the migration (you may skip this step if you wish to keep the importer down on the source for the length of the migration)
+6. If you created a clone in step 5, you may now restart the importer on the source.
+7. Populate correct values for the source and target configuration in the
+   [migration.config](/hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.config). The source should be the source from step 5.
+8. Run the [migration.sh](/hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.sh) script. Due to the time it will take to complete the migration,
    it is recommended to run the script in a way that doesn't require your terminal session to remain open (e.g. `./migration.sh > migration.log 2> migration-error.log & disown`)
-7. Update the mirror node configuration to point to the new Citus DB and start it.
+9. Update the mirror node configuration to point to the new Citus DB and enable the importer. If you did not create a clone in step 5, you may now restart the importer process on the source.
 
 ## Citus Backup and Restore
 

diff --git a/.../hedera/mirror/importer/parser/record/transactionhandler/TokenWipeTransactionHandler.java b/.../hedera/mirror/importer/parser/record/transactionhandler/TokenWipeTransactionHandler.java
@@ -16,22 +16,29 @@
 
 package com.hedera.mirror.importer.parser.record.transactionhandler;
 
+import static java.util.function.Predicate.not;
+
 import com.google.common.collect.Range;
 import com.hedera.mirror.common.domain.entity.EntityId;
 import com.hedera.mirror.common.domain.token.Nft;
 import com.hedera.mirror.common.domain.token.Token;
 import com.hedera.mirror.common.domain.transaction.RecordItem;
 import com.hedera.mirror.common.domain.transaction.Transaction;
 import com.hedera.mirror.common.domain.transaction.TransactionType;
+import com.hedera.mirror.importer.domain.EntityIdService;
 import com.hedera.mirror.importer.parser.record.entity.EntityListener;
 import com.hedera.mirror.importer.parser.record.entity.EntityProperties;
+import com.hedera.mirror.importer.util.Utility;
 import jakarta.inject.Named;
+import lombok.CustomLog;
 import lombok.RequiredArgsConstructor;
 
+@CustomLog
 @Named
 @RequiredArgsConstructor
 class TokenWipeTransactionHandler extends AbstractTransactionHandler {
 
+    private final EntityIdService entityIdService;
     private final EntityListener entityListener;
     private final EntityProperties entityProperties;
 
@@ -71,6 +78,13 @@ protected void doUpdateTransaction(Transaction transaction, RecordItem recordIte
             entityListener.onNft(nft);
         });
 
-        recordItem.addEntityId(EntityId.of(transactionBody.getAccount()));
+        var account = transactionBody.getAccount();
+        entityIdService
+                .lookup(account)
+                .filter(not(EntityId::isEmpty))
+                .ifPresentOrElse(
+                        recordItem::addEntityId,
+                        () -> Utility.handleRecoverableError(
+                                "Invalid TokenWipe account {} at {}", account, consensusTimestamp));
     }
 }
diff --git a/...ror-importer/src/main/resources/db/migration/v1/V1.97.0__remove_bootstrap_addressbook.sql b/...ror-importer/src/main/resources/db/migration/v1/V1.97.0__remove_bootstrap_addressbook.sql
@@ -0,0 +1 @@
+delete from file_data where consensus_timestamp=0 and entity_id=102;
diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.sh b/hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.sh
@@ -147,18 +147,21 @@ getCursorConditions() {
   local pctPerCursor=$(echo "scale=10; 100 / ${ASYNC_TABLE_SPLITS}" | bc)
   local totalPct=0
   local -n cursors=$1 # Init array to hold cursor definitions
+  log "Will split data into ${pctPerCursor}% chunks"
   local stats=$(PGPASSWORD="${SOURCE_DB_PASSWORD}" psql -q --csv -t -h "${SOURCE_DB_HOST}" -d "${SOURCE_DB_NAME}" -p "${SOURCE_DB_PORT}" -U "${SOURCE_DB_USER}" -c "${query}")
-
   while IFS="," read -r pct_of_total lower_bound upper_bound; do
     upperBound=$((upper_bound > upperBound ? upper_bound : upperBound))
-    local groupRange=$((upperBound - lower_bound))
-    local rangeToTake=$(echo "scale=10; (${pctPerCursor} / 100) * ${groupRange}" | bc)
     totalPct=$(echo "(${totalPct} + ${pct_of_total})" | bc)
+    local groupRange=$((upperBound - lower_bound))
+    local rangeToTake=$(echo "scale=10; ${groupRange} / (${totalPct} / ${pctPerCursor})" | bc)
     while (($(echo "${totalPct} >= ${pctPerCursor}" | bc) )); do
-      totalPct=$(echo "${totalPct} - ${pctPerCursor}" | bc)
       cursorLowerBound=$(echo "scale=0; (${upperBound} - ${rangeToTake}) / 1" | bc)
+      if (( $(echo "${cursorLowerBound} < ${lower_bound}" | bc) )); then
+        break
+      fi
+      totalPct=$(echo "${totalPct} - ${pctPerCursor}" | bc)
       cursors+=("\"(${cursorLowerBound},${upperBound}]\"")
-      upperBound=$cursorLowerBound
+      upperBound="${cursorLowerBound}"
     done
   done <<< "${stats}"
 

diff --git a/...test/java/com/hedera/mirror/importer/domain/ContractResultServiceImplIntegrationTest.java b/...test/java/com/hedera/mirror/importer/domain/ContractResultServiceImplIntegrationTest.java
@@ -385,15 +385,9 @@ void processGasConsumedCalculationContractCall() {
         process(recordItem);
 
         // Then
-        final var contractResult = contractResultRepository
-                .findById(recordItem.getConsensusTimestamp())
-                .orElse(null);
-
-        assertThat(contractResult)
-                .isNotNull()
-                .extracting(ContractResult::getGasConsumed)
-                .isNotNull()
-                .isEqualTo(22074L);
+        assertThat(contractResultRepository.findById(recordItem.getConsensusTimestamp()))
+                .get()
+                .returns(22074L, ContractResult::getGasConsumed);
     }
 
     @Test
@@ -839,7 +833,7 @@ private void assertContractLogs(RecordItem recordItem) {
         var contractFunctionResult = getFunctionResult(recordItem);
         var listAssert = assertThat(contractLogRepository.findAll()).hasSize(contractFunctionResult.getLogInfoCount());
         var transactionHash = getTransactionHash(recordItem);
-        Integer transactionIndex = transaction.getIndex();
+        var transactionIndex = recordItem.getTransactionIndex();
 
         if (contractFunctionResult.getLogInfoCount() > 0) {
             var blooms = new ArrayList<byte[]>();

diff --git a/...or-importer/src/test/java/com/hedera/mirror/importer/parser/domain/RecordItemBuilder.java b/...or-importer/src/test/java/com/hedera/mirror/importer/parser/domain/RecordItemBuilder.java
@@ -256,7 +256,7 @@ public Builder<ContractCallTransactionBody.Builder> contractCall(ContractID cont
         ContractCallTransactionBody.Builder transactionBody = ContractCallTransactionBody.newBuilder()
                 .setAmount(5_000L)
                 .setContractID(contractId)
-                .setFunctionParameters(bytes(64))
+                .setFunctionParameters(nonZeroBytes(64))
                 .setGas(10_000L);
 
         return new Builder<>(TransactionType.CONTRACTCALL, transactionBody)
@@ -850,6 +850,16 @@ public ByteString bytes(int length) {
         return ByteString.copyFrom(bytes);
     }
 
+    public ByteString nonZeroBytes(int length) {
+        byte[] bytes = randomBytes(length);
+        for (int i = 0; i < length; i++) {
+            if (bytes[i] == 0) {
+                bytes[i] = (byte) random.nextInt(1, Byte.MAX_VALUE);
+            }
+        }
+        return ByteString.copyFrom(bytes);
+    }
+
     // Helper methods
     private AccountAmount accountAmount(AccountID accountID, long amount) {
         return AccountAmount.newBuilder()

diff --git a/...irror/importer/parser/record/transactionhandler/ContractCreateTransactionHandlerTest.java b/...irror/importer/parser/record/transactionhandler/ContractCreateTransactionHandlerTest.java
@@ -692,10 +692,12 @@ private ObjectAssert<Entity> assertEntity(EntityId contractId, long timestamp) {
     @SuppressWarnings("deprecation")
     private Map<Long, EntityTransaction> getExpectedEntityTransactions(RecordItem recordItem, Transaction transaction) {
         var body = recordItem.getTransactionBody().getContractCreateInstance();
+        var autoRenewAccountId =
+                !body.getAutoRenewAccountId().hasAlias() ? EntityId.of(body.getAutoRenewAccountId()) : EntityId.EMPTY;
         return getExpectedEntityTransactions(
                 recordItem,
                 transaction,
-                EntityId.of(body.getAutoRenewAccountId()),
+                autoRenewAccountId,
                 EntityId.of(body.getFileID()),
                 EntityId.of(body.getProxyAccountID()),
                 EntityId.of(body.getStakedAccountId()));

diff --git a/...irror/importer/parser/record/transactionhandler/ContractUpdateTransactionHandlerTest.java b/...irror/importer/parser/record/transactionhandler/ContractUpdateTransactionHandlerTest.java
@@ -395,7 +395,8 @@ private void assertContractUpdate(long timestamp, EntityId contractId, Consumer<
     private Map<Long, EntityTransaction> getExpectedEntityTransactions(
             EntityId autoRenewAccountId, RecordItem recordItem, Transaction transaction) {
         var body = recordItem.getTransactionBody().getContractUpdateInstance();
-        if (EntityId.isEmpty(autoRenewAccountId)) {
+        if (EntityId.isEmpty(autoRenewAccountId)
+                && !body.getAutoRenewAccountId().hasAlias()) {
             autoRenewAccountId = EntityId.of(body.getAutoRenewAccountId());
         }