Merge remote-tracking branch 'origin/main' into 8000-rest-cache

Signed-off-by: Steven Sheehy <steven.sheehy@swirldslabs.com>

build.gradle.kts

@@ -99,7 +99,7 @@ dependencies {
         api("org.springframework.cloud:spring-cloud-dependencies:2023.0.1")
         api("org.testcontainers:junit-jupiter:1.19.7")
         api("org.mockito:mockito-inline:5.2.0")
-        api("software.amazon.awssdk:bom:2.25.35")
+        api("software.amazon.awssdk:bom:2.25.40")
         api("uk.org.webcompere:system-stubs-jupiter:2.1.6")
     }
 }

buildSrc/build.gradle.kts

@@ -36,15 +36,15 @@ dependencies {
     implementation("com.github.johnrengelman:shadow:8.1.1")
     implementation("com.github.node-gradle:gradle-node-plugin:7.0.2")
     implementation("com.google.protobuf:protobuf-gradle-plugin:0.9.4")
-    implementation("com.gorylenko.gradle-git-properties:gradle-git-properties:2.4.1")
+    implementation("com.gorylenko.gradle-git-properties:gradle-git-properties:2.4.2")
     implementation("com.graphql-java-generator:graphql-gradle-plugin3:2.4")
     implementation("gradle.plugin.io.snyk.gradle.plugin:snyk:0.6.1")
     implementation("gradle.plugin.org.flywaydb:gradle-plugin-publishing:$flywayVersion")
     implementation("io.freefair.gradle:lombok-plugin:8.6")
     implementation("io.spring.gradle:dependency-management-plugin:1.1.4")
     implementation("org.apache.commons:commons-compress:1.26.1")
     implementation("org.flywaydb:flyway-database-postgresql:$flywayVersion")
-    implementation("org.gradle:test-retry-gradle-plugin:1.5.8")
+    implementation("org.gradle:test-retry-gradle-plugin:1.5.9")
     implementation("org.jooq:jooq-codegen-gradle:$jooqVersion")
     implementation("org.jooq:jooq-meta:$jooqVersion")
     implementation("org.openapitools:openapi-generator-gradle-plugin:7.5.0")

charts/hedera-mirror/ci/v2-values.yaml

@@ -5,7 +5,6 @@ graphql:
 importer:
   env:
     SPRING_FLYWAY_PLACEHOLDERS_HASHSHARDCOUNT: "2"
-    SPRING_FLYWAY_PLACEHOLDERS_IDPARTITIONSIZE: "1000000000000000"
     SPRING_FLYWAY_PLACEHOLDERS_PARTITIONTIMEINTERVAL: "'10 years'"
     SPRING_FLYWAY_PLACEHOLDERS_SHARDCOUNT: "2"
     SPRING_PROFILES_ACTIVE: v2

docs/database/README.md

@@ -268,14 +268,14 @@ Following are the prerequisites and steps for migrating V1 data to V2.
 
 1. Create a Citus cluster with enough resources (Disk, CPU and memory). For GKE, an e2-custom-6-32768 can be used.
 2. Ensure the source and target schemas are compatible by deploying the same version to both.
-3. Create a VM with a disk big enough to contain the source data and checkout the source code to it.
-4. Populate correct values for the source and target configuration in the
+3. Populate correct values for the source and target configuration in the
    [migration.config](/hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.config).
-5. Get the correct version of [flyway](https://flywaydb.org/documentation/usage/commandline/) based on your OS and
+4. Get the correct version of [flyway](https://flywaydb.org/documentation/usage/commandline/) based on your OS and
    update it in the `FLYWAY_URL` field in the `migration.config` file. The default is set to the linux version.
-6. Stop the [Importer](/docs/importer/README.md) process.
-7. Run the [migration.sh](/hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.sh) script.
-8. Update the mirror node configuration to point to the new Citus DB and start it.
+5. Stop the [Importer](/docs/importer/README.md) process. 
+6. Run the [migration.sh](/hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.sh) script. Due to the time it will take to complete the migration, 
+   it is recommended to run the script in a way that doesn't require your terminal session to remain open (e.g. `./migration.sh > migration.log 2> migration-error.log & disown`)
+7. Update the mirror node configuration to point to the new Citus DB and start it.
 
 ## Citus Backup and Restore
 

hedera-mirror-importer/src/main/resources/db/scripts/v2/migration.config

@@ -1,6 +1,22 @@
-CONCURRENCY=
-EXCLUDED_TABLES=("'entity_state_start'","'flyway_schema_history'","'transaction_hash'")
+# Tables to migrate asynchronously. These tables will run last and must contain an indexed consensus_timestamp column
+ASYNC_TABLES=("'crypto_transfer'","'transaction'")
+
+# Number of copy jobs to create for each table
+ASYNC_TABLE_SPLITS=1000
+
+# Number of tables allowed to be copied concurrently
+CONCURRENCY=5
+
+#Number of copy jobs allowed to run concurrently for each table. Only applies to ASYNC_TABLES
+CONCURRENT_COPIES_PER_TABLE=5
+
+# Tables to exclude from migration by this script
+EXCLUDED_TABLES=("'entity_state_start'","'entity_transaction'","'flyway_schema_history'","'transaction_hash'")
+
+# Download url for flyway
 FLYWAY_URL=https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/9.22.3/flyway-commandline-9.22.3-linux-x64.tar.gz
+
+# Maxmium timestamp to copy. If not set, will attempt to load ./migration.env and will fall back to 'select max(consensus_end) from record_file'
 MAX_TIMESTAMP=
 SOURCE_DB_HOST=
 SOURCE_DB_NAME=