close ITestMetastoreDatabase in all tests (#3946)

apache · May 8, 2024 · 8eeb743 · 8eeb743
1 parent c1a09f0
commit 8eeb743
Show file tree

Hide file tree

Showing 15 changed files with 140 additions and 36 deletions.
diff --git a/...a/org/apache/gobblin/data/management/retention/CleanableMysqlDatasetStoreDatasetTest.java b/...a/org/apache/gobblin/data/management/retention/CleanableMysqlDatasetStoreDatasetTest.java
@@ -23,6 +23,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.testng.Assert;
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
@@ -108,6 +109,13 @@ public void setUp() throws Exception {
     this.dbDatasetStateStore.delete(TEST_JOB_NAME2);
   }
 
+  @AfterClass(alwaysRun = true)
+  public void tearDown() throws Exception {
+    if (testMetastoreDatabase != null) {
+      testMetastoreDatabase.close();
+    }
+  }
+
   /**
    * Test cleanup of the job state store
    * @throws IOException

diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
@@ -332,5 +332,8 @@ public void testDeleteDatasetJobState() throws IOException {
   public void tearDown() throws IOException {
     dbJobStateStore.delete(TEST_JOB_NAME);
     dbDatasetStateStore.delete(TEST_JOB_NAME);
+    if (testMetastoreDatabase != null) {
+      testMetastoreDatabase.close();
+    }
   }
 }
diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/cli/JobStateStoreCliTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/cli/JobStateStoreCliTest.java
@@ -21,6 +21,11 @@
 import java.io.FileOutputStream;
 import java.io.OutputStream;
 
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
 import com.zaxxer.hikari.HikariDataSource;
 
 import org.apache.gobblin.config.ConfigBuilder;
@@ -36,9 +41,6 @@
 import org.apache.gobblin.runtime.TaskState;
 import org.apache.gobblin.util.ClassAliasResolver;
 import org.apache.gobblin.util.ConfigUtils;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
 
 
 @Test(groups = { "gobblin.runtime" })
@@ -174,4 +176,10 @@ public void testCliDeleteSingle() throws Exception {
     Assert.assertNull(jobState);
   }
 
+  @AfterClass(alwaysRun = true)
+  public void tearDown() throws Exception {
+    if (this.testMetastoreDatabase != null) {
+      this.testMetastoreDatabase.close();
+    }
+  }
 }
diff --git a/...lin-runtime/src/test/java/org/apache/gobblin/runtime/job_catalog/TestMysqlJobCatalog.java b/...lin-runtime/src/test/java/org/apache/gobblin/runtime/job_catalog/TestMysqlJobCatalog.java
@@ -17,10 +17,18 @@
 
 package org.apache.gobblin.runtime.job_catalog;
 
-import com.google.common.base.Predicates;
-import com.typesafe.config.Config;
 import java.net.URI;
 import java.util.concurrent.TimeUnit;
+
+import org.mockito.Mockito;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Predicates;
+import com.typesafe.config.Config;
+
 import org.apache.gobblin.config.ConfigBuilder;
 import org.apache.gobblin.configuration.ConfigurationKeys;
 import org.apache.gobblin.instrumented.GobblinMetricsKeys;
@@ -30,10 +38,6 @@
 import org.apache.gobblin.runtime.api.JobCatalog;
 import org.apache.gobblin.runtime.api.JobCatalogListener;
 import org.apache.gobblin.runtime.api.JobSpec;
-import org.mockito.Mockito;
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
 
 
 /** Verify {@link MysqlJobCatalog} [modeled on {@link TestInMemoryJobCatalog}] */
@@ -43,11 +47,12 @@ public class TestMysqlJobCatalog {
   private static final String TABLE = "job_catalog";
 
   private MysqlJobCatalog cat;
+  private static ITestMetastoreDatabase testDb;
 
   /** create a new DB/`JobCatalog` for each test, so they're completely independent */
   @BeforeMethod
   public void setUp() throws Exception {
-    ITestMetastoreDatabase testDb = TestMetastoreDatabaseFactory.get();
+    testDb = TestMetastoreDatabaseFactory.get();
 
     Config config = ConfigBuilder.create()
         .addPrimitive(ConfigurationKeys.METRICS_ENABLED_KEY, "true")
@@ -209,4 +214,11 @@ public void testMetrics() throws Exception {
     cat.stopAsync();
     cat.awaitTerminated(1, TimeUnit.SECONDS);
   }
+
+  @AfterClass(alwaysRun = true)
+  public void tearDown() throws Exception {
+    if (testDb != null) {
+      testDb.close();
+    }
+  }
 }
diff --git a/...n-runtime/src/test/java/org/apache/gobblin/runtime/spec_store/MysqlBaseSpecStoreTest.java b/...n-runtime/src/test/java/org/apache/gobblin/runtime/spec_store/MysqlBaseSpecStoreTest.java
@@ -17,26 +17,30 @@
 
 package org.apache.gobblin.runtime.spec_store;
 
-import com.google.common.collect.Iterators;
-import com.typesafe.config.Config;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
+
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.Iterators;
+import com.typesafe.config.Config;
+
 import org.apache.gobblin.config.ConfigBuilder;
 import org.apache.gobblin.configuration.ConfigurationKeys;
 import org.apache.gobblin.metastore.testing.ITestMetastoreDatabase;
 import org.apache.gobblin.metastore.testing.TestMetastoreDatabaseFactory;
-import org.apache.gobblin.runtime.api.TopologySpec;
 import org.apache.gobblin.runtime.api.FlowSpecSearchObject;
 import org.apache.gobblin.runtime.api.Spec;
+import org.apache.gobblin.runtime.api.TopologySpec;
 import org.apache.gobblin.runtime.spec_executorInstance.MockedSpecExecutor;
 import org.apache.gobblin.runtime.spec_serde.JavaSpecSerDe;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
 
 
 public class MysqlBaseSpecStoreTest {
@@ -48,14 +52,15 @@ public class MysqlBaseSpecStoreTest {
   private final URI uri1 = new URI(new TopologySpec.Builder().getDefaultTopologyCatalogURI().toString() + "1");
   private final URI uri2 = new URI(new TopologySpec.Builder().getDefaultTopologyCatalogURI().toString() + "2");
   private TopologySpec topoSpec1, topoSpec2;
+  private static ITestMetastoreDatabase testDb;
 
   public MysqlBaseSpecStoreTest()
       throws URISyntaxException { // (based on `uri1` and other initializations just above)
   }
 
   @BeforeClass
   public void setUp() throws Exception {
-    ITestMetastoreDatabase testDb = TestMetastoreDatabaseFactory.get();
+    testDb = TestMetastoreDatabaseFactory.get();
 
     // prefix keys to demonstrate disambiguation mechanism used to side-step intentially-sabatoged non-prefixed, 'fallback'
     Config config = ConfigBuilder.create()
@@ -86,6 +91,13 @@ public void setUp() throws Exception {
         .build();
   }
 
+  @AfterClass(alwaysRun = true)
+  public void tearDown() throws Exception {
+    if (testDb != null) {
+      testDb.close();
+    }
+  }
+
   @Test(expectedExceptions = UnsupportedOperationException.class)
   public void testSpecSearchUnsupported() throws Exception {
     FlowSpecSearchObject flowSpecSearchObject = FlowSpecSearchObject.builder().build();

diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/spec_store/MysqlSpecStoreTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/spec_store/MysqlSpecStoreTest.java
@@ -30,6 +30,7 @@
 
 import org.apache.commons.lang3.ArrayUtils;
 import org.testng.Assert;
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
@@ -64,14 +65,15 @@ public class MysqlSpecStoreTest {
   private final URI uri3 = FlowSpec.Utils.createFlowSpecUri(new FlowId().setFlowName("fg3").setFlowGroup("fn3"));
   private final URI uri4 = FlowSpec.Utils.createFlowSpecUri(new FlowId().setFlowName("fg4").setFlowGroup("fn4"));
   private FlowSpec flowSpec1, flowSpec2, flowSpec3, flowSpec4;
+  private static ITestMetastoreDatabase testDb;
 
   public MysqlSpecStoreTest()
       throws URISyntaxException { // (based on `uri1` and other initializations just above)
   }
 
   @BeforeClass
   public void setUp() throws Exception {
-    ITestMetastoreDatabase testDb = TestMetastoreDatabaseFactory.get();
+    testDb = TestMetastoreDatabaseFactory.get();
 
     Config config = ConfigBuilder.create()
         .addPrimitive(ConfigurationKeys.STATE_STORE_DB_URL_KEY, testDb.getJdbcUrl())
@@ -130,6 +132,13 @@ public void setUp() throws Exception {
         .build();
   }
 
+  @AfterClass(alwaysRun = true)
+  public void tearDown() throws Exception {
+    if (testDb != null) {
+      testDb.close();
+    }
+  }
+
   @Test(expectedExceptions = IOException.class)
   public void testSpecSearch() throws Exception {
     // empty FlowSpecSearchObject should throw an error

diff --git a/...ime/src/test/java/org/apache/gobblin/runtime/spec_store/MysqlSpecStoreWithUpdateTest.java b/...ime/src/test/java/org/apache/gobblin/runtime/spec_store/MysqlSpecStoreWithUpdateTest.java
@@ -30,6 +30,7 @@
 
 import org.apache.commons.lang3.ArrayUtils;
 import org.testng.Assert;
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
@@ -64,14 +65,15 @@ public class MysqlSpecStoreWithUpdateTest {
   private final URI uri3 = FlowSpec.Utils.createFlowSpecUri(new FlowId().setFlowName("fg3").setFlowGroup("fn3"));
   private final URI uri4 = FlowSpec.Utils.createFlowSpecUri(new FlowId().setFlowName("fg4").setFlowGroup("fn4"));
   private FlowSpec flowSpec1, flowSpec2, flowSpec3, flowSpec4, flowSpec4_update;
+  private static ITestMetastoreDatabase testDb;
 
   public MysqlSpecStoreWithUpdateTest()
       throws URISyntaxException { // (based on `uri1` and other initializations just above)
   }
 
   @BeforeClass
   public void setUp() throws Exception {
-    ITestMetastoreDatabase testDb = TestMetastoreDatabaseFactory.get();
+    testDb = TestMetastoreDatabaseFactory.get();
 
     Config config = ConfigBuilder.create()
         .addPrimitive(ConfigurationKeys.STATE_STORE_DB_URL_KEY, testDb.getJdbcUrl())
@@ -140,6 +142,13 @@ public void setUp() throws Exception {
         .build();
   }
 
+  @AfterClass(alwaysRun = true)
+  public void tearDown() throws Exception {
+    if (testDb != null) {
+      testDb.close();
+    }
+  }
+
   @Test(expectedExceptions = IOException.class)
   public void testSpecSearch() throws Exception {
     // empty FlowSpecSearchObject should throw an error

diff --git a/...n-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceHATest.java b/...n-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceHATest.java
@@ -102,6 +102,7 @@ public class GobblinServiceHATest {
   private TestingServer testingZKServer;
 
   private MySQLContainer mysql;
+  private static ITestMetastoreDatabase testMetastoreDatabase;
 
   @BeforeClass
   public void setup() throws Exception {
@@ -122,7 +123,7 @@ public void setup() throws Exception {
     HelixUtils.createGobblinHelixCluster(testingZKServer.getConnectString(), TEST_HELIX_CLUSTER_NAME);
 
 
-    ITestMetastoreDatabase testMetastoreDatabase = TestMetastoreDatabaseFactory.get();
+    testMetastoreDatabase = TestMetastoreDatabaseFactory.get();
 
     Properties commonServiceCoreProperties = new Properties();
 
@@ -199,8 +200,11 @@ private void cleanUpDir(String dir) throws Exception {
     }
   }
 
-  @AfterClass
+  @AfterClass(alwaysRun = true)
   public void cleanUp() throws Exception {
+    if (testMetastoreDatabase != null) {
+      testMetastoreDatabase.close();
+    }
     // Shutdown Node 1
     try {
       logger.info("+++++++++++++++++++ start shutdown noad1");

diff --git a/...ice/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceRedirectTest.java b/...ice/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceRedirectTest.java
@@ -109,6 +109,7 @@ public class GobblinServiceRedirectTest {
   private Properties node2ServiceCoreProperties;
 
   private MySQLContainer mysql;
+  private static ITestMetastoreDatabase testMetastoreDatabase;
 
   @BeforeClass
   public void setup() throws Exception {
@@ -123,7 +124,7 @@ public void setup() throws Exception {
     logger.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());
     HelixUtils.createGobblinHelixCluster(testingZKServer.getConnectString(), TEST_HELIX_CLUSTER_NAME);
 
-    ITestMetastoreDatabase testMetastoreDatabase = TestMetastoreDatabaseFactory.get();
+    testMetastoreDatabase = TestMetastoreDatabaseFactory.get();
 
     Properties commonServiceCoreProperties = new Properties();
 
@@ -223,6 +224,9 @@ public void cleanUp() throws Exception {
     }
 
     mysql.stop();
+    if (testMetastoreDatabase != null) {
+      testMetastoreDatabase.close();
+    }
   }
 
   @Test

diff --git a/...ce/src/test/java/org/apache/gobblin/service/modules/orchestration/DagManagerFlowTest.java b/...ce/src/test/java/org/apache/gobblin/service/modules/orchestration/DagManagerFlowTest.java
@@ -26,8 +26,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import org.apache.gobblin.runtime.spec_catalog.FlowCatalog;
-import org.apache.gobblin.service.monitoring.FlowStatusGenerator;
 import org.mockito.Mockito;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
@@ -47,10 +45,12 @@
 import org.apache.gobblin.metastore.testing.TestMetastoreDatabaseFactory;
 import org.apache.gobblin.runtime.api.FlowSpec;
 import org.apache.gobblin.runtime.api.TopologySpec;
+import org.apache.gobblin.runtime.spec_catalog.FlowCatalog;
 import org.apache.gobblin.service.ExecutionStatus;
 import org.apache.gobblin.service.FlowId;
 import org.apache.gobblin.service.modules.flowgraph.Dag;
 import org.apache.gobblin.service.modules.spec.JobExecutionPlan;
+import org.apache.gobblin.service.monitoring.FlowStatusGenerator;
 import org.apache.gobblin.service.monitoring.JobStatusRetriever;
 import org.apache.gobblin.testing.AssertWithBackoff;
 import org.apache.gobblin.util.ConfigUtils;
@@ -74,12 +74,13 @@ public class DagManagerFlowTest {
   private static final String flowExecutionId = "12345677";
   private static final String flowExecutionId_2 = "12345678";
   private DagActionStore dagActionStore;
+  private static ITestMetastoreDatabase testDb;
 
   @BeforeClass
   public void setUp() throws Exception {
     Properties props = new Properties();
     props.put(DagManager.JOB_STATUS_POLLING_INTERVAL_KEY, 1);
-    ITestMetastoreDatabase testDb = TestMetastoreDatabaseFactory.get();
+    testDb = TestMetastoreDatabaseFactory.get();
 
     Config config = ConfigBuilder.create()
         .addPrimitive("MysqlDagActionStore." + ConfigurationKeys.STATE_STORE_DB_URL_KEY, testDb.getJdbcUrl())
@@ -102,6 +103,9 @@ public void setUp() throws Exception {
   public void cleanUp() throws Exception {
     dagManager.setActive(false);
     Assert.assertEquals(dagManager.getHouseKeepingThreadPool().isShutdown(), true);
+    if (testDb != null) {
+      testDb.close();
+    }
   }
 
   @Test