Merge pull request #109 from KenSuenobu/ticket-102

Ticket-102
KenSuenobu · Jun 24, 2018 · e166da4 · e166da4
2 parents b433423 + 33a0503
commit e166da4
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 6 deletions.
diff --git a/...sphere-core/src/main/scala/io/buildfactory/scattersphere/core/util/spark/SparkCache.scala b/...sphere-core/src/main/scala/io/buildfactory/scattersphere/core/util/spark/SparkCache.scala
@@ -42,6 +42,15 @@ object SparkCache {
       throw new NullPointerException("Missing SparkConf object.")
     }
 
+    // Appends scattersphere-base to the jars list if not already added.
+    var currentJars: Seq[String] = conf.get("spark.jars").split(",")
+
+    if (!currentJars.contains("scattersphere-base")) {
+      currentJars = currentJars :+ "../scattersphere-base/target/scattersphere-base-0.2.1.jar"
+    }
+
+    conf.setJars(currentJars)
+
     SPARK_CONF_CACHE.put(key, conf)
   }
 

diff --git a/...re-core/src/test/scala/io/buildfactory/scattersphere/core/util/spark/SparkCacheTest.scala b/...re-core/src/test/scala/io/buildfactory/scattersphere/core/util/spark/SparkCacheTest.scala
@@ -33,8 +33,7 @@ class SparkCacheTest extends FlatSpec with Matchers with SimpleLogger {
     SparkCache.save("test", new SparkConf()
       .setMaster(Properties.envOrElse("SPARK_MASTER", "local[*]"))
       .setAppName("local pi test")
-      .setJars(Array("target/scattersphere-core-0.2.1-tests.jar",
-        "../scattersphere-base/target/scattersphere-base-0.2.1.jar"))
+      .setJars(Array("target/scattersphere-core-0.2.1-tests.jar"))
       .set("spark.ui.enabled", "false"))
     val spark: SparkSession = SparkCache.getSession("test")
     val sContext: SparkContext = spark.sparkContext

diff --git a/...e-tasks/src/test/scala/io/buildfactory/scattersphere/tasks/spark/SparkRealWorldTest.scala b/...e-tasks/src/test/scala/io/buildfactory/scattersphere/tasks/spark/SparkRealWorldTest.scala
@@ -36,8 +36,7 @@ class SparkRealWorldTest extends FlatSpec with Matchers with SimpleLogger {
   SparkCache.save("realWorldTest", new SparkConf()
     .setMaster(Properties.envOrElse("SPARK_MASTER", "local[*]"))
     .setAppName("local pi test")
-    .setJars(Array("target/scattersphere-tasks-0.2.1-tests.jar",
-      "../scattersphere-base/target/scattersphere-base-0.2.1.jar"))
+    .setJars(Array("target/scattersphere-tasks-0.2.1-tests.jar"))
     .set("spark.ui.enabled", "false"))
 
   "Spark Real World Test" should "be able to run in Spark" in {

diff --git a/...here-tasks/src/test/scala/io/buildfactory/scattersphere/tasks/spark/SparkTaskPiTest.scala b/...here-tasks/src/test/scala/io/buildfactory/scattersphere/tasks/spark/SparkTaskPiTest.scala
@@ -29,8 +29,7 @@ class SparkTaskPiTest extends FlatSpec with Matchers with SimpleLogger {
   SparkCache.save("sparkPiTestCache", new SparkConf()
     .setMaster(Properties.envOrElse("SPARK_MASTER", "local[*]"))
     .setAppName("local pi test")
-    .setJars(Array("target/scattersphere-tasks-0.2.1-tests.jar",
-      "../scattersphere-base/target/scattersphere-base-0.2.1.jar"))
+    .setJars(Array("target/scattersphere-tasks-0.2.1-tests.jar"))
     .set("spark.ui.enabled", "false"))
 
   "Spark task pi test" should "calculate Pi in the form of a task" in {