replaced sqlContext with sparkSessesion

MD2Korg · Jun 26, 2020 · 74b64e5 · 74b64e5
1 parent a398aa3
commit 74b64e5
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 5 deletions.
diff --git a/cerebralcortex/algorithms/utils/mprov_helper.py b/cerebralcortex/algorithms/utils/mprov_helper.py
@@ -47,7 +47,8 @@ def CC_MProvAgg(in_stream_name,op,out_stream_name,in_stream_key=['index'],out_st
         connection_key = MProvConnectionCache.Key(user=os.getenv("MPROV_USER"), password=os.getenv("MPROV_PASSWORD"),
                                                   host=os.getenv("MPROV_HOST"), graph=graph_name)
         mprov_conn = MProvConnectionCache.get_connection(connection_key)
-        mprov_conn.create_or_reset_graph()
+        if mprov_conn:
+            mprov_conn.create_or_reset_graph()
         return MProvAgg(in_stream_name=in_stream_name, op=op, out_stream_name=out_stream_name, in_stream_key=in_stream_key, out_stream_key=out_stream_key, map=map, connection_key=connection_key)
     else:
         return MProvAgg_empty()
diff --git a/cerebralcortex/core/config_manager/default.yml b/cerebralcortex/core/config_manager/default.yml
@@ -43,7 +43,7 @@ visualization_storage: none
 #########################################################################
 #                          Data Provenance Storage                      #
 #########################################################################
-mprov: pennprov
+mprov: none
 
 pennprov:
   host: http://127.0.0.1:8088

diff --git a/cerebralcortex/core/util/spark_helper.py b/cerebralcortex/core/util/spark_helper.py
@@ -56,14 +56,15 @@ def get_or_create_sc(type="sparkContext", name="CerebralCortex-Kernal", enable_s
     sc.setLogLevel("ERROR")
 
 
-    sqlContext = SQLContext(sc)
+    #sqlContext = SQLContext(sc)
+
     if type=="SparkSessionBuilder":
         return sc
     elif type=="sparkContext":
         return sc
     elif type=="sparkSession":
         return sparkSession
     elif type=="sqlContext":
-        return sqlContext
+        return sparkSession
     else:
         raise ValueError("Unknown type.")
diff --git a/setup.py b/setup.py
@@ -40,6 +40,9 @@
 
     version='3.2.1r4',
 
+    package_data={'': ['default.yml']},
+    include_package_data=True,
+
     description='Backend data analytics platform for MD2K software',
     long_description=long_description,
 
@@ -85,5 +88,5 @@
         ]
     },
 
-    #data_files=[('/etc/rsyslog.d', ['cerebralcortex/core/resources/20-cerebralcortex.conf']), ('/etc/logrotate.d', ['cerebralcortex/core/resources/cerebralcortex']),]
+    data_files=[('/etc/rsyslog.d', ['cerebralcortex/core/resources/20-cerebralcortex.conf']), ('/etc/logrotate.d', ['cerebralcortex/core/resources/cerebralcortex']),]
 )