Implementing customized exports for PDK data types.

generators/pdk_app_event.py

@@ -1,6 +1,56 @@
+import calendar
+import csv
+import json
+import tempfile
+
+from ..models import DataPoint, install_supports_jsonfield
 
 def extract_secondary_identifier(properties):
     if 'event_name' in properties:
         return properties['event_name']
 
     return None
+
+def compile_report(generator, sources):
+    filename = tempfile.gettempdir() + '/pdk_' + generator + '.txt'
+
+    with open(filename, 'w') as outfile:
+        writer = csv.writer(outfile, delimiter='\t')
+
+        writer.writerow([
+            'Source',
+            'Created Timestamp',
+            'Created Date',
+            'Event Name',
+            'Event Properties'
+        ])
+
+        for source in sources:
+            points = DataPoint.objects.filter(source=source, generator_identifier=generator).order_by('created') # pylint: disable=no-member,line-too-long
+
+            index = 0
+            count = points.count()
+
+            while index < count:
+                for point in points[index:(index + 5000)]:
+                    row = []
+
+                    row.append(point.source)
+                    row.append(calendar.timegm(point.created.utctimetuple()))
+                    row.append(point.created.isoformat())
+
+                    properties = {}
+
+                    if install_supports_jsonfield():
+                        properties = point.properties
+                    else:
+                        properties = json.loads(point.properties)
+
+                    row.append(properties['event_name'])
+                    row.append(json.dumps(properties['event_details']))
+
+                    writer.writerow(row)
+
+                index += 5000
+
+    return filename

management/commands/compile_reports.py

@@ -6,6 +6,7 @@
 import json
 import os
 import tempfile
+# import traceback
 
 from zipfile import ZipFile
 
@@ -16,8 +17,8 @@
 from django.template.loader import render_to_string
 from django.utils import timezone
 
-from passive_data_kit.decorators import handle_lock
-from passive_data_kit.models import DataPoint, ReportJob
+from ...decorators import handle_lock
+from ...models import DataPoint, ReportJob, install_supports_jsonfield
 
 class Command(BaseCommand):
     help = 'Compiles data reports requested by end users.'
@@ -48,12 +49,19 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b
             report.started = timezone.now()
             report.save()
 
-            sources = report.parameters['sources']
-            generators = report.parameters['generators']
+            parameters = {}
+
+            if install_supports_jsonfield():
+                parameters = report.parameters
+            else:
+                parameters = json.loads(report.parameters)
+
+            sources = parameters['sources']
+            generators = parameters['generators']
 
             raw_json = False
 
-            if ('raw_data' in report.parameters) and report.parameters['raw_data'] is True:
+            if ('raw_data' in parameters) and parameters['raw_data'] is True:
                 raw_json = True
 
             filename = tempfile.gettempdir() + '/pdk_export_' + str(report.pk) + '.zip'
@@ -115,10 +123,10 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b
 
                                     output_file = pdk_api.compile_report(generator, sources)
                                 except ImportError:
-    #                                traceback.print_exc()
+#                                    traceback.print_exc()
                                     output_file = None
                                 except AttributeError:
-    #                                traceback.print_exc()
+#                                    traceback.print_exc()
                                     output_file = None
 
                         if output_file is not None:
@@ -142,7 +150,10 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b
                 'url': settings.SITE_URL
             })
 
-            host = settings.SITE_URL.split('/')[-2]
+            host = settings.SITE_URL.split('/')[-1]
+
+            if len(settings.SITE_URL.split('/')) > 2:
+                host = settings.SITE_URL.split('/')[-1]
 
             send_mail(subject, \
                       message, \

models.py

@@ -185,5 +185,9 @@ class ReportJob(models.Model):
 @receiver(post_delete, sender=ReportJob)
 def report_job_post_delete_handler(sender, **kwargs): # pylint: disable=unused-argument
     job = kwargs['instance']
-    storage, path = job.report.storage, job.report.path
-    storage.delete(path)
+
+    try:
+        storage, path = job.report.storage, job.report.path
+        storage.delete(path)
+    except ValueError:
+        pass

pdk_api.py

@@ -1,5 +1,7 @@
+# pylint: disable=line-too-long
 import calendar
 import csv
+import importlib
 import json
 import tempfile
 
@@ -28,6 +30,18 @@
 #    return None
 
 def compile_report(generator, sources):
+    try:
+        generator_module = importlib.import_module('.generators.' + generator.replace('-', '_'), package='passive_data_kit')
+
+        output_file = generator_module.compile_report(generator, sources)
+
+        if output_file is not None:
+            return output_file
+    except ImportError:
+        pass
+    except AttributeError:
+        pass
+
     filename = tempfile.gettempdir() + '/pdk_' + generator + '.txt'
 
     with open(filename, 'w') as outfile:
@@ -53,8 +67,6 @@ def compile_report(generator, sources):
             count = points.count()
 
             while index < count:
-#                print(source + '/' + generator + ': ' + str(index) + ' / ' + str(count))
-#
                 for point in points[index:(index + 5000)]:
                     row = []
 

views.py

@@ -9,7 +9,7 @@
 from django.core.management import call_command
 from django.http import HttpResponse, HttpResponseNotAllowed, JsonResponse, HttpResponseNotFound, \
                         FileResponse
-from django.shortcuts import render_to_response
+from django.shortcuts import render
 from django.utils import timezone
 from django.views.decorators.csrf import csrf_exempt
 
@@ -202,7 +202,7 @@ def pdk_home(request):
     context['groups'] = DataSourceGroup.objects.order_by('name')
     context['solo_sources'] = DataSource.objects.filter(group=None).order_by('name')
 
-    return render_to_response('pdk_home.html', context)
+    return render(request, 'pdk_home.html', context=context)
 
 @staff_member_required
 def pdk_source(request, source_id): # pylint: disable=unused-argument
@@ -216,7 +216,7 @@ def pdk_source(request, source_id): # pylint: disable=unused-argument
 
     context['source'] = source
 
-    return render_to_response('pdk_source.html', context)
+    return render(request, 'pdk_source.html', context=context)
 
 @staff_member_required
 def pdk_source_generator(request, source_id, generator_id): # pylint: disable=unused-argument
@@ -244,7 +244,7 @@ def pdk_source_generator(request, source_id, generator_id): # pylint: disable=un
         except AttributeError:
             pass
 
-    return render_to_response('pdk_source_generator.html', context)
+    return render(request, 'pdk_source_generator.html', context=context)
 
 @staff_member_required
 def unmatched_sources(request): # pylint: disable=unused-argument
@@ -287,7 +287,7 @@ def pdk_download_report(request, report_id): # pylint: disable=unused-argument
     return response
 
 @staff_member_required
-def pdk_export(request):
+def pdk_export(request): # pylint: disable=too-many-branches
     context = {}
 
     context['sources'] = DataPoint.objects.all().order_by('source')\
@@ -339,11 +339,14 @@ def pdk_export(request):
             if 'export_raw_json' in request.POST and request.POST['export_raw_json']:
                 params['raw_data'] = True
 
-            job.parameters = params
+            if install_supports_jsonfield():
+                job.parameters = params
+            else:
+                job.parameters = json.dumps(params, indent=2)
 
             job.save()
 
             context['message_type'] = 'ok'
             context['message'] = 'Export job queued. Check your e-mail for a link to the output when the export is complete.' # pylint: disable=line-too-long
 
-    return render_to_response('pdk_export.html', context)
+    return render(request, 'pdk_export.html', context=context)