diff --git a/generators/pdk_app_event.py b/generators/pdk_app_event.py index 15afe18..7ef86d4 100644 --- a/generators/pdk_app_event.py +++ b/generators/pdk_app_event.py @@ -1,6 +1,56 @@ +import calendar +import csv +import json +import tempfile + +from ..models import DataPoint, install_supports_jsonfield def extract_secondary_identifier(properties): if 'event_name' in properties: return properties['event_name'] return None + +def compile_report(generator, sources): + filename = tempfile.gettempdir() + '/pdk_' + generator + '.txt' + + with open(filename, 'w') as outfile: + writer = csv.writer(outfile, delimiter='\t') + + writer.writerow([ + 'Source', + 'Created Timestamp', + 'Created Date', + 'Event Name', + 'Event Properties' + ]) + + for source in sources: + points = DataPoint.objects.filter(source=source, generator_identifier=generator).order_by('created') # pylint: disable=no-member,line-too-long + + index = 0 + count = points.count() + + while index < count: + for point in points[index:(index + 5000)]: + row = [] + + row.append(point.source) + row.append(calendar.timegm(point.created.utctimetuple())) + row.append(point.created.isoformat()) + + properties = {} + + if install_supports_jsonfield(): + properties = point.properties + else: + properties = json.loads(point.properties) + + row.append(properties['event_name']) + row.append(json.dumps(properties['event_details'])) + + writer.writerow(row) + + index += 5000 + + return filename diff --git a/management/commands/compile_reports.py b/management/commands/compile_reports.py index 02e573f..ecf7ceb 100644 --- a/management/commands/compile_reports.py +++ b/management/commands/compile_reports.py @@ -6,6 +6,7 @@ import json import os import tempfile +# import traceback from zipfile import ZipFile @@ -16,8 +17,8 @@ from django.template.loader import render_to_string from django.utils import timezone -from passive_data_kit.decorators import handle_lock -from passive_data_kit.models import DataPoint, ReportJob +from ...decorators import handle_lock +from ...models import DataPoint, ReportJob, install_supports_jsonfield class Command(BaseCommand): help = 'Compiles data reports requested by end users.' @@ -48,12 +49,19 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b report.started = timezone.now() report.save() - sources = report.parameters['sources'] - generators = report.parameters['generators'] + parameters = {} + + if install_supports_jsonfield(): + parameters = report.parameters + else: + parameters = json.loads(report.parameters) + + sources = parameters['sources'] + generators = parameters['generators'] raw_json = False - if ('raw_data' in report.parameters) and report.parameters['raw_data'] is True: + if ('raw_data' in parameters) and parameters['raw_data'] is True: raw_json = True filename = tempfile.gettempdir() + '/pdk_export_' + str(report.pk) + '.zip' @@ -115,10 +123,10 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b output_file = pdk_api.compile_report(generator, sources) except ImportError: - # traceback.print_exc() +# traceback.print_exc() output_file = None except AttributeError: - # traceback.print_exc() +# traceback.print_exc() output_file = None if output_file is not None: @@ -142,7 +150,10 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b 'url': settings.SITE_URL }) - host = settings.SITE_URL.split('/')[-2] + host = settings.SITE_URL.split('/')[-1] + + if len(settings.SITE_URL.split('/')) > 2: + host = settings.SITE_URL.split('/')[-1] send_mail(subject, \ message, \ diff --git a/models.py b/models.py index f568e01..23418ea 100644 --- a/models.py +++ b/models.py @@ -185,5 +185,9 @@ class ReportJob(models.Model): @receiver(post_delete, sender=ReportJob) def report_job_post_delete_handler(sender, **kwargs): # pylint: disable=unused-argument job = kwargs['instance'] - storage, path = job.report.storage, job.report.path - storage.delete(path) + + try: + storage, path = job.report.storage, job.report.path + storage.delete(path) + except ValueError: + pass diff --git a/pdk_api.py b/pdk_api.py index df5787e..96237e0 100644 --- a/pdk_api.py +++ b/pdk_api.py @@ -1,5 +1,7 @@ +# pylint: disable=line-too-long import calendar import csv +import importlib import json import tempfile @@ -28,6 +30,18 @@ # return None def compile_report(generator, sources): + try: + generator_module = importlib.import_module('.generators.' + generator.replace('-', '_'), package='passive_data_kit') + + output_file = generator_module.compile_report(generator, sources) + + if output_file is not None: + return output_file + except ImportError: + pass + except AttributeError: + pass + filename = tempfile.gettempdir() + '/pdk_' + generator + '.txt' with open(filename, 'w') as outfile: @@ -53,8 +67,6 @@ def compile_report(generator, sources): count = points.count() while index < count: -# print(source + '/' + generator + ': ' + str(index) + ' / ' + str(count)) -# for point in points[index:(index + 5000)]: row = [] diff --git a/views.py b/views.py index 22e05a0..05b6ecf 100644 --- a/views.py +++ b/views.py @@ -9,7 +9,7 @@ from django.core.management import call_command from django.http import HttpResponse, HttpResponseNotAllowed, JsonResponse, HttpResponseNotFound, \ FileResponse -from django.shortcuts import render_to_response +from django.shortcuts import render from django.utils import timezone from django.views.decorators.csrf import csrf_exempt @@ -202,7 +202,7 @@ def pdk_home(request): context['groups'] = DataSourceGroup.objects.order_by('name') context['solo_sources'] = DataSource.objects.filter(group=None).order_by('name') - return render_to_response('pdk_home.html', context) + return render(request, 'pdk_home.html', context=context) @staff_member_required def pdk_source(request, source_id): # pylint: disable=unused-argument @@ -216,7 +216,7 @@ def pdk_source(request, source_id): # pylint: disable=unused-argument context['source'] = source - return render_to_response('pdk_source.html', context) + return render(request, 'pdk_source.html', context=context) @staff_member_required def pdk_source_generator(request, source_id, generator_id): # pylint: disable=unused-argument @@ -244,7 +244,7 @@ def pdk_source_generator(request, source_id, generator_id): # pylint: disable=un except AttributeError: pass - return render_to_response('pdk_source_generator.html', context) + return render(request, 'pdk_source_generator.html', context=context) @staff_member_required def unmatched_sources(request): # pylint: disable=unused-argument @@ -287,7 +287,7 @@ def pdk_download_report(request, report_id): # pylint: disable=unused-argument return response @staff_member_required -def pdk_export(request): +def pdk_export(request): # pylint: disable=too-many-branches context = {} context['sources'] = DataPoint.objects.all().order_by('source')\ @@ -339,11 +339,14 @@ def pdk_export(request): if 'export_raw_json' in request.POST and request.POST['export_raw_json']: params['raw_data'] = True - job.parameters = params + if install_supports_jsonfield(): + job.parameters = params + else: + job.parameters = json.dumps(params, indent=2) job.save() context['message_type'] = 'ok' context['message'] = 'Export job queued. Check your e-mail for a link to the output when the export is complete.' # pylint: disable=line-too-long - return render_to_response('pdk_export.html', context) + return render(request, 'pdk_export.html', context=context)