diff --git a/admin.py b/admin.py index 0f8e5f2..4616534 100644 --- a/admin.py +++ b/admin.py @@ -30,11 +30,27 @@ class DataSourceAdmin(admin.OSMGeoAdmin): list_display = ('name', 'identifier', 'group') list_filter = ('group',) + +def reset_report_jobs(modeladmin, request, queryset): # pylint: disable=unused-argument + for job in queryset: + job.started = None + job.completed = None + + if job.report is not None: + job.report.delete() + job.report = None + + job.save() + +reset_report_jobs.description = 'Reset report jobs' + @admin.register(ReportJob) class ReportJobAdmin(admin.OSMGeoAdmin): list_display = ('requester', 'requested', 'started', 'completed') list_filter = ('requested', 'started', 'completed',) + actions = [reset_report_jobs] + @admin.register(DataSourceAlert) class DataSourceAlertAdmin(admin.OSMGeoAdmin): list_display = ('alert_name', 'data_source', 'generator_identifier', 'active', 'alert_level', \ diff --git a/generators/pdk_sensor_accelerometer.py b/generators/pdk_sensor_accelerometer.py new file mode 100644 index 0000000..c9d8b0e --- /dev/null +++ b/generators/pdk_sensor_accelerometer.py @@ -0,0 +1,166 @@ +# pylint: disable=line-too-long, no-member + +import calendar +import csv +import datetime +import os +import tempfile +import time + +from zipfile import ZipFile + +import arrow + +from django.template.loader import render_to_string +from django.utils import timezone +from django.utils.text import slugify + +from ..models import DataPoint + +WINDOW_SIZE = 300 + +def fetch_values(source, generator, start, end): + values = [] + + index = start + current_value = None + + for point in DataPoint.objects.filter(source=source.identifier, generator_identifier=generator, created__gt=start, created__lte=end).order_by('created'): + if (point.created - index).total_seconds() > WINDOW_SIZE: + if current_value is not None and current_value['min_value'] != -1: # pylint: disable=unsubscriptable-object + values.append(current_value) + + current_value = None + index = index + datetime.timedelta(seconds=WINDOW_SIZE) + + if current_value is None: + current_value = { + 'min_value': -1, + 'max_value': -1, + 'timestamp': time.mktime(index.timetuple()), + 'created': index, + 'duration': WINDOW_SIZE + } + + properties = point.fetch_properties() + + for level in properties['sensor_data']['light_level']: + if current_value['min_value'] == -1 or level < current_value['min_value']: + current_value['min_value'] = level + + if current_value['max_value'] == -1 or level > current_value['max_value']: + current_value['max_value'] = level + +# duration = properties['sensor_data']['observed'][-1] - properties['sensor_data']['observed'][0] +# timestamp = properties['sensor_data']['observed'][0] + (duration / 2) +# +# value['min_value'] = min_value +# value['max_value'] = max_value +# value['duration'] = duration / (1000 * 1000 * 1000) +# value['timestamp'] = timestamp / (1000 * 1000 * 1000) +# value['created'] = point.created +# +# values.append(value) + + return values + + +def generator_name(identifier): # pylint: disable=unused-argument + return 'Accelerometer Sensor' + +def visualization_todo(source, generator): + context = {} + context['source'] = source + context['generator_identifier'] = generator + + end = timezone.now() + start = end - datetime.timedelta(days=1) + + context['values'] = fetch_values(source, generator, start, end) + + context['start'] = time.mktime(start.timetuple()) + context['end'] = time.mktime(end.timetuple()) + + render_to_string('pdk_sensor_light_template.html', context) + +def visualization(source, generator): # pylint: disable=unused-argument + return 'TODO' + +def data_table_todo(source, generator): + context = {} + context['source'] = source + context['generator_identifier'] = generator + + end = timezone.now() + start = end - datetime.timedelta(days=1) + + context['values'] = fetch_values(source, generator, start, end) + + return render_to_string('pdk_sensor_light_table_template.html', context) + +def data_table(source, generator): # pylint: disable=unused-argument + return 'TODO' + +def compile_report(generator, sources): # pylint: disable=too-many-locals + filename = tempfile.gettempdir() + '/pdk_export_' + str(arrow.get().timestamp) + '.zip' + + with ZipFile(filename, 'w') as export_file: + for source in sources: + identifier = slugify(generator + '__' + source) + + secondary_filename = tempfile.gettempdir() + '/' + identifier + '.txt' + + with open(secondary_filename, 'w') as outfile: + writer = csv.writer(outfile, delimiter='\t') + + columns = [ + 'Source', + 'Created Timestamp', + 'Created Date', + 'Recorded Timestamp', + 'Recorded Date', + 'Raw Timestamp', + 'Normalized Timestamp', + 'X', + 'Y', + 'Z', + 'Accuracy' + ] + + writer.writerow(columns) + + points = DataPoint.objects.filter(source=source, generator_identifier=generator).order_by('created') + + index = 0 + count = points.count() + + while index < count: + for point in points[index:(index + 500)]: + properties = point.fetch_properties() + + for i in range(0, len(properties['sensor_data']['observed'])): + row = [] + + row.append(point.source) + row.append(calendar.timegm(point.created.utctimetuple())) + row.append(point.created.isoformat()) + + row.append(calendar.timegm(point.recorded.utctimetuple())) + row.append(point.recorded.isoformat()) + + row.append(properties['sensor_data']['raw_timestamp'][i]) + row.append(properties['sensor_data']['observed'][i]) + row.append(properties['sensor_data']['x'][i]) + row.append(properties['sensor_data']['y'][i]) + row.append(properties['sensor_data']['z'][i]) + row.append(properties['sensor_data']['accuracy'][i]) + + writer.writerow(row) + + index += 500 + + export_file.write(secondary_filename, slugify(generator) + '/' + slugify(source) + '.txt') + + os.remove(secondary_filename) + + return filename diff --git a/generators/pdk_sensor_light.py b/generators/pdk_sensor_light.py index f05a4b2..d9bbf9a 100644 --- a/generators/pdk_sensor_light.py +++ b/generators/pdk_sensor_light.py @@ -1,10 +1,19 @@ # pylint: disable=line-too-long, no-member +import calendar +import csv import datetime +import os +import tempfile import time +from zipfile import ZipFile + +import arrow + from django.template.loader import render_to_string from django.utils import timezone +from django.utils.text import slugify from ..models import DataPoint @@ -86,54 +95,62 @@ def data_table(source, generator): return render_to_string('pdk_sensor_light_table_template.html', context) -# def compile_report(generator, sources): # pylint: disable=too-many-locals -# filename = tempfile.gettempdir() + '/pdk_export_' + str(arrow.get().timestamp) + '.zip' -# -# with ZipFile(filename, 'w') as export_file: -# for secondary_identifier in SECONDARY_FIELDS: -# secondary_filename = tempfile.gettempdir() + '/' + generator + '-' + \ -# secondary_identifier + '.txt' -# -# with open(secondary_filename, 'w') as outfile: -# writer = csv.writer(outfile, delimiter='\t') -# -# columns = [ -# 'Source', -# 'Created Timestamp', -# 'Created Date', -# ] -# -# for column in SECONDARY_FIELDS[secondary_identifier]: -# columns.append(column) -# -# writer.writerow(columns) -# -# for source in sources: -# points = DataPoint.objects.filter(source=source, generator_identifier=generator, secondary_identifier=secondary_identifier).order_by('source', 'created') # pylint: disable=no-member,line-too-long -# -# index = 0 -# count = points.count() -# -# while index < count: -# for point in points[index:(index + 5000)]: -# row = [] -# -# row.append(point.source) -# row.append(calendar.timegm(point.created.utctimetuple())) -# row.append(point.created.isoformat()) -# -# properties = point.fetch_properties() -# -# for column in SECONDARY_FIELDS[secondary_identifier]: -# if column in properties: -# row.append(properties[column]) -# else: -# row.append('') -# -# writer.writerow(row) -# -# index += 5000 -# -# export_file.write(secondary_filename, secondary_filename.split('/')[-1]) -# -# return filename +def compile_report(generator, sources): # pylint: disable=too-many-locals + filename = tempfile.gettempdir() + '/pdk_export_' + str(arrow.get().timestamp) + '.zip' + + with ZipFile(filename, 'w') as export_file: + for source in sources: + identifier = slugify(generator + '__' + source) + + secondary_filename = tempfile.gettempdir() + '/' + identifier + '.txt' + + with open(secondary_filename, 'w') as outfile: + writer = csv.writer(outfile, delimiter='\t') + + columns = [ + 'Source', + 'Created Timestamp', + 'Created Date', + 'Recorded Timestamp', + 'Recorded Date', + 'Raw Timestamp', + 'Normalized Timestamp', + 'Light Level', + 'Accuracy' + ] + + writer.writerow(columns) + + points = DataPoint.objects.filter(source=source, generator_identifier=generator).order_by('created') + + index = 0 + count = points.count() + + while index < count: + for point in points[index:(index + 500)]: + properties = point.fetch_properties() + + for i in range(0, len(properties['sensor_data']['observed'])): + row = [] + + row.append(point.source) + row.append(calendar.timegm(point.created.utctimetuple())) + row.append(point.created.isoformat()) + + row.append(calendar.timegm(point.recorded.utctimetuple())) + row.append(point.recorded.isoformat()) + + row.append(properties['sensor_data']['raw_timestamp'][i]) + row.append(properties['sensor_data']['observed'][i]) + row.append(properties['sensor_data']['light_level'][i]) + row.append(properties['sensor_data']['accuracy'][i]) + + writer.writerow(row) + + index += 500 + + export_file.write(secondary_filename, slugify(generator) + '/' + slugify(source) + '.txt') + + os.remove(secondary_filename) + + return filename diff --git a/generators/pdk_withings_device.py b/generators/pdk_withings_device.py index 95b36d5..beb2311 100644 --- a/generators/pdk_withings_device.py +++ b/generators/pdk_withings_device.py @@ -3,6 +3,7 @@ import calendar import csv import datetime +import os import tempfile from zipfile import ZipFile @@ -118,6 +119,8 @@ def compile_report(generator, sources): # pylint: disable=too-many-locals export_file.write(secondary_filename, secondary_filename.split('/')[-1]) + os.remove(secondary_filename) + return filename def data_table(source, generator): diff --git a/management/commands/compile_reports.py b/management/commands/compile_reports.py index 2742d17..1be9cf9 100644 --- a/management/commands/compile_reports.py +++ b/management/commands/compile_reports.py @@ -7,8 +7,7 @@ import os import tempfile # import traceback - -from zipfile import ZipFile +import zipfile from django.conf import settings from django.core.files import File @@ -66,7 +65,7 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b filename = tempfile.gettempdir() + '/pdk_export_' + str(report.pk) + '.zip' - with ZipFile(filename, 'w') as export_file: + with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED) as export_file: for generator in generators: # pylint: disable=too-many-nested-blocks if raw_json: for source in sources: @@ -124,11 +123,12 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b output_file = pdk_api.compile_report(generator, sources) if output_file.lower().endswith('.zip'): - with ZipFile(output_file, 'r') as source_file: + with zipfile.ZipFile(output_file, 'r') as source_file: for name in source_file.namelist(): export_file.writestr(name, source_file.open(name).read()) # pylint: disable=line-too-long os.remove(output_file) + output_file = None except ImportError: # traceback.print_exc() @@ -148,6 +148,8 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b report.completed = timezone.now() report.save() + os.remove(filename) + subject = render_to_string('pdk_report_subject.txt', { 'report': report, 'url': settings.SITE_URL