Updates to data exporters.

* Implemented new exporter for the accelerometer sensor.
* Changed the output generated by sensor exporters.
* Cleaned up temp files used for generating output.

admin.py

@@ -30,11 +30,27 @@ class DataSourceAdmin(admin.OSMGeoAdmin):
     list_display = ('name', 'identifier', 'group')
     list_filter = ('group',)
 
+
+def reset_report_jobs(modeladmin, request, queryset): # pylint: disable=unused-argument
+    for job in queryset:
+        job.started = None
+        job.completed = None
+
+        if job.report is not None:
+            job.report.delete()
+            job.report = None
+
+        job.save()
+
+reset_report_jobs.description = 'Reset report jobs'
+
 @admin.register(ReportJob)
 class ReportJobAdmin(admin.OSMGeoAdmin):
     list_display = ('requester', 'requested', 'started', 'completed')
     list_filter = ('requested', 'started', 'completed',)
 
+    actions = [reset_report_jobs]
+
 @admin.register(DataSourceAlert)
 class DataSourceAlertAdmin(admin.OSMGeoAdmin):
     list_display = ('alert_name', 'data_source', 'generator_identifier', 'active', 'alert_level', \

generators/pdk_sensor_accelerometer.py

@@ -0,0 +1,166 @@
+# pylint: disable=line-too-long, no-member
+
+import calendar
+import csv
+import datetime
+import os
+import tempfile
+import time
+
+from zipfile import ZipFile
+
+import arrow
+
+from django.template.loader import render_to_string
+from django.utils import timezone
+from django.utils.text import slugify
+
+from ..models import DataPoint
+
+WINDOW_SIZE = 300
+
+def fetch_values(source, generator, start, end):
+    values = []
+
+    index = start
+    current_value = None
+
+    for point in DataPoint.objects.filter(source=source.identifier, generator_identifier=generator, created__gt=start, created__lte=end).order_by('created'):
+        if (point.created - index).total_seconds() > WINDOW_SIZE:
+            if current_value is not None and current_value['min_value'] != -1: # pylint: disable=unsubscriptable-object
+                values.append(current_value)
+
+            current_value = None
+            index = index + datetime.timedelta(seconds=WINDOW_SIZE)
+
+        if current_value is None:
+            current_value = {
+                'min_value': -1,
+                'max_value': -1,
+                'timestamp': time.mktime(index.timetuple()),
+                'created': index,
+                'duration': WINDOW_SIZE
+            }
+
+        properties = point.fetch_properties()
+
+        for level in properties['sensor_data']['light_level']:
+            if current_value['min_value'] == -1 or level < current_value['min_value']:
+                current_value['min_value'] = level
+
+            if current_value['max_value'] == -1 or level > current_value['max_value']:
+                current_value['max_value'] = level
+
+#        duration = properties['sensor_data']['observed'][-1] - properties['sensor_data']['observed'][0]
+#        timestamp = properties['sensor_data']['observed'][0] + (duration / 2)
+#
+#        value['min_value'] = min_value
+#        value['max_value'] = max_value
+#        value['duration'] = duration / (1000 * 1000 * 1000)
+#        value['timestamp'] = timestamp / (1000 * 1000 * 1000)
+#        value['created'] = point.created
+#
+#        values.append(value)
+
+    return values
+
+
+def generator_name(identifier): # pylint: disable=unused-argument
+    return 'Accelerometer Sensor'
+
+def visualization_todo(source, generator):
+    context = {}
+    context['source'] = source
+    context['generator_identifier'] = generator
+
+    end = timezone.now()
+    start = end - datetime.timedelta(days=1)
+
+    context['values'] = fetch_values(source, generator, start, end)
+
+    context['start'] = time.mktime(start.timetuple())
+    context['end'] = time.mktime(end.timetuple())
+
+    render_to_string('pdk_sensor_light_template.html', context)
+
+def visualization(source, generator): # pylint: disable=unused-argument
+    return 'TODO'
+
+def data_table_todo(source, generator):
+    context = {}
+    context['source'] = source
+    context['generator_identifier'] = generator
+
+    end = timezone.now()
+    start = end - datetime.timedelta(days=1)
+
+    context['values'] = fetch_values(source, generator, start, end)
+
+    return render_to_string('pdk_sensor_light_table_template.html', context)
+
+def data_table(source, generator): # pylint: disable=unused-argument
+    return 'TODO'
+
+def compile_report(generator, sources): # pylint: disable=too-many-locals
+    filename = tempfile.gettempdir() + '/pdk_export_' + str(arrow.get().timestamp) + '.zip'
+
+    with ZipFile(filename, 'w') as export_file:
+        for source in sources:
+            identifier = slugify(generator + '__' + source)
+
+            secondary_filename = tempfile.gettempdir() + '/' + identifier + '.txt'
+
+            with open(secondary_filename, 'w') as outfile:
+                writer = csv.writer(outfile, delimiter='\t')
+
+                columns = [
+                    'Source',
+                    'Created Timestamp',
+                    'Created Date',
+                    'Recorded Timestamp',
+                    'Recorded Date',
+                    'Raw Timestamp',
+                    'Normalized Timestamp',
+                    'X',
+                    'Y',
+                    'Z',
+                    'Accuracy'
+                ]
+
+                writer.writerow(columns)
+
+                points = DataPoint.objects.filter(source=source, generator_identifier=generator).order_by('created')
+
+                index = 0
+                count = points.count()
+
+                while index < count:
+                    for point in points[index:(index + 500)]:
+                        properties = point.fetch_properties()
+
+                        for i in range(0, len(properties['sensor_data']['observed'])):
+                            row = []
+
+                            row.append(point.source)
+                            row.append(calendar.timegm(point.created.utctimetuple()))
+                            row.append(point.created.isoformat())
+
+                            row.append(calendar.timegm(point.recorded.utctimetuple()))
+                            row.append(point.recorded.isoformat())
+
+                            row.append(properties['sensor_data']['raw_timestamp'][i])
+                            row.append(properties['sensor_data']['observed'][i])
+                            row.append(properties['sensor_data']['x'][i])
+                            row.append(properties['sensor_data']['y'][i])
+                            row.append(properties['sensor_data']['z'][i])
+                            row.append(properties['sensor_data']['accuracy'][i])
+
+                            writer.writerow(row)
+
+                    index += 500
+
+            export_file.write(secondary_filename, slugify(generator) + '/' + slugify(source) + '.txt')
+
+            os.remove(secondary_filename)
+
+    return filename

generators/pdk_sensor_light.py

@@ -1,10 +1,19 @@
 # pylint: disable=line-too-long, no-member
 
+import calendar
+import csv
 import datetime
+import os
+import tempfile
 import time
 
+from zipfile import ZipFile
+
+import arrow
+
 from django.template.loader import render_to_string
 from django.utils import timezone
+from django.utils.text import slugify
 
 from ..models import DataPoint
 
@@ -86,54 +95,62 @@ def data_table(source, generator):
 
     return render_to_string('pdk_sensor_light_table_template.html', context)
 
-# def compile_report(generator, sources): # pylint: disable=too-many-locals
-#    filename = tempfile.gettempdir() + '/pdk_export_' + str(arrow.get().timestamp) + '.zip'
-#
-#    with ZipFile(filename, 'w') as export_file:
-#        for secondary_identifier in SECONDARY_FIELDS:
-#            secondary_filename = tempfile.gettempdir() + '/' + generator + '-' + \
-#                                 secondary_identifier + '.txt'
-#
-#            with open(secondary_filename, 'w') as outfile:
-#                writer = csv.writer(outfile, delimiter='\t')
-#
-#                columns = [
-#                    'Source',
-#                    'Created Timestamp',
-#                    'Created Date',
-#                ]
-#
-#                for column in SECONDARY_FIELDS[secondary_identifier]:
-#                    columns.append(column)
-#
-#                writer.writerow(columns)
-#
-#                for source in sources:
-#                    points = DataPoint.objects.filter(source=source, generator_identifier=generator, secondary_identifier=secondary_identifier).order_by('source', 'created') # pylint: disable=no-member,line-too-long
-#
-#                    index = 0
-#                    count = points.count()
-#
-#                    while index < count:
-#                        for point in points[index:(index + 5000)]:
-#                            row = []
-#
-#                            row.append(point.source)
-#                            row.append(calendar.timegm(point.created.utctimetuple()))
-#                            row.append(point.created.isoformat())
-#
-#                            properties = point.fetch_properties()
-#
-#                            for column in SECONDARY_FIELDS[secondary_identifier]:
-#                                if column in properties:
-#                                    row.append(properties[column])
-#                                else:
-#                                    row.append('')
-#
-#                            writer.writerow(row)
-#
-#                        index += 5000
-#
-#            export_file.write(secondary_filename, secondary_filename.split('/')[-1])
-#
-#    return filename
+def compile_report(generator, sources): # pylint: disable=too-many-locals
+    filename = tempfile.gettempdir() + '/pdk_export_' + str(arrow.get().timestamp) + '.zip'
+
+    with ZipFile(filename, 'w') as export_file:
+        for source in sources:
+            identifier = slugify(generator + '__' + source)
+
+            secondary_filename = tempfile.gettempdir() + '/' + identifier + '.txt'
+
+            with open(secondary_filename, 'w') as outfile:
+                writer = csv.writer(outfile, delimiter='\t')
+
+                columns = [
+                    'Source',
+                    'Created Timestamp',
+                    'Created Date',
+                    'Recorded Timestamp',
+                    'Recorded Date',
+                    'Raw Timestamp',
+                    'Normalized Timestamp',
+                    'Light Level',
+                    'Accuracy'
+                ]
+
+                writer.writerow(columns)
+
+                points = DataPoint.objects.filter(source=source, generator_identifier=generator).order_by('created')
+
+                index = 0
+                count = points.count()
+
+                while index < count:
+                    for point in points[index:(index + 500)]:
+                        properties = point.fetch_properties()
+
+                        for i in range(0, len(properties['sensor_data']['observed'])):
+                            row = []
+
+                            row.append(point.source)
+                            row.append(calendar.timegm(point.created.utctimetuple()))
+                            row.append(point.created.isoformat())
+
+                            row.append(calendar.timegm(point.recorded.utctimetuple()))
+                            row.append(point.recorded.isoformat())
+
+                            row.append(properties['sensor_data']['raw_timestamp'][i])
+                            row.append(properties['sensor_data']['observed'][i])
+                            row.append(properties['sensor_data']['light_level'][i])
+                            row.append(properties['sensor_data']['accuracy'][i])
+
+                            writer.writerow(row)
+
+                    index += 500
+
+            export_file.write(secondary_filename, slugify(generator) + '/' + slugify(source) + '.txt')
+
+            os.remove(secondary_filename)
+
+    return filename

generators/pdk_withings_device.py

@@ -3,6 +3,7 @@
 import calendar
 import csv
 import datetime
+import os
 import tempfile
 
 from zipfile import ZipFile
@@ -118,6 +119,8 @@ def compile_report(generator, sources): # pylint: disable=too-many-locals
 
             export_file.write(secondary_filename, secondary_filename.split('/')[-1])
 
+            os.remove(secondary_filename)
+
     return filename
 
 def data_table(source, generator):

management/commands/compile_reports.py

@@ -7,8 +7,7 @@
 import os
 import tempfile
 # import traceback
-
-from zipfile import ZipFile
+import zipfile
 
 from django.conf import settings
 from django.core.files import File
@@ -66,7 +65,7 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b
 
             filename = tempfile.gettempdir() + '/pdk_export_' + str(report.pk) + '.zip'
 
-            with ZipFile(filename, 'w') as export_file:
+            with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED) as export_file:
                 for generator in generators: # pylint: disable=too-many-nested-blocks
                     if raw_json:
                         for source in sources:
@@ -124,11 +123,12 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b
                                     output_file = pdk_api.compile_report(generator, sources)
 
                                     if output_file.lower().endswith('.zip'):
-                                        with ZipFile(output_file, 'r') as source_file:
+                                        with zipfile.ZipFile(output_file, 'r') as source_file:
                                             for name in source_file.namelist():
                                                 export_file.writestr(name, source_file.open(name).read()) # pylint: disable=line-too-long
 
                                         os.remove(output_file)
+
                                         output_file = None
                                 except ImportError:
 #                                    traceback.print_exc()
@@ -148,6 +148,8 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b
             report.completed = timezone.now()
             report.save()
 
+            os.remove(filename)
+
             subject = render_to_string('pdk_report_subject.txt', {
                 'report': report,
                 'url': settings.SITE_URL