From f6805be80464499511b4bcd9ea89134c59f22500 Mon Sep 17 00:00:00 2001 From: "Chris J. Karr" Date: Mon, 31 Jul 2017 23:27:10 +0000 Subject: [PATCH] Updated web interface to use cached performance data. * Renamed management commands to use pdk_ prefix. --- admin.py | 6 +- generators/pdk_sensor_accelerometer.py | 2 +- ...pile_reports.py => pdk_compile_reports.py} | 2 +- ...tions.py => pdk_compile_visualizations.py} | 0 ...gs.py => pdk_delete_redundant_readings.py} | 0 ... => pdk_fetch_historical_withings_data.py} | 25 ++-- ...cess_bundles.py => pdk_process_bundles.py} | 2 +- ...check_withings_device_check_last_upload.py | 4 +- .../pdk_update_performance_metadata.py | 18 +++ migrations/0020_auto_20170731_1939.py | 21 +++ migrations/0021_auto_20170731_2011.py | 21 +++ migrations/0022_auto_20170731_2133.py | 37 +++++ migrations/0023_auto_20170731_2137.py | 21 +++ .../0024_datasource_performance_metadata.py | 33 +++++ ...datasource_performance_metadata_updated.py | 22 +++ models.py | 139 +++++++++++++++--- templates/tag_generators_table.html | 8 +- 17 files changed, 316 insertions(+), 45 deletions(-) rename management/commands/{compile_reports.py => pdk_compile_reports.py} (99%) rename management/commands/{compile_visualizations.py => pdk_compile_visualizations.py} (100%) rename management/commands/{delete_redundant_readings.py => pdk_delete_redundant_readings.py} (100%) rename management/commands/{fetch_historical_withings_data.py => pdk_fetch_historical_withings_data.py} (91%) rename management/commands/{process_bundles.py => pdk_process_bundles.py} (98%) create mode 100644 management/commands/pdk_update_performance_metadata.py create mode 100644 migrations/0020_auto_20170731_1939.py create mode 100644 migrations/0021_auto_20170731_2011.py create mode 100644 migrations/0022_auto_20170731_2133.py create mode 100644 migrations/0023_auto_20170731_2137.py create mode 100644 migrations/0024_datasource_performance_metadata.py create mode 100644 migrations/0025_datasource_performance_metadata_updated.py diff --git a/admin.py b/admin.py index 4616534..a40d892 100644 --- a/admin.py +++ b/admin.py @@ -14,7 +14,11 @@ class DataPointAdmin(admin.OSMGeoAdmin): list_display = ('source', 'generator_identifier', 'secondary_identifier', 'created', \ 'recorded',) - list_filter = ('created', 'recorded', 'generator_identifier', 'secondary_identifier',) + list_filter = ( + 'created', + 'recorded', + 'generator_identifier', + ) @admin.register(DataBundle) class DataBundleAdmin(admin.OSMGeoAdmin): diff --git a/generators/pdk_sensor_accelerometer.py b/generators/pdk_sensor_accelerometer.py index c9d8b0e..339d11d 100644 --- a/generators/pdk_sensor_accelerometer.py +++ b/generators/pdk_sensor_accelerometer.py @@ -104,7 +104,7 @@ def data_table(source, generator): # pylint: disable=unused-argument def compile_report(generator, sources): # pylint: disable=too-many-locals filename = tempfile.gettempdir() + '/pdk_export_' + str(arrow.get().timestamp) + '.zip' - with ZipFile(filename, 'w') as export_file: + with ZipFile(filename, 'w', allowZip64=True) as export_file: for source in sources: identifier = slugify(generator + '__' + source) diff --git a/management/commands/compile_reports.py b/management/commands/pdk_compile_reports.py similarity index 99% rename from management/commands/compile_reports.py rename to management/commands/pdk_compile_reports.py index 1be9cf9..0926cb3 100644 --- a/management/commands/compile_reports.py +++ b/management/commands/pdk_compile_reports.py @@ -65,7 +65,7 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b filename = tempfile.gettempdir() + '/pdk_export_' + str(report.pk) + '.zip' - with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED) as export_file: + with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as export_file: # pylint: disable=line-too-long for generator in generators: # pylint: disable=too-many-nested-blocks if raw_json: for source in sources: diff --git a/management/commands/compile_visualizations.py b/management/commands/pdk_compile_visualizations.py similarity index 100% rename from management/commands/compile_visualizations.py rename to management/commands/pdk_compile_visualizations.py diff --git a/management/commands/delete_redundant_readings.py b/management/commands/pdk_delete_redundant_readings.py similarity index 100% rename from management/commands/delete_redundant_readings.py rename to management/commands/pdk_delete_redundant_readings.py diff --git a/management/commands/fetch_historical_withings_data.py b/management/commands/pdk_fetch_historical_withings_data.py similarity index 91% rename from management/commands/fetch_historical_withings_data.py rename to management/commands/pdk_fetch_historical_withings_data.py index 4cdf403..cba6eba 100644 --- a/management/commands/fetch_historical_withings_data.py +++ b/management/commands/pdk_fetch_historical_withings_data.py @@ -54,27 +54,28 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b for source in sources: data_point = DataPoint.objects.filter(source=source, generator_identifier='pdk-withings-device').order_by('-created').first() - properties = data_point.fetch_properties() + if data_point is not None: + properties = data_point.fetch_properties() - if 'oauth_user_token' in properties and 'oauth_user_secret' in properties and 'oauth_user_id' in properties: - index_date = start_date + if 'oauth_user_token' in properties and 'oauth_user_secret' in properties and 'oauth_user_id' in properties: + index_date = start_date - while index_date < end_date: - next_day = index_date.replace(days=+1) + while index_date < end_date: + next_day = index_date.replace(days=+1) -# print('FETCHING INTRADAY FOR ' + source + ': ' + str(index_date) + ': ' + str(next_day)) + # print('FETCHING INTRADAY FOR ' + source + ': ' + str(index_date) + ': ' + str(next_day)) - fetch_intraday(source, properties, index_date, next_day) + fetch_intraday(source, properties, index_date, next_day) - time.sleep(1) + time.sleep(1) -# print('FETCHING SLEEP MEASURES FOR ' + source + ': ' + str(index_date) + ': ' + str(next_day)) + # print('FETCHING SLEEP MEASURES FOR ' + source + ': ' + str(index_date) + ': ' + str(next_day)) - fetch_sleep_measures(source, properties, index_date, next_day) + fetch_sleep_measures(source, properties, index_date, next_day) - time.sleep(1) + time.sleep(1) - index_date = next_day + index_date = next_day def fetch_intraday(user_id, properties, start_date, end_date): # pylint: disable=too-many-locals, too-many-statements, too-many-branches diff --git a/management/commands/process_bundles.py b/management/commands/pdk_process_bundles.py similarity index 98% rename from management/commands/process_bundles.py rename to management/commands/pdk_process_bundles.py index b6d9cd1..0dc5441 100644 --- a/management/commands/process_bundles.py +++ b/management/commands/pdk_process_bundles.py @@ -23,7 +23,7 @@ def add_arguments(self, parser): parser.add_argument('--count', type=int, dest='bundle_count', - default=100, + default=10, help='Number of bundles to process in a single run') @handle_lock diff --git a/management/commands/pdk_status_check_withings_device_check_last_upload.py b/management/commands/pdk_status_check_withings_device_check_last_upload.py index e7f9485..78d496f 100644 --- a/management/commands/pdk_status_check_withings_device_check_last_upload.py +++ b/management/commands/pdk_status_check_withings_device_check_last_upload.py @@ -27,9 +27,9 @@ def handle(self, *args, **options): # pylint: disable=too-many-branches, too-man alert_details = {} alert_level = 'info' - delta = now - last_upload.created - if last_upload is not None: + delta = now - last_upload.created + if delta.days >= WARNING_DAYS: alert_name = 'Withings upload is overdue' alert_details['message'] = 'Latest Withings upload was 1 day ago.' diff --git a/management/commands/pdk_update_performance_metadata.py b/management/commands/pdk_update_performance_metadata.py new file mode 100644 index 0000000..19cf00e --- /dev/null +++ b/management/commands/pdk_update_performance_metadata.py @@ -0,0 +1,18 @@ +# pylint: disable=no-member + +from django.core.management.base import BaseCommand + +from ...decorators import handle_lock +from ...models import DataSource + +class Command(BaseCommand): + help = 'Updates each user performance metadata measurements on a round-robin basis' + + @handle_lock + def handle(self, *args, **options): + source = DataSource.objects.filter(performance_metadata_updated=None).first() + + if source is None: + source = DataSource.objects.all().order_by('performance_metadata_updated').first() + + source.update_performance_metadata() diff --git a/migrations/0020_auto_20170731_1939.py b/migrations/0020_auto_20170731_1939.py new file mode 100644 index 0000000..66844f7 --- /dev/null +++ b/migrations/0020_auto_20170731_1939.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-31 19:39 +# pylint: skip-file + +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('passive_data_kit', '0019_datasourcealert_alert_level'), + ] + + operations = [ + migrations.AlterIndexTogether( + name='datapoint', + index_together=set([('source', 'generator_identifier', 'created'), ('source', 'generator_identifier')]), + ), + ] diff --git a/migrations/0021_auto_20170731_2011.py b/migrations/0021_auto_20170731_2011.py new file mode 100644 index 0000000..7930338 --- /dev/null +++ b/migrations/0021_auto_20170731_2011.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-31 20:11 +# pylint: skip-file + +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('passive_data_kit', '0020_auto_20170731_1939'), + ] + + operations = [ + migrations.AlterIndexTogether( + name='datapoint', + index_together=set([('source', 'generator_identifier', 'secondary_identifier'), ('source', 'generator_identifier', 'secondary_identifier', 'created'), ('generator_identifier', 'secondary_identifier', 'recorded'), ('source', 'generator_identifier', 'secondary_identifier', 'recorded'), ('generator_identifier', 'secondary_identifier'), ('generator_identifier', 'created', 'recorded'), ('source', 'generator_identifier', 'created'), ('source', 'generator_identifier'), ('source', 'generator_identifier', 'created', 'recorded'), ('generator_identifier', 'secondary_identifier', 'created', 'recorded'), ('generator_identifier', 'created'), ('generator_identifier', 'recorded'), ('source', 'generator_identifier', 'secondary_identifier', 'created', 'recorded'), ('generator_identifier', 'secondary_identifier', 'created'), ('source', 'generator_identifier', 'recorded')]), + ), + ] diff --git a/migrations/0022_auto_20170731_2133.py b/migrations/0022_auto_20170731_2133.py new file mode 100644 index 0000000..4613ca9 --- /dev/null +++ b/migrations/0022_auto_20170731_2133.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-31 21:33 +# pylint: skip-file + +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('passive_data_kit', '0021_auto_20170731_2011'), + ] + + operations = [ + migrations.AlterField( + model_name='datasourcealert', + name='active', + field=models.BooleanField(db_index=True, default=True), + ), + migrations.AlterField( + model_name='datasourcealert', + name='alert_level', + field=models.CharField(choices=[('info', 'Informative'), ('warning', 'Warning'), ('critical', 'Critical')], db_index=True, default='info', max_length=64), + ), + migrations.AlterField( + model_name='datasourcealert', + name='created', + field=models.DateTimeField(db_index=True), + ), + migrations.AlterField( + model_name='datasourcealert', + name='updated', + field=models.DateTimeField(blank=True, db_index=True, null=True), + ), + ] diff --git a/migrations/0023_auto_20170731_2137.py b/migrations/0023_auto_20170731_2137.py new file mode 100644 index 0000000..a0c20f7 --- /dev/null +++ b/migrations/0023_auto_20170731_2137.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-31 21:37 +# pylint: skip-file + +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('passive_data_kit', '0022_auto_20170731_2133'), + ] + + operations = [ + migrations.AlterIndexTogether( + name='datapoint', + index_together=set([('source', 'generator_identifier', 'secondary_identifier'), ('generator_identifier', 'recorded'), ('source', 'generator_identifier'), ('generator_identifier', 'secondary_identifier', 'recorded'), ('source', 'created'), ('generator_identifier', 'secondary_identifier'), ('source', 'generator_identifier', 'secondary_identifier', 'recorded'), ('source', 'generator_identifier', 'secondary_identifier', 'created'), ('generator_identifier', 'created', 'recorded'), ('source', 'generator_identifier', 'created'), ('generator_identifier', 'secondary_identifier', 'created'), ('source', 'generator_identifier', 'created', 'recorded'), ('generator_identifier', 'created'), ('generator_identifier', 'secondary_identifier', 'created', 'recorded'), ('source', 'generator_identifier', 'secondary_identifier', 'created', 'recorded'), ('source', 'generator_identifier', 'recorded')]), + ), + ] diff --git a/migrations/0024_datasource_performance_metadata.py b/migrations/0024_datasource_performance_metadata.py new file mode 100644 index 0000000..acc780b --- /dev/null +++ b/migrations/0024_datasource_performance_metadata.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-31 21:50 +# pylint: skip-file + +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + +from ..models import install_supports_jsonfield + +class Migration(migrations.Migration): + + dependencies = [ + ('passive_data_kit', '0023_auto_20170731_2137'), + ] + + if install_supports_jsonfield(): + operations = [ + migrations.AddField( + model_name='datasource', + name='performance_metadata', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + ] + else: + operations = [ + migrations.AddField( + model_name='datasource', + name='performance_metadata', + field=models.TextField(max_length=(32 * 1024 * 1024 * 1024), blank=True, null=True) + ), + ] diff --git a/migrations/0025_datasource_performance_metadata_updated.py b/migrations/0025_datasource_performance_metadata_updated.py new file mode 100644 index 0000000..1d7d4f2 --- /dev/null +++ b/migrations/0025_datasource_performance_metadata_updated.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-31 22:13 +# pylint: skip-file + +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('passive_data_kit', '0024_datasource_performance_metadata'), + ] + + operations = [ + migrations.AddField( + model_name='datasource', + name='performance_metadata_updated', + field=models.DateTimeField(blank=True, db_index=True, null=True), + ), + ] diff --git a/models.py b/models.py index c3f06df..b7123f1 100644 --- a/models.py +++ b/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals +import calendar import json import importlib @@ -12,8 +13,11 @@ from django.db import connection from django.db.models.signals import post_delete from django.dispatch.dispatcher import receiver +from django.utils import timezone from django.utils.text import slugify +DB_SUPPORTS_JSON = None + def generator_label(identifier): for app in settings.INSTALLED_APPS: try: @@ -34,10 +38,34 @@ def generator_slugify(str_obj): return slugify(str_obj.replace('.', ' ')).replace('-', '_') def install_supports_jsonfield(): - return connection.pg_version >= 90400 + global DB_SUPPORTS_JSON # pylint: disable=global-statement + + if True and DB_SUPPORTS_JSON is None: + DB_SUPPORTS_JSON = connection.pg_version >= 90400 + return DB_SUPPORTS_JSON class DataPoint(models.Model): + class Meta: # pylint: disable=old-style-class, no-init, too-few-public-methods + index_together = [ + ['source', 'created'], + ['source', 'generator_identifier'], + ['source', 'generator_identifier', 'created'], + ['source', 'generator_identifier', 'recorded'], + ['source', 'generator_identifier', 'created', 'recorded'], + ['source', 'generator_identifier', 'secondary_identifier'], + ['source', 'generator_identifier', 'secondary_identifier', 'created'], + ['source', 'generator_identifier', 'secondary_identifier', 'recorded'], + ['source', 'generator_identifier', 'secondary_identifier', 'created', 'recorded'], + ['generator_identifier', 'created'], + ['generator_identifier', 'recorded'], + ['generator_identifier', 'created', 'recorded'], + ['generator_identifier', 'secondary_identifier'], + ['generator_identifier', 'secondary_identifier', 'created'], + ['generator_identifier', 'secondary_identifier', 'recorded'], + ['generator_identifier', 'secondary_identifier', 'created', 'recorded'], + ] + source = models.CharField(max_length=1024, db_index=True) generator = models.CharField(max_length=1024, db_index=True) generator_identifier = models.CharField(max_length=1024, db_index=True, default='unknown-generator') @@ -113,29 +141,50 @@ class DataSource(models.Model): group = models.ForeignKey(DataSourceGroup, related_name='sources', null=True, on_delete=models.SET_NULL) + if install_supports_jsonfield(): + performance_metadata = JSONField(null=True, blank=True) + else: + performance_metadata = models.TextField(max_length=(32 * 1024 * 1024 * 1024), null=True, blank=True) + + performance_metadata_updated = models.DateTimeField(db_index=True, null=True, blank=True) + def __unicode__(self): return self.name + ' (' + self.identifier + ')' - def latest_point(self): - return DataPoint.objects.filter(source=self.identifier).order_by('-created').first() + def fetch_performance_metadata(self): + if self.performance_metadata is not None: + if install_supports_jsonfield(): + return self.performance_metadata - def point_count(self): - return DataPoint.objects.filter(source=self.identifier).count() + return json.loads(self.performance_metadata) - def point_frequency(self): - count = self.point_count() + return {} - if count > 0: - first = DataPoint.objects.filter(source=self.identifier).order_by('created').first() - last = DataPoint.objects.filter(source=self.identifier).order_by('created').last() + def update_performance_metadata(self): + metadata = self.fetch_performance_metadata() - seconds = (last.created - first.created).total_seconds() + # Update latest_point - return count / seconds + latest_point = DataPoint.objects.filter(source=self.identifier).order_by('-created').first() - return 0 + if latest_point is not None: + metadata['latest_point'] = latest_point.pk + + # Update point_count + + metadata['point_count'] = DataPoint.objects.filter(source=self.identifier).count() + + # Update point_frequency + + if metadata['point_count'] > 1: + earliest_point = DataPoint.objects.filter(source=self.identifier).order_by('created').first() + + seconds = (latest_point.created - earliest_point.created).total_seconds() + + metadata['point_frequency'] = metadata['point_count'] / seconds + else: + metadata['point_frequency'] = 0 - def generator_statistics(self): generators = [] identifiers = DataPoint.objects.filter(source=self.identifier).order_by('generator_identifier').values_list('generator_identifier', flat=True).distinct() @@ -153,15 +202,59 @@ def generator_statistics(self): last_point = DataPoint.objects.filter(source=self.identifier, generator_identifier=identifier).order_by('-created').first() last_recorded = DataPoint.objects.filter(source=self.identifier, generator_identifier=identifier).order_by('-recorded').first() - generator['last_recorded'] = last_recorded.recorded - generator['first_created'] = first_point.created - generator['last_created'] = last_point.created + generator['last_recorded'] = calendar.timegm(last_recorded.recorded.timetuple()) + generator['first_created'] = calendar.timegm(first_point.created.timetuple()) + generator['last_created'] = calendar.timegm(last_point.created.timetuple()) - generator['frequency'] = float(generator['points_count']) / (last_point.created - first_point.created).total_seconds() + if generator['points_count'] > 1: + generator['frequency'] = float(generator['points_count']) / (last_point.created - first_point.created).total_seconds() + else: + generator['frequency'] = 0 generators.append(generator) - return generators + metadata['generator_statistics'] = generators + + if install_supports_jsonfield(): + self.performance_metadata = metadata + else: + self.performance_metadata = json.dumps(metadata, indent=2) + + self.performance_metadata_updated = timezone.now() + + self.save() + + def latest_point(self): + metadata = self.fetch_performance_metadata() + + if 'latest_point' in metadata: + return DataPoint.objects.get(pk=metadata['latest_point']) + + return None + + def point_count(self): + metadata = self.fetch_performance_metadata() + + if 'point_count' in metadata: + return metadata['point_count'] + + return None + + def point_frequency(self): + metadata = self.fetch_performance_metadata() + + if 'point_frequency' in metadata: + return metadata['point_frequency'] + + return None + + def generator_statistics(self): + metadata = self.fetch_performance_metadata() + + if 'generator_statistics' in metadata: + return metadata['generator_statistics'] + + return [] ALERT_LEVEL_CHOICES = ( ('info', 'Informative'), @@ -171,7 +264,7 @@ def generator_statistics(self): class DataSourceAlert(models.Model): alert_name = models.CharField(max_length=1024) - alert_level = models.CharField(max_length=64, choices=ALERT_LEVEL_CHOICES, default='info') + alert_level = models.CharField(max_length=64, choices=ALERT_LEVEL_CHOICES, default='info', db_index=True) if install_supports_jsonfield(): alert_details = JSONField() @@ -181,10 +274,10 @@ class DataSourceAlert(models.Model): data_source = models.ForeignKey(DataSource, related_name='alerts') generator_identifier = models.CharField(max_length=1024, null=True, blank=True) - created = models.DateTimeField() - updated = models.DateTimeField(null=True, blank=True) + created = models.DateTimeField(db_index=True) + updated = models.DateTimeField(null=True, blank=True, db_index=True) - active = models.BooleanField(default=True) + active = models.BooleanField(default=True, db_index=True) def fetch_alert_details(self): if install_supports_jsonfield(): diff --git a/templates/tag_generators_table.html b/templates/tag_generators_table.html index cb34759..45a9808 100644 --- a/templates/tag_generators_table.html +++ b/templates/tag_generators_table.html @@ -14,12 +14,12 @@ {% generator_name generator.label %} {{ generator.points_count }} - {{ generator.last_recorded }} + {{ generator.last_recorded|to_datetime }} - {% if generator.first_created.date == generator.last_created.date %} - {{ generator.first_created.date }} + {% if generator.first_created == generator.last_created %} + {{ generator.first_created|to_datetime }} {% else %} - {{ generator.first_created.date }} - {{ generator.last_created.date }} + {{ generator.first_created|to_datetime }} - {{ generator.last_created|to_datetime }} {% endif %} {% to_hz generator.frequency %}