/
compile_reports.py
151 lines (117 loc) · 6.22 KB
/
compile_reports.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
# -*- coding: utf-8 -*-
# pylint: disable=no-member
import datetime
import importlib
import json
import os
import tempfile
from zipfile import ZipFile
from django.conf import settings
from django.core.files import File
from django.core.mail import send_mail
from django.core.management.base import BaseCommand
from django.template.loader import render_to_string
from django.utils import timezone
from passive_data_kit.decorators import handle_lock
from passive_data_kit.models import DataPoint, ReportJob
class Command(BaseCommand):
help = 'Compiles data reports requested by end users.'
def add_arguments(self, parser):
pass
# parser.add_argument('--delete',
# action='store_true',
# dest='delete',
# default=False,
# help='Delete data bundles after processing')
#
# parser.add_argument('--count',
# type=int,
# dest='bundle_count',
# default=100,
# help='Number of bundles to process in a single run')
@handle_lock
def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
os.umask(000)
report = ReportJob.objects.filter(started=None, completed=None)\
.order_by('requested')\
.first()
if report is not None:
report.started = timezone.now()
report.save()
sources = report.parameters['sources']
generators = report.parameters['generators']
raw_json = False
if ('raw_data' in report.parameters) and report.parameters['raw_data'] is True:
raw_json = True
filename = tempfile.gettempdir() + '/pdk_export_' + str(report.pk) + '.zip'
with ZipFile(filename, 'w') as export_file:
for generator in generators: # pylint: disable=too-many-nested-blocks
if raw_json:
for source in sources:
first = DataPoint.objects.filter(source=source, generator_identifier=generator).first() # pylint: disable=line-too-long
last = DataPoint.objects.filter(source=source, generator_identifier=generator).last() # pylint: disable=line-too-long
if first is not None:
first_create = first.created
last_create = last.created
start = datetime.datetime(first_create.year, \
first_create.month, \
first_create.day, \
0, \
0, \
0, \
0, \
first_create.tzinfo)
end = datetime.datetime(last_create.year, \
last_create.month, \
last_create.day, \
0, \
0, \
0, \
0, \
first_create.tzinfo) + \
datetime.timedelta(days=1)
while start <= end:
day_end = start + datetime.timedelta(days=1)
day_filename = source + '__' + generator + '__' + \
start.date().isoformat() + '.json'
points = DataPoint.objects.filter(source=source, generator_identifier=generator, created__gte=start, created__lt=day_end).order_by('created') # pylint: disable=line-too-long
out_points = []
for point in points:
out_points.append(point.properties)
if out_points:
export_file.writestr(day_filename, unicode(json.dumps(out_points, indent=2)).encode("utf-8")) # pylint: disable=line-too-long
start = day_end
else:
output_file = None
for app in settings.INSTALLED_APPS:
if output_file is None:
try:
pdk_api = importlib.import_module(app + '.pdk_api')
output_file = pdk_api.compile_report(generator, sources)
except ImportError:
# traceback.print_exc()
output_file = None
except AttributeError:
# traceback.print_exc()
output_file = None
if output_file is not None:
export_file.write(output_file, output_file.split('/')[-1])
os.remove(output_file)
export_file.close()
report.report.save(filename.split('/')[-1], File(open(filename, 'r')))
report.completed = timezone.now()
report.save()
subject = render_to_string('pdk_report_subject.txt', {
'report': report,
'url': settings.SITE_URL
})
message = render_to_string('pdk_report_message.txt', {
'report': report,
'url': settings.SITE_URL
})
host = settings.SITE_URL.split('/')[-2]
send_mail(subject, \
message, \
'Petey Kay <noreply@' + host + '>', \
[report.requester.email], \
fail_silently=False)