Skip to content

Commit

Permalink
Implemented raw JSON export
Browse files Browse the repository at this point in the history
  • Loading branch information
@audaciouscode
audaciouscode committed Aug 22, 2016
1 parent 0ea9abb commit f9f3801
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 102 deletions.
64 changes: 49 additions & 15 deletions management/commands/compile_reports.py
View file
Expand Up @@ -51,29 +51,63 @@ def handle(self, *args, **options):
sources = report.parameters['sources']
generators = report.parameters['generators']

raw_json = False

if ('raw_data' in report.parameters) and report.parameters['raw_data'] is True:
raw_json = True

filename = '/tmp/pdk_export_' + str(report.pk) + '.zip'

with ZipFile(filename, 'w') as export_file:
for generator in generators:
output_file = None
if raw_json:
for source in sources:
first = DataPoint.objects.filter(source=source, generator_identifier=generator).first()
last = DataPoint.objects.filter(source=source, generator_identifier=generator).last()

if first is not None:
first_create = first.created
last_create = last.created

start = datetime.datetime(first_create.year, first_create.month, first_create.day, 0, 0, 0, 0, first_create.tzinfo)
end = datetime.datetime(last_create.year, last_create.month, last_create.day, 0, 0, 0, 0, first_create.tzinfo) + datetime.timedelta(days=1)

while start <= end:
day_end = start + datetime.timedelta(days=1)

day_filename = source + '__' + generator + '__' + start.date().isoformat() + '.json'

points = DataPoint.objects.filter(source=source, generator_identifier=generator, created__gte=start, created__lt=day_end).order_by('created')

out_points = []

for point in points:
out_points.append(point.properties)

if len(out_points) > 0:
export_file.writestr(day_filename, unicode(json.dumps(out_points, indent=2)).encode("utf-8"))

start = day_end
else:
output_file = None

for app in settings.INSTALLED_APPS:
if output_file is None:
try:
pdk_api = importlib.import_module(app + '.pdk_api')
for app in settings.INSTALLED_APPS:
if output_file is None:
try:
pdk_api = importlib.import_module(app + '.pdk_api')

output_file = pdk_api.compile_report(generator, sources)
except ImportError:
traceback.print_exc()
output_file = None
except AttributeError:
traceback.print_exc()
output_file = None
output_file = pdk_api.compile_report(generator, sources)
except ImportError:
# traceback.print_exc()
output_file = None
except AttributeError:
# traceback.print_exc()
output_file = None

if output_file is not None:
export_file.write(output_file, output_file.split('/')[-1])
if output_file is not None:
export_file.write(output_file, output_file.split('/')[-1])

os.remove(output_file)
os.remove(output_file)

export_file.close()

Expand Down
179 changes: 92 additions & 87 deletions templates/pdk_export.html
View file
Expand Up @@ -3,98 +3,103 @@
{% load passive_data_kit %}

{% block sidebar %}
<ul class="nav nav-sidebar">
<li><a href="#alerts">Alerts <span class="badge pull-right">?</span></a></li>
</ul>
<ul class="nav nav-sidebar">
<li><label>TODO</label></li>
<li><a href="#generators">Generators</a></li>
</ul>
<ul class="nav nav-sidebar">
<li><label>Actions</label></li>
<li><a href="#add_source">Add Data Source</a></li>
</ul>
<ul class="nav nav-sidebar">
<li><a href="#alerts">Alerts <span class="badge pull-right">?</span></a></li>
</ul>
<ul class="nav nav-sidebar">
<li><label>TODO</label></li>
<li><a href="#generators">Generators</a></li>
</ul>
<ul class="nav nav-sidebar">
<li><label>Actions</label></li>
<li><a href="#add_source">Add Data Source</a></li>
</ul>
{% endblock %}

{% block content %}
<div class="row">
<div class="col-md-12">
<h1 class="sub-header">Export Data</h1>
</div>
</div>
<div class="row">
<div class="col-md-12">
<h1 class="sub-header">Export Data</h1>
</div>
</div>

<div class="row">
<div class="col-md-12">
<p>
Please select the relevant users and data generators to export.
Passive Data Kit will compile the requested data and e-mail you a download link.
</p>
</div>
</div>
{% if message != '' %}
<div class="row">
<div class="col-md-12">
{% if message_type == 'ok' %}
<div class="alert alert-info" role="alert">{{ message }}</div>
{% else %}
<div class="alert alert-danger" role="alert">{{ message }}</div>
{% endif %}
</div>
</div>
{% endif %}
<div class="row">
<div class="col-md-12">
<p>
Please select the relevant users and data generators to export.
Passive Data Kit will compile the requested data and e-mail you a download link.
</p>
</div>
</div>
{% if message != '' %}
<div class="row">
<div class="col-md-12">
{% if message_type == 'ok' %}
<div class="alert alert-info" role="alert">{{ message }}</div>
{% else %}
<div class="alert alert-danger" role="alert">{{ message }}</div>
{% endif %}
</div>
</div>
{% endif %}

<form action="{% url 'pdk_export' %}" method="POST">
<div class="row">
{% csrf_token %}
<div class="col-md-6">
<h3>Sources</h3>
<p>
<input type="checkbox" id="sources_select_all"> Select All Sources
</p>
<p>
{% for source in sources %}
<input type="checkbox" name="source_{{ source }}" class="source_checkbox"> {{ source }}<br/>
{% endfor %}
</p>
</div>
<div class="col-md-6">
<h3>Data Generators</h3>
<p>
<input type="checkbox" id="generators_select_all"> Select All Generators
</p>
<p>
{% for generator in generators %}
<input type="checkbox" name="generator_{{ generator }}" class="generator_checkbox"> {{ generator }}<br/>
{% endfor %}
</p>
</div>
</div>
<form action="{% url 'pdk_export' %}" method="POST">
<div class="row">
{% csrf_token %}
<div class="col-md-6">
<h3>Sources</h3>
<p>
<input type="checkbox" id="sources_select_all"> Select All Sources
</p>
<p>
{% for source in sources %}
<input type="checkbox" name="source_{{ source }}" class="source_checkbox"> {{ source }}<br/>
{% endfor %}
</p>
</div>
<div class="col-md-6">
<h3>Data Generators</h3>
<p>
<input type="checkbox" id="generators_select_all"> Select All Generators
</p>
<p>
{% for generator in generators %}
<input type="checkbox" name="generator_{{ generator }}" class="generator_checkbox"> {{ generator }}<br/>
{% endfor %}
</p>
</div>
</div>

<div class="row">
<div class="col-md-12">
<button type="submit" class="btn btn-default">Export Data</button>
</div>
</div>
</form>

<script>
window.setTimeout(function()
{
$('#sources_select_all').change(function()
{
$(".source_checkbox").prop("checked", $(this).is(":checked"));
});
<div class="row">
<div class="col-md-6">
</div>
<div class="col-md-3">
<input type="checkbox" name="export_raw_json" /> Export raw data as JSON
</div>
<div class="col-md-3">
<button type="submit" class="btn btn-default pull-right">Export Data</button>
</div>
</div>
</form>

<script>
window.setTimeout(function()
{
$('#sources_select_all').change(function()
{
$(".source_checkbox").prop("checked", $(this).is(":checked"));
});

$('#generators_select_all').change(function()
{
$(".generator_checkbox").prop("checked", $(this).is(":checked"));
});
}, 500);
</script>
<script data-main="{% static 'pdk/js/generator.js' %}" src="{% static 'pdk/js/lib/require.js' %}"></script>
$('#generators_select_all').change(function()
{
$(".generator_checkbox").prop("checked", $(this).is(":checked"));
});
}, 500);
</script>
<script data-main="{% static 'pdk/js/generator.js' %}" src="{% static 'pdk/js/lib/require.js' %}"></script>
{% endblock %}
3 changes: 3 additions & 0 deletions views.py
View file
Expand Up @@ -277,6 +277,9 @@ def pdk_export(request):

params['sources'] = export_sources
params['generators'] = export_generators

if 'export_raw_json' in request.POST and request.POST['export_raw_json']:
params['raw_data'] = True

job.parameters = params

Expand Down

0 comments on commit f9f3801

Please sign in to comment.