/
mode_dashboard_queries_extractor.py
104 lines (82 loc) · 4.68 KB
/
mode_dashboard_queries_extractor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
# Copyright Contributors to the Amundsen project.
# SPDX-License-Identifier: Apache-2.0
import logging
from typing import Any, List
from pyhocon import ConfigFactory, ConfigTree
from databuilder import Scoped
from databuilder.extractor.base_extractor import Extractor
from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
from databuilder.rest_api.rest_api_query import RestApiQuery
from databuilder.transformer.base_transformer import ChainedTransformer, Transformer
from databuilder.transformer.dict_to_model import MODEL_CLASS, DictToModel
from databuilder.transformer.regex_str_replace_transformer import (
ATTRIBUTE_NAME, REGEX_REPLACE_TUPLE_LIST, RegexStrReplaceTransformer,
)
from databuilder.transformer.template_variable_substitution_transformer import (
FIELD_NAME, TEMPLATE, TemplateVariableSubstitutionTransformer,
)
LOGGER = logging.getLogger(__name__)
class ModeDashboardQueriesExtractor(Extractor):
"""
A Extractor that extracts Query information
"""
def init(self, conf: ConfigTree) -> None:
self._conf = conf
restapi_query = self._build_restapi_query()
self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
restapi_query=restapi_query,
conf=self._conf
)
# Constructing URL using several ID via TemplateVariableSubstitutionTransformer
transformers: List[Transformer] = []
variable_substitution_transformer = TemplateVariableSubstitutionTransformer()
variable_substitution_transformer.init(
conf=Scoped.get_scoped_conf(self._conf,
variable_substitution_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict({FIELD_NAME: 'url',
TEMPLATE: 'https://app.mode.com/{organization}'
'/reports/{dashboard_id}/queries/{query_id}'})))
transformers.append(variable_substitution_transformer)
# Escape backslash as it breaks Cypher statement.
replace_transformer = RegexStrReplaceTransformer()
replace_transformer.init(
conf=Scoped.get_scoped_conf(self._conf, replace_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict(
{REGEX_REPLACE_TUPLE_LIST: [('\\', '\\\\')], ATTRIBUTE_NAME: 'query_text'})))
transformers.append(replace_transformer)
dict_to_model_transformer = DictToModel()
dict_to_model_transformer.init(
conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict(
{MODEL_CLASS: 'databuilder.models.dashboard.dashboard_query.DashboardQuery'})))
transformers.append(dict_to_model_transformer)
self._transformer = ChainedTransformer(transformers=transformers)
def extract(self) -> Any:
record = self._extractor.extract()
if not record:
return None
return self._transformer.transform(record=record)
def get_scope(self) -> str:
return 'extractor.mode_dashboard_query'
def _build_restapi_query(self) -> RestApiQuery:
"""
Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
API, and queries API) joining together.
:return: A RestApiQuery that provides Mode Dashboard execution (run)
"""
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
json_path = '(_embedded.reports[*].token)'
field_names = ['dashboard_id']
reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
json_path=json_path, field_names=field_names, skip_no_result=True)
queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
json_path = '_embedded.queries[*].[token,name,raw_query]'
field_names = ['query_id', 'query_name', 'query_text']
query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params,
json_path=json_path, field_names=field_names, skip_no_result=True)
return query_names_query