From c0557a5c768a647032bccfa24f2218bbc13b8c04 Mon Sep 17 00:00:00 2001 From: Junda Yang Date: Thu, 22 Apr 2021 11:50:16 -0700 Subject: [PATCH] fix: Switch to discovery api for mode spaces (#481) * use pagenated rest api query for getting all Mode spaces Signed-off-by: Junda Yang * nit Signed-off-by: Junda Yang * update comment Signed-off-by: Junda Yang --- .../mode_analytics/mode_dashboard_utils.py | 25 ++++++++++++------- setup.py | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_utils.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_utils.py index ecd963395..92cbeea41 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_utils.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_utils.py @@ -14,7 +14,7 @@ REST_API_QUERY, STATIC_RECORD_DICT, RestAPIExtractor, ) from databuilder.rest_api.base_rest_api_query import BaseRestApiQuery, RestApiQuerySeed -from databuilder.rest_api.rest_api_query import RestApiQuery +from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery class ModeDashboardUtils(object): @@ -28,21 +28,28 @@ def get_spaces_query_api(conf: ConfigTree) -> BaseRestApiQuery: :return: """ - # https://mode.com/developer/api-reference/management/spaces/#listSpaces - spaces_url_template = 'https://app.mode.com/api/{organization}/spaces?filter=all' + # https://mode.com/developer/discovery-api/analytics/spaces + spaces_url_template = 'https://app.mode.com/batch/{organization}/spaces' # Seed query record for next query api to join with seed_record = [{'organization': conf.get_string(ORGANIZATION)}] seed_query = RestApiQuerySeed(seed_record=seed_record) - # Spaces - params = {'auth': HTTPBasicAuth(conf.get_string(MODE_ACCESS_TOKEN), - conf.get_string(MODE_PASSWORD_TOKEN))} + # mode_bearer_token must be provided in the conf + # the token is required to access discovery endpoint + # https://mode.com/developer/discovery-api/introduction/ + params = ModeDashboardUtils.get_auth_params(conf=conf, discover_auth=True) - json_path = '_embedded.spaces[*].[token,name,description]' + json_path = 'spaces[*].[token,name,description]' field_names = ['dashboard_group_id', 'dashboard_group', 'dashboard_group_description'] - spaces_query = RestApiQuery(query_to_join=seed_query, url=spaces_url_template, params=params, - json_path=json_path, field_names=field_names) + + # based on https://mode.com/developer/discovery-api/analytics/spaces/#listSpacesForAccount + pagination_json_path = 'spaces[*]' + max_per_page = 1000 + spaces_query = ModePaginatedRestApiQuery(pagination_json_path=pagination_json_path, + max_record_size=max_per_page, query_to_join=seed_query, + url=spaces_url_template, params=params, json_path=json_path, + field_names=field_names) return spaces_query diff --git a/setup.py b/setup.py index e9be3be94..4e238d724 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages, setup -__version__ = '4.3.0' +__version__ = '4.3.1' requirements = [