fix: Switch to discovery api for mode spaces (#481)

* use pagenated rest api query for getting all Mode spaces

Signed-off-by: Junda Yang <youngyjd@gmail.com>

* nit

Signed-off-by: Junda Yang <youngyjd@gmail.com>

* update comment

Signed-off-by: Junda Yang <youngyjd@gmail.com>

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_utils.py

@@ -14,7 +14,7 @@
     REST_API_QUERY, STATIC_RECORD_DICT, RestAPIExtractor,
 )
 from databuilder.rest_api.base_rest_api_query import BaseRestApiQuery, RestApiQuerySeed
-from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
 
 
 class ModeDashboardUtils(object):
@@ -28,21 +28,28 @@ def get_spaces_query_api(conf: ConfigTree) -> BaseRestApiQuery:
         :return:
         """
 
-        # https://mode.com/developer/api-reference/management/spaces/#listSpaces
-        spaces_url_template = 'https://app.mode.com/api/{organization}/spaces?filter=all'
+        # https://mode.com/developer/discovery-api/analytics/spaces
+        spaces_url_template = 'https://app.mode.com/batch/{organization}/spaces'
 
         # Seed query record for next query api to join with
         seed_record = [{'organization': conf.get_string(ORGANIZATION)}]
         seed_query = RestApiQuerySeed(seed_record=seed_record)
 
-        # Spaces
-        params = {'auth': HTTPBasicAuth(conf.get_string(MODE_ACCESS_TOKEN),
-                                        conf.get_string(MODE_PASSWORD_TOKEN))}
+        # mode_bearer_token must be provided in the conf
+        # the token is required to access discovery endpoint
+        # https://mode.com/developer/discovery-api/introduction/
+        params = ModeDashboardUtils.get_auth_params(conf=conf, discover_auth=True)
 
-        json_path = '_embedded.spaces[*].[token,name,description]'
+        json_path = 'spaces[*].[token,name,description]'
         field_names = ['dashboard_group_id', 'dashboard_group', 'dashboard_group_description']
-        spaces_query = RestApiQuery(query_to_join=seed_query, url=spaces_url_template, params=params,
-                                    json_path=json_path, field_names=field_names)
+
+        # based on https://mode.com/developer/discovery-api/analytics/spaces/#listSpacesForAccount
+        pagination_json_path = 'spaces[*]'
+        max_per_page = 1000
+        spaces_query = ModePaginatedRestApiQuery(pagination_json_path=pagination_json_path,
+                                                 max_record_size=max_per_page, query_to_join=seed_query,
+                                                 url=spaces_url_template, params=params, json_path=json_path,
+                                                 field_names=field_names)
 
         return spaces_query
 

setup.py

@@ -4,7 +4,7 @@
 
 from setuptools import find_packages, setup
 
-__version__ = '4.3.0'
+__version__ = '4.3.1'
 
 
 requirements = [