chore: fix rest_api_query to do basic check

Signed-off-by: feng-tao <fengtao04@gmail.com>
amundsen-io · Aug 19, 2020 · c1b57fb · c1b57fb
1 parent bf4c6ab
commit c1b57fb
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/databuilder/rest_api/rest_api_query.py b/databuilder/rest_api/rest_api_query.py
@@ -153,7 +153,7 @@ def execute(self) -> Iterator[Dict[str, Any]]:  # noqa: C901
                 result_list: List[Any] = [match.value for match in self._jsonpath_expr.find(response_json)]
 
                 if not result_list:
-                    log_msg = 'No result from URL: {url}  , JSONPATH: {json_path} , response payload: {response}' \
+                    log_msg = 'No result from URL: {url}, JSONPATH: {json_path} , response payload: {response}' \
                         .format(url=self._url, json_path=self._json_path, response=response_json)
                     LOGGER.info(log_msg)
 
@@ -172,6 +172,9 @@ def execute(self) -> Iterator[Dict[str, Any]]:  # noqa: C901
                                                                 json_path_contains_or=self._json_path_contains_or)
 
                 for sub_record in sub_records:
+                    if not sub_record or len(sub_record) != len(self._field_names):
+                        # skip the record
+                        continue
                     record_dict = copy.deepcopy(record_dict)
                     for field_name in self._field_names:
                         record_dict[field_name] = sub_record.pop(0)