Error in Airbyte.connections.list_connections with offset equal to the number of connections available

[henriquemeloo]

As described in the following code:

import airbyte


airbyte_workspace = airbyte.Airbyte(
    server_url=SERVER_URL
)


response = airbyte_workspace.connections.list_connections(
    airbyte.models.operations.ListConnectionsRequest(
        workspace_ids=[WORKSPACE_ID],
        offset=0, limit=100
    )
)
# There are only 20 connections
assert len(response.connections_response.data) == 20

response = airbyte_workspace.connections.list_connections(
    airbyte.models.operations.ListConnectionsRequest(
        workspace_ids=[WORKSPACE_ID],
        offset=0, limit=20
    )
)
# Still, with a limit of 20, a "next" page is provided
assert response.connections_response.next is not None

# But it breaks if we try to fetch data from it
airbyte_workspace.connections.list_connections(
    airbyte.models.operations.ListConnectionsRequest(
        workspace_ids=[WORKSPACE_ID],
        offset=20, limit=20
    )
)

which raises:

KeyError                                  Traceback (most recent call last)
     28 assert response.connections_response.next is not None
     30 # But it breaks if we try to fetch data from it
---> 31 airbyte_workspace.connections.list_connections(
     32     airbyte.models.operations.ListConnectionsRequest(
     33         workspace_ids=[WORKSPACE_ID],
     34         offset=20, limit=20
     35     )
     36 )

File .../.venv/lib/python3.11/site-packages/airbyte/connections.py:135, in Connections.list_connections(self, request)
    133 if http_res.status_code == 200:
    134     if utils.match_content_type(content_type, 'application/json'):
--> 135         out = utils.unmarshal_json(http_res.text, Optional[shared.ConnectionsResponse])
    136         res.connections_response = out
    137     else:

File .../.venv/lib/python3.11/site-packages/airbyte/utils/utils.py:695, in unmarshal_json(data, typ, decoder)
    693 json_dict = json.loads(data)
    694 try:
--> 695     out = unmarshal.from_dict({"res": json_dict})
    696 except AttributeError as attr_err:
    697     raise AttributeError(
    698         f'unable to unmarshal {data} as {typ} - {attr_err}') from attr_err

File .../.venv/lib/python3.11/site-packages/dataclasses_json/api.py:70, in DataClassJsonMixin.from_dict(cls, kvs, infer_missing)
     65 @classmethod
     66 def from_dict(cls: Type[A],
     67               kvs: Json,
     68               *,
     69               infer_missing=False) -> A:
---> 70     return _decode_dataclass(cls, kvs, infer_missing)

File .../.venv/lib/python3.11/site-packages/dataclasses_json/core.py:220, in _decode_dataclass(cls, kvs, infer_missing)
    218     init_kwargs[field.name] = value
    219 elif _is_supported_generic(field_type) and field_type != str:
--> 220     init_kwargs[field.name] = _decode_generic(field_type,
    221                                               field_value,
    222                                               infer_missing)
    223 else:
    224     init_kwargs[field.name] = _support_extended_types(field_type,
    225                                                       field_value)

File .../.venv/lib/python3.11/site-packages/dataclasses_json/core.py:309, in _decode_generic(type_, value, infer_missing)
    307 type_arg = _get_type_arg_param(type_, 0)
    308 if is_dataclass(type_arg) or is_dataclass(value):
--> 309     res = _decode_dataclass(type_arg, value, infer_missing)
    310 elif _is_supported_generic(type_arg):
    311     res = _decode_generic(type_arg, value, infer_missing)

File .../.venv/lib/python3.11/site-packages/dataclasses_json/core.py:172, in _decode_dataclass(cls, kvs, infer_missing)
    169 if not field.init:
    170     continue
--> 172 field_value = kvs[field.name]
    173 field_type = types[field.name]
    174 if field_value is None:

KeyError: 'data'

[henriquemeloo]

When reaching an "empty page", the response has only a "previous" key, and the SDK does not handle it correctly:

import requests


response = requests.get(
    SERVER_URL + "/connections",
    headers={"accept": "application/json"},
    params={
        "workspaceIds": WORKSPACE_ID,
        "offset": 20, "limit": 20
    }
)

assert list(response.json().keys()) == ["previous"]