Obscure error message ValueError: Request obj does not match any template when required field is not provided

[parthea]

In PR googleapis/python-compute#147, I had to apply this change to the samples tests because a required field usage_export_location_resource was not provided. I've opened this issue because the error message ValueError: Request obj does not match any template could be improved. We should provide more information to help users debug the issue as I found it difficult to find the root cause.

See build log here and stack trace below.

=================================== FAILURES ===================================
_____________________ test_set_usage_export_bucket_default _____________________

capsys = <_pytest.capture.CaptureFixture object at 0x7fee8a8dd690>
temp_bucket = <Bucket: teste4bc9ab375>

    def test_set_usage_export_bucket_default(capsys: typing.Any,
                                             temp_bucket: storage.Bucket) -> None:
        set_usage_export_bucket(project_id=PROJECT, bucket_name=temp_bucket.name)
        time.sleep(5)  # To make sure the settings are properly updated
        uel = get_usage_export_bucket(project_id=PROJECT)
        assert(uel.bucket_name == temp_bucket.name)
        assert(uel.report_name_prefix == 'usage_gce')
        out, _ = capsys.readouterr()
        assert('default prefix of `usage_gce`.' in out)

>       disable_usage_export(project_id=PROJECT)

test_sample_default_values.py:48:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
sample_default_values.py:118: in disable_usage_export
    project=project_id, usage_export_location_resource=None)
../../google/cloud/compute_v1/services/projects/client.py:1399: in set_usage_export_bucket
    response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)
.nox/py-3-7/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py:154: in __call__
    return wrapped_func(*args, **kwargs)
.nox/py-3-7/lib/python3.7/site-packages/google/api_core/grpc_helpers.py:66: in error_remapped_callable
    return callable_(*args, **kwargs)
../../google/cloud/compute_v1/services/projects/transports/rest.py:1352: in _set_usage_export_bucket
    transcoded_request = path_template.transcode(http_options, **request_kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

http_options = [{'body': 'usage_export_location_resource', 'method': 'post', 'uri': '/compute/v1/projects/{project}/setUsageExportBucket'}]
request_kwargs = {'project': 'python-docs-samples-tests-py37'}
http_option = {'body': 'usage_export_location_resource', 'method': 'post', 'uri': '/compute/v1/projects/{project}/setUsageExportBucket'}
request = {'uri': '/compute/v1/projects/python-docs-samples-tests-py37/setUsageExportBucket'}
uri_template = '/compute/v1/projects/{project}/setUsageExportBucket'
path_fields = ['project']
path_args = {'project': 'python-docs-samples-tests-py37'}, leftovers = {}
path_field = 'project'

    def transcode(http_options, **request_kwargs):
        """Transcodes a grpc request pattern into a proper HTTP request following the rules outlined here,
           https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312

            Args:
                http_options (list(dict)): A list of dicts which consist of these keys,
                    'method'    (str): The http method
                    'uri'       (str): The path template
                    'body'      (str): The body field name (optional)
                    (This is a simplified representation of the proto option `google.api.http`)

                request_kwargs (dict) : A dict representing the request object

            Returns:
                dict: The transcoded request with these keys,
                    'method'        (str)   : The http method
                    'uri'           (str)   : The expanded uri
                    'body'          (dict)  : A dict representing the body (optional)
                    'query_params'  (dict)  : A dict mapping query parameter variables and values

            Raises:
                ValueError: If the request does not match the given template.
        """
        for http_option in http_options:
            request = {}

            # Assign path
            uri_template = http_option["uri"]
            path_fields = [
                match.group("name") for match in _VARIABLE_RE.finditer(uri_template)
            ]
            path_args = {field: get_field(request_kwargs, field) for field in path_fields}
            request["uri"] = expand(uri_template, **path_args)

            # Remove fields used in uri path from request
            leftovers = copy.deepcopy(request_kwargs)
            for path_field in path_fields:
                delete_field(leftovers, path_field)

            if not validate(uri_template, request["uri"]) or not all(path_args.values()):
                continue

            # Assign body and query params
            body = http_option.get("body")

            if body:
                if body == "*":
                    request["body"] = leftovers
                    request["query_params"] = {}
                else:
                    try:
                        request["body"] = leftovers.pop(body)
                    except KeyError:
                        continue
                    request["query_params"] = leftovers
            else:
                request["query_params"] = leftovers
            request["method"] = http_option["method"]
            return request

>       raise ValueError("Request obj does not match any template")
E       ValueError: Request obj does not match any template

.nox/py-3-7/lib/python3.7/site-packages/google/api_core/path_template.py:300: ValueError
_____________________ test_set_usage_export_bucket_custom ______________________

capsys = <_pytest.capture.CaptureFixture object at 0x7fee8a74ba90>
temp_bucket = <Bucket: teste4bc9ab375>

    def test_set_usage_export_bucket_custom(capsys: typing.Any,
                                            temp_bucket: storage.Bucket) -> None:
        set_usage_export_bucket(project_id=PROJECT, bucket_name=temp_bucket.name,
                                report_name_prefix=TEST_PREFIX)
        time.sleep(5)  # To make sure the settings are properly updated
        uel = get_usage_export_bucket(project_id=PROJECT)
        assert(uel.bucket_name == temp_bucket.name)
        assert(uel.report_name_prefix == TEST_PREFIX)
        out, _ = capsys.readouterr()
        assert('usage_gce' not in out)

>       disable_usage_export(project_id=PROJECT)

test_sample_default_values.py:66:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
sample_default_values.py:118: in disable_usage_export
    project=project_id, usage_export_location_resource=None)
../../google/cloud/compute_v1/services/projects/client.py:1399: in set_usage_export_bucket
    response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)
.nox/py-3-7/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py:154: in __call__
    return wrapped_func(*args, **kwargs)
.nox/py-3-7/lib/python3.7/site-packages/google/api_core/grpc_helpers.py:66: in error_remapped_callable
    return callable_(*args, **kwargs)
../../google/cloud/compute_v1/services/projects/transports/rest.py:1352: in _set_usage_export_bucket
    transcoded_request = path_template.transcode(http_options, **request_kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

http_options = [{'body': 'usage_export_location_resource', 'method': 'post', 'uri': '/compute/v1/projects/{project}/setUsageExportBucket'}]
request_kwargs = {'project': 'python-docs-samples-tests-py37'}
http_option = {'body': 'usage_export_location_resource', 'method': 'post', 'uri': '/compute/v1/projects/{project}/setUsageExportBucket'}
request = {'uri': '/compute/v1/projects/python-docs-samples-tests-py37/setUsageExportBucket'}
uri_template = '/compute/v1/projects/{project}/setUsageExportBucket'
path_fields = ['project']
path_args = {'project': 'python-docs-samples-tests-py37'}, leftovers = {}
path_field = 'project'

    def transcode(http_options, **request_kwargs):
        """Transcodes a grpc request pattern into a proper HTTP request following the rules outlined here,
           https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312

            Args:
                http_options (list(dict)): A list of dicts which consist of these keys,
                    'method'    (str): The http method
                    'uri'       (str): The path template
                    'body'      (str): The body field name (optional)
                    (This is a simplified representation of the proto option `google.api.http`)

                request_kwargs (dict) : A dict representing the request object

            Returns:
                dict: The transcoded request with these keys,
                    'method'        (str)   : The http method
                    'uri'           (str)   : The expanded uri
                    'body'          (dict)  : A dict representing the body (optional)
                    'query_params'  (dict)  : A dict mapping query parameter variables and values

            Raises:
                ValueError: If the request does not match the given template.
        """
        for http_option in http_options:
            request = {}

            # Assign path
            uri_template = http_option["uri"]
            path_fields = [
                match.group("name") for match in _VARIABLE_RE.finditer(uri_template)
            ]
            path_args = {field: get_field(request_kwargs, field) for field in path_fields}
            request["uri"] = expand(uri_template, **path_args)

            # Remove fields used in uri path from request
            leftovers = copy.deepcopy(request_kwargs)
            for path_field in path_fields:
                delete_field(leftovers, path_field)

            if not validate(uri_template, request["uri"]) or not all(path_args.values()):
                continue

            # Assign body and query params
            body = http_option.get("body")

            if body:
                if body == "*":
                    request["body"] = leftovers
                    request["query_params"] = {}
                else:
                    try:
                        request["body"] = leftovers.pop(body)
                    except KeyError:
                        continue
                    request["query_params"] = leftovers
            else:
                request["query_params"] = leftovers
            request["method"] = http_option["method"]
            return request

>       raise ValueError("Request obj does not match any template")
E       ValueError: Request obj does not match any template

.nox/py-3-7/lib/python3.7/site-packages/google/api_core/path_template.py:300: ValueError

[software-dov]

This bug really belongs to python api core. Transferring there.

[atulep]

The exception is raised when the transcoded URI doesn't match the path_template from http_options, or when GRPC request has unset values for path fields from the path_template. I think you hit the latter case (i.e. usage_export_location_resource is present in the path_template, but unset in the request). Ref:

python-api-core/google/api_core/path_template.py

Lines 279 to 280 in 4422cce

	if not validate(uri_template, request["uri"]) or not all(path_args.values()):
	continue

I think we can improve the error message by showing all path_templates in http_options and including the request.