tests.system.test_instance_admin: test_instance_create_prod failed

[flaky-bot]

Note: #457 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.

---

commit: d2bfca6
buildURL: Build Status, Sponge
status: failed

Test output

target = functools.partial(>)
predicate = .if_exception_type_predicate at 0x7f92971659d0>
sleep_generator = 
deadline = 60, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:


          return target()


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:190:

self = <google.api_core.operation.Operation object at 0x7f92945716d0>

retry = <google.api_core.retry.Retry object at 0x7f929716a340>
def _done_or_raise(self, retry=DEFAULT_RETRY):
    """Check if the future is done and raise if it's not."""
    kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}

    if not self.done(**kwargs):


      raise _OperationNotComplete()


E           google.api_core.future.polling._OperationNotComplete
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:89: _OperationNotComplete
The above exception was the direct cause of the following exception:
self = <google.api_core.operation.Operation object at 0x7f92945716d0>

timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f929716a340>
def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}


      retry_(self._done_or_raise)(**kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110:

args = (), kwargs = {}

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f92945716d0>>)

sleep_generator = <generator object exponential_sleep_generator at 0x7f9295602e40>
@functools.wraps(func)
def retry_wrapped_func(*args, **kwargs):
    """A wrapper that calls target function with retry."""
    target = functools.partial(func, *args, **kwargs)
    sleep_generator = exponential_sleep_generator(
        self._initial, self._maximum, multiplier=self._multiplier
    )


  return retry_target(


        target,
        self._predicate,
        sleep_generator,
        self._deadline,
        on_error=on_error,
    )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:283:

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f92945716d0>>)

predicate = <function if_exception_type..if_exception_type_predicate at 0x7f92971659d0>

sleep_generator = <generator object exponential_sleep_generator at 0x7f9295602e40>

deadline = 60, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:
            return target()

        # pylint: disable=broad-except
        # This function explicitly must deal with broad exceptions.
        except Exception as exc:
            if not predicate(exc):
                raise
            last_exc = exc
            if on_error is not None:
                on_error(exc)

        now = datetime_helpers.utcnow()

        if deadline_datetime is not None:
            if deadline_datetime <= now:


              raise exceptions.RetryError(


                    "Deadline of {:.1f}s exceeded while calling target function".format(
                        deadline
                    ),
                    last_exc,
                ) from last_exc

E                   google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling target function, last exception:
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:205: RetryError
During handling of the above exception, another exception occurred:
admin_client = <google.cloud.bigtable.client.Client object at 0x7f9296eb1d60>

unique_suffix = '-1661370419528', location_id = 'us-central1-c'

instance_labels = {'python-system': '2022-08-24t19-46-59'}

instances_to_delete = [<google.cloud.bigtable.instance.Instance object at 0x7f9294571640>]

skip_on_emulator = None
def test_instance_create_prod(
    admin_client,
    unique_suffix,
    location_id,
    instance_labels,
    instances_to_delete,
    skip_on_emulator,
):
    from google.cloud.bigtable import enums

    alt_instance_id = f"ndef{unique_suffix}"
    instance = admin_client.instance(alt_instance_id, labels=instance_labels)
    alt_cluster_id = f"{alt_instance_id}-cluster"
    serve_nodes = 1
    cluster = instance.cluster(
        alt_cluster_id,
        location_id=location_id,
        serve_nodes=serve_nodes,
    )

    operation = instance.create(clusters=[cluster])
    instances_to_delete.append(instance)


  operation.result(timeout=60)  # Ensure the operation completes.


tests/system/test_instance_admin.py:166:

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:132: in result

self._blocking_poll(timeout=timeout, **kwargs)

self = <google.api_core.operation.Operation object at 0x7f92945716d0>

timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f929716a340>
def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
        retry_(self._done_or_raise)(**kwargs)
    except exceptions.RetryError:


      raise concurrent.futures.TimeoutError(


            "Operation did not complete within the designated " "timeout."
        )

E           concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:112: TimeoutError

[flaky-bot]

Looks like this issue is flaky. 😟

I'm going to leave this open and stop commenting.

A human should fix and close this.

---

When run at the same commit (d2bfca6), this test passed in one build (Build Status, Sponge) and failed in another build (Build Status, Sponge).

[Mariatta]

Last failed test was Oct 28.