From 5273f34d83b72c963db3b545acb8399d28c5c999 Mon Sep 17 00:00:00 2001 From: Igor Berntein Date: Fri, 18 Nov 2022 13:15:37 -0500 Subject: [PATCH 1/2] fix: make internal rst_stream errors retriable --- google/cloud/bigtable/row_data.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/google/cloud/bigtable/row_data.py b/google/cloud/bigtable/row_data.py index 0e07ff268..009d77bf5 100644 --- a/google/cloud/bigtable/row_data.py +++ b/google/cloud/bigtable/row_data.py @@ -18,6 +18,7 @@ import copy import time +import google.api_core.exceptions import six import grpc @@ -507,7 +508,16 @@ def _read_next(self): try: return six.next(self.response_iterator) except grpc.RpcError as grpc_error: - raise exceptions.from_grpc_error(grpc_error) + # TODO: this needs to be moved to a more general location (ie interceptor) + e = exceptions.from_grpc_error(grpc_error) + # Sometimes GOAWAYs are surfaced as INTERNAL errors, which makes + # them unretriable. This patches that behavior + if ( + e.grpc_status_code == grpc.StatusCode.INTERNAL + and "RST_STREAM" in e.message + ): + raise google.api_core.exceptions.ServiceUnavailable(e.message) + raise e def _read_next_response(self): """Helper for :meth:`__iter__`.""" From 2798e8d0efcaba8cc9d94f81716a791063f11046 Mon Sep 17 00:00:00 2001 From: Igor Berntein Date: Fri, 18 Nov 2022 13:33:11 -0500 Subject: [PATCH 2/2] catch more rst stream variants --- google/cloud/bigtable/row_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigtable/row_data.py b/google/cloud/bigtable/row_data.py index 009d77bf5..0a0bd96e4 100644 --- a/google/cloud/bigtable/row_data.py +++ b/google/cloud/bigtable/row_data.py @@ -512,9 +512,8 @@ def _read_next(self): e = exceptions.from_grpc_error(grpc_error) # Sometimes GOAWAYs are surfaced as INTERNAL errors, which makes # them unretriable. This patches that behavior - if ( - e.grpc_status_code == grpc.StatusCode.INTERNAL - and "RST_STREAM" in e.message + if e.grpc_status_code == grpc.StatusCode.INTERNAL and ( + "rst_stream" in e.message.lower() or "rst stream" in e.message.lower() ): raise google.api_core.exceptions.ServiceUnavailable(e.message) raise e