Merge pull request #2795 from murgatroid99/grpc-js_retry_limit_option

grpc-js: Add a channel option to configure retry attempt limits
grpc · Jul 12, 2024 · d60f516 · d60f516
2 parents b35896b + 8ee8e99
commit d60f516
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 5 deletions.
diff --git a/packages/grpc-js/src/channel-options.ts b/packages/grpc-js/src/channel-options.ts
@@ -63,6 +63,7 @@ export interface ChannelOptions {
    */
   'grpc-node.tls_enable_trace'?: number;
   'grpc.lb.ring_hash.ring_size_cap'?: number;
+  'grpc-node.retry_max_attempts_limit'?: number;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   [key: string]: any;
 }
@@ -99,6 +100,7 @@ export const recognizedOptions = {
   'grpc.client_idle_timeout_ms': true,
   'grpc-node.tls_enable_trace': true,
   'grpc.lb.ring_hash.ring_size_cap': true,
+  'grpc-node.retry_max_attempts_limit': true,
 };
 
 export function channelOptionsEqual(

diff --git a/packages/grpc-js/src/internal-channel.ts b/packages/grpc-js/src/internal-channel.ts
@@ -867,4 +867,8 @@ export class InternalChannel {
       propagateFlags
     );
   }
+
+  getOptions() {
+    return this.options;
+  }
 }
diff --git a/packages/grpc-js/src/retrying-call.ts b/packages/grpc-js/src/retrying-call.ts
@@ -172,6 +172,8 @@ interface WriteBufferEntry {
 
 const PREVIONS_RPC_ATTEMPTS_METADATA_KEY = 'grpc-previous-rpc-attempts';
 
+const DEFAULT_MAX_ATTEMPTS_LIMIT = 5;
+
 export class RetryingCall implements Call, DeadlineInfoProvider {
   private state: RetryingCallState;
   private listener: InterceptingListener | null = null;
@@ -201,6 +203,7 @@ export class RetryingCall implements Call, DeadlineInfoProvider {
   private initialRetryBackoffSec = 0;
   private nextRetryBackoffSec = 0;
   private startTime: Date;
+  private maxAttempts: number;
   constructor(
     private readonly channel: InternalChannel,
     private readonly callConfig: CallConfig,
@@ -212,6 +215,7 @@ export class RetryingCall implements Call, DeadlineInfoProvider {
     private readonly bufferTracker: MessageBufferTracker,
     private readonly retryThrottler?: RetryThrottler
   ) {
+    const maxAttemptsLimit = channel.getOptions()['grpc-node.retry_max_attempts_limit'] ?? DEFAULT_MAX_ATTEMPTS_LIMIT;
     if (callConfig.methodConfig.retryPolicy) {
       this.state = 'RETRY';
       const retryPolicy = callConfig.methodConfig.retryPolicy;
@@ -221,10 +225,13 @@ export class RetryingCall implements Call, DeadlineInfoProvider {
           retryPolicy.initialBackoff.length - 1
         )
       );
+      this.maxAttempts = Math.min(retryPolicy.maxAttempts, maxAttemptsLimit);
     } else if (callConfig.methodConfig.hedgingPolicy) {
       this.state = 'HEDGING';
+      this.maxAttempts = Math.min(callConfig.methodConfig.hedgingPolicy.maxAttempts, maxAttemptsLimit);
     } else {
       this.state = 'TRANSPARENT_ONLY';
+      this.maxAttempts = 1;
     }
     this.startTime = new Date();
   }
@@ -419,8 +426,7 @@ export class RetryingCall implements Call, DeadlineInfoProvider {
       callback(false);
       return;
     }
-    const retryPolicy = this.callConfig!.methodConfig.retryPolicy!;
-    if (this.attempts >= Math.min(retryPolicy.maxAttempts, 5)) {
+    if (this.attempts >= this.maxAttempts) {
       callback(false);
       return;
     }
@@ -596,8 +602,7 @@ export class RetryingCall implements Call, DeadlineInfoProvider {
     if (!this.callConfig.methodConfig.hedgingPolicy) {
       return;
     }
-    const hedgingPolicy = this.callConfig.methodConfig.hedgingPolicy;
-    if (this.attempts >= Math.min(hedgingPolicy.maxAttempts, 5)) {
+    if (this.attempts >= this.maxAttempts) {
       return;
     }
     this.attempts += 1;
@@ -616,7 +621,7 @@ export class RetryingCall implements Call, DeadlineInfoProvider {
       return;
     }
     const hedgingPolicy = this.callConfig.methodConfig.hedgingPolicy;
-    if (this.attempts >= Math.min(hedgingPolicy.maxAttempts, 5)) {
+    if (this.attempts >= this.maxAttempts) {
       return;
     }
     const hedgingDelayString = hedgingPolicy.hedgingDelay ?? '0s';

diff --git a/packages/grpc-js/test/test-retry.ts b/packages/grpc-js/test/test-retry.ts
@@ -281,6 +281,48 @@ describe('Retries', () => {
         }
       );
     });
+
+    it('Should be able to make more than 5 attempts with a channel argument', done => {
+      const serviceConfig = {
+        loadBalancingConfig: [],
+        methodConfig: [
+          {
+            name: [
+              {
+                service: 'EchoService',
+              },
+            ],
+            retryPolicy: {
+              maxAttempts: 10,
+              initialBackoff: '0.1s',
+              maxBackoff: '10s',
+              backoffMultiplier: 1.2,
+              retryableStatusCodes: [14, 'RESOURCE_EXHAUSTED'],
+            },
+          },
+        ],
+      };
+      const client2 = new EchoService(
+        `localhost:${port}`,
+        grpc.credentials.createInsecure(),
+        {
+          'grpc.service_config': JSON.stringify(serviceConfig),
+          'grpc-node.retry_max_attempts_limit': 8
+        }
+      );
+      const metadata = new grpc.Metadata();
+      metadata.set('succeed-on-retry-attempt', '7');
+      metadata.set('respond-with-status', `${grpc.status.RESOURCE_EXHAUSTED}`);
+      client2.echo(
+        { value: 'test value', value2: 3 },
+        metadata,
+        (error: grpc.ServiceError, response: any) => {
+          assert.ifError(error);
+          assert.deepStrictEqual(response, { value: 'test value', value2: 3 });
+          done();
+        }
+      );
+    });
   });
 
   describe('Client with hedging configured', () => {