From 730fe5164211321315fb1ea250495d387e1e3598 Mon Sep 17 00:00:00 2001
From: Mathew Kamkar <578302+matkam@users.noreply.github.com>
Date: Wed, 4 Sep 2024 23:18:31 -0700
Subject: [PATCH] exponential backoff when calling CreateRoom (#2977)

* exponential backoff when calling CreateRoom

* dz+bb review: check ctx cancellation, remove retry max

* bb review: fix loop condition

* raja review: use timer, bring back max tries
---
 magefile.go               |  6 ++++--
 pkg/service/rtcservice.go | 28 +++++++++++++++++++++++-----
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/magefile.go b/magefile.go
index 1b3fdfa81..629e0e6af 100644
--- a/magefile.go
+++ b/magefile.go
@@ -39,8 +39,10 @@ const (
 
 // Default target to run when none is specified
 // If not set, running mage will list available targets
-var Default = Build
-var checksummer = mageutil.NewChecksummer(".", goChecksumFile, ".go", ".mod")
+var (
+	Default     = Build
+	checksummer = mageutil.NewChecksummer(".", goChecksumFile, ".go", ".mod")
+)
 
 func init() {
 	checksummer.IgnoredPaths = []string{
diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go
index fad638de9..a2feb19ef 100644
--- a/pkg/service/rtcservice.go
+++ b/pkg/service/rtcservice.go
@@ -42,6 +42,13 @@ import (
 	"github.com/livekit/psrpc"
 )
 
+const (
+	// TODO: make these configurable
+	initialBackoffSeconds   = 1
+	maxBackoffSecondsPow    = 6 // 64 seconds
+	startConnectionMaxTries = 3
+)
+
 type RTCService struct {
 	router        routing.MessageRouter
 	roomAllocator RoomAllocator
@@ -229,15 +236,26 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// give it a few attempts to start session
 	var cr connectionResult
 	var initialResponse *livekit.SignalResponse
-	for i := 0; i < 3; i++ {
-		connectionTimeout := 3 * time.Second * time.Duration(i+1)
-		ctx := utils.ContextWithAttempt(r.Context(), i)
+	for attempt := 0; attempt < startConnectionMaxTries && r.Context().Err() == nil; attempt++ {
+		connectionTimeout := 3 * time.Second * time.Duration(attempt+1)
+		ctx := utils.ContextWithAttempt(r.Context(), attempt)
 		cr, initialResponse, err = s.startConnection(ctx, roomName, pi, connectionTimeout)
 		if err == nil || errors.Is(err, context.Canceled) {
 			break
 		}
-		if i < 2 {
-			pLogger.Warnw("failed to start connection, retrying", err, "attempt", i)
+
+		if attempt < startConnectionMaxTries-1 {
+			// exponential backoff delay. powers of 2.
+			backoffDelay := time.NewTimer(time.Duration(initialBackoffSeconds<<min(attempt, maxBackoffSecondsPow)) * time.Second)
+			pLogger.Warnw("failed to start connection, retrying", err,
+				"attempt", attempt,
+				"backoffDelay", backoffDelay,
+			)
+			select {
+			case <-backoffDelay.C: // wait for backoff
+			case <-ctx.Done():
+				backoffDelay.Stop()
+			}
 		}
 	}