exponential backoff when calling CreateRoom (#2977)

* exponential backoff when calling CreateRoom

* dz+bb review: check ctx cancellation, remove retry max

* bb review: fix loop condition

* raja review: use timer, bring back max tries

magefile.go

@@ -39,8 +39,10 @@ const (
 
 // Default target to run when none is specified
 // If not set, running mage will list available targets
-var Default = Build
-var checksummer = mageutil.NewChecksummer(".", goChecksumFile, ".go", ".mod")
+var (
+	Default     = Build
+	checksummer = mageutil.NewChecksummer(".", goChecksumFile, ".go", ".mod")
+)
 
 func init() {
 	checksummer.IgnoredPaths = []string{

pkg/service/rtcservice.go

@@ -42,6 +42,13 @@ import (
 	"github.com/livekit/psrpc"
 )
 
+const (
+	// TODO: make these configurable
+	initialBackoffSeconds   = 1
+	maxBackoffSecondsPow    = 6 // 64 seconds
+	startConnectionMaxTries = 3
+)
+
 type RTCService struct {
 	router        routing.MessageRouter
 	roomAllocator RoomAllocator
@@ -229,15 +236,26 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// give it a few attempts to start session
 	var cr connectionResult
 	var initialResponse *livekit.SignalResponse
-	for i := 0; i < 3; i++ {
-		connectionTimeout := 3 * time.Second * time.Duration(i+1)
-		ctx := utils.ContextWithAttempt(r.Context(), i)
+	for attempt := 0; attempt < startConnectionMaxTries && r.Context().Err() == nil; attempt++ {
+		connectionTimeout := 3 * time.Second * time.Duration(attempt+1)
+		ctx := utils.ContextWithAttempt(r.Context(), attempt)
 		cr, initialResponse, err = s.startConnection(ctx, roomName, pi, connectionTimeout)
 		if err == nil || errors.Is(err, context.Canceled) {
 			break
 		}
-		if i < 2 {
-			pLogger.Warnw("failed to start connection, retrying", err, "attempt", i)
+
+		if attempt < startConnectionMaxTries-1 {
+			// exponential backoff delay. powers of 2.
+			backoffDelay := time.NewTimer(time.Duration(initialBackoffSeconds<<min(attempt, maxBackoffSecondsPow)) * time.Second)
+			pLogger.Warnw("failed to start connection, retrying", err,
+				"attempt", attempt,
+				"backoffDelay", backoffDelay,
+			)
+			select {
+			case <-backoffDelay.C: // wait for backoff
+			case <-ctx.Done():
+				backoffDelay.Stop()
+			}
 		}
 	}