diff --git a/config-sample.yaml b/config-sample.yaml
index 1e44c6c2b..aa2b01889 100644
--- a/config-sample.yaml
+++ b/config-sample.yaml
@@ -156,7 +156,6 @@ keys:
 #   playout_delay:
 #     enabled: true
 #     min: 100
-#     max: 300
 
 # Webhooks
 # when configured, LiveKit notifies your URL handler with room events
diff --git a/go.mod b/go.mod
index f282c501a..6a67fb3f4 100644
--- a/go.mod
+++ b/go.mod
@@ -18,7 +18,7 @@ require (
 	github.com/jxskiss/base62 v1.1.0
 	github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1
 	github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a
-	github.com/livekit/protocol v1.5.11-0.20230803070907-caf65ebc47a0
+	github.com/livekit/protocol v1.5.11-0.20230803081122-d791f31b6e0f
 	github.com/livekit/psrpc v0.3.2
 	github.com/mackerelio/go-osstat v0.2.4
 	github.com/magefile/mage v1.15.0
diff --git a/go.sum b/go.sum
index c75e6a8c0..597d1bc43 100644
--- a/go.sum
+++ b/go.sum
@@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD
 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ=
 github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a h1:JWpPHcMFuw0fP4swE89CfMgeUXiSN5IKvCJL/5HLI3A=
 github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a/go.mod h1:xirUXW8xnLGmfCwUeAv/nj1VGo1OO1BmgxrYP7jK/14=
-github.com/livekit/protocol v1.5.11-0.20230803070907-caf65ebc47a0 h1:zoHo3GmUmfZrfZBKF8l+P07ntNitqkEZ0NBjwNdHbhk=
-github.com/livekit/protocol v1.5.11-0.20230803070907-caf65ebc47a0/go.mod h1:SUS9foM1xBzw/AFrgTJuFX/oSuwlnIbHmpdiPdCvwEM=
+github.com/livekit/protocol v1.5.11-0.20230803081122-d791f31b6e0f h1:nhkzJVWnRePDo6imcFnuIljwU8MTBjcRdzfdLaVdWXw=
+github.com/livekit/protocol v1.5.11-0.20230803081122-d791f31b6e0f/go.mod h1:SUS9foM1xBzw/AFrgTJuFX/oSuwlnIbHmpdiPdCvwEM=
 github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew=
 github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U=
 github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs=
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 923bab6e7..d05eceab7 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -200,7 +200,6 @@ type StreamTrackersConfig struct {
 type PlayoutDelayConfig struct {
 	Enabled bool `yaml:"enabled,omitempty"`
 	Min     int  `yaml:"min,omitempty"`
-	Max     int  `yaml:"max,omitempty"`
 }
 
 type VideoConfig struct {
diff --git a/pkg/service/roomallocator.go b/pkg/service/roomallocator.go
index f07545c52..e7fbf878f 100644
--- a/pkg/service/roomallocator.go
+++ b/pkg/service/roomallocator.go
@@ -85,6 +85,12 @@ func (r *StandardRoomAllocator) CreateRoom(ctx context.Context, req *livekit.Cre
 	if req.Egress != nil && req.Egress.Tracks != nil {
 		internal = &livekit.RoomInternal{TrackEgress: req.Egress.Tracks}
 	}
+	if req.MinPlayoutDelay > 0 {
+		rm.PlayoutDelay = &livekit.PlayoutDelay{
+			Enabled: true,
+			Min:     req.MinPlayoutDelay,
+		}
+	}
 
 	if err = r.roomStore.StoreRoom(ctx, rm, internal); err != nil {
 		return nil, err
@@ -154,6 +160,5 @@ func applyDefaultRoomConfig(room *livekit.Room, conf *config.RoomConfig) {
 	room.PlayoutDelay = &livekit.PlayoutDelay{
 		Enabled: conf.PlayoutDelay.Enabled,
 		Min:     uint32(conf.PlayoutDelay.Min),
-		Max:     uint32(conf.PlayoutDelay.Max),
 	}
 }
diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go
index 70a8071bc..eabfa7090 100644
--- a/pkg/sfu/downtrack.go
+++ b/pkg/sfu/downtrack.go
@@ -330,7 +330,7 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) {
 	if d.params.PlayoutDelayLimit.GetEnabled() && d.params.PlayoutDelayLimit.GetMin() > 0 {
 		delay := rtpextension.PlayoutDelayFromValue(
 			uint16(d.params.PlayoutDelayLimit.GetMin()),
-			uint16(d.params.PlayoutDelayLimit.GetMax()),
+			rtpextension.PlayoutDelayDefaultMax,
 		)
 		b, err := delay.Marshal()
 		if err == nil {
diff --git a/pkg/sfu/rtpextension/playoutdelay.go b/pkg/sfu/rtpextension/playoutdelay.go
index 2cb719269..d42322be7 100644
--- a/pkg/sfu/rtpextension/playoutdelay.go
+++ b/pkg/sfu/rtpextension/playoutdelay.go
@@ -6,7 +6,8 @@ import (
 )
 
 const (
-	PlayoutDelayURI = "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay"
+	PlayoutDelayURI        = "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay"
+	PlayoutDelayDefaultMax = 4000 // 4s
 
 	playoutDelayExtensionSize = 3
 )