diff --git a/pkg/config/config.go b/pkg/config/config.go
index f8c572354..ffc3fc9cf 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -78,7 +78,8 @@ func NewConfig(confString string) (*Config, error) {
 				"stun.l.google.com:19302",
 				"stun1.l.google.com:19302",
 			},
-			MaxBitrate: 3 * 1024 * 1024, // 3 mbps
+			MaxBitrate:       3 * 1024 * 1024, // 3 mbps
+			PacketBufferSize: 500,
 		},
 		Audio: AudioConfig{
 			ActiveLevel:    40,
diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go
index 14aa9b73f..307a0c986 100644
--- a/pkg/rtc/config.go
+++ b/pkg/rtc/config.go
@@ -6,7 +6,6 @@ import (
 	"net"
 	"syscall"
 
-	"github.com/go-logr/zapr"
 	"github.com/pion/ice/v2"
 	"github.com/pion/ion-sfu/pkg/buffer"
 	"github.com/pion/logging"
@@ -64,8 +63,6 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err
 	if rtcConf.PacketBufferSize == 0 {
 		rtcConf.PacketBufferSize = 500
 	}
-	bufferFactory := buffer.NewBufferFactory(rtcConf.PacketBufferSize, zapr.NewLogger(logger.GetLogger()))
-	s.BufferFactory = bufferFactory.GetOrNew
 
 	networkTypes := make([]webrtc.NetworkType, 0, 4)
 	if !rtcConf.ForceTCP {
@@ -135,7 +132,6 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err
 	return &WebRTCConfig{
 		Configuration: c,
 		SettingEngine: s,
-		BufferFactory: bufferFactory,
 		Receiver: ReceiverConfig{
 			packetBufferSize: rtcConf.PacketBufferSize,
 			maxBitrate:       rtcConf.MaxBitrate,
@@ -146,6 +142,11 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err
 	}, nil
 }
 
+func (c *WebRTCConfig) SetBufferFactory(factory *buffer.Factory) {
+	c.BufferFactory = factory
+	c.SettingEngine.BufferFactory = factory.GetOrNew
+}
+
 func checkUDPReadBuffer() (int, error) {
 	conn, err := net.ListenUDP("udp4", nil)
 	if err != nil {
diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go
index 214da4aaa..342958afc 100644
--- a/pkg/rtc/room.go
+++ b/pkg/rtc/room.go
@@ -6,7 +6,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/go-logr/zapr"
 	"github.com/livekit/protocol/utils"
+	"github.com/pion/ion-sfu/pkg/buffer"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/livekit/livekit-server/pkg/logger"
@@ -27,6 +29,7 @@ type Room struct {
 	// map of identity -> Participant
 	participants    map[string]types.Participant
 	participantOpts map[string]*ParticipantOptions
+	bufferFactory   *buffer.Factory
 
 	// time the first participant joined the room
 	joinedAt atomic.Value
@@ -57,6 +60,7 @@ func NewRoom(room *livekit.Room, config WebRTCConfig, iceServers []*livekit.ICES
 		statsReporter:       NewRoomStatsReporter(room.Name),
 		participants:        make(map[string]types.Participant),
 		participantOpts:     make(map[string]*ParticipantOptions),
+		bufferFactory:       buffer.NewBufferFactory(config.Receiver.packetBufferSize, zapr.NewLogger(logger.GetLogger())),
 	}
 	if r.Room.EmptyTimeout == 0 {
 		r.Room.EmptyTimeout = DefaultEmptyTimeout
@@ -110,6 +114,10 @@ func (r *Room) GetStatsReporter() *RoomStatsReporter {
 	return r.statsReporter
 }
 
+func (r *Room) GetBufferFactor() *buffer.Factory {
+	return r.bufferFactory
+}
+
 func (r *Room) FirstJoinedAt() int64 {
 	j := r.joinedAt.Load()
 	if t, ok := j.(int64); ok {
diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go
index 6c639fc91..14a452365 100644
--- a/pkg/service/roommanager.go
+++ b/pkg/service/roommanager.go
@@ -250,6 +250,7 @@ func (r *RoomManager) StartSession(roomName string, pi routing.ParticipantInit,
 
 	pv := types.ProtocolVersion(pi.ProtocolVersion)
 	rtcConf := *r.rtcConfig
+	rtcConf.SetBufferFactory(room.GetBufferFactor())
 	if pi.UsePlanB {
 		rtcConf.Configuration.SDPSemantics = webrtc.SDPSemanticsPlanB
 	}