diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go
index 0e094bdad..818562722 100644
--- a/pkg/sfu/streamallocator/channelobserver.go
+++ b/pkg/sfu/streamallocator/channelobserver.go
@@ -61,6 +61,7 @@ type ChannelObserverParams struct {
 	EstimateDownwardTrendThreshold float64
 	EstimateCollapseThreshold      time.Duration
 	EstimateValidityWindow         time.Duration
+	NackMinPackets                 uint32
 	NackWindowMinDuration          time.Duration
 	NackWindowMaxDuration          time.Duration
 	NackRatioThreshold             float64
@@ -72,10 +73,6 @@ type ChannelObserver struct {
 
 	estimateTrend *TrendDetector
 	nackTracker   *NackTracker
-
-	nackWindowStartTime time.Time
-	packets             uint32
-	repeatedNacks       uint32
 }
 
 func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *ChannelObserver {
@@ -93,6 +90,7 @@ func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *Cha
 		nackTracker: NewNackTracker(NackTrackerParams{
 			Name:              params.Name + "-nack",
 			Logger:            logger,
+			MinPackets:        params.NackMinPackets,
 			WindowMinDuration: params.NackWindowMinDuration,
 			WindowMaxDuration: params.NackWindowMaxDuration,
 			RatioThreshold:    params.NackRatioThreshold,
diff --git a/pkg/sfu/streamallocator/nacktracker.go b/pkg/sfu/streamallocator/nacktracker.go
index 74104d625..46182fa05 100644
--- a/pkg/sfu/streamallocator/nacktracker.go
+++ b/pkg/sfu/streamallocator/nacktracker.go
@@ -12,6 +12,7 @@ import (
 type NackTrackerParams struct {
 	Name              string
 	Logger            logger.Logger
+	MinPackets        uint32
 	WindowMinDuration time.Duration
 	WindowMaxDuration time.Duration
 	RatioThreshold    float64
@@ -62,7 +63,7 @@ func (n *NackTracker) Add(packets uint32, repeatedNacks uint32) {
 
 func (n *NackTracker) GetRatio() float64 {
 	ratio := 0.0
-	if n.packets != 0 {
+	if n.packets != 0 && n.packets >= n.params.MinPackets {
 		ratio = float64(n.repeatedNacks) / float64(n.packets)
 		if ratio > 1.0 {
 			ratio = 1.0
diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go
index 475b38e94..207118f76 100644
--- a/pkg/sfu/streamallocator/streamallocator.go
+++ b/pkg/sfu/streamallocator/streamallocator.go
@@ -46,6 +46,7 @@ var (
 		EstimateDownwardTrendThreshold: 0.0,
 		EstimateCollapseThreshold:      0,
 		EstimateValidityWindow:         10 * time.Second,
+		NackMinPackets:                 0,
 		NackWindowMinDuration:          500 * time.Millisecond,
 		NackWindowMaxDuration:          1 * time.Second,
 		NackRatioThreshold:             0.04,
@@ -57,6 +58,7 @@ var (
 		EstimateDownwardTrendThreshold: -0.5,
 		EstimateCollapseThreshold:      500 * time.Millisecond,
 		EstimateValidityWindow:         10 * time.Second,
+		NackMinPackets:                 50,
 		NackWindowMinDuration:          1 * time.Second,
 		NackWindowMaxDuration:          2 * time.Second,
 		NackRatioThreshold:             0.08,