diff --git a/cmd/server/main.go b/cmd/server/main.go index 3e8e3344d..427702954 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -272,7 +272,9 @@ func startServer(c *cli.Context) error { return err } - prometheus.Init(currentNode.Id, currentNode.Type, conf.Environment) + if err := prometheus.Init(currentNode.Id, currentNode.Type); err != nil { + return err + } server, err := service.InitializeServer(conf, currentNode) if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 241bb8f0d..78cd0f1e1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -61,7 +61,6 @@ type Config struct { Port uint32 `yaml:"port,omitempty"` BindAddresses []string `yaml:"bind_addresses,omitempty"` PrometheusPort uint32 `yaml:"prometheus_port,omitempty"` - Environment string `yaml:"environment,omitempty"` RTC RTCConfig `yaml:"rtc,omitempty"` Redis redisLiveKit.RedisConfig `yaml:"redis,omitempty"` Audio AudioConfig `yaml:"audio,omitempty"` @@ -574,10 +573,6 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c conf.Logging.ComponentLevels["pion"] = conf.Logging.PionLevel } - if conf.Development { - conf.Environment = "dev" - } - return &conf, nil } diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index c73b5fa5c..40a8b30ad 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -38,7 +38,7 @@ import ( ) func init() { - prometheus.Init("test", livekit.NodeType_SERVER, "test") + prometheus.Init("test", livekit.NodeType_SERVER) } const ( diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index 951837176..9da50679f 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -35,7 +35,7 @@ import ( ) func init() { - prometheus.Init("node", livekit.NodeType_CONTROLLER, "test") + prometheus.Init("node", livekit.NodeType_CONTROLLER) } func TestSignal(t *testing.T) { diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 08d493b13..80b8bd8c0 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index ba6dd4507..703615693 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -24,6 +24,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/utils/hwstats" ) const ( @@ -41,11 +42,13 @@ var ( sysDroppedPacketsStart uint32 promSysPacketGauge *prometheus.GaugeVec promSysDroppedPacketPctGauge prometheus.Gauge + + cpuStats *hwstats.CPUStats ) -func Init(nodeID string, nodeType livekit.NodeType, env string) { +func Init(nodeID string, nodeType livekit.NodeType) error { if initialized.Swap(true) { - return + return nil } MessageCounter = prometheus.NewCounterVec( @@ -53,7 +56,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "messages", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"type", "status"}, ) @@ -63,7 +66,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "service_operation", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"type", "status", "error_type"}, ) @@ -73,7 +76,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "twirp_request_status", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"service", "method", "status", "code"}, ) @@ -83,7 +86,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "packet_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Help: "System level packet count. Count starts at 0 when service is first started.", }, []string{"type"}, @@ -94,7 +97,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "dropped_packets", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Help: "System level dropped outgoing packet percentage.", }, ) @@ -107,10 +110,18 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats() - initPacketStats(nodeID, nodeType, env) - initRoomStats(nodeID, nodeType, env) - rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}) - initQualityStats(nodeID, nodeType, env) + initPacketStats(nodeID, nodeType) + initRoomStats(nodeID, nodeType) + rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}) + initQualityStats(nodeID, nodeType) + + var err error + cpuStats, err = hwstats.NewCPUStats(nil) + if err != nil { + return err + } + + return nil } func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { @@ -119,9 +130,10 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats return nil, false, err } - cpuLoad, numCPUs, err := getCPUStats() - if err != nil { - return nil, false, err + var cpuLoad float64 + cpuIdle := cpuStats.GetCPUIdle() + if cpuIdle > 0 { + cpuLoad = 1 - (cpuIdle / cpuStats.NumCPU()) } // On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin @@ -198,8 +210,8 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats ParticipantSignalConnectedPerSec: prevAverage.ParticipantSignalConnectedPerSec, ParticipantRtcInitPerSec: prevAverage.ParticipantRtcInitPerSec, ParticipantRtcConnectedPerSec: prevAverage.ParticipantRtcConnectedPerSec, - NumCpus: numCPUs, - CpuLoad: cpuLoad, + NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer + CpuLoad: float32(cpuLoad), MemoryTotal: memTotal, MemoryUsed: memUsed, LoadAvgLast1Min: float32(loadAvg.Loadavg1), diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 6b643e903..3efd1eb28 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -67,55 +67,55 @@ var ( promPacketBytesOutgoingRetransmit prometheus.Counter ) -func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) { +func initPacketStats(nodeID string, nodeType livekit.NodeType) { promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promPacketLabels) promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "bytes", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promPacketLabels) promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "nack", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "pli", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "fir", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promPacketLossTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet_loss", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promStreamLabels) promPacketLoss = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "packet_loss", Name: "percent", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100}, }, promStreamLabels) promJitter = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "jitter", Name: "us", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, // 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000}, @@ -124,20 +124,20 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "rtt", Name: "ms", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{50, 100, 150, 200, 250, 500, 750, 1000, 5000, 10000}, }, promStreamLabels) promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "participant_join", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"state"}) promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "connection", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) prometheus.MustRegister(promPacketTotal) diff --git a/pkg/telemetry/prometheus/quality.go b/pkg/telemetry/prometheus/quality.go index 481b7558f..c9eaf6b65 100644 --- a/pkg/telemetry/prometheus/quality.go +++ b/pkg/telemetry/prometheus/quality.go @@ -26,26 +26,26 @@ var ( qualityDrop *prometheus.CounterVec ) -func initQualityStats(nodeID string, nodeType livekit.NodeType, env string) { +func initQualityStats(nodeID string, nodeType livekit.NodeType) { qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "quality", Name: "rating", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{0, 1, 2}, }) qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "quality", Name: "score", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5}, }) qualityDrop = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "quality", Name: "drop", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"direction"}) prometheus.MustRegister(qualityRating) diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index 91c7cc2ba..6c7540d78 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -47,18 +47,18 @@ var ( promSessionStartTime *prometheus.HistogramVec ) -func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { +func initRoomStats(nodeID string, nodeType livekit.NodeType) { promRoomCurrent = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }) promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "duration_seconds", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{ 5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60, }, @@ -67,37 +67,37 @@ func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "participant", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }) promTrackPublishedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "published_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) promTrackSubscribedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "subscribed_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) promTrackPublishCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "publish_counter", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind", "state"}) promTrackSubscribeCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "subscribe_counter", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"state", "error"}) promSessionStartTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "session", Name: "start_time_ms", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: prometheus.ExponentialBucketsRange(100, 10000, 15), }, []string{"protocol_version"}) diff --git a/pkg/telemetry/stats_test.go b/pkg/telemetry/stats_test.go index 1758c8b45..89051ee92 100644 --- a/pkg/telemetry/stats_test.go +++ b/pkg/telemetry/stats_test.go @@ -29,7 +29,7 @@ import ( ) func init() { - prometheus.Init("test", livekit.NodeType_SERVER, "test") + prometheus.Init("test", livekit.NodeType_SERVER) } type telemetryServiceFixture struct { diff --git a/test/integration_helpers.go b/test/integration_helpers.go index 0ce079c23..81db35ed4 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -58,7 +58,7 @@ var roomClient livekit.RoomService func init() { config.InitLoggerFromConfig(&config.DefaultConfig.Logging) - prometheus.Init("test", livekit.NodeType_SERVER, "test") + prometheus.Init("test", livekit.NodeType_SERVER) } func setupSingleNodeTest(name string) (*service.LivekitServer, func()) {