dandri/livekit
1
0
Fork 0
mirror of https://github.com/livekit/livekit.git synced 2026-07-29 14:09:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

get cpu stats from cgroup, remove env (#2636)

Browse Source 
* get cpu stats from cgroup, remove env

* undo rand seed removal

* tests
This commit is contained in:
Mathew Kamkar
2024-04-08 21:15:17 -07:00
committed by GitHub
parent 196fed605e
commit 10c8582a6b
11 changed files with 61 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/server/main.go
+3 -1
View File
@@ -272,7 +272,9 @@ func startServer(c *cli.Context) error {
return err
}
prometheus.Init(currentNode.Id, currentNode.Type, conf.Environment)
if err := prometheus.Init(currentNode.Id, currentNode.Type); err != nil {
return err
}
server, err := service.InitializeServer(conf, currentNode)
if err != nil {
pkg/config/config.go
-5
View File
@@ -61,7 +61,6 @@ type Config struct {
Port uint32 `yaml:"port,omitempty"`
BindAddresses []string `yaml:"bind_addresses,omitempty"`
PrometheusPort uint32 `yaml:"prometheus_port,omitempty"`
Environment string `yaml:"environment,omitempty"`
RTC RTCConfig `yaml:"rtc,omitempty"`
Redis redisLiveKit.RedisConfig `yaml:"redis,omitempty"`
Audio AudioConfig `yaml:"audio,omitempty"`
@@ -574,10 +573,6 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c
conf.Logging.ComponentLevels["pion"] = conf.Logging.PionLevel
}
if conf.Development {
conf.Environment = "dev"
}
return &conf, nil
}
pkg/rtc/room_test.go
+1 -1
View File
@@ -38,7 +38,7 @@ import (
)
func init() {
prometheus.Init("test", livekit.NodeType_SERVER, "test")
prometheus.Init("test", livekit.NodeType_SERVER)
}
const (
pkg/service/signal_test.go
+1 -1
View File
@@ -35,7 +35,7 @@ import (
)
func init() {
prometheus.Init("node", livekit.NodeType_CONTROLLER, "test")
prometheus.Init("node", livekit.NodeType_CONTROLLER)
}
func TestSignal(t *testing.T) {
pkg/service/wire_gen.go
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by Wire. DO NOT EDIT.
//go:generate go run -mod=mod github.com/google/wire/cmd/wire
//go:generate go run github.com/google/wire/cmd/wire
//go:build !wireinject
// +build !wireinject
pkg/telemetry/prometheus/node.go
+28 -16
View File
@@ -24,6 +24,7 @@ import (
"github.com/livekit/livekit-server/pkg/config"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/rpc"
"github.com/livekit/protocol/utils/hwstats"
)
const (
@@ -41,11 +42,13 @@ var (
sysDroppedPacketsStart uint32
promSysPacketGauge *prometheus.GaugeVec
promSysDroppedPacketPctGauge prometheus.Gauge
cpuStats *hwstats.CPUStats
)
func Init(nodeID string, nodeType livekit.NodeType, env string) {
func Init(nodeID string, nodeType livekit.NodeType) error {
if initialized.Swap(true) {
return
return nil
}
MessageCounter = prometheus.NewCounterVec(
@@ -53,7 +56,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) {
Namespace: livekitNamespace,
Subsystem: "node",
Name: "messages",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
},
[]string{"type", "status"},
)
@@ -63,7 +66,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) {
Namespace: livekitNamespace,
Subsystem: "node",
Name: "service_operation",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
},
[]string{"type", "status", "error_type"},
)
@@ -73,7 +76,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) {
Namespace: livekitNamespace,
Subsystem: "node",
Name: "twirp_request_status",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
},
[]string{"service", "method", "status", "code"},
)
@@ -83,7 +86,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) {
Namespace: livekitNamespace,
Subsystem: "node",
Name: "packet_total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Help: "System level packet count. Count starts at 0 when service is first started.",
},
[]string{"type"},
@@ -94,7 +97,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) {
Namespace: livekitNamespace,
Subsystem: "node",
Name: "dropped_packets",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Help: "System level dropped outgoing packet percentage.",
},
)
@@ -107,10 +110,18 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) {
sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats()
initPacketStats(nodeID, nodeType, env)
initRoomStats(nodeID, nodeType, env)
rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env})
initQualityStats(nodeID, nodeType, env)
initPacketStats(nodeID, nodeType)
initRoomStats(nodeID, nodeType)
rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()})
initQualityStats(nodeID, nodeType)
var err error
cpuStats, err = hwstats.NewCPUStats(nil)
if err != nil {
return err
}
return nil
}
func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) {
@@ -119,9 +130,10 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats
return nil, false, err
}
cpuLoad, numCPUs, err := getCPUStats()
if err != nil {
return nil, false, err
var cpuLoad float64
cpuIdle := cpuStats.GetCPUIdle()
if cpuIdle > 0 {
cpuLoad = 1 - (cpuIdle / cpuStats.NumCPU())
}
// On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin
@@ -198,8 +210,8 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats
ParticipantSignalConnectedPerSec: prevAverage.ParticipantSignalConnectedPerSec,
ParticipantRtcInitPerSec: prevAverage.ParticipantRtcInitPerSec,
ParticipantRtcConnectedPerSec: prevAverage.ParticipantRtcConnectedPerSec,
NumCpus: numCPUs,
CpuLoad: cpuLoad,
NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer
CpuLoad: float32(cpuLoad),
MemoryTotal: memTotal,
MemoryUsed: memUsed,
LoadAvgLast1Min: float32(loadAvg.Loadavg1),
pkg/telemetry/prometheus/packets.go
+12 -12
View File
@@ -67,55 +67,55 @@ var (
promPacketBytesOutgoingRetransmit prometheus.Counter
)
func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) {
func initPacketStats(nodeID string, nodeType livekit.NodeType) {
promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "packet",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promPacketLabels)
promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "packet",
Name: "bytes",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promPacketLabels)
promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "nack",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promRTCPLabels)
promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "pli",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promRTCPLabels)
promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "fir",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promRTCPLabels)
promPacketLossTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "packet_loss",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promStreamLabels)
promPacketLoss = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "packet_loss",
Name: "percent",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100},
}, promStreamLabels)
promJitter = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "jitter",
Name: "us",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
// 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s
Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000},
@@ -124,20 +124,20 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) {
Namespace: livekitNamespace,
Subsystem: "rtt",
Name: "ms",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{50, 100, 150, 200, 250, 500, 750, 1000, 5000, 10000},
}, promStreamLabels)
promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "participant_join",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"state"})
promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "connection",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"kind"})
prometheus.MustRegister(promPacketTotal)
pkg/telemetry/prometheus/quality.go
+4 -4
View File
@@ -26,26 +26,26 @@ var (
qualityDrop *prometheus.CounterVec
)
func initQualityStats(nodeID string, nodeType livekit.NodeType, env string) {
func initQualityStats(nodeID string, nodeType livekit.NodeType) {
qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "quality",
Name: "rating",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{0, 1, 2},
})
qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "quality",
Name: "score",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5},
})
qualityDrop = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "quality",
Name: "drop",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"direction"})
prometheus.MustRegister(qualityRating)
pkg/telemetry/prometheus/rooms.go
+9 -9
View File
@@ -47,18 +47,18 @@ var (
promSessionStartTime *prometheus.HistogramVec
)
func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) {
func initRoomStats(nodeID string, nodeType livekit.NodeType) {
promRoomCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "room",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
})
promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "room",
Name: "duration_seconds",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{
5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60,
},
@@ -67,37 +67,37 @@ func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) {
Namespace: livekitNamespace,
Subsystem: "participant",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
})
promTrackPublishedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "track",
Name: "published_total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"kind"})
promTrackSubscribedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "track",
Name: "subscribed_total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"kind"})
promTrackPublishCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "track",
Name: "publish_counter",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"kind", "state"})
promTrackSubscribeCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "track",
Name: "subscribe_counter",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"state", "error"})
promSessionStartTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "session",
Name: "start_time_ms",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: prometheus.ExponentialBucketsRange(100, 10000, 15),
}, []string{"protocol_version"})
pkg/telemetry/stats_test.go
+1 -1
View File
@@ -29,7 +29,7 @@ import (
)
func init() {
prometheus.Init("test", livekit.NodeType_SERVER, "test")
prometheus.Init("test", livekit.NodeType_SERVER)
}
type telemetryServiceFixture struct {
test/integration_helpers.go
+1 -1
View File
@@ -58,7 +58,7 @@ var roomClient livekit.RoomService
func init() {
config.InitLoggerFromConfig(&config.DefaultConfig.Logging)
prometheus.Init("test", livekit.NodeType_SERVER, "test")
prometheus.Init("test", livekit.NodeType_SERVER)
}
func setupSingleNodeTest(name string) (*service.LivekitServer, func()) {