diff --git a/cmd/server/commands.go b/cmd/server/commands.go index 0611dcf97..927668a0a 100644 --- a/cmd/server/commands.go +++ b/cmd/server/commands.go @@ -171,17 +171,18 @@ func listNodes(c *cli.Context) error { table.SetHeader([]string{ "ID", "IP Address", "Region", "CPUs", "CPU Usage\nLoad Avg", + "Memory Used/Total", "Rooms", "Clients\nTracks In/Out", - "Bytes/s In/Out\nBytes Total", "Packets/s In/Out\nPackets Total", "System Dropped Pkts/s\nPkts/s Out / Dropped", + "Bytes/s In/Out\nBytes Total", "Packets/s In/Out\nPackets Total", "System Dropped Pkts/s\nPkts/s Out/Dropped", "Nack/s\nNack Total", "Retrans/s\nRetrans Total", "Started At\nUpdated At", }) table.SetColumnAlignment([]int{ tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, + tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, - tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, - tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, + tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_CENTER, }) @@ -196,6 +197,7 @@ func listNodes(c *cli.Context) error { cpus := strconv.Itoa(int(stats.NumCpus)) cpuUsageAndLoadAvg := fmt.Sprintf("%.2f %%\n%.2f %.2f %.2f", stats.CpuLoad*100, stats.LoadAvgLast1Min, stats.LoadAvgLast5Min, stats.LoadAvgLast15Min) + memUsage := fmt.Sprintf("%s / %s", humanize.Bytes(stats.MemoryUsed), humanize.Bytes(stats.MemoryTotal)) // Room stats rooms := strconv.Itoa(int(stats.NumRooms)) @@ -217,6 +219,7 @@ func listNodes(c *cli.Context) error { table.Append([]string{ idAndState, node.Ip, node.Region, cpus, cpuUsageAndLoadAvg, + memUsage, rooms, clientsAndTracks, bytes, packets, sysPackets, nacks, retransmit, diff --git a/go.mod b/go.mod index 254c20093..3cb7e5361 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20221221221243-f361fbe40290 github.com/livekit/mediatransportutil v0.0.0-20221007030528-7440725c362b - github.com/livekit/protocol v1.3.2-0.20221228091932-fa87d56355d2 + github.com/livekit/protocol v1.3.2-0.20230110201647-34cae0997a36 github.com/livekit/psrpc v0.2.1 github.com/livekit/rtcscore-go v0.0.0-20220815072451-20ee10ae1995 github.com/mackerelio/go-osstat v0.2.3 diff --git a/go.sum b/go.sum index 1c9af48a2..1eae3cc40 100644 --- a/go.sum +++ b/go.sum @@ -233,8 +233,8 @@ github.com/livekit/mageutil v0.0.0-20221221221243-f361fbe40290 h1:ZVsQUuUOM9G7O3 github.com/livekit/mageutil v0.0.0-20221221221243-f361fbe40290/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20221007030528-7440725c362b h1:RBNV8TckETSkIkKxcD12d8nZKVkB9GSY/sQlMoaruP4= github.com/livekit/mediatransportutil v0.0.0-20221007030528-7440725c362b/go.mod h1:1Dlx20JPoIKGP45eo+yuj0HjeE25zmyeX/EWHiPCjFw= -github.com/livekit/protocol v1.3.2-0.20221228091932-fa87d56355d2 h1:Kj7tg1YxRZFeTQLJNm8x8KIdhFTgLCQ6QFNDio/2bOM= -github.com/livekit/protocol v1.3.2-0.20221228091932-fa87d56355d2/go.mod h1:gwCG03nKlHlC9hTjL4pXQpn783ALhmbyhq65UZxqbb8= +github.com/livekit/protocol v1.3.2-0.20230110201647-34cae0997a36 h1:SlWsB3XEt4fYYkcCeCES2V8CFkRYcX0ThdkNqWP4MPg= +github.com/livekit/protocol v1.3.2-0.20230110201647-34cae0997a36/go.mod h1:gwCG03nKlHlC9hTjL4pXQpn783ALhmbyhq65UZxqbb8= github.com/livekit/psrpc v0.2.1 h1:ph/4egUMueUPoh5PZ/Aw4v6SH3wAbA+2t/GyCbpPKTg= github.com/livekit/psrpc v0.2.1/go.mod h1:MCe0xLdFPXmzogPiLrM94JIJbctb9+fAv5qYPkY2DXw= github.com/livekit/rtcscore-go v0.0.0-20220815072451-20ee10ae1995 h1:vOaY2qvfLihDyeZtnGGN1Law9wRrw8BMGCr1TygTvMw= diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index aef4155ff..06de3fc15 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -85,18 +85,6 @@ func Init(nodeID string, nodeType livekit.NodeType) { initRoomStats(nodeID, nodeType) } -func getMemoryStats() (memoryLoad float32, err error) { - memInfo, err := memory.Get() - if err != nil { - return - } - - if memInfo.Total != 0 { - memoryLoad = float32(memInfo.Used) / float32(memInfo.Total) - } - return -} - func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { loadAvg, err := loadavg.Get() if err != nil { @@ -108,9 +96,9 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats return nil, false, err } - memoryLoad, _ := getMemoryStats() // On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin // So, do not error out. Use the information if it is available. + memInfo, _ := memory.Get() sysPackets, sysDroppedPackets, err := getTCStats() if err != nil { @@ -166,12 +154,13 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats ParticipantJoinPerSec: prevAverage.ParticipantJoinPerSec, NumCpus: numCPUs, CpuLoad: cpuLoad, + MemoryTotal: memInfo.Total, + MemoryUsed: memInfo.Used, LoadAvgLast1Min: float32(loadAvg.Loadavg1), LoadAvgLast5Min: float32(loadAvg.Loadavg5), LoadAvgLast15Min: float32(loadAvg.Loadavg15), SysPacketsOut: sysPackets, SysPacketsDropped: sysDroppedPackets, - MemoryLoad: memoryLoad, } // update stats