diff --git a/pkg/service/twirp.go b/pkg/service/twirp.go index 1dbc0bc0c..030659144 100644 --- a/pkg/service/twirp.go +++ b/pkg/service/twirp.go @@ -143,9 +143,10 @@ func loggerResponseSent(ctx context.Context, twirpLoggerPool *sync.Pool) { r.fields = append(r.fields, "code", r.error.Code()) } + statusFamily, _ := twirpResponseStatus(ctx, r.error) serviceMethod := "API " + r.service + "." + r.method + prometheus.RecordTwirpRequestLatency(r.service, r.method, duration, statusFamily) utils.GetLogger(ctx).WithComponent(utils.ComponentAPI).Infow(serviceMethod, r.fields...) - prometheus.RecordTwirpRequestLatency(r.service, r.method, duration) // reset fields and return to pool r.reset() @@ -197,31 +198,38 @@ func statusReporterRequestRouted(ctx context.Context) (context.Context, error) { return ctx, nil } +func twirpResponseStatus(ctx context.Context, err twirp.Error) (statusFamily string, code twirp.ErrorCode) { + if statusCode, ok := twirp.StatusCode(ctx); ok { + statusFamily = httpStatusFamily(statusCode) + } + if err != nil { + code = err.Code() + } + return +} + +func httpStatusFamily(statusCode string) string { + var statusFamily string + if status, err := strconv.Atoi(statusCode); err == nil { + switch { + case status >= 400 && status <= 499: + statusFamily = "4xx" + case status >= 500 && status <= 599: + statusFamily = "5xx" + default: + statusFamily = statusCode + } + } + return statusFamily +} + func statusReporterResponseSent(ctx context.Context) { r, ok := ctx.Value(statusReporterKey{}).(*twirpRequestFields) if !ok || r == nil { return } - var statusFamily string - if statusCode, ok := twirp.StatusCode(ctx); ok { - if status, err := strconv.Atoi(statusCode); err == nil { - switch { - case status >= 400 && status <= 499: - statusFamily = "4xx" - case status >= 500 && status <= 599: - statusFamily = "5xx" - default: - statusFamily = statusCode - } - } - } - - var code twirp.ErrorCode - if r.error != nil { - code = r.error.Code() - } - + statusFamily, code := twirpResponseStatus(ctx, r.error) prometheus.RecordTwirpRequestStatus(r.service, r.method, statusFamily, code) } diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 0214d8e8e..fb540fb30 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -90,7 +90,7 @@ func Init(nodeID string, nodeType livekit.NodeType) error { ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000}, }, - []string{"service", "method"}, + []string{"service", "method", "status"}, ) promSysPacketGauge = prometheus.NewGaugeVec( @@ -305,6 +305,6 @@ func RecordTwirpRequestStatus(service string, method string, statusFamily string promTwirpRequestStatusCounter.WithLabelValues(service, method, statusFamily, string(code)).Add(1) } -func RecordTwirpRequestLatency(service, method string, duration time.Duration) { - promTwirpRequestLatency.WithLabelValues(service, method).Observe(float64(duration.Milliseconds())) +func RecordTwirpRequestLatency(service, method string, duration time.Duration, statusFamily string) { + promTwirpRequestLatency.WithLabelValues(service, method, statusFamily).Observe(float64(duration.Milliseconds())) }