Record publish time on participant close for pending tracks. (#4738)

With https://github.com/livekit/livekit/pull/4706, there was a case of
some downstream component taking a long time while lock was held. While
the underlying cause of holding a lock while doing callback was removed
in that PR, to catch such cases, some publish side metric anomaly would
be useful to monitor and alert on.

Adding a publish time record for pending tracks on participant close.
That would inflate the publish time for participants not being able to
publish and can be alerted on as it will spike up the value at node
level and at cluster level if multiple nodes have the issue.

pkg/rtc/participant.go

@@ -1506,6 +1506,7 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea
 		p.supervisor.Stop()
 	}
 
+	sdk := p.GetClientInfo().GetSdk()
 	p.pendingTracksLock.Lock()
 	if p.IsConnectionCanceled(reason) {
 		for _, pti := range p.pendingTracks {
@@ -1514,6 +1515,23 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea
 			}
 			prometheus.RecordTrackPublishCancels(pti.trackInfos[0].Type.String(), int32(len(pti.trackInfos)))
 		}
+	} else {
+		// record close time as publish time for non canceled pending tracks,
+		// this will inflate publish time and can signal a node wide issue if
+		// enough participants are closing without publishing tracks.
+		for _, pti := range p.pendingTracks {
+			if len(pti.trackInfos) == 0 || pti.migrated {
+				continue
+			}
+			prometheus.RecordPublishTime(
+				p.params.Country,
+				pti.trackInfos[0].Source,
+				pti.trackInfos[0].Type,
+				time.Since(pti.createdAt),
+				sdk,
+				p.Kind(),
+			)
+		}
 	}
 	p.pendingTracks = make(map[string]*pendingTrackInfo)
 	p.pendingPublishingTracks = make(map[livekit.TrackID]*pendingTrackInfo)