agent: release JobTerminate handler when job ends before registration (#4908)

* agent: release JobTerminate handler when job ends before registration

JobRequest registered the job's JobTerminate handler after AssignJob
returned, so an ended UpdateJobStatus or worker disconnect handled in
between deregistered nothing and the handler leaked. AssignJob also sent
the assignment before recording the job as running, so an immediate
ended update was dropped.

Record the job before sending the assignment, and after registering the
handler recheck under h.mu that the job is still running and its worker
still registered, deregistering otherwise.

Fixes #4901

* replace utils.CloneProto with proto.CloneOf

* agent: wait on worker readiness and poll for released handlers in test

pkg/routing/node.go

@@ -19,6 +19,8 @@ import (
 	"sync"
 	"time"
 
+	"google.golang.org/protobuf/proto"
+
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/utils"
 	"github.com/livekit/protocol/utils/guid"
@@ -76,14 +78,14 @@ func NewLocalNode(conf *config.Config) (*LocalNodeImpl, error) {
 }
 
 func NewLocalNodeFromNodeProto(node *livekit.Node) (*LocalNodeImpl, error) {
-	return &LocalNodeImpl{node: utils.CloneProto(node)}, nil
+	return &LocalNodeImpl{node: proto.CloneOf(node)}, nil
 }
 
 func (l *LocalNodeImpl) Clone() *livekit.Node {
 	l.lock.RLock()
 	defer l.lock.RUnlock()
 
-	return utils.CloneProto(l.node)
+	return proto.CloneOf(l.node)
 }
 
 // for testing only
@@ -134,7 +136,7 @@ func (l *LocalNodeImpl) SetStats(stats *livekit.NodeStats) {
 	l.lock.Lock()
 	defer l.lock.Unlock()
 
-	l.node.Stats = utils.CloneProto(stats)
+	l.node.Stats = proto.CloneOf(stats)
 }
 
 func (l *LocalNodeImpl) UpdateNodeStats() bool {