From 070c5e5f92eea3eaa9292dd67fa22f997277b710 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 27 Sep 2022 15:07:49 -0700 Subject: [PATCH] Mageutil (#1047) * Mageutil * update mageutil --- go.mod | 3 +- go.sum | 8 +- magefile.go | 249 +++++----------------------------------------------- 3 files changed, 29 insertions(+), 231 deletions(-) diff --git a/go.mod b/go.mod index bf3947d45..637616511 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/golang-lru v0.5.4 + github.com/livekit/mageutil v0.0.0-20220927214055-ff37ecf1f093 github.com/livekit/protocol v1.1.3-0.20220926180122-bafacf7041a5 github.com/livekit/rtcscore-go v0.0.0-20220815072451-20ee10ae1995 github.com/mackerelio/go-osstat v0.2.3 @@ -45,7 +46,7 @@ require ( github.com/urfave/negroni v1.0.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.23.0 - golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde + golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 34a35b800..81f8399eb 100644 --- a/go.sum +++ b/go.sum @@ -240,8 +240,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= -github.com/livekit/protocol v1.1.3-0.20220922204413-19ead11853f1 h1:HEh5B8U66t0h0QkMt65Udon3oldBJGfl7THeKf/9bmw= -github.com/livekit/protocol v1.1.3-0.20220922204413-19ead11853f1/go.mod h1:eburCdz6ZtbgKSKYkAeCdWP1z33DB9clTphz7uNaxp0= +github.com/livekit/mageutil v0.0.0-20220927214055-ff37ecf1f093 h1:a5Pzvr9EC+qLVDOVxM/008CCBogusc9pNIfCrXCTQtw= +github.com/livekit/mageutil v0.0.0-20220927214055-ff37ecf1f093/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/protocol v1.1.3-0.20220926180122-bafacf7041a5 h1:ct0ellNBvJ67i1VSxI9NIYAypBGRBdaJqucVm5PWIE0= github.com/livekit/protocol v1.1.3-0.20220926180122-bafacf7041a5/go.mod h1:eburCdz6ZtbgKSKYkAeCdWP1z33DB9clTphz7uNaxp0= github.com/livekit/rtcscore-go v0.0.0-20220815072451-20ee10ae1995 h1:vOaY2qvfLihDyeZtnGGN1Law9wRrw8BMGCr1TygTvMw= @@ -556,8 +556,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 h1:ZrnxWX62AgTKOSagEqxvb3ffipvEDX2pl7E1TdqLqIc= +golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/magefile.go b/magefile.go index 6fe4bf992..5d9198b48 100644 --- a/magefile.go +++ b/magefile.go @@ -4,21 +4,17 @@ package main import ( - "crypto/sha1" + "context" "errors" "fmt" - "go/build" - "io/ioutil" - "log" "os" "os/exec" - "path/filepath" - "sort" "strings" "github.com/magefile/mage/mg" "github.com/livekit/livekit-server/version" + "github.com/livekit/mageutil" ) const ( @@ -29,7 +25,7 @@ const ( // Default target to run when none is specified // If not set, running mage will list available targets var Default = Build -var checksummer = NewChecksummer(".", goChecksumFile, ".go", ".mod") +var checksummer = mageutil.NewChecksummer(".", goChecksumFile, ".go", ".mod") func init() { checksummer.IgnoredPaths = []string{ @@ -55,10 +51,7 @@ func Build() error { if err := os.MkdirAll("bin", 0755); err != nil { return err } - cmd := exec.Command("go", "build", "-o", "../../bin/livekit-server") - cmd.Dir = "cmd/server" - connectStd(cmd) - if err := cmd.Run(); err != nil { + if err := mageutil.RunDir(context.Background(), "cmd/server", "go build -o ../../bin/livekit-server"); err != nil { return err } @@ -78,15 +71,13 @@ func BuildLinux() error { if err := os.MkdirAll("bin", 0755); err != nil { return err } - cmd := exec.Command("go", "build", "-buildvcs=false", "-o", "../../bin/livekit-server-amd64") + cmd := mageutil.CommandDir(context.Background(), "cmd/server", "go build -buildvcs=false -o ../../bin/livekit-server-amd64") cmd.Env = []string{ "GOOS=linux", "GOARCH=amd64", "HOME=" + os.Getenv("HOME"), "GOPATH=" + os.Getenv("GOPATH"), } - cmd.Dir = "cmd/server" - connectStd(cmd) if err := cmd.Run(); err != nil { return err } @@ -96,38 +87,39 @@ func BuildLinux() error { } func Deadlock() error { - if err := installTool("golang.org/x/tools/cmd/goimports", "latest", false); err != nil { + ctx := context.Background() + if err := mageutil.InstallTool("golang.org/x/tools/cmd/goimports", "latest", false); err != nil { return err } - if err := run("go get github.com/sasha-s/go-deadlock"); err != nil { + if err := mageutil.Run(ctx, "go get github.com/sasha-s/go-deadlock"); err != nil { return err } - if err := pipe("grep -rl sync.Mutex ./pkg", "xargs sed -i -e s/sync.Mutex/deadlock.Mutex/g"); err != nil { + if err := mageutil.Pipe("grep -rl sync.Mutex ./pkg", "xargs sed -i -e s/sync.Mutex/deadlock.Mutex/g"); err != nil { return err } - if err := pipe("grep -rl sync.RWMutex ./pkg", "xargs sed -i -e s/sync.RWMutex/deadlock.RWMutex/g"); err != nil { + if err := mageutil.Pipe("grep -rl sync.RWMutex ./pkg", "xargs sed -i -e s/sync.RWMutex/deadlock.RWMutex/g"); err != nil { return err } - if err := pipe("grep -rl deadlock.Mutex\\|deadlock.RWMutex ./pkg", "xargs goimports -w"); err != nil { + if err := mageutil.Pipe("grep -rl deadlock.Mutex\\|deadlock.RWMutex ./pkg", "xargs goimports -w"); err != nil { return err } - if err := run("go mod tidy"); err != nil { + if err := mageutil.Run(ctx, "go mod tidy"); err != nil { return err } return nil } func Sync() error { - if err := pipe("grep -rl deadlock.Mutex ./pkg", "xargs sed -i -e s/deadlock.Mutex/sync.Mutex/g"); err != nil { + if err := mageutil.Pipe("grep -rl deadlock.Mutex ./pkg", "xargs sed -i -e s/deadlock.Mutex/sync.Mutex/g"); err != nil { return err } - if err := pipe("grep -rl deadlock.RWMutex ./pkg", "xargs sed -i -e s/deadlock.RWMutex/sync.RWMutex/g"); err != nil { + if err := mageutil.Pipe("grep -rl deadlock.RWMutex ./pkg", "xargs sed -i -e s/deadlock.RWMutex/sync.RWMutex/g"); err != nil { return err } - if err := pipe("grep -rl sync.Mutex\\|sync.RWMutex ./pkg", "xargs goimports -w"); err != nil { + if err := mageutil.Pipe("grep -rl sync.Mutex\\|sync.RWMutex ./pkg", "xargs goimports -w"); err != nil { return err } - if err := run("go mod tidy"); err != nil { + if err := mageutil.Run(context.Background(), "go mod tidy"); err != nil { return err } return nil @@ -145,7 +137,7 @@ func PublishDocker() error { "--push", "--platform", "linux/amd64,linux/arm64", "--tag", versionImg, ".") - connectStd(cmd) + mageutil.ConnectStd(cmd) if err := cmd.Run(); err != nil { return err } @@ -155,15 +147,13 @@ func PublishDocker() error { // run unit tests, skipping integration func Test() error { mg.Deps(generateWire, setULimit) - cmd := exec.Command("go", "test", "-short", "./...", "-count=1") - connectStd(cmd) - return cmd.Run() + return mageutil.Run(context.Background(), "go test -short ./... -count=1") } // run all tests including integration func TestAll() error { mg.Deps(generateWire, setULimit) - return run("go test ./... -count=1 -timeout=4m -v") + return mageutil.Run(context.Background(), "go test ./... -count=1 -timeout=4m -v") } // cleans up builds @@ -178,7 +168,7 @@ func Generate() error { mg.Deps(installDeps, generateWire) fmt.Println("generating...") - return run("go generate ./...") + return mageutil.Run(context.Background(), "go generate ./...") } // code generation for wiring @@ -190,13 +180,13 @@ func generateWire() error { fmt.Println("wiring...") - wire, err := getToolPath("wire") + wire, err := mageutil.GetToolPath("wire") if err != nil { return err } cmd := exec.Command(wire) cmd.Dir = "pkg/service" - connectStd(cmd) + mageutil.ConnectStd(cmd) if err := cmd.Run(); err != nil { return err } @@ -214,202 +204,9 @@ func installTools(force bool) error { "github.com/google/wire/cmd/wire": "latest", } for t, v := range tools { - if err := installTool(t, v, force); err != nil { + if err := mageutil.InstallTool(t, v, force); err != nil { return err } } return nil } - -func installTool(url, version string, force bool) error { - name := filepath.Base(url) - if !force { - _, err := getToolPath(name) - if err == nil { - // already installed - return nil - } - } - - fmt.Printf("installing %s %s\n", name, version) - urlWithVersion := fmt.Sprintf("%s@%s", url, version) - cmd := exec.Command("go", "install", urlWithVersion) - connectStd(cmd) - if err := cmd.Run(); err != nil { - return err - } - - // check - _, err := getToolPath(name) - return err -} - -// helpers - -func getToolPath(name string) (string, error) { - if p, err := exec.LookPath(name); err == nil { - return p, nil - } - // check under gopath - gopath := os.Getenv("GOPATH") - if gopath == "" { - gopath = build.Default.GOPATH - } - p := filepath.Join(gopath, "bin", name) - if _, err := os.Stat(p); err != nil { - return "", err - } - return p, nil -} - -func connectStd(cmd *exec.Cmd) { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr -} - -// A helper checksum library that generates a fast, non-portable checksum over a directory of files -// it's designed as a quick way to bypass -type Checksummer struct { - dir string - file string - checksum string - allExts bool - extMap map[string]bool - IgnoredPaths []string -} - -func NewChecksummer(dir string, checksumfile string, exts ...string) *Checksummer { - c := &Checksummer{ - dir: dir, - file: checksumfile, - extMap: make(map[string]bool), - } - if len(exts) == 0 { - c.allExts = true - } else { - for _, ext := range exts { - c.extMap[ext] = true - } - } - - return c -} - -func (c *Checksummer) IsChanged() bool { - // default changed - if err := c.computeChecksum(); err != nil { - log.Println("could not compute checksum", err) - return true - } - // read - existing, err := c.ReadChecksum() - if err != nil { - // may not be there - return true - } - - return existing != c.checksum -} - -func (c *Checksummer) ReadChecksum() (string, error) { - b, err := ioutil.ReadFile(filepath.Join(c.dir, c.file)) - if err != nil { - return "", err - } - return string(b), nil -} - -func (c *Checksummer) WriteChecksum() error { - if err := c.computeChecksum(); err != nil { - return err - } - return ioutil.WriteFile(filepath.Join(c.dir, c.file), []byte(c.checksum), 0644) -} - -func (c *Checksummer) computeChecksum() error { - if c.checksum != "" { - return nil - } - - entries := make([]string, 0) - ignoredMap := make(map[string]bool) - for _, f := range c.IgnoredPaths { - ignoredMap[f] = true - } - err := filepath.Walk(c.dir, func(path string, info os.FileInfo, err error) error { - if path == c.dir { - return nil - } - if strings.HasPrefix(info.Name(), ".") || ignoredMap[path] { - if info.IsDir() { - return filepath.SkipDir - } else { - return nil - } - } - if info.IsDir() { - entries = append(entries, fmt.Sprintf("%s %d", path, info.ModTime().Unix())) - } else if c.allExts || c.extMap[filepath.Ext(info.Name())] { - entries = append(entries, fmt.Sprintf("%s %d %d", path, info.Size(), info.ModTime().Unix())) - } - return nil - }) - if err != nil { - return err - } - - sort.Strings(entries) - - h := sha1.New() - for _, e := range entries { - h.Write([]byte(e)) - } - c.checksum = fmt.Sprintf("%x", h.Sum(nil)) - - return nil -} - -func run(commands ...string) error { - for _, command := range commands { - args := strings.Split(command, " ") - cmd := exec.Command(args[0], args[1:]...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return err - } - } - return nil -} - -func pipe(first, second string) error { - a1 := strings.Split(first, " ") - c1 := exec.Command(a1[0], a1[1:]...) - - c1.Stderr = os.Stderr - p, err := c1.StdoutPipe() - if err != nil { - return err - } - - a2 := strings.Split(second, " ") - c2 := exec.Command(a2[0], a2[1:]...) - - c2.Stdin = p - c2.Stdout = os.Stdout - c2.Stderr = os.Stderr - - if err = c1.Start(); err != nil { - return err - } - if err = c2.Start(); err != nil { - return err - } - if err = c1.Wait(); err != nil { - return err - } - if err = c2.Wait(); err != nil { - return err - } - return nil -}