From 94ad55ac2efc0a241e5369dc0808c1d6ee02df14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BENEDEK=20L=C3=A1szl=C3=B3?= Date: Sat, 5 Oct 2024 01:43:59 +0200 Subject: [PATCH] init --- .gitignore | 2 + buildx/build.go | 52 +++++++++++++++++++++++ buildx/command.go | 26 ++++++++++++ buildx/load.go | 105 ++++++++++++++++++++++++++++++++++++++++++++++ buildx/tag.go | 62 +++++++++++++++++++++++++++ buildx/types.go | 38 +++++++++++++++++ config/config.go | 41 ++++++++++++++++++ go.mod | 10 +++++ go.sum | 13 ++++++ main.go | 88 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 437 insertions(+) create mode 100644 .gitignore create mode 100644 buildx/build.go create mode 100644 buildx/command.go create mode 100644 buildx/load.go create mode 100644 buildx/tag.go create mode 100644 buildx/types.go create mode 100644 config/config.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d414612 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +buildx-manager +test.yml \ No newline at end of file diff --git a/buildx/build.go b/buildx/build.go new file mode 100644 index 0000000..d458ef5 --- /dev/null +++ b/buildx/build.go @@ -0,0 +1,52 @@ +package buildx + +import ( + "path/filepath" + + "git.tek.govt.hu/dowerx/buildx-manager/config" +) + +func RepoToCommands(repo *Repository) ([]Command, error) { + cfg := config.GetConfig() + + workdir, err := filepath.Abs(repo.Path) + if err != nil { + return nil, err + } + + globalArgs := make([]string, 0) + for _, arg := range repo.GlobalArguments { + globalArgs = append(globalArgs, "--build-arg", arg.Key+"="+arg.Value) + } + + commands := make([]Command, 0) + + // builds + for _, build := range repo.Builds { + id := uid() + + cmd := Command{Program: cfg.DockerExecutable, Arguments: []string{"buildx", "build", "-f", build.Dockerfile, "--platform", "linux/" + build.Architecture, "--tag", id, cfg.Action}, WorkingDirectory: workdir} + + // args + cmd.Arguments = append(cmd.Arguments, globalArgs...) + for _, arg := range build.Arguments { + cmd.Arguments = append(cmd.Arguments, "--build-arg", arg.Key+"="+arg.Value) + } + + // tags + for _, tag := range build.Tags { + addUniqueTag(id, repo.Library+"/"+tag) + } + + commands = append(commands, cmd) + } + + // tags + for _, rtag := range repo.Tags { + for _, tag := range rtag.Tags { + addGroupTag(repo.Library+"/"+rtag.Name, repo.Library+"/"+tag) + } + } + + return commands, err +} diff --git a/buildx/command.go b/buildx/command.go new file mode 100644 index 0000000..2b2a03e --- /dev/null +++ b/buildx/command.go @@ -0,0 +1,26 @@ +package buildx + +import ( + "os" + "os/exec" + + "git.tek.govt.hu/dowerx/buildx-manager/config" +) + +type Command struct { + Program string + Arguments []string + WorkingDirectory string +} + +func (c *Command) Run() error { + cmd := exec.Command(c.Program, c.Arguments...) + cmd.Dir = c.WorkingDirectory + + if config.GetConfig().Verbose { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } + + return cmd.Run() +} diff --git a/buildx/load.go b/buildx/load.go new file mode 100644 index 0000000..9dfb31a --- /dev/null +++ b/buildx/load.go @@ -0,0 +1,105 @@ +package buildx + +import ( + "os" + + "github.com/drone/envsubst" + "gopkg.in/yaml.v3" +) + +func subst(ref *string) error { + result, err := envsubst.EvalEnv(*ref) + *ref = result + return err +} + +func LoadJob(path string) (*Job, error) { + var job Job + + file, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(file, &job) + if err != nil { + return nil, err + } + + // envsubst + + // registries + for i := range job.Registries { + if err = subst(&job.Registries[i]); err != nil { + return nil, err + } + } + + // repos + for i := range job.Repositories { + // name + if err = subst(&job.Repositories[i].Name); err != nil { + return nil, err + } + + // library + if err = subst(&job.Repositories[i].Library); err != nil { + return nil, err + } + + // path + if err = subst(&job.Repositories[i].Path); err != nil { + return nil, err + } + + // global args + for j := range job.Repositories[i].GlobalArguments { + // key + if err = subst(&job.Repositories[i].GlobalArguments[j].Key); err != nil { + return nil, err + } + + // value + if err = subst(&job.Repositories[i].GlobalArguments[j].Value); err != nil { + return nil, err + } + } + + // tags + for j := range job.Repositories[i].Tags { + // name + if err = subst(&job.Repositories[i].Tags[j].Name); err != nil { + return nil, err + } + + // tags + for u := range job.Repositories[i].Tags[j].Tags { + if err = subst(&job.Repositories[i].Tags[j].Tags[u]); err != nil { + return nil, err + } + } + } + + // builds + for j := range job.Repositories[i].Builds { + // arch + if err = subst(&job.Repositories[i].Builds[j].Architecture); err != nil { + return nil, err + } + + // dockerfile + if err = subst(&job.Repositories[i].Builds[j].Dockerfile); err != nil { + return nil, err + } + + // tags + for u := range job.Repositories[i].Builds[j].Tags { + if err = subst(&job.Repositories[i].Builds[j].Tags[u]); err != nil { + return nil, err + } + } + } + } + + return &job, err +} diff --git a/buildx/tag.go b/buildx/tag.go new file mode 100644 index 0000000..fd25583 --- /dev/null +++ b/buildx/tag.go @@ -0,0 +1,62 @@ +package buildx + +import ( + "git.tek.govt.hu/dowerx/buildx-manager/config" + "golang.org/x/exp/rand" +) + +var uniqueTags map[string][]string = make(map[string][]string) +var groupTags map[string][]string = make(map[string][]string) +var runes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func uid() string { + for { + id := make([]rune, 16) + for i := range id { + id[i] = runes[rand.Intn(len(runes))] + } + + _, exists := uniqueTags[string(id)] + if !exists { + return string(id) + } + } +} + +func addUniqueTag(id string, tag string) { + uniqueTags[id] = append(uniqueTags[id], tag) +} + +func addGroupTag(id string, tag string) { + groupTags[id] = append(groupTags[id], tag) +} + +func TagsToCommands(registries []string) []Command { + cfg := config.GetConfig() + + commands := make([]Command, 0) + + for _, registry := range registries { + + // unique tags + for id, tags := range uniqueTags { + for _, tag := range tags { + cmd := Command{Program: cfg.DockerExecutable, Arguments: []string{"buildx", "imagetools", "create", "-t", registry + "/" + tag, id}} + commands = append(commands, cmd) + } + } + + // group tags + for newTag, tags := range groupTags { + cmd := Command{Program: cfg.DockerExecutable, Arguments: []string{"buildx", "imagetools", "create", "-t", registry + "/" + newTag}} + + for _, tag := range tags { + cmd.Arguments = append(cmd.Arguments, registry+"/"+tag) + } + + commands = append(commands, cmd) + } + } + + return commands +} diff --git a/buildx/types.go b/buildx/types.go new file mode 100644 index 0000000..5b55505 --- /dev/null +++ b/buildx/types.go @@ -0,0 +1,38 @@ +package buildx + +const ( + AMD64 string = "amd64" + ARM64 string = "arm64" + ARMv7 string = "armv7" +) + +type Argument struct { + Key string `yaml:"key"` + Value string `yaml:"value"` +} + +type Tag struct { + Name string `yaml:"name"` + Tags []string `yaml:"tags"` +} + +type Build struct { + Architecture string `yaml:"arch"` + Dockerfile string `yaml:"dockerfile" default:"Dockerfile"` + Tags []string `yaml:"tags"` + Arguments []Argument `yaml:"args"` +} + +type Repository struct { + Name string `yaml:"name"` + Library string `yaml:"library"` + Path string `yaml:"path"` + GlobalArguments []Argument `yaml:"args"` + Tags []Tag `yaml:"tags"` + Builds []Build `yaml:"builds"` +} + +type Job struct { + Registries []string `yaml:"registries"` + Repositories []Repository `yaml:"repos"` +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..e32eca4 --- /dev/null +++ b/config/config.go @@ -0,0 +1,41 @@ +package config + +import "flag" + +var config *Config + +type Config struct { + File string + Action string + Verbose bool + Parallel bool + Dryrun bool + DockerExecutable string +} + +func GetConfig() Config { + if config != nil { + return *config + } + + config = &Config{} + + flag.StringVar(&config.File, "f", "", "job definition") + flag.StringVar(&config.Action, "a", "", "action") + flag.BoolVar(&config.Verbose, "v", false, "verbose") + flag.BoolVar(&config.Parallel, "p", false, "parallel") + flag.BoolVar(&config.Dryrun, "dry", false, "dryrun") + flag.StringVar(&config.DockerExecutable, "docker", "docker", "docker executable") + + flag.Parse() + + if config.File == "" { + panic("missing job definition") + } + + if config.Action != "load" && config.Action != "push" { + panic("action must be \"load\" or \"push\"") + } + + return *config +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9373ffc --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module git.tek.govt.hu/dowerx/buildx-manager + +go 1.23.0 + +require ( + github.com/drone/envsubst v1.0.3 // indirect + github.com/sanity-io/litter v1.5.5 // indirect + golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c96dba1 --- /dev/null +++ b/go.sum @@ -0,0 +1,13 @@ +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= +github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= +github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= +github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 h1:1wqE9dj9NpSm04INVsJhhEUzhuDVjbcyKH91sVyPATw= +golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..302a949 --- /dev/null +++ b/main.go @@ -0,0 +1,88 @@ +package main + +import ( + "fmt" + "time" + + "git.tek.govt.hu/dowerx/buildx-manager/buildx" + "git.tek.govt.hu/dowerx/buildx-manager/config" + "github.com/sanity-io/litter" + "golang.org/x/exp/rand" +) + +func main() { + rand.Seed(uint64(time.Now().Unix())) + + cfg := config.GetConfig() + + job, err := buildx.LoadJob(cfg.File) + if err != nil { + panic(err) + } + + buildCommands := make([]buildx.Command, 0) + + for _, repo := range job.Repositories { + cmd, err := buildx.RepoToCommands(&repo) + if err != nil { + panic(err) + } + buildCommands = append(buildCommands, cmd...) + } + + tagCommands := buildx.TagsToCommands(job.Registries) + + if cfg.Verbose { + fmt.Println("config:") + litter.Dump(cfg) + + fmt.Println("\njob loaded:") + litter.Dump(job) + + fmt.Println("\nbuild commands:", len(buildCommands)) + litter.Dump(buildCommands) + + fmt.Println("\ntag commands:", len(tagCommands)) + litter.Dump(tagCommands) + } + + if cfg.Dryrun { + return + } + + // build + if cfg.Parallel { + errors := make(chan error) + + // start builds + for _, cmd := range buildCommands { + go func() { + errors <- cmd.Run() + }() + } + + // wait for builds + for i := 0; i < len(buildCommands); i++ { + for err := range errors { + if err != nil { + panic(err) + } + } + } + } else { + for _, cmd := range buildCommands { + err = cmd.Run() + if err != nil { + panic(err) + } + } + } + + // tag + for _, cmd := range tagCommands { + err = cmd.Run() + if err != nil { + panic(err) + } + } +}