From b97df057c1d02127f0ea3da63ef7886189b2180f Mon Sep 17 00:00:00 2001 From: Pieter Vincken <4153479+pietervincken@users.noreply.github.com> Date: Tue, 9 Jun 2020 16:35:09 +0200 Subject: [PATCH 1/5] feat: re-introduce shallow git clones Making full git clones slows down the kustomize build significantly for big repositories. --- api/internal/git/cloner.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/api/internal/git/cloner.go b/api/internal/git/cloner.go index e3696db3e..8b420b62b 100644 --- a/api/internal/git/cloner.go +++ b/api/internal/git/cloner.go @@ -4,7 +4,6 @@ package git import ( - "log" "os/exec" "github.com/pkg/errors" @@ -33,25 +32,36 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { cmd := exec.Command( gitProgram, "clone", + "--depth=1", repoSpec.CloneSpec(), repoSpec.Dir.String()) - out, err := cmd.CombinedOutput() + _, err = cmd.CombinedOutput() if err != nil { - log.Printf("Error cloning git repo: %s", out) return errors.Wrapf( err, "trouble cloning git repo %v in %s", repoSpec.CloneSpec(), repoSpec.Dir.String()) } + cmd = exec.Command( + gitProgram, + "fetch", + "--depth=1", + "origin", + repoSpec.Ref) + cmd.Dir = repoSpec.Dir.String() + _, err = cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "trouble fetching %s", repoSpec.Ref) + } + cmd = exec.Command( gitProgram, "checkout", repoSpec.Ref) cmd.Dir = repoSpec.Dir.String() - out, err = cmd.CombinedOutput() + _, err = cmd.CombinedOutput() if err != nil { - log.Printf("Error checking out ref: %s", out) return errors.Wrapf(err, "trouble checking out %s", repoSpec.Ref) } @@ -62,9 +72,8 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { "--init", "--recursive") cmd.Dir = repoSpec.Dir.String() - out, err = cmd.CombinedOutput() + _, err = cmd.CombinedOutput() if err != nil { - log.Printf("Error fetching submodules: %s", out) return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec()) } From e630334837bdf553df30eb42bd24d1eec4c96bdc Mon Sep 17 00:00:00 2001 From: Pieter Vincken <4153479+pietervincken@users.noreply.github.com> Date: Tue, 9 Jun 2020 16:56:20 +0200 Subject: [PATCH 2/5] fix: re-add log outputs --- api/internal/git/cloner.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/api/internal/git/cloner.go b/api/internal/git/cloner.go index 8b420b62b..6a9078fab 100644 --- a/api/internal/git/cloner.go +++ b/api/internal/git/cloner.go @@ -4,6 +4,7 @@ package git import ( + "log" "os/exec" "github.com/pkg/errors" @@ -35,8 +36,9 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { "--depth=1", repoSpec.CloneSpec(), repoSpec.Dir.String()) - _, err = cmd.CombinedOutput() + out, err = cmd.CombinedOutput() if err != nil { + log.Printf("Error cloning git repo: %s", out) return errors.Wrapf( err, "trouble cloning git repo %v in %s", @@ -50,8 +52,9 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { "origin", repoSpec.Ref) cmd.Dir = repoSpec.Dir.String() - _, err = cmd.CombinedOutput() + out, err = cmd.CombinedOutput() if err != nil { + log.Printf("Error fetching ref: %s", out) return errors.Wrapf(err, "trouble fetching %s", repoSpec.Ref) } @@ -60,8 +63,9 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { "checkout", repoSpec.Ref) cmd.Dir = repoSpec.Dir.String() - _, err = cmd.CombinedOutput() + out, err = cmd.CombinedOutput() if err != nil { + log.Printf("Error checking out ref: %s", out) return errors.Wrapf(err, "trouble checking out %s", repoSpec.Ref) } @@ -72,8 +76,9 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { "--init", "--recursive") cmd.Dir = repoSpec.Dir.String() - _, err = cmd.CombinedOutput() + out, err = cmd.CombinedOutput() if err != nil { + log.Printf("Error fetching submodules: %s", out) return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec()) } From 28045399f3d8ff37b35e3aed08bff56e22220d49 Mon Sep 17 00:00:00 2001 From: Pieter Vincken <4153479+pietervincken@users.noreply.github.com> Date: Tue, 9 Jun 2020 16:57:01 +0200 Subject: [PATCH 3/5] fix: add missing assignment --- api/internal/git/cloner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/internal/git/cloner.go b/api/internal/git/cloner.go index 6a9078fab..8ac26dab9 100644 --- a/api/internal/git/cloner.go +++ b/api/internal/git/cloner.go @@ -36,7 +36,7 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { "--depth=1", repoSpec.CloneSpec(), repoSpec.Dir.String()) - out, err = cmd.CombinedOutput() + out, err := cmd.CombinedOutput() if err != nil { log.Printf("Error cloning git repo: %s", out) return errors.Wrapf( From cd25740b618a768ded5fbf54c6fe1facaa0e0e09 Mon Sep 17 00:00:00 2001 From: Pieter Vincken <4153479+pietervincken@users.noreply.github.com> Date: Tue, 9 Jun 2020 19:56:04 +0200 Subject: [PATCH 4/5] fix: solve issues with tags not being loaded --- api/internal/git/cloner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/internal/git/cloner.go b/api/internal/git/cloner.go index 8ac26dab9..40568174a 100644 --- a/api/internal/git/cloner.go +++ b/api/internal/git/cloner.go @@ -50,7 +50,7 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { "fetch", "--depth=1", "origin", - repoSpec.Ref) + "FETCH_HEAD") cmd.Dir = repoSpec.Dir.String() out, err = cmd.CombinedOutput() if err != nil { From 58c2df2dec14eb13e0bc42d3dfd524c2b73fb369 Mon Sep 17 00:00:00 2001 From: Pieter Vincken <4153479+pietervincken@users.noreply.github.com> Date: Tue, 9 Jun 2020 20:22:28 +0200 Subject: [PATCH 5/5] fix: add the fetch head in the correct location --- api/internal/git/cloner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/internal/git/cloner.go b/api/internal/git/cloner.go index 40568174a..6f377fb19 100644 --- a/api/internal/git/cloner.go +++ b/api/internal/git/cloner.go @@ -50,7 +50,7 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { "fetch", "--depth=1", "origin", - "FETCH_HEAD") + repoSpec.Ref) cmd.Dir = repoSpec.Dir.String() out, err = cmd.CombinedOutput() if err != nil { @@ -61,7 +61,7 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error { cmd = exec.Command( gitProgram, "checkout", - repoSpec.Ref) + "FETCH_HEAD") cmd.Dir = repoSpec.Dir.String() out, err = cmd.CombinedOutput() if err != nil {