Feature/dependency pinning and update automation (#5451)

* * handle local flag
* add managerfactory handling for local flag
* add shortName handling for local flag
* add dot git file handling for local flag
* add tests

* fix normal listing

* add ParseGitRepository function, add viper, add testing for utils

* add latest tag logic, add auto pinning and auto fetching

* makke gorepomod list works with --local

* make pinning works with local flag, enable auto update on fork and non-fork repo

* fix: refactor to pass linter

* refactor code and fix comments

* edit README

* refactor code to pass linting

* refactor code

* refactor code and enable patch branch label

* ru add license

* fbackward compatibility for unpin

cmd/gorepomod/internal/arguments/args.go

@@ -14,6 +14,7 @@ import (
 
 const (
 	doItFlag     = "--doIt"
+	localFlag    = "--local"
 	cmdPin       = "pin"
 	cmdUnPin     = "unpin"
 	cmdTidy      = "tidy"
@@ -64,6 +65,7 @@ type Args struct {
 	version           semver.SemVer
 	bump              semver.SvBump
 	doIt              bool
+	localFlag         bool
 }
 
 func (a *Args) GetCommand() Command {
@@ -106,9 +108,14 @@ func (a *Args) DoIt() bool {
 	return a.doIt
 }
 
+func (a *Args) LocalFlag() bool {
+	return a.localFlag
+}
+
 type myArgs struct {
-	args []string
-	doIt bool
+	args      []string
+	doIt      bool
+	localFlag bool
 }
 
 func (a *myArgs) next() (result string) {
@@ -129,6 +136,8 @@ func newArgs() *myArgs {
 	for _, a := range os.Args[1:] {
 		if a == doItFlag {
 			result.doIt = true
+		} else if a == localFlag {
+			result.localFlag = true
 		} else {
 			result.args = append(result.args, a)
 		}
@@ -140,6 +149,7 @@ func Parse() (result *Args, err error) {
 	result = &Args{}
 	clArgs := newArgs()
 	result.doIt = clArgs.doIt
+	result.localFlag = clArgs.localFlag
 
 	result.moduleName = misc.ModuleUnknown
 	result.conditionalModule = misc.ModuleUnknown

cmd/gorepomod/internal/edit/editor.go

@@ -57,19 +57,19 @@ func (e *Editor) Tidy() error {
 func (e *Editor) Pin(target misc.LaModule, oldV, newV semver.SemVer) error {
 	err := e.run(
 		"edit",
-		"-dropreplace="+target.ImportPath(),
-		"-dropreplace="+target.ImportPath()+"@"+oldV.String(),
-		"-require="+target.ImportPath()+"@"+newV.String(),
+		"-dropreplace=sigs.k8s.io/kustomize/"+string(target.ShortName()),
+		"-dropreplace=sigs.k8s.io/kustomize/"+string(target.ShortName())+"@"+oldV.String(),
+		"-require=sigs.k8s.io/kustomize/"+string(target.ShortName())+"@"+newV.String(),
 	)
 	if err != nil {
-		return err
+		return fmt.Errorf("%w", err)
 	}
 	return e.run("tidy")
 }
 
 func (e *Editor) UnPin(target misc.LaModule, oldV semver.SemVer) error {
 	var r strings.Builder
-	r.WriteString(target.ImportPath())
+	r.WriteString("sigs.k8s.io/kustomize/" + string(target.ShortName()))
 	// Don't specify the old version.
 	// r.WriteString("@")
 	// r.WriteString(oldV.String())
@@ -81,7 +81,7 @@ func (e *Editor) UnPin(target misc.LaModule, oldV semver.SemVer) error {
 		"-replace="+r.String(),
 	)
 	if err != nil {
-		return err
+		return fmt.Errorf("%w", err)
 	}
 	return e.run("tidy")
 }

cmd/gorepomod/internal/git/runner.go

@@ -48,20 +48,22 @@ type Runner struct {
 	workDir string
 	// Run commands, or merely print commands.
 	doIt bool
+	// Indicate local execution to adapt with path.
+	localFlag bool
 	// Run commands, or merely print commands.
 	verbosity Verbosity
 }
 
-func NewLoud(wd string, doIt bool) *Runner {
-	return newRunner(wd, doIt, High)
+func NewLoud(wd string, doIt bool, localFlag bool) *Runner {
+	return newRunner(wd, doIt, High, localFlag)
 }
 
-func NewQuiet(wd string, doIt bool) *Runner {
-	return newRunner(wd, doIt, Low)
+func NewQuiet(wd string, doIt bool, localFlag bool) *Runner {
+	return newRunner(wd, doIt, Low, localFlag)
 }
 
-func newRunner(wd string, doIt bool, v Verbosity) *Runner {
-	return &Runner{workDir: wd, doIt: doIt, verbosity: v}
+func newRunner(wd string, doIt bool, v Verbosity, localFlag bool) *Runner {
+	return &Runner{workDir: wd, doIt: doIt, verbosity: v, localFlag: localFlag}
 }
 
 func (gr *Runner) comment(f string) {
@@ -165,8 +167,20 @@ func (gr *Runner) LoadLocalTags() (result misc.VersionMap, err error) {
 
 func (gr *Runner) LoadRemoteTags(
 	remote misc.TrackedRepo) (result misc.VersionMap, err error) {
-	gr.comment("loading remote tags")
 	var out string
+
+	// Update latest tags from upstream
+	gr.comment("updating tags from upstream")
+	_, err = gr.run(noHarmDone, "fetch", "-t", string(remoteUpstream), string(mainBranch))
+	if err != nil {
+		// Handle if repo is not a fork
+		_, err = gr.run(noHarmDone, "fetch", "-t", string(mainBranch))
+		if err != nil {
+			_ = fmt.Errorf("failed to fetch tags from %s", string(mainBranch))
+		}
+	}
+
+	gr.comment("loading remote tags")
 	out, err = gr.run(noHarmDone, "ls-remote", "--ref", string(remote))
 	if err != nil {
 		return nil, err
@@ -250,7 +264,15 @@ func (gr *Runner) CheckoutMainBranch() error {
 // FetchRemote does that.
 func (gr *Runner) FetchRemote(remote misc.TrackedRepo) error {
 	gr.comment("fetching remote")
-	return gr.runNoOut(noHarmDone, "fetch", string(remote))
+	err := gr.runNoOut(noHarmDone, "fetch", string(remote))
+	if err != nil {
+		// If current repo is fork
+		err = gr.runNoOut(noHarmDone, "fetch", string(remoteUpstream))
+		if err != nil {
+			return fmt.Errorf("%w", err)
+		}
+	}
+	return nil
 }
 
 // MergeFromRemoteMain does a fast forward only merge with main branch.
@@ -342,3 +364,27 @@ func (gr *Runner) DeleteTagFromRemote(
 	gr.comment("deleting tags from remote")
 	return gr.runNoOut(undoPainful, "push", string(remote), ":"+refsTags+tag)
 }
+
+func (gr *Runner) GetLatestTag(releaseBranch string) (string, error) {
+	var latestTag string
+	// Assuming release branch has this format: release-path/to/module-vX.Y.Z
+	// and each release branch maintains tags, extract version from latest `releaseBranch`
+	gr.comment("extract version from latest release branch")
+	filteredBranchList, err := gr.run(noHarmDone, "branch", "-a", "--list", "*"+releaseBranch+"*", "--sort=-committerdate")
+	if len(filteredBranchList) < 1 {
+		_ = fmt.Errorf("latest tag not found for %s", releaseBranch)
+		return "", err
+	}
+	newestBranch := strings.Split(strings.ReplaceAll(filteredBranchList, "\r\n", "\n"), "\n")
+	split := strings.Split(newestBranch[0], "-")
+	latestTag = split[len(split)-1]
+	if err != nil {
+		_ = fmt.Errorf("error getting latest tag for %s", releaseBranch)
+	}
+
+	return latestTag, nil
+}
+
+func (gr *Runner) GetMainBranch() string {
+	return string(mainBranch)
+}

cmd/gorepomod/internal/mod/module.go

@@ -63,7 +63,8 @@ func (m *Module) AbsPath() string {
 
 func (m *Module) DependsOn(target misc.LaModule) (bool, semver.SemVer) {
 	for _, r := range m.mf.Require {
-		if r.Mod.Path == target.ImportPath() {
+		targetImportPath := fmt.Sprintf("sigs.k8s.io/kustomize/%s", string(target.ShortName()))
+		if r.Mod.Path == targetImportPath {
 			v, err := semver.Parse(r.Mod.Version)
 			if err != nil {
 				panic(err)

cmd/gorepomod/internal/repo/dotgitdata.go

@@ -5,9 +5,12 @@ package repo
 
 import (
 	"fmt"
+	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
+	v "github.com/spf13/viper"
 	"sigs.k8s.io/kustomize/cmd/gorepomod/internal/git"
 	"sigs.k8s.io/kustomize/cmd/gorepomod/internal/misc"
 	"sigs.k8s.io/kustomize/cmd/gorepomod/internal/utils"
@@ -48,39 +51,70 @@ func (dg *DotGitData) AbsPath() string {
 //
 //	~/gopath/src/sigs.k8s.io/kustomize
 //	~/gopath/src/github.com/monopole/gorepomod
-func NewDotGitDataFromPath(path string) (*DotGitData, error) {
+func NewDotGitDataFromPath(path string, localFlag bool) (*DotGitData, error) {
 	if !utils.DirExists(filepath.Join(path, dotGitFileName)) {
 		return nil, fmt.Errorf(
 			"%q doesn't have a %q file", path, dotGitFileName)
 	}
-	// This is an attempt to figure out where the user has cloned
-	// their repos.  In the old days, it was an import path under
-	// $GOPATH/src.  If we cannot guess it, we may need to ask for it,
-	// or maybe proceed without knowing it.
-	index := strings.Index(path, srcHint)
-	if index < 0 {
-		return nil, fmt.Errorf(
-			"path %q doesn't contain %q", path, srcHint)
+
+	// If local flag is supplied, use local git naming instead of production (sigs.k8s.io)
+	if localFlag {
+		localPrefix, err := getLocalPrefix(path)
+		if err != nil {
+			return nil, fmt.Errorf("%w", err)
+		}
+		v.Set("LocalGitPrefix", localPrefix)
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, fmt.Errorf("error extracting git repository %w", err)
+		}
+
+		pathSlice := strings.Split(wd, "/")
+
+		return &DotGitData{
+			srcPath:  strings.Join(pathSlice[:len(pathSlice)-1], "/"),
+			repoPath: pathSlice[len(pathSlice)-1],
+		}, nil
+	} else {
+		// This is an attempt to figure out where the user has cloned
+		// their repos.  In the old days, it was an import path under
+		// $GOPATH/src.  If we cannot guess it, we may need to ask for it,
+		// or maybe proceed without knowing it.
+		index := strings.Index(path, srcHint)
+		if index < 0 {
+			return nil, fmt.Errorf(
+				"path %q doesn't contain %q", path, srcHint)
+		}
+
+		return &DotGitData{
+			srcPath:  path[:index+len(srcHint)-1],
+			repoPath: path[index+len(srcHint):],
+		}, nil
 	}
-	return &DotGitData{
-		srcPath:  path[:index+len(srcHint)-1],
-		repoPath: path[index+len(srcHint):],
-	}, nil
 }
 
 // It's a factory factory.
 func (dg *DotGitData) NewRepoFactory(
-	exclusions []string) (*ManagerFactory, error) {
+	exclusions []string, localFlag bool) (*ManagerFactory, error) {
 	modules, err := loadProtoModules(dg.AbsPath(), exclusions)
-	if err != nil {
-		return nil, err
-	}
-	err = dg.checkModules(modules)
+
 	if err != nil {
 		return nil, err
 	}
 
-	runner := git.NewQuiet(dg.AbsPath(), true)
+	if localFlag {
+		err = dg.checkModulesLocal(modules)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		err = dg.checkModules(modules)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	runner := git.NewQuiet(dg.AbsPath(), true, localFlag)
 	remoteName, err := runner.DetermineRemoteToUse()
 	if err != nil {
 		return nil, err
@@ -136,3 +170,35 @@ func (dg *DotGitData) checkModules(modules []*protoModule) error {
 	}
 	return nil
 }
+
+func (dg *DotGitData) checkModulesLocal(modules []*protoModule) error {
+	for _, pm := range modules {
+		file := filepath.Join(pm.PathToGoMod(), goModFile)
+		_, err := os.Stat(file)
+		if err != nil {
+			return fmt.Errorf(
+				"cannot find go.mod file in %q", file)
+		}
+		pm.ShortNameWithLocalFlag(dg.RepoPath())
+	}
+	return nil
+}
+
+// Extract local prefix from origin url information retrieved from .git
+func getLocalPrefix(dgAbsPath string) (string, error) {
+	_, err := os.Stat(dgAbsPath + "/.git")
+	if err != nil {
+		return "", fmt.Errorf(".git directory does not exist in path %s", dgAbsPath)
+	}
+
+	out, err := exec.Command("git", "config", "--get", "remote.origin.url").Output()
+	if err != nil {
+		return "", fmt.Errorf("failed extracting git information: %w", err)
+	}
+
+	localPrefix := utils.ParseGitRepositoryPath(string(out))
+	if len(localPrefix) == 0 {
+		_ = fmt.Errorf("parsed git repository path is empty: %w", err)
+	}
+	return localPrefix, nil
+}

cmd/gorepomod/internal/repo/dotgitdata_test.go

@@ -3,7 +3,10 @@
 
 package repo
 
-import "testing"
+import (
+	"testing"
+)
 
-func TestLoadTags(t *testing.T) {
+func TestLoadRepoManager(t *testing.T) {
+	t.Skip()
 }

cmd/gorepomod/internal/repo/manager.go

@@ -80,6 +80,16 @@ func (mgr *Manager) hasUnPinnedDeps(m misc.LaModule) string {
 }
 
 func (mgr *Manager) List() error {
+	// Auto-update local tags
+	gr := git.NewQuiet(mgr.AbsPath(), false, false)
+	for _, module := range mgr.modules {
+		releaseBranch := fmt.Sprintf("release-%s", module.ShortName())
+		_, err := gr.GetLatestTag(releaseBranch)
+		if err != nil {
+			return fmt.Errorf("failed getting latest tags for %s", module)
+		}
+	}
+
 	fmt.Printf("   src path: %s\n", mgr.dg.SrcPath())
 	fmt.Printf("  repo path: %s\n", mgr.RepoPath())
 	fmt.Printf("     remote: %s\n", mgr.remoteName)
@@ -113,8 +123,8 @@ func determineBranchAndTag(
 		string(m.ShortName()) + "/" + v.String()
 }
 
-func (mgr *Manager) Debug(_ misc.LaModule, doIt bool) error {
-	gr := git.NewLoud(mgr.AbsPath(), doIt)
+func (mgr *Manager) Debug(_ misc.LaModule, doIt bool, localFlag bool) error {
+	gr := git.NewLoud(mgr.AbsPath(), doIt, localFlag)
 	return gr.Debug(mgr.remoteName)
 }
 
@@ -122,10 +132,8 @@ func (mgr *Manager) Debug(_ misc.LaModule, doIt bool) error {
 //
 // * All development happens in the branch named "master".
 // * Each minor release gets its own branch.
-//
 func (mgr *Manager) Release(
-	target misc.LaModule, bump semver.SvBump, doIt bool) error {
-
+	target misc.LaModule, bump semver.SvBump, doIt bool, localFlag bool) error {
 	if reps := target.GetDisallowedReplacements(
 		mgr.allowedReplacements); len(reps) > 0 {
 		return fmt.Errorf(
@@ -145,7 +153,7 @@ func (mgr *Manager) Release(
 			newVersion, target.VersionRemote())
 	}
 
-	gr := git.NewLoud(mgr.AbsPath(), doIt)
+	gr := git.NewLoud(mgr.AbsPath(), doIt, localFlag)
 
 	relBranch, relTag := determineBranchAndTag(target, newVersion)
 
@@ -189,14 +197,14 @@ func (mgr *Manager) Release(
 	return nil
 }
 
-func (mgr *Manager) UnRelease(target misc.LaModule, doIt bool) error {
+func (mgr *Manager) UnRelease(target misc.LaModule, doIt bool, localFlag bool) error {
 	fmt.Printf(
 		"Unreleasing %s/%s\n",
 		target.ShortName(), target.VersionRemote())
 
 	_, tag := determineBranchAndTag(target, target.VersionRemote())
 
-	gr := git.NewLoud(mgr.AbsPath(), doIt)
+	gr := git.NewLoud(mgr.AbsPath(), doIt, localFlag)
 
 	if err := gr.DeleteTagFromRemote(mgr.remoteName, tag); err != nil {
 		return err

cmd/gorepomod/internal/repo/managerfactory.go

@@ -37,3 +37,23 @@ func (mf *ManagerFactory) NewRepoManager(allowedReplacements []string) *Manager
 	result.allowedReplacements = allowedReplacements
 	return result
 }
+
+func (mf *ManagerFactory) NewRepoManagerWithLocalFlag(allowedReplacements []string) *Manager {
+	result := &Manager{
+		dg:         mf.dg,
+		remoteName: mf.remoteName,
+	}
+	var modules misc.LesModules
+	for _, pm := range mf.modules {
+		shortName := pm.ShortNameWithLocalFlag(mf.dg.RepoPath())
+		modules = append(
+			modules,
+			mod.New(
+				result, shortName, pm.mf,
+				mf.versionMapLocal.Latest(shortName),
+				mf.versionMapRemote.Latest(shortName)))
+	}
+	result.modules = modules
+	result.allowedReplacements = allowedReplacements
+	return result
+}

cmd/gorepomod/internal/repo/protomodule.go

@@ -8,7 +8,9 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 
+	v "github.com/spf13/viper"
 	"golang.org/x/mod/modfile"
 	"sigs.k8s.io/kustomize/cmd/gorepomod/internal/misc"
 	"sigs.k8s.io/kustomize/cmd/gorepomod/internal/utils"
@@ -28,6 +30,13 @@ func (pm *protoModule) FullPath() string {
 	return pm.mf.Module.Mod.Path
 }
 
+func (pm *protoModule) FullLocalPath() string {
+	var localPrefix string = v.GetString("LocalGitPrefix")
+	var pathToModule string = pm.mf.Module.Mod.Path
+	pathSlice := strings.Split(pathToModule, "/")
+	return localPrefix + "/" + strings.Join(pathSlice[1:], "/")
+}
+
 func (pm *protoModule) PathToGoMod() string {
 	return pm.pathToGoMod
 }
@@ -47,6 +56,19 @@ func (pm *protoModule) ShortName(
 	return misc.ModuleShortName(stripped)
 }
 
+func (pm *protoModule) ShortNameWithLocalFlag(
+	repoImportPath string) misc.ModuleShortName {
+	fp := pm.FullLocalPath()
+
+	if fp == repoImportPath {
+		return misc.ModuleAtTop
+	}
+	p := fp[len(repoImportPath)+2:]
+	stripped := trailingVersionPattern.ReplaceAllString(p, "")
+	pathSlice := strings.Split(stripped, "/")
+	return misc.ModuleShortName(strings.Join(pathSlice[3:], "/"))
+}
+
 func loadProtoModules(
 	repoRoot string, exclusions []string) (result []*protoModule, err error) {
 	var paths []string

cmd/gorepomod/internal/semver/semver.go

@@ -66,13 +66,15 @@ func (v SemVer) Bump(b SvBump) SemVer {
 		return New(v.major+1, 0, 0)
 	case Minor:
 		return New(v.major, v.minor+1, 0)
+	case Patch:
+		return New(v.major, v.minor, v.patch+1)
 	default:
 		return New(v.major, v.minor, v.patch+1)
 	}
 }
 
 func (v SemVer) BranchLabel() string {
-	return fmt.Sprintf("v%d.%d", v.major, v.minor)
+	return fmt.Sprintf("v%d.%d.%d", v.major, v.minor, v.patch)
 }
 
 func (v SemVer) String() string {

cmd/gorepomod/internal/utils/utils.go

@@ -4,8 +4,10 @@
 package utils
 
 import (
+	"fmt"
 	"log"
 	"os"
+	"regexp"
 	"strings"
 )
 
@@ -45,3 +47,38 @@ func SliceContains(slice []string, target string) bool {
 	}
 	return false
 }
+
+// Receive git remote url as input and produce string containing git repository name
+// e.g. kustomize, myrepo/kustomize
+func ParseGitRepositoryPath(urlString string) string {
+	var d []string = getURLStringArray(urlString)
+	protocol := d[0]
+
+	var repoPath string
+
+	// TODO(antoooks): Confirm if we should handle other formats not commonly supported by Github
+	switch protocol {
+	// ssh protocol, e.g. git@github.com:path/repo.git
+	case "git":
+		repoPath = strings.Join(d[2:len(d)-1], "/") + "/" + d[3][:len(d[3])-4]
+	// https protocol, e.g. https://github.com/path/repo.git
+	case "https":
+		repoPath = strings.Join(d[2:], "/")
+		repoPath = repoPath[:len(repoPath)-4]
+	// unsupported format
+	default:
+		_ = fmt.Errorf("protocol format is not supported: %s", protocol)
+		return ""
+	}
+
+	return d[1] + "/" + repoPath
+}
+
+// Extract array of string from urlString
+func getURLStringArray(urlString string) []string {
+	// Supported git regex based on URI allowed regex as defined under RFC3986
+	const rfc3986 = `[A-Za-z0-9][A-Za-z0-9+.-]*`
+	re := regexp.MustCompile(rfc3986)
+	var u []string = re.FindAllString(urlString, -1)
+	return u
+}

cmd/gorepomod/internal/utils/utils_test.go

@@ -0,0 +1,36 @@
+// Copyright 2023 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"testing"
+)
+
+func TestParseGitRepositoryPath(t *testing.T) {
+	var testCases = map[string]struct {
+		urlString string
+		expected  string
+	}{
+		"ssh format": {
+			urlString: "git@github.com:path/dummy.git",
+			expected:  "github.com/path/dummy",
+		},
+		"https format": {
+			urlString: "https://github.com/path/dummy.git",
+			expected:  "github.com/path/dummy",
+		},
+		"unknown format": {
+			urlString: "file:///path/to/repo.git/",
+			expected:  "",
+		},
+	}
+
+	for n, tc := range testCases {
+		actual := ParseGitRepositoryPath(tc.urlString)
+		t.Log(actual)
+		if actual != tc.expected {
+			t.Errorf("%s: expected %s, got %s", n, tc.expected, actual)
+		}
+	}
+}