diff --git a/pkg/git/cloner_tmp.go b/pkg/git/cloner_tmp.go new file mode 100644 index 000000000..d2f296974 --- /dev/null +++ b/pkg/git/cloner_tmp.go @@ -0,0 +1,210 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "bytes" + "io/ioutil" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/pkg/errors" +) + +// Cloner is a function that can clone a git repo. +type Cloner func(url string) ( + // Directory where the repo is cloned to. + checkoutDir string, + // Relative path in the checkoutDir to location + // of kustomization file. + pathInCoDir string, + // Any error encountered when cloning. + err error) + +// IsRepoUrl checks if a string is likely a github repo Url. +func IsRepoUrl(arg string) bool { + arg = strings.ToLower(arg) + return !filepath.IsAbs(arg) && + (strings.HasPrefix(arg, "git::") || + strings.HasPrefix(arg, "gh:") || + strings.HasPrefix(arg, "ssh:") || + strings.HasPrefix(arg, "github.com") || + strings.HasPrefix(arg, "git@") || + strings.Index(arg, "github.com/") > -1 || + isAzureHost(arg) || isAWSHost(arg)) +} + +func makeTmpDir() (string, error) { + return ioutil.TempDir("", "kustomize-") +} + +func ClonerUsingGitExec(spec string) ( + checkoutDir string, pathInCoDir string, err error) { + gitProgram, err := exec.LookPath("git") + if err != nil { + return "", "", errors.Wrap(err, "no 'git' program on path") + } + checkoutDir, err = makeTmpDir() + if err != nil { + return + } + repo, pathInCoDir, gitRef, err := parseUrl(spec) + if err != nil { + return + } + cmd := exec.Command( + gitProgram, + "clone", + repo, + checkoutDir) + var out bytes.Buffer + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return "", "", + errors.Wrapf(err, "trouble cloning %s", spec) + } + if gitRef == "" { + return + } + cmd = exec.Command(gitProgram, "checkout", gitRef) + cmd.Dir = checkoutDir + err = cmd.Run() + if err != nil { + return "", "", + errors.Wrapf(err, "trouble checking out href %s", gitRef) + } + return checkoutDir, pathInCoDir, nil +} + +func parseUrl(n string) ( + repo string, path string, gitRef string, err error) { + host, repo, path, gitRef, err := parseGithubUrl(n) + if err != nil { + return + } + if isAzureHost(host) || isAWSHost(host) { + repo = host + repo + return + } + repo = host + repo + ".git" + return +} + +const ( + refQuery = "?ref=" + gitSuffix = ".git" +) + +// From strings like git@github.com:someOrg/someRepo.git or +// https://github.com/someOrg/someRepo?ref=someHash, extract +// the parts. +func parseGithubUrl(n string) ( + host string, repo string, path string, gitRef string, err error) { + host, n = parseHostSpec(n) + host = normalizeGitHostSpec(host) + + if strings.HasSuffix(n, gitSuffix) { + repo = n[0 : len(n)-len(gitSuffix)] + return + } + if strings.Contains(n, gitSuffix) { + index := strings.Index(n, gitSuffix) + repo = n[0:index] + n = n[index+len(gitSuffix):] + path, gitRef = peelQuery(n) + return + } + i := strings.Index(n, "/") + if i < 1 { + return "", "", "", "", errors.New("no separator") + } + j := strings.Index(n[i+1:], "/") + if j >= 0 { + j += i + 1 + repo = n[:j] + path, gitRef = peelQuery(n[j+1:]) + } else { + path = "" + repo, gitRef = peelQuery(n) + } + return +} + +func peelQuery(arg string) (string, string) { + j := strings.Index(arg, refQuery) + if j >= 0 { + return arg[:j], arg[j+len(refQuery):] + } + return arg, "" +} + +func parseHostSpec(n string) (string, string) { + var host string + for _, p := range []string{ + // Order matters here. + "git::", "gh:", "ssh://", "https://", "http://", + "git@", "github.com:", "github.com/"} { + if strings.ToLower(n[:len(p)]) == p { + n = n[len(p):] + host = host + p + } + } + + // If host is a http(s) or ssh URL, grab the domain part. + for _, p := range []string{ + "ssh://", "https://", "http://"} { + if strings.HasSuffix(strings.ToLower(host), p) { + index := regexp.MustCompile("^(.*?)/").FindStringIndex(n) + if len(index) > 0 { + host = host + n[0:index[len(index)-1]] + n = n[index[len(index)-1]:] + } + } + } + return host, n +} + +func normalizeGitHostSpec(host string) string { + s := strings.ToLower(host) + if strings.Contains(s, "github.com") { + if strings.Contains(s, "git@") || strings.Contains(s, "ssh:") { + host = "git@github.com:" + } else { + host = "https://github.com/" + } + } + if strings.HasPrefix(s, "git::") { + host = strings.TrimLeft(s, "git::") + } + return host +} + +// The format of Azure repo URL is documented +// https://docs.microsoft.com/en-us/azure/devops/repos/git/clone?view=vsts&tabs=visual-studio#clone_url +func isAzureHost(host string) bool { + return strings.Contains(host, "dev.azure.com") || + strings.Contains(host, "visualstudio.com") +} + +// The format of AWS repo URL is documented +// https://docs.aws.amazon.com/codecommit/latest/userguide/regions.html +func isAWSHost(host string) bool { + return strings.Contains(host, "amazonaws.com") +} diff --git a/pkg/git/cloner_tmp_test.go b/pkg/git/cloner_tmp_test.go new file mode 100644 index 000000000..67f5e9c04 --- /dev/null +++ b/pkg/git/cloner_tmp_test.go @@ -0,0 +1,293 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "fmt" + "path/filepath" + "testing" +) + +func TestIsRepoURL(t *testing.T) { + + testcases := []struct { + input string + expected bool + }{ + { + input: "https://github.com/org/repo", + expected: true, + }, + { + input: "github.com/org/repo", + expected: true, + }, + { + input: "git@github.com:org/repo", + expected: true, + }, + { + input: "gh:org/repo", + expected: true, + }, + { + input: "git::https://gitlab.com/org/repo", + expected: true, + }, + { + input: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?ref=v0.1.0", + expected: true, + }, + { + input: "git@bitbucket.org:org/repo.git", + expected: true, + }, + { + input: "git::http://git.example.com/org/repo.git", + expected: true, + }, + { + input: "git::https://git.example.com/org/repo.git", + expected: true, + }, + { + input: "ssh://git.example.com:7999/org/repo.git", + expected: true, + }, + { + input: "/github.com/org/repo", + expected: false, + }, + { + input: "/abs/path/to/file", + expected: false, + }, + { + input: "../relative", + expected: false, + }, + { + input: "foo", + expected: false, + }, + { + input: ".", + expected: false, + }, + { + input: "", + expected: false, + }, + } + for _, tc := range testcases { + actual := IsRepoUrl(tc.input) + if actual != tc.expected { + t.Errorf("unexpected error: unexpected result %t for input %s", actual, tc.input) + } + } +} + +var repoNames = []string{"someOrg/someRepo", "kubernetes/website"} + +var paths = []string{"README.md", "foo/krusty.txt", ""} + +var hrefArgs = []string{"someBranch", ""} + +var extractFmts = map[string]string{ + "gh:%s": "gh:", + "GH:%s": "gh:", + "gitHub.com/%s": "https://github.com/", + "https://github.com/%s": "https://github.com/", + "hTTps://github.com/%s": "https://github.com/", + "git::https://gitlab.com/%s": "https://gitlab.com/", + "github.com:%s": "https://github.com/", + "git::http://git.example.com/%s": "http://git.example.com/", + "git::https://git.example.com/%s": "https://git.example.com/", + "ssh://git.example.com:7999/%s": "ssh://git.example.com:7999/", +} + +func TestParseGithubUrl(t *testing.T) { + for _, repoName := range repoNames { + for _, pathName := range paths { + for extractFmt, hostSpec := range extractFmts { + for _, hrefArg := range hrefArgs { + spec := repoName + if len(pathName) > 0 { + spec = filepath.Join(spec, pathName) + } + input := fmt.Sprintf(extractFmt, spec) + if hrefArg != "" { + input = input + refQuery + hrefArg + } + if !IsRepoUrl(input) { + t.Errorf("Should smell like github arg: %s\n", input) + continue + } + host, repo, path, gitRef, err := parseGithubUrl(input) + if err != nil { + t.Errorf("problem %v", err) + } + if host != hostSpec { + t.Errorf("\n"+ + " from %s\n"+ + " actual host %s\n"+ + "expected host %s\n", input, host, hostSpec) + } + if repo != repoName { + t.Errorf("\n"+ + " from %s\n"+ + " actual Repo %s\n"+ + "expected Repo %s\n", input, repo, repoName) + } + if path != pathName { + t.Errorf("\n"+ + " from %s\n"+ + " actual Path %s\n"+ + "expected Path %s\n", input, path, pathName) + } + if gitRef != hrefArg { + t.Errorf("\n"+ + " from %s\n"+ + " actual Href %s\n"+ + "expected Href %s\n", input, gitRef, hrefArg) + } + } + } + } + } +} + +func TestParseUrl(t *testing.T) { + testcases := []struct { + input string + repo string + path string + ref string + }{ + { + input: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir", + repo: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo", + path: "somedir", + ref: "", + }, + { + input: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir?ref=testbranch", + repo: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo", + path: "somedir", + ref: "testbranch", + }, + { + input: "https://fabrikops2.visualstudio.com/someorg/somerepo?ref=master", + repo: "https://fabrikops2.visualstudio.com/someorg/somerepo", + path: "", + ref: "master", + }, + { + input: "http://github.com/someorg/somerepo/somedir", + repo: "https://github.com/someorg/somerepo.git", + path: "somedir", + ref: "", + }, + { + input: "git@github.com:someorg/somerepo/somedir", + repo: "git@github.com:someorg/somerepo.git", + path: "somedir", + ref: "", + }, + { + input: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?ref=v0.1.0", + repo: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git", + path: "", + ref: "v0.1.0", + }, + } + for _, testcase := range testcases { + repo, path, ref, err := parseUrl(testcase.input) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if repo != testcase.repo { + t.Errorf("repo expected to be %v, but got %v on %s", testcase.repo, repo, testcase.input) + } + if path != testcase.path { + t.Errorf("path expected to be %v, but got %v on %s", testcase.path, path, testcase.input) + } + if ref != testcase.ref { + t.Errorf("ref expected to be %v, but got %v on %s", testcase.ref, ref, testcase.input) + } + } +} + +func TestIsAzureHost(t *testing.T) { + testcases := []struct { + input string + expect bool + }{ + { + input: "https://git-codecommit.us-east-2.amazonaws.com", + expect: false, + }, + { + input: "ssh://git-codecommit.us-east-2.amazonaws.com", + expect: false, + }, + { + input: "https://fabrikops2.visualstudio.com/", + expect: true, + }, + { + input: "https://dev.azure.com/myorg/myproject/", + expect: true, + }, + } + for _, testcase := range testcases { + actual := isAzureHost(testcase.input) + if actual != testcase.expect { + t.Errorf("IsAzureHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input) + } + } +} + +func TestIsAWSHost(t *testing.T) { + testcases := []struct { + input string + expect bool + }{ + { + input: "https://git-codecommit.us-east-2.amazonaws.com", + expect: true, + }, + { + input: "ssh://git-codecommit.us-east-2.amazonaws.com", + expect: true, + }, + { + input: "git@github.com:", + expect: false, + }, + { + input: "http://github.com/", + expect: false, + }, + } + for _, testcase := range testcases { + actual := isAWSHost(testcase.input) + if actual != testcase.expect { + t.Errorf("IsAWSHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input) + } + } +}