From 259cecd4b830bc674941ca5e1eef02ea4eb36149 Mon Sep 17 00:00:00 2001 From: Jingfang Liu Date: Mon, 10 Dec 2018 10:50:31 -0800 Subject: [PATCH] add handling function for remote url hostname --- pkg/loader/gitcloner.go | 91 ++++++++++++++++++---- pkg/loader/gitcloner_test.go | 141 ++++++++++++++++++++++++++++++++--- 2 files changed, 209 insertions(+), 23 deletions(-) diff --git a/pkg/loader/gitcloner.go b/pkg/loader/gitcloner.go index 6171aedb1..98a732a8f 100644 --- a/pkg/loader/gitcloner.go +++ b/pkg/loader/gitcloner.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os/exec" "path/filepath" + "regexp" "strings" "github.com/pkg/errors" @@ -44,7 +45,8 @@ func isRepoUrl(arg string) bool { strings.HasPrefix(arg, "gh:") || strings.HasPrefix(arg, "github.com") || strings.HasPrefix(arg, "git@github.com:") || - strings.Index(arg, "github.com/") > -1) + strings.Index(arg, "github.com/") > -1 || + isAzureHost(arg) || isAWSHost(arg)) } func makeTmpDir() (string, error) { @@ -61,14 +63,14 @@ func simpleGitCloner(spec string) ( if err != nil { return } - repo, pathInCoDir, gitRef, err := parseGithubUrl(spec) + repo, pathInCoDir, gitRef, err := parseUrl(spec) if err != nil { return } cmd := exec.Command( gitProgram, "clone", - "https://github.com/"+repo+".git", + repo, checkoutDir) var out bytes.Buffer cmd.Stdout = &out @@ -90,27 +92,36 @@ func simpleGitCloner(spec string) ( return checkoutDir, pathInCoDir, nil } +func parseUrl(n string) ( + repo string, path string, gitRef string, err error) { + host, repo, path, gitRef, err := parseGithubUrl(n) + if err != nil { + return + } + if isAzureHost(host) || isAWSHost(host) { + repo = host + repo + return + } + repo = host + repo + ".git" + return +} + const refQuery = "?ref=" // From strings like git@github.com:someOrg/someRepo.git or // https://github.com/someOrg/someRepo?ref=someHash, extract // the parts. func parseGithubUrl(n string) ( - repo string, path string, gitRef string, err error) { - for _, p := range []string{ - // Order matters here. - "git::", "gh:", "https://", "http://", - "git@", "github.com:", "github.com/", "gitlab.com/"} { - if strings.ToLower(n[:len(p)]) == p { - n = n[len(p):] - } - } + host string, repo string, path string, gitRef string, err error) { + host, n = parseHostSpec(n) + host = normalizeGitHostSpec(host) + if strings.HasSuffix(n, ".git") { n = n[0 : len(n)-len(".git")] } i := strings.Index(n, string(filepath.Separator)) if i < 1 { - return "", "", "", errors.New("no separator") + return "", "", "", "", errors.New("no separator") } j := strings.Index(n[i+1:], string(filepath.Separator)) if j >= 0 { @@ -131,3 +142,57 @@ func peelQuery(arg string) (string, string) { } return arg, "" } + +func parseHostSpec(n string) (string, string) { + var host string + for _, p := range []string{ + // Order matters here. + "git::", "gh:", "ssh://", "https://", "http://", + "git@", "github.com:", "github.com/", "gitlab.com/"} { + if strings.ToLower(n[:len(p)]) == p { + n = n[len(p):] + host = host + p + } + } + for _, p := range []string{ + "git-codecommit.[a-z0-9-]*.amazonaws.com/", + "dev.azure.com/", + ".*visualstudio.com/"} { + index := regexp.MustCompile(p).FindStringIndex(n) + if len(index) > 0 { + host = host + n[0:index[len(index)-1]] + n = n[index[len(index)-1]:] + } + } + return host, n +} + +func normalizeGitHostSpec(host string) string { + s := strings.ToLower(host) + if strings.Contains(s, "github.com") { + if strings.Contains(s, "git@") || strings.Contains(s, "ssh:") { + host = "git@github.com:" + } else { + host = "https://github.com/" + } + } + if strings.Contains(s, "gitlab") { + if strings.HasPrefix(s, "git::") { + host = strings.TrimLeft(s, "git::") + } + } + return host +} + +// The format of Azure repo URL is documented +// https://docs.microsoft.com/en-us/azure/devops/repos/git/clone?view=vsts&tabs=visual-studio#clone_url +func isAzureHost(host string) bool { + return strings.Contains(host, "dev.azure.com") || + strings.Contains(host, "visualstudio.com") +} + +// The format of AWS repo URL is documented +// https://docs.aws.amazon.com/codecommit/latest/userguide/regions.html +func isAWSHost(host string) bool { + return strings.Contains(host, "amazonaws.com") +} diff --git a/pkg/loader/gitcloner_test.go b/pkg/loader/gitcloner_test.go index 984c25682..732d074ee 100644 --- a/pkg/loader/gitcloner_test.go +++ b/pkg/loader/gitcloner_test.go @@ -185,20 +185,20 @@ var paths = []string{"README.md", "foo/krusty.txt", ""} var hrefArgs = []string{"someBranch", ""} -var extractFmts = []string{ - "gh:%s", - "GH:%s", - "gitHub.com/%s", - "https://github.com/%s", - "hTTps://github.com/%s", - "git::https://gitlab.com/%s", - "github.com:%s", +var extractFmts = map[string]string{ + "gh:%s": "gh:", + "GH:%s": "gh:", + "gitHub.com/%s": "https://github.com/", + "https://github.com/%s": "https://github.com/", + "hTTps://github.com/%s": "https://github.com/", + "git::https://gitlab.com/%s": "https://gitlab.com/", + "github.com:%s": "https://github.com/", } func TestParseGithubUrl(t *testing.T) { for _, repoName := range repoNames { for _, pathName := range paths { - for _, extractFmt := range extractFmts { + for extractFmt, hostSpec := range extractFmts { for _, hrefArg := range hrefArgs { spec := repoName if len(pathName) > 0 { @@ -212,10 +212,16 @@ func TestParseGithubUrl(t *testing.T) { t.Errorf("Should smell like github arg: %s\n", input) continue } - repo, path, gitRef, err := parseGithubUrl(input) + host, repo, path, gitRef, err := parseGithubUrl(input) if err != nil { t.Errorf("problem %v", err) } + if host != hostSpec { + t.Errorf("\n"+ + " from %s\n"+ + " actual host %s\n"+ + "expected host %s\n", input, host, hostSpec) + } if repo != repoName { t.Errorf("\n"+ " from %s\n"+ @@ -239,3 +245,118 @@ func TestParseGithubUrl(t *testing.T) { } } } + +func TestParseUrl(t *testing.T) { + testcases := []struct { + input string + repo string + path string + ref string + }{ + { + input: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir", + repo: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo", + path: "somedir", + ref: "", + }, + { + input: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir?ref=testbranch", + repo: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo", + path: "somedir", + ref: "testbranch", + }, + { + input: "https://fabrikops2.visualstudio.com/someorg/somerepo?ref=master", + repo: "https://fabrikops2.visualstudio.com/someorg/somerepo", + path: "", + ref: "master", + }, + { + input: "http://github.com/someorg/somerepo/somedir", + repo: "https://github.com/someorg/somerepo.git", + path: "somedir", + ref: "", + }, + { + input: "git@github.com:someorg/somerepo/somedir", + repo: "git@github.com:someorg/somerepo.git", + path: "somedir", + ref: "", + }, + } + for _, testcase := range testcases { + repo, path, ref, err := parseUrl(testcase.input) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if repo != testcase.repo { + t.Errorf("repo expected to be %v, but got %v on %s", testcase.repo, repo, testcase.input) + } + if path != testcase.path { + t.Errorf("path expected to be %v, but got %v on %s", testcase.path, path, testcase.input) + } + if ref != testcase.ref { + t.Errorf("ref expected to be %v, but got %v on %s", testcase.ref, ref, testcase.input) + } + } +} + +func TestIsAzureHost(t *testing.T) { + testcases := []struct { + input string + expect bool + }{ + { + input: "https://git-codecommit.us-east-2.amazonaws.com", + expect: false, + }, + { + input: "ssh://git-codecommit.us-east-2.amazonaws.com", + expect: false, + }, + { + input: "https://fabrikops2.visualstudio.com/", + expect: true, + }, + { + input: "https://dev.azure.com/myorg/myproject/", + expect: true, + }, + } + for _, testcase := range testcases { + actual := isAzureHost(testcase.input) + if actual != testcase.expect { + t.Errorf("IsAzureHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input) + } + } +} + +func TestIsAWSHost(t *testing.T) { + testcases := []struct { + input string + expect bool + }{ + { + input: "https://git-codecommit.us-east-2.amazonaws.com", + expect: true, + }, + { + input: "ssh://git-codecommit.us-east-2.amazonaws.com", + expect: true, + }, + { + input: "git@github.com:", + expect: false, + }, + { + input: "http://github.com/", + expect: false, + }, + } + for _, testcase := range testcases { + actual := isAWSHost(testcase.input) + if actual != testcase.expect { + t.Errorf("IsAWSHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input) + } + } +}