Move git code to its own pkg.

This commit is contained in:
jregan
2019-01-26 13:36:23 -08:00
parent d953eca630
commit 1d9a20b391
5 changed files with 129 additions and 128 deletions

View File

@@ -23,6 +23,7 @@ import (
"strings"
"sigs.k8s.io/kustomize/pkg/fs"
"sigs.k8s.io/kustomize/pkg/git"
"sigs.k8s.io/kustomize/pkg/ifc"
)
@@ -91,7 +92,7 @@ type fileLoader struct {
// File system utilities.
fSys fs.FileSystem
// Used to clone repositories.
cloner gitCloner
cloner git.Cloner
// Used to clean up, as needed.
cleaner func() error
}
@@ -113,18 +114,18 @@ func (l *fileLoader) Root() string {
}
func newLoaderOrDie(fSys fs.FileSystem, path string) *fileLoader {
l, err := newFileLoaderAt(
path, fSys, nil, simpleGitCloner)
l, err := newLoaderAtConfirmedDir(
path, fSys, nil, git.ClonerUsingGitExec)
if err != nil {
log.Fatalf("unable to make loader at '%s'; %v", path, err)
}
return l
}
// newFileLoaderAt returns a new fileLoader with given root.
func newFileLoaderAt(
// newLoaderAtConfirmedDir returns a new fileLoader with given root.
func newLoaderAtConfirmedDir(
possibleRoot string, fSys fs.FileSystem,
referrer *fileLoader, cloner gitCloner) (*fileLoader, error) {
referrer *fileLoader, cloner git.Cloner) (*fileLoader, error) {
if possibleRoot == "" {
return nil, fmt.Errorf(
"loader root cannot be empty")
@@ -159,25 +160,25 @@ func (l *fileLoader) New(path string) (ifc.Loader, error) {
if path == "" {
return nil, fmt.Errorf("new root cannot be empty")
}
if isRepoUrl(path) {
if git.IsRepoUrl(path) {
// Avoid cycles.
if err := l.errIfPreviouslySeenUri(path); err != nil {
return nil, err
}
return newGitLoader(path, l.fSys, l.referrer, l.cloner)
return newLoaderAtGitClone(path, l.fSys, l.referrer, l.cloner)
}
if filepath.IsAbs(path) {
return nil, fmt.Errorf("new root '%s' cannot be absolute", path)
}
return newFileLoaderAt(
return newLoaderAtConfirmedDir(
l.root.Join(path), l.fSys, l, l.cloner)
}
// newGitLoader returns a new Loader pinned to a temporary
// newLoaderAtGitClone returns a new Loader pinned to a temporary
// directory holding a cloned git repo.
func newGitLoader(
func newLoaderAtGitClone(
uri string, fSys fs.FileSystem,
referrer *fileLoader, cloner gitCloner) (ifc.Loader, error) {
referrer *fileLoader, cloner git.Cloner) (ifc.Loader, error) {
tmpDirForRepo, pathInRepo, err := cloner(uri)
if err != nil {
return nil, err

View File

@@ -25,6 +25,9 @@ import (
"strings"
"testing"
"sigs.k8s.io/kustomize/pkg/constants"
"sigs.k8s.io/kustomize/pkg/git"
"sigs.k8s.io/kustomize/pkg/fs"
"sigs.k8s.io/kustomize/pkg/ifc"
)
@@ -61,17 +64,17 @@ func MakeFakeFs(td []testData) fs.FileSystem {
return fSys
}
func TestNewFileLoaderAt_DemandsDirectory(t *testing.T) {
func TestNewLoaderAtConfirmedDir_DemandsDirectory(t *testing.T) {
fSys := MakeFakeFs(testCases)
_, err := newFileLoaderAt("/foo", fSys, nil, nil)
_, err := newLoaderAtConfirmedDir("/foo", fSys, nil, nil)
if err != nil {
t.Fatalf("Unexpected error - a directory should work.")
}
_, err = newFileLoaderAt("/foo/project", fSys, nil, nil)
_, err = newLoaderAtConfirmedDir("/foo/project", fSys, nil, nil)
if err != nil {
t.Fatalf("Unexpected error - a directory should work.")
}
_, err = newFileLoaderAt("/foo/project/fileA.yaml", fSys, nil, nil)
_, err = newLoaderAtConfirmedDir("/foo/project/fileA.yaml", fSys, nil, nil)
if err == nil {
t.Fatalf("Expected error - a file should not work.")
}
@@ -321,3 +324,97 @@ func TestRestrictedLoadingInRealLoader(t *testing.T) {
t.Fatalf("unexpected err: %v", err)
}
}
func splitOnNthSlash(v string, n int) (string, string) {
left := ""
for i := 0; i < n; i++ {
k := strings.Index(v, "/")
if k < 0 {
break
}
left = left + v[:k+1]
v = v[k+1:]
}
return left[:len(left)-1], v
}
func TestSplit(t *testing.T) {
path := "a/b/c/d/e/f/g"
if left, right := splitOnNthSlash(path, 2); left != "a/b" || right != "c/d/e/f/g" {
t.Fatalf("got left='%s', right='%s'", left, right)
}
if left, right := splitOnNthSlash(path, 3); left != "a/b/c" || right != "d/e/f/g" {
t.Fatalf("got left='%s', right='%s'", left, right)
}
if left, right := splitOnNthSlash(path, 6); left != "a/b/c/d/e/f" || right != "g" {
t.Fatalf("got left='%s', right='%s'", left, right)
}
}
// makeFakeGitCloner returns a cloner that ignores the
// URL argument and returns a path in a fake file system
// that should already hold the 'repo' contents.
func makeFakeGitCloner(t *testing.T, fSys fs.FileSystem, coRoot string) git.Cloner {
if !fSys.IsDir(coRoot) {
t.Fatalf("expecting a directory at '%s'", coRoot)
}
return func(url string) (
checkoutDir string, pathInCoDir string, err error) {
_, path := splitOnNthSlash(url, 3)
if !fSys.IsDir(coRoot + "/" + path) {
t.Fatalf("expecting a directory at '%s'/'%s'",
coRoot, path)
}
return coRoot, path, nil
}
}
func TestNewLoaderAtGitClone(t *testing.T) {
rootUrl := "github.com/someOrg/someRepo"
pathInRepo := "foo/base"
url := rootUrl + "/" + pathInRepo
if !git.IsRepoUrl(url) {
t.Fatalf("'%s' should be accepted as a repo url", url)
}
coRoot := "/tmp"
fSys := fs.MakeFakeFS()
fSys.MkdirAll(coRoot)
fSys.MkdirAll(coRoot + "/" + pathInRepo)
fSys.WriteFile(
coRoot+"/"+pathInRepo+"/"+constants.KustomizationFileNames[0],
[]byte(`
whatever
`))
l, err := newLoaderAtGitClone(
url, fSys, nil,
makeFakeGitCloner(t, fSys, coRoot))
if err != nil {
t.Fatalf("unexpected err: %v\n", err)
}
if coRoot+"/"+pathInRepo != l.Root() {
t.Fatalf("expected root '%s', got '%s'\n",
coRoot+"/"+pathInRepo, l.Root())
}
if _, err = l.New(url); err == nil {
t.Fatalf("expected cycle error 1")
}
if _, err = l.New(rootUrl + "/" + "foo"); err == nil {
t.Fatalf("expected cycle error 2")
}
pathInRepo = "foo/overlay"
fSys.MkdirAll(coRoot + "/" + pathInRepo)
url = rootUrl + "/" + pathInRepo
if !git.IsRepoUrl(url) {
t.Fatalf("'%s' should be accepted as a repo url", url)
}
l2, err := l.New(url)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if coRoot+"/"+pathInRepo != l2.Root() {
t.Fatalf("expected root '%s', got '%s'\n",
coRoot+"/"+pathInRepo, l2.Root())
}
}

View File

@@ -1,210 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package loader
import (
"bytes"
"io/ioutil"
"os/exec"
"path/filepath"
"regexp"
"strings"
"github.com/pkg/errors"
)
// gitCloner is a function that can clone a git repo.
type gitCloner func(url string) (
// Directory where the repo is cloned to.
checkoutDir string,
// Relative path in the checkoutDir to location
// of kustomization file.
pathInCoDir string,
// Any error encountered when cloning.
err error)
// isRepoUrl checks if a string is likely a github repo Url.
func isRepoUrl(arg string) bool {
arg = strings.ToLower(arg)
return !filepath.IsAbs(arg) &&
(strings.HasPrefix(arg, "git::") ||
strings.HasPrefix(arg, "gh:") ||
strings.HasPrefix(arg, "ssh:") ||
strings.HasPrefix(arg, "github.com") ||
strings.HasPrefix(arg, "git@") ||
strings.Index(arg, "github.com/") > -1 ||
isAzureHost(arg) || isAWSHost(arg))
}
func makeTmpDir() (string, error) {
return ioutil.TempDir("", "kustomize-")
}
func simpleGitCloner(spec string) (
checkoutDir string, pathInCoDir string, err error) {
gitProgram, err := exec.LookPath("git")
if err != nil {
return "", "", errors.Wrap(err, "no 'git' program on path")
}
checkoutDir, err = makeTmpDir()
if err != nil {
return
}
repo, pathInCoDir, gitRef, err := parseUrl(spec)
if err != nil {
return
}
cmd := exec.Command(
gitProgram,
"clone",
repo,
checkoutDir)
var out bytes.Buffer
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
return "", "",
errors.Wrapf(err, "trouble cloning %s", spec)
}
if gitRef == "" {
return
}
cmd = exec.Command(gitProgram, "checkout", gitRef)
cmd.Dir = checkoutDir
err = cmd.Run()
if err != nil {
return "", "",
errors.Wrapf(err, "trouble checking out href %s", gitRef)
}
return checkoutDir, pathInCoDir, nil
}
func parseUrl(n string) (
repo string, path string, gitRef string, err error) {
host, repo, path, gitRef, err := parseGithubUrl(n)
if err != nil {
return
}
if isAzureHost(host) || isAWSHost(host) {
repo = host + repo
return
}
repo = host + repo + ".git"
return
}
const (
refQuery = "?ref="
gitSuffix = ".git"
)
// From strings like git@github.com:someOrg/someRepo.git or
// https://github.com/someOrg/someRepo?ref=someHash, extract
// the parts.
func parseGithubUrl(n string) (
host string, repo string, path string, gitRef string, err error) {
host, n = parseHostSpec(n)
host = normalizeGitHostSpec(host)
if strings.HasSuffix(n, gitSuffix) {
repo = n[0 : len(n)-len(gitSuffix)]
return
}
if strings.Contains(n, gitSuffix) {
index := strings.Index(n, gitSuffix)
repo = n[0:index]
n = n[index+len(gitSuffix):]
path, gitRef = peelQuery(n)
return
}
i := strings.Index(n, "/")
if i < 1 {
return "", "", "", "", errors.New("no separator")
}
j := strings.Index(n[i+1:], "/")
if j >= 0 {
j += i + 1
repo = n[:j]
path, gitRef = peelQuery(n[j+1:])
} else {
path = ""
repo, gitRef = peelQuery(n)
}
return
}
func peelQuery(arg string) (string, string) {
j := strings.Index(arg, refQuery)
if j >= 0 {
return arg[:j], arg[j+len(refQuery):]
}
return arg, ""
}
func parseHostSpec(n string) (string, string) {
var host string
for _, p := range []string{
// Order matters here.
"git::", "gh:", "ssh://", "https://", "http://",
"git@", "github.com:", "github.com/"} {
if strings.ToLower(n[:len(p)]) == p {
n = n[len(p):]
host = host + p
}
}
// If host is a http(s) or ssh URL, grab the domain part.
for _, p := range []string{
"ssh://", "https://", "http://"} {
if strings.HasSuffix(strings.ToLower(host), p) {
index := regexp.MustCompile("^(.*?)/").FindStringIndex(n)
if len(index) > 0 {
host = host + n[0:index[len(index)-1]]
n = n[index[len(index)-1]:]
}
}
}
return host, n
}
func normalizeGitHostSpec(host string) string {
s := strings.ToLower(host)
if strings.Contains(s, "github.com") {
if strings.Contains(s, "git@") || strings.Contains(s, "ssh:") {
host = "git@github.com:"
} else {
host = "https://github.com/"
}
}
if strings.HasPrefix(s, "git::") {
host = strings.TrimLeft(s, "git::")
}
return host
}
// The format of Azure repo URL is documented
// https://docs.microsoft.com/en-us/azure/devops/repos/git/clone?view=vsts&tabs=visual-studio#clone_url
func isAzureHost(host string) bool {
return strings.Contains(host, "dev.azure.com") ||
strings.Contains(host, "visualstudio.com")
}
// The format of AWS repo URL is documented
// https://docs.aws.amazon.com/codecommit/latest/userguide/regions.html
func isAWSHost(host string) bool {
return strings.Contains(host, "amazonaws.com")
}

View File

@@ -1,391 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package loader
import (
"fmt"
"path/filepath"
"strings"
"testing"
"sigs.k8s.io/kustomize/pkg/constants"
"sigs.k8s.io/kustomize/pkg/fs"
)
func TestIsRepoURL(t *testing.T) {
testcases := []struct {
input string
expected bool
}{
{
input: "https://github.com/org/repo",
expected: true,
},
{
input: "github.com/org/repo",
expected: true,
},
{
input: "git@github.com:org/repo",
expected: true,
},
{
input: "gh:org/repo",
expected: true,
},
{
input: "git::https://gitlab.com/org/repo",
expected: true,
},
{
input: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?ref=v0.1.0",
expected: true,
},
{
input: "git@bitbucket.org:org/repo.git",
expected: true,
},
{
input: "git::http://git.example.com/org/repo.git",
expected: true,
},
{
input: "git::https://git.example.com/org/repo.git",
expected: true,
},
{
input: "ssh://git.example.com:7999/org/repo.git",
expected: true,
},
{
input: "/github.com/org/repo",
expected: false,
},
{
input: "/abs/path/to/file",
expected: false,
},
{
input: "../relative",
expected: false,
},
{
input: "foo",
expected: false,
},
{
input: ".",
expected: false,
},
{
input: "",
expected: false,
},
}
for _, tc := range testcases {
actual := isRepoUrl(tc.input)
if actual != tc.expected {
t.Errorf("unexpected error: unexpected result %t for input %s", actual, tc.input)
}
}
}
func splitOnNthSlash(v string, n int) (string, string) {
left := ""
for i := 0; i < n; i++ {
k := strings.Index(v, "/")
if k < 0 {
break
}
left = left + v[:k+1]
v = v[k+1:]
}
return left[:len(left)-1], v
}
func TestSplit(t *testing.T) {
path := "a/b/c/d/e/f/g"
if left, right := splitOnNthSlash(path, 2); left != "a/b" || right != "c/d/e/f/g" {
t.Fatalf("got left='%s', right='%s'", left, right)
}
if left, right := splitOnNthSlash(path, 3); left != "a/b/c" || right != "d/e/f/g" {
t.Fatalf("got left='%s', right='%s'", left, right)
}
if left, right := splitOnNthSlash(path, 6); left != "a/b/c/d/e/f" || right != "g" {
t.Fatalf("got left='%s', right='%s'", left, right)
}
}
// makeFakeGitCloner returns a cloner that ignores the
// URL argument and returns a path in a fake file system
// that should already hold the 'repo' contents.
func makeFakeGitCloner(t *testing.T, fSys fs.FileSystem, coRoot string) gitCloner {
if !fSys.IsDir(coRoot) {
t.Fatalf("expecting a directory at '%s'", coRoot)
}
return func(url string) (
checkoutDir string, pathInCoDir string, err error) {
_, path := splitOnNthSlash(url, 3)
if !fSys.IsDir(coRoot + "/" + path) {
t.Fatalf("expecting a directory at '%s'/'%s'",
coRoot, path)
}
return coRoot, path, nil
}
}
func TestGitLoader(t *testing.T) {
rootUrl := "github.com/someOrg/someRepo"
pathInRepo := "foo/base"
url := rootUrl + "/" + pathInRepo
if !isRepoUrl(url) {
t.Fatalf("'%s' should be accepted as a repo url", url)
}
coRoot := "/tmp"
fSys := fs.MakeFakeFS()
fSys.MkdirAll(coRoot)
fSys.MkdirAll(coRoot + "/" + pathInRepo)
fSys.WriteFile(
coRoot+"/"+pathInRepo+"/"+constants.KustomizationFileNames[0],
[]byte(`
whatever
`))
l, err := newGitLoader(
url, fSys, nil,
makeFakeGitCloner(t, fSys, coRoot))
if err != nil {
t.Fatalf("unexpected err: %v\n", err)
}
if coRoot+"/"+pathInRepo != l.Root() {
t.Fatalf("expected root '%s', got '%s'\n",
coRoot+"/"+pathInRepo, l.Root())
}
if _, err = l.New(url); err == nil {
t.Fatalf("expected cycle error 1")
}
if _, err = l.New(rootUrl + "/" + "foo"); err == nil {
t.Fatalf("expected cycle error 2")
}
pathInRepo = "foo/overlay"
fSys.MkdirAll(coRoot + "/" + pathInRepo)
url = rootUrl + "/" + pathInRepo
if !isRepoUrl(url) {
t.Fatalf("'%s' should be accepted as a repo url", url)
}
l2, err := l.New(url)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if coRoot+"/"+pathInRepo != l2.Root() {
t.Fatalf("expected root '%s', got '%s'\n",
coRoot+"/"+pathInRepo, l2.Root())
}
}
var repoNames = []string{"someOrg/someRepo", "kubernetes/website"}
var paths = []string{"README.md", "foo/krusty.txt", ""}
var hrefArgs = []string{"someBranch", ""}
var extractFmts = map[string]string{
"gh:%s": "gh:",
"GH:%s": "gh:",
"gitHub.com/%s": "https://github.com/",
"https://github.com/%s": "https://github.com/",
"hTTps://github.com/%s": "https://github.com/",
"git::https://gitlab.com/%s": "https://gitlab.com/",
"github.com:%s": "https://github.com/",
"git::http://git.example.com/%s": "http://git.example.com/",
"git::https://git.example.com/%s": "https://git.example.com/",
"ssh://git.example.com:7999/%s": "ssh://git.example.com:7999/",
}
func TestParseGithubUrl(t *testing.T) {
for _, repoName := range repoNames {
for _, pathName := range paths {
for extractFmt, hostSpec := range extractFmts {
for _, hrefArg := range hrefArgs {
spec := repoName
if len(pathName) > 0 {
spec = filepath.Join(spec, pathName)
}
input := fmt.Sprintf(extractFmt, spec)
if hrefArg != "" {
input = input + refQuery + hrefArg
}
if !isRepoUrl(input) {
t.Errorf("Should smell like github arg: %s\n", input)
continue
}
host, repo, path, gitRef, err := parseGithubUrl(input)
if err != nil {
t.Errorf("problem %v", err)
}
if host != hostSpec {
t.Errorf("\n"+
" from %s\n"+
" actual host %s\n"+
"expected host %s\n", input, host, hostSpec)
}
if repo != repoName {
t.Errorf("\n"+
" from %s\n"+
" actual Repo %s\n"+
"expected Repo %s\n", input, repo, repoName)
}
if path != pathName {
t.Errorf("\n"+
" from %s\n"+
" actual Path %s\n"+
"expected Path %s\n", input, path, pathName)
}
if gitRef != hrefArg {
t.Errorf("\n"+
" from %s\n"+
" actual Href %s\n"+
"expected Href %s\n", input, gitRef, hrefArg)
}
}
}
}
}
}
func TestParseUrl(t *testing.T) {
testcases := []struct {
input string
repo string
path string
ref string
}{
{
input: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir",
repo: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo",
path: "somedir",
ref: "",
},
{
input: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir?ref=testbranch",
repo: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo",
path: "somedir",
ref: "testbranch",
},
{
input: "https://fabrikops2.visualstudio.com/someorg/somerepo?ref=master",
repo: "https://fabrikops2.visualstudio.com/someorg/somerepo",
path: "",
ref: "master",
},
{
input: "http://github.com/someorg/somerepo/somedir",
repo: "https://github.com/someorg/somerepo.git",
path: "somedir",
ref: "",
},
{
input: "git@github.com:someorg/somerepo/somedir",
repo: "git@github.com:someorg/somerepo.git",
path: "somedir",
ref: "",
},
{
input: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?ref=v0.1.0",
repo: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git",
path: "",
ref: "v0.1.0",
},
}
for _, testcase := range testcases {
repo, path, ref, err := parseUrl(testcase.input)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
if repo != testcase.repo {
t.Errorf("repo expected to be %v, but got %v on %s", testcase.repo, repo, testcase.input)
}
if path != testcase.path {
t.Errorf("path expected to be %v, but got %v on %s", testcase.path, path, testcase.input)
}
if ref != testcase.ref {
t.Errorf("ref expected to be %v, but got %v on %s", testcase.ref, ref, testcase.input)
}
}
}
func TestIsAzureHost(t *testing.T) {
testcases := []struct {
input string
expect bool
}{
{
input: "https://git-codecommit.us-east-2.amazonaws.com",
expect: false,
},
{
input: "ssh://git-codecommit.us-east-2.amazonaws.com",
expect: false,
},
{
input: "https://fabrikops2.visualstudio.com/",
expect: true,
},
{
input: "https://dev.azure.com/myorg/myproject/",
expect: true,
},
}
for _, testcase := range testcases {
actual := isAzureHost(testcase.input)
if actual != testcase.expect {
t.Errorf("IsAzureHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input)
}
}
}
func TestIsAWSHost(t *testing.T) {
testcases := []struct {
input string
expect bool
}{
{
input: "https://git-codecommit.us-east-2.amazonaws.com",
expect: true,
},
{
input: "ssh://git-codecommit.us-east-2.amazonaws.com",
expect: true,
},
{
input: "git@github.com:",
expect: false,
},
{
input: "http://github.com/",
expect: false,
},
}
for _, testcase := range testcases {
actual := isAWSHost(testcase.input)
if actual != testcase.expect {
t.Errorf("IsAWSHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input)
}
}
}

View File

@@ -19,15 +19,16 @@ package loader
import (
"sigs.k8s.io/kustomize/pkg/fs"
"sigs.k8s.io/kustomize/pkg/git"
"sigs.k8s.io/kustomize/pkg/ifc"
)
// NewLoader returns a Loader.
func NewLoader(root string, fSys fs.FileSystem) (ifc.Loader, error) {
if isRepoUrl(root) {
return newGitLoader(
root, fSys, nil, simpleGitCloner)
if git.IsRepoUrl(root) {
return newLoaderAtGitClone(
root, fSys, nil, git.ClonerUsingGitExec)
}
return newFileLoaderAt(
root, fSys, nil, simpleGitCloner)
return newLoaderAtConfirmedDir(
root, fSys, nil, git.ClonerUsingGitExec)
}