Merge pull request #1793 from haiyanmeng/fix-git

Fix two broken deps to allow tests under hack/crawl to pass

hack/crawl/crawler/crawler_test.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	"sigs.k8s.io/kustomize/api/pgmconfig"
+	"sigs.k8s.io/kustomize/api/konfig"
 	"sigs.k8s.io/kustomize/hack/crawl/doc"
 )
 
@@ -37,7 +37,7 @@ func (c testCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
-	for _, suffix := range pgmconfig.RecognizedKustomizationFileNames() {
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
 		fmt.Println(d.ID(), "/", suffix)
 		i, ok := c.lukp[d.ID()+"/"+suffix]
 		if !ok {
@@ -76,7 +76,7 @@ func (c testCrawler) Crawl(ctx context.Context,
 
 	for i, d := range c.docs {
 		isResource := true
-		for _, suffix := range pgmconfig.RecognizedKustomizationFileNames() {
+		for _, suffix := range konfig.RecognizedKustomizationFileNames() {
 			if strings.HasSuffix(d.FilePath, suffix) {
 				isResource = false
 				break

hack/crawl/crawler/git/cloner.go

@@ -0,0 +1,121 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package git
+
+import (
+	"bytes"
+	"log"
+	"os/exec"
+
+	"github.com/pkg/errors"
+	"sigs.k8s.io/kustomize/api/filesys"
+)
+
+// Cloner is a function that can clone a git repo.
+type Cloner func(repoSpec *RepoSpec) error
+
+// ClonerUsingGitExec uses a local git install, as opposed
+// to say, some remote API, to obtain a local clone of
+// a remote repo.
+func ClonerUsingGitExec(repoSpec *RepoSpec) error {
+	gitProgram, err := exec.LookPath("git")
+	if err != nil {
+		return errors.Wrap(err, "no 'git' program on path")
+	}
+	repoSpec.Dir, err = filesys.NewTmpConfirmedDir()
+	if err != nil {
+		return err
+	}
+	cmd := exec.Command(
+		gitProgram,
+		"init",
+		repoSpec.Dir.String())
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	err = cmd.Run()
+	if err != nil {
+		log.Printf("Error initializing empty git repo: %s", out.String())
+		return errors.Wrapf(
+			err,
+			"trouble initializing empty git repo in %s",
+			repoSpec.Dir.String())
+	}
+
+	cmd = exec.Command(
+		gitProgram,
+		"remote",
+		"add",
+		"origin",
+		repoSpec.CloneSpec())
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	cmd.Dir = repoSpec.Dir.String()
+	err = cmd.Run()
+	if err != nil {
+		log.Printf("Error setting git remote: %s", out.String())
+		return errors.Wrapf(
+			err,
+			"trouble adding remote %s",
+			repoSpec.CloneSpec())
+	}
+	if repoSpec.Ref == "" {
+		repoSpec.Ref = "master"
+	}
+	cmd = exec.Command(
+		gitProgram,
+		"fetch",
+		"--depth=1",
+		"origin",
+		repoSpec.Ref)
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	cmd.Dir = repoSpec.Dir.String()
+	err = cmd.Run()
+	if err != nil {
+		log.Printf("Error performing git fetch: %s", out.String())
+		return errors.Wrapf(err, "trouble fetching %s", repoSpec.Ref)
+	}
+
+	cmd = exec.Command(
+		gitProgram,
+		"reset",
+		"--hard",
+		"FETCH_HEAD")
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	cmd.Dir = repoSpec.Dir.String()
+	err = cmd.Run()
+	if err != nil {
+		log.Printf("Error performing git reset: %s", out.String())
+		return errors.Wrapf(
+			err, "trouble hard resetting empty repository to %s", repoSpec.Ref)
+	}
+
+	cmd = exec.Command(
+		gitProgram,
+		"submodule",
+		"update",
+		"--init",
+		"--recursive")
+	cmd.Stdout = &out
+	cmd.Dir = repoSpec.Dir.String()
+	err = cmd.Run()
+	if err != nil {
+		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.Ref)
+	}
+
+	return nil
+}
+
+// DoNothingCloner returns a cloner that only sets
+// cloneDir field in the repoSpec.  It's assumed that
+// the cloneDir is associated with some fake filesystem
+// used in a test.
+func DoNothingCloner(dir filesys.ConfirmedDir) Cloner {
+	return func(rs *RepoSpec) error {
+		rs.Dir = dir
+		return nil
+	}
+}

hack/crawl/crawler/git/repospec.go

@@ -0,0 +1,219 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package git
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/filesys"
+)
+
+// Used as a temporary non-empty occupant of the cloneDir
+// field, as something distinguishable from the empty string
+// in various outputs (especially tests). Not using an
+// actual directory name here, as that's a temporary directory
+// with a unique name that isn't created until clone time.
+const notCloned = filesys.ConfirmedDir("/notCloned")
+
+// RepoSpec specifies a git repository and a branch and path therein.
+type RepoSpec struct {
+	// Raw, original spec, used to look for cycles.
+	// TODO(monopole): Drop raw, use processed fields instead.
+	raw string
+
+	// Host, e.g. github.com
+	Host string
+
+	// orgRepo name (organization/repoName),
+	// e.g. kubernetes-sigs/kustomize
+	OrgRepo string
+
+	// Dir where the orgRepo is cloned to.
+	Dir filesys.ConfirmedDir
+
+	// Relative path in the repository, and in the cloneDir,
+	// to a Kustomization.
+	Path string
+
+	// Branch or tag reference.
+	Ref string
+
+	// e.g. .git or empty in case of _git is present
+	GitSuffix string
+}
+
+// CloneSpec returns a string suitable for "git clone {spec}".
+func (x *RepoSpec) CloneSpec() string {
+	if isAzureHost(x.Host) || isAWSHost(x.Host) {
+		return x.Host + x.OrgRepo
+	}
+	return x.Host + x.OrgRepo + x.GitSuffix
+}
+
+func (x *RepoSpec) CloneDir() filesys.ConfirmedDir {
+	return x.Dir
+}
+
+func (x *RepoSpec) Raw() string {
+	return x.raw
+}
+
+func (x *RepoSpec) AbsPath() string {
+	return x.Dir.Join(x.Path)
+}
+
+func (x *RepoSpec) Cleaner(fSys filesys.FileSystem) func() error {
+	return func() error { return fSys.RemoveAll(x.Dir.String()) }
+}
+
+// From strings like git@github.com:someOrg/someRepo.git or
+// https://github.com/someOrg/someRepo?ref=someHash, extract
+// the parts.
+func NewRepoSpecFromUrl(n string) (*RepoSpec, error) {
+	if filepath.IsAbs(n) {
+		return nil, fmt.Errorf("uri looks like abs path: %s", n)
+	}
+	host, orgRepo, path, gitRef, gitSuffix := parseGitUrl(n)
+	if orgRepo == "" {
+		return nil, fmt.Errorf("url lacks orgRepo: %s", n)
+	}
+	if host == "" {
+		return nil, fmt.Errorf("url lacks host: %s", n)
+	}
+	return &RepoSpec{
+		raw: n, Host: host, OrgRepo: orgRepo,
+		Dir: notCloned, Path: path, Ref: gitRef, GitSuffix: gitSuffix}, nil
+}
+
+const (
+	refQuery      = "?ref="
+	refQueryRegex = "\\?(version|ref)="
+	gitSuffix     = ".git"
+	gitDelimiter  = "_git/"
+)
+
+// From strings like git@github.com:someOrg/someRepo.git or
+// https://github.com/someOrg/someRepo?ref=someHash, extract
+// the parts.
+func parseGitUrl(n string) (
+	host string, orgRepo string, path string, gitRef string, gitSuff string) {
+
+	if strings.Contains(n, gitDelimiter) {
+		index := strings.Index(n, gitDelimiter)
+		// Adding _git/ to host
+		host = normalizeGitHostSpec(n[:index+len(gitDelimiter)])
+		orgRepo = strings.Split(strings.Split(n[index+len(gitDelimiter):], "/")[0], "?")[0]
+		path, gitRef = peelQuery(n[index+len(gitDelimiter)+len(orgRepo):])
+		return
+	}
+	host, n = parseHostSpec(n)
+	gitSuff = gitSuffix
+	if strings.Contains(n, gitSuffix) {
+		index := strings.Index(n, gitSuffix)
+		orgRepo = n[0:index]
+		n = n[index+len(gitSuffix):]
+		path, gitRef = peelQuery(n)
+		return
+	}
+
+	i := strings.Index(n, "/")
+	if i < 1 {
+		return "", "", "", "", ""
+	}
+	j := strings.Index(n[i+1:], "/")
+	if j >= 0 {
+		j += i + 1
+		orgRepo = n[:j]
+		path, gitRef = peelQuery(n[j+1:])
+		return
+	}
+	path = ""
+	orgRepo, gitRef = peelQuery(n)
+	return host, orgRepo, path, gitRef, gitSuff
+}
+
+func peelQuery(arg string) (string, string) {
+
+	r, _ := regexp.Compile(refQueryRegex)
+	j := r.FindStringIndex(arg)
+
+	if len(j) > 0 {
+		return arg[:j[0]], arg[j[0]+len(r.FindString(arg)):]
+	}
+	return arg, ""
+}
+
+func parseHostSpec(n string) (string, string) {
+	var host string
+	// Start accumulating the host part.
+	for _, p := range []string{
+		// Order matters here.
+		"git::", "gh:", "ssh://", "https://", "http://",
+		"git@", "github.com:", "github.com/"} {
+		if len(p) < len(n) && strings.ToLower(n[:len(p)]) == p {
+			n = n[len(p):]
+			host += p
+		}
+	}
+	if host == "git@" {
+		i := strings.Index(n, "/")
+		if i > -1 {
+			host += n[:i+1]
+			n = n[i+1:]
+		} else {
+			i = strings.Index(n, ":")
+			if i > -1 {
+				host += n[:i+1]
+				n = n[i+1:]
+			}
+		}
+		return host, n
+	}
+
+	// If host is a http(s) or ssh URL, grab the domain part.
+	for _, p := range []string{
+		"ssh://", "https://", "http://"} {
+		if strings.HasSuffix(host, p) {
+			i := strings.Index(n, "/")
+			if i > -1 {
+				host = host + n[0:i+1]
+				n = n[i+1:]
+			}
+			break
+		}
+	}
+
+	return normalizeGitHostSpec(host), n
+}
+
+func normalizeGitHostSpec(host string) string {
+	s := strings.ToLower(host)
+	if strings.Contains(s, "github.com") {
+		if strings.Contains(s, "git@") || strings.Contains(s, "ssh:") {
+			host = "git@github.com:"
+		} else {
+			host = "https://github.com/"
+		}
+	}
+	if strings.HasPrefix(s, "git::") {
+		host = strings.TrimPrefix(s, "git::")
+	}
+	return host
+}
+
+// The format of Azure repo URL is documented
+// https://docs.microsoft.com/en-us/azure/devops/repos/git/clone?view=vsts&tabs=visual-studio#clone_url
+func isAzureHost(host string) bool {
+	return strings.Contains(host, "dev.azure.com") ||
+		strings.Contains(host, "visualstudio.com")
+}
+
+// The format of AWS repo URL is documented
+// https://docs.aws.amazon.com/codecommit/latest/userguide/regions.html
+func isAWSHost(host string) bool {
+	return strings.Contains(host, "amazonaws.com")
+}

hack/crawl/crawler/git/repospec_test.go

@@ -0,0 +1,288 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package git
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+var orgRepos = []string{"someOrg/someRepo", "kubernetes/website"}
+
+var pathNames = []string{"README.md", "foo/krusty.txt", ""}
+
+var hrefArgs = []string{"someBranch", "master", "v0.1.0", ""}
+
+var hostNamesRawAndNormalized = [][]string{
+	{"gh:", "gh:"},
+	{"GH:", "gh:"},
+	{"gitHub.com/", "https://github.com/"},
+	{"github.com:", "https://github.com/"},
+	{"http://github.com/", "https://github.com/"},
+	{"https://github.com/", "https://github.com/"},
+	{"hTTps://github.com/", "https://github.com/"},
+	{"https://git-codecommit.us-east-2.amazonaws.com/", "https://git-codecommit.us-east-2.amazonaws.com/"},
+	{"https://fabrikops2.visualstudio.com/", "https://fabrikops2.visualstudio.com/"},
+	{"ssh://git.example.com:7999/", "ssh://git.example.com:7999/"},
+	{"git::https://gitlab.com/", "https://gitlab.com/"},
+	{"git::http://git.example.com/", "http://git.example.com/"},
+	{"git::https://git.example.com/", "https://git.example.com/"},
+	{"git@github.com:", "git@github.com:"},
+	{"git@github.com/", "git@github.com:"},
+	{"git@gitlab2.sqtools.ru:10022/", "git@gitlab2.sqtools.ru:10022/"},
+}
+
+func makeUrl(hostFmt, orgRepo, path, href string) string {
+	if len(path) > 0 {
+		orgRepo = filepath.Join(orgRepo, path)
+	}
+	url := hostFmt + orgRepo
+	if href != "" {
+		url += refQuery + href
+	}
+	return url
+}
+
+func TestNewRepoSpecFromUrl(t *testing.T) {
+	var bad [][]string
+	for _, tuple := range hostNamesRawAndNormalized {
+		hostRaw := tuple[0]
+		hostSpec := tuple[1]
+		for _, orgRepo := range orgRepos {
+			for _, pathName := range pathNames {
+				for _, hrefArg := range hrefArgs {
+					uri := makeUrl(hostRaw, orgRepo, pathName, hrefArg)
+					rs, err := NewRepoSpecFromUrl(uri)
+					if err != nil {
+						t.Errorf("problem %v", err)
+					}
+					if rs.Host != hostSpec {
+						bad = append(bad, []string{"host", uri, rs.Host, hostSpec})
+					}
+					if rs.OrgRepo != orgRepo {
+						bad = append(bad, []string{"orgRepo", uri, rs.OrgRepo, orgRepo})
+					}
+					if rs.Path != pathName {
+						bad = append(bad, []string{"path", uri, rs.Path, pathName})
+					}
+					if rs.Ref != hrefArg {
+						bad = append(bad, []string{"ref", uri, rs.Ref, hrefArg})
+					}
+				}
+			}
+		}
+	}
+	if len(bad) > 0 {
+		for _, tuple := range bad {
+			fmt.Printf("\n"+
+				"     from uri: %s\n"+
+				"  actual %4s: %s\n"+
+				"expected %4s: %s\n",
+				tuple[1], tuple[0], tuple[2], tuple[0], tuple[3])
+		}
+		t.Fail()
+	}
+}
+
+var badData = [][]string{
+	{"/tmp", "uri looks like abs path"},
+	{"iauhsdiuashduas", "url lacks orgRepo"},
+	{"htxxxtp://github.com/", "url lacks host"},
+	{"ssh://git.example.com", "url lacks orgRepo"},
+	{"git::___", "url lacks orgRepo"},
+}
+
+func TestNewRepoSpecFromUrlErrors(t *testing.T) {
+	for _, tuple := range badData {
+		_, err := NewRepoSpecFromUrl(tuple[0])
+		if err == nil {
+			t.Error("expected error")
+		}
+		if !strings.Contains(err.Error(), tuple[1]) {
+			t.Errorf("unexpected error: %s", err)
+		}
+	}
+}
+
+func TestNewRepoSpecFromUrl_CloneSpecs(t *testing.T) {
+	testcases := []struct {
+		input     string
+		cloneSpec string
+		absPath   string
+		ref       string
+	}{
+		{
+			input:     "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir",
+			cloneSpec: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo",
+			absPath:   notCloned.Join("somedir"),
+			ref:       "",
+		},
+		{
+			input:     "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir?ref=testbranch",
+			cloneSpec: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo",
+			absPath:   notCloned.Join("somedir"),
+			ref:       "testbranch",
+		},
+		{
+			input:     "https://fabrikops2.visualstudio.com/someorg/somerepo?ref=master",
+			cloneSpec: "https://fabrikops2.visualstudio.com/someorg/somerepo",
+			absPath:   notCloned.String(),
+			ref:       "master",
+		},
+		{
+			input:     "http://github.com/someorg/somerepo/somedir",
+			cloneSpec: "https://github.com/someorg/somerepo.git",
+			absPath:   notCloned.Join("somedir"),
+			ref:       "",
+		},
+		{
+			input:     "git@github.com:someorg/somerepo/somedir",
+			cloneSpec: "git@github.com:someorg/somerepo.git",
+			absPath:   notCloned.Join("somedir"),
+			ref:       "",
+		},
+		{
+			input:     "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?ref=v0.1.0",
+			cloneSpec: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git",
+			absPath:   notCloned.String(),
+			ref:       "v0.1.0",
+		},
+		{
+			input:     "git@bitbucket.org:company/project.git//path?ref=branch",
+			cloneSpec: "git@bitbucket.org:company/project.git",
+			absPath:   notCloned.Join("path"),
+			ref:       "branch",
+		},
+		{
+			input:     "https://itfs.mycompany.com/collection/project/_git/somerepos",
+			cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos",
+			absPath:   notCloned.String(),
+			ref:       "",
+		},
+		{
+			input:     "https://itfs.mycompany.com/collection/project/_git/somerepos?version=v1.0.0",
+			cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos",
+			absPath:   notCloned.String(),
+			ref:       "v1.0.0",
+		},
+		{
+			input:     "https://itfs.mycompany.com/collection/project/_git/somerepos/somedir?version=v1.0.0",
+			cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos",
+			absPath:   notCloned.Join("somedir"),
+			ref:       "v1.0.0",
+		},
+		{
+			input:     "git::https://itfs.mycompany.com/collection/project/_git/somerepos",
+			cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos",
+			absPath:   notCloned.String(),
+			ref:       "",
+		},
+	}
+	for _, testcase := range testcases {
+		rs, err := NewRepoSpecFromUrl(testcase.input)
+		if err != nil {
+			t.Errorf("Unexpected error: %v", err)
+		}
+		if rs.CloneSpec() != testcase.cloneSpec {
+			t.Errorf("CloneSpec expected to be %v, but got %v on %s",
+				testcase.cloneSpec, rs.CloneSpec(), testcase.input)
+		}
+		if rs.AbsPath() != testcase.absPath {
+			t.Errorf("AbsPath expected to be %v, but got %v on %s",
+				testcase.absPath, rs.AbsPath(), testcase.input)
+		}
+		if rs.Ref != testcase.ref {
+			t.Errorf("ref expected to be %v, but got %v on %s",
+				testcase.ref, rs.Ref, testcase.input)
+		}
+	}
+}
+
+func TestIsAzureHost(t *testing.T) {
+	testcases := []struct {
+		input  string
+		expect bool
+	}{
+		{
+			input:  "https://git-codecommit.us-east-2.amazonaws.com",
+			expect: false,
+		},
+		{
+			input:  "ssh://git-codecommit.us-east-2.amazonaws.com",
+			expect: false,
+		},
+		{
+			input:  "https://fabrikops2.visualstudio.com/",
+			expect: true,
+		},
+		{
+			input:  "https://dev.azure.com/myorg/myproject/",
+			expect: true,
+		},
+	}
+	for _, testcase := range testcases {
+		actual := isAzureHost(testcase.input)
+		if actual != testcase.expect {
+			t.Errorf("IsAzureHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input)
+		}
+	}
+}
+
+func TestPeelQuery(t *testing.T) {
+	testcases := []struct {
+		input  string
+		expect [2]string
+	}{
+		{
+			input:  "somerepos?ref=v1.0.0",
+			expect: [2]string{"somerepos", "v1.0.0"},
+		},
+		{
+			input:  "somerepos?version=master",
+			expect: [2]string{"somerepos", "master"},
+		},
+		{
+			input:  "somerepos",
+			expect: [2]string{"somerepos", ""},
+		},
+	}
+	for _, testcase := range testcases {
+		path, ref := peelQuery(testcase.input)
+		if path != testcase.expect[0] || ref != testcase.expect[1] {
+			t.Errorf("peelQuery: expected (%s, %s) got (%s, %s) on %s", testcase.expect[0], testcase.expect[1], path, ref, testcase.input)
+		}
+	}
+}
+
+func TestIsAWSHost(t *testing.T) {
+	testcases := []struct {
+		input  string
+		expect bool
+	}{
+		{
+			input:  "https://git-codecommit.us-east-2.amazonaws.com",
+			expect: true,
+		},
+		{
+			input:  "ssh://git-codecommit.us-east-2.amazonaws.com",
+			expect: true,
+		},
+		{
+			input:  "git@github.com:",
+			expect: false,
+		},
+		{
+			input:  "http://github.com/",
+			expect: false,
+		},
+	}
+	for _, testcase := range testcases {
+		actual := isAWSHost(testcase.input)
+		if actual != testcase.expect {
+			t.Errorf("IsAWSHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input)
+		}
+	}
+}

hack/crawl/crawler/github/crawler.go

@@ -16,9 +16,9 @@ import (
 	"strings"
 	"time"
 
-	"sigs.k8s.io/kustomize/api/git"
+	"sigs.k8s.io/kustomize/api/konfig"
-	"sigs.k8s.io/kustomize/api/pgmconfig"
 	"sigs.k8s.io/kustomize/hack/crawl/crawler"
+	"sigs.k8s.io/kustomize/hack/crawl/crawler/git"
 	"sigs.k8s.io/kustomize/hack/crawl/doc"
 	"sigs.k8s.io/kustomize/hack/crawl/httpclient"
 )
@@ -122,7 +122,7 @@ func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) erro
 		return nil
 	}
 
-	for _, file := range pgmconfig.RecognizedKustomizationFileNames() {
+	for _, file := range konfig.RecognizedKustomizationFileNames() {
 		resp, err = gc.client.GetRawUserContent(url + "/" + file)
 		err := handle(resp, err, "/"+file)
 		if err != nil {

hack/crawl/doc/doc.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 
 	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
-	"sigs.k8s.io/kustomize/api/pgmconfig"
+	"sigs.k8s.io/kustomize/api/konfig"
 	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
@@ -46,7 +46,7 @@ type set map[string]struct{}
 // Implements the CrawlerDocument interface.
 func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 	isResource := true
-	for _, suffix := range pgmconfig.RecognizedKustomizationFileNames() {
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
 		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
 			isResource = false
 		}
@@ -85,7 +85,7 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
 	data := []byte(doc.DocumentData)
 
-	for _, suffix := range pgmconfig.RecognizedKustomizationFileNames() {
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
 		if !strings.HasSuffix(doc.FilePath, "/"+suffix) {
 			continue
 		}

hack/crawl/doc/docname.go

@@ -4,7 +4,7 @@ import (
 	"path"
 	"time"
 
-	"sigs.k8s.io/kustomize/api/git"
+	"sigs.k8s.io/kustomize/hack/crawl/crawler/git"
 )
 
 type Document struct {

hack/crawl/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/gorilla/mux v1.7.3
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
+	github.com/pkg/errors v0.8.1
 	github.com/rs/cors v1.7.0
 	sigs.k8s.io/kustomize/api v0.2.0
 	sigs.k8s.io/yaml v1.1.0

hack/crawl/go.sum

@@ -41,16 +41,13 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/elastic/go-elasticsearch/v6 v6.8.2 h1:rp5DGrd63V5c6nHLjF6QEXUpZSvs0+QM3ld7m9VhV2g=
 github.com/elastic/go-elasticsearch/v6 v6.8.2/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
-github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -92,11 +89,9 @@ github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -124,7 +119,6 @@ github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNu
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
@@ -150,7 +144,6 @@ github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:Fecb
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -159,7 +152,6 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -307,11 +299,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
 golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -330,9 +320,8 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
 golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
-golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -371,7 +360,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
@@ -384,29 +372,20 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-k8s.io/api v0.0.0-20191016225839-816a9b7df678 h1:z/0BV/tMBIvdwZvqBH/f7TWjQX9y3dj1nMNhrSK0h/8=
-k8s.io/api v0.0.0-20191016225839-816a9b7df678/go.mod h1:LZQaT8MvVpl7Bg2lYFcQm7+Mpdxq8p1NFl3yh+5DCwY=
-k8s.io/apimachinery v0.0.0-20191016225534-b1267f8c42b4/go.mod h1:92mWDd8Ji2sw2157KIgino5wCxffA8KSvhW2oY4ypdw=
-k8s.io/apimachinery v0.0.0-20191020214737-6c8691705fc5 h1:r3/YL3+t1U46lJF5zUSArskUpnLyWuM28rQDpM1qQPI=
-k8s.io/apimachinery v0.0.0-20191020214737-6c8691705fc5/go.mod h1:92mWDd8Ji2sw2157KIgino5wCxffA8KSvhW2oY4ypdw=
-k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
-k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/kube-openapi v0.0.0-20190918143330-0270cf2f1c1d/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.1.1 h1:W2dWXex2MhF4/EZNokZllvet2RejCHqdAFklufN7VTg=
+sigs.k8s.io/kustomize/api v0.2.0 h1:e++6JpysnnlUbHmFrv6jvfF5rFlgQ103bS1DO7r5bWA=
-sigs.k8s.io/kustomize/api v0.1.1/go.mod h1:FyfJD1q1QMjC/TvK78b6cCtZB+mbpnGIo9YOvbucJes=
 sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4=
+sigs.k8s.io/kustomize/pseudo/k8s v0.1.0 h1:otg4dLFc03c3gzl+2CV8GPGcd1kk8wjXwD+UhhcCn5I=
 sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=