Merge pull request #2074 from monopole/incApiReleaseTo032

increment api patch release to 0.3.2

.travis.yml

@@ -34,9 +34,9 @@ before_install:
 install: true
 
 script:
-  - ./travis/verify-deps.sh
   - make verify-kustomize
   - ./travis/kyaml-pre-commit.sh
+  - ./travis/check-go-mod.sh
 
 # TBD. Suppressing for now.
 notifications:

Makefile

@@ -152,6 +152,14 @@ $(pGen)/%.go: $(MYGOBIN)/pluginator
 .PHONY: generate-kustomize-builtin-plugins
 generate-kustomize-builtin-plugins: $(builtinplugins)
 
+.PHONY: kustomize-external-go-plugin-build
+kustomize-external-go-plugin-build:
+	./hack/buildExternalGoPlugins.sh ./plugin
+
+.PHONY: kustomize-external-go-plugin-clean
+kustomize-external-go-plugin-clean:
+	./hack/buildExternalGoPlugins.sh ./plugin clean
+
 ### End kustomize plugin rules.
 
 .PHONY: lint-kustomize
@@ -230,7 +238,8 @@ $(MYGOBIN)/helm:
 	)
 
 .PHONY: clean
-clean:
+clean: kustomize-external-go-plugin-clean
+	go clean --cache
 	rm -f $(builtinplugins)
 	rm -f $(MYGOBIN)/pluginator
 	rm -f $(MYGOBIN)/kustomize

README.md

@@ -28,9 +28,9 @@ Since [v1.14][kubectl announcement] the kustomize build system has been included
 
 | kubectl version | kustomize version |
 |---------|--------|
-| v1.16.0 | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
-| v1.15.x | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
-| v1.14.x | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
+| v1.16.0 | [v2.0.3](/../../tree/v2.0.3) |
+| v1.15.x | [v2.0.3](/../../tree/v2.0.3) |
+| v1.14.x | [v2.0.3](/../../tree/v2.0.3) |
 
 For examples and guides for using the kubectl integration please see the [kubectl book] or the [kubernetes documentation].
 
@@ -167,7 +167,7 @@ is governed by the [Kubernetes Code of Conduct].
 [eschewed feature list]: docs/eschewedFeatures.md
 [imageBase]: docs/images/base.jpg
 [imageOverlay]: docs/images/overlay.jpg
-[kind/feature]: https://github.com/kubernetes-sigs/kustomize/labels/kind%2Ffeature
+[kind/feature]: /../../labels/kind%2Ffeature
 [kubectl announcement]: https://kubernetes.io/blog/2019/03/25/kubernetes-1-14-release-announcement
 [kubectl book]: https://kubectl.docs.kubernetes.io/pages/app_customization/introduction.html
 [kubernetes documentation]: https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/
@@ -175,12 +175,12 @@ is governed by the [Kubernetes Code of Conduct].
 [kustomization]: docs/glossary.md#kustomization
 [overlay]: docs/glossary.md#overlay
 [overlays]: docs/glossary.md#overlay
-[release page]: https://github.com/kubernetes-sigs/kustomize/releases
+[release page]: /../../releases
 [resource]: docs/glossary.md#resource
 [resources]: docs/glossary.md#resource
 [sig-cli]: https://github.com/kubernetes/community/blob/master/sig-cli/README.md
 [variant]: docs/glossary.md#variant
 [variants]: docs/glossary.md#variant
-[v2.0.3]: https://github.com/kubernetes-sigs/kustomize/releases/tag/v2.0.3
-[v2.1.0]: https://github.com/kubernetes-sigs/kustomize/releases/tag/v2.1.0
+[v2.0.3]: /../../releases/tag/v2.0.3
+[v2.1.0]: /../../releases/tag/v2.1.0
 [workflows]: docs/workflows.md

api/builtins/AnnotationsTransformer.go

@@ -17,7 +17,7 @@ type AnnotationsTransformerPlugin struct {
 }
 
 func (p *AnnotationsTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Annotations = nil
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)

api/builtins/HashTransformer.go

@@ -15,7 +15,7 @@ type HashTransformerPlugin struct {
 }
 
 func (p *HashTransformerPlugin) Config(
-	h *resmap.PluginHelpers, config []byte) (err error) {
+	h *resmap.PluginHelpers, _ []byte) (err error) {
 	p.hasher = h.ResmapFactory().RF().Hasher()
 	return nil
 }

api/builtins/ImageTagTransformer.go

@@ -23,7 +23,7 @@ type ImageTagTransformerPlugin struct {
 }
 
 func (p *ImageTagTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.ImageTag = types.Image{}
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)
@@ -82,7 +82,7 @@ func (p *ImageTagTransformerPlugin) findAndReplaceImage(obj map[string]interface
 	updated := false
 	for _, path := range paths {
 		containers, found := obj[path]
-		if found {
+		if found && containers != nil {
 			if _, err := p.updateContainers(containers); err != nil {
 				return err
 			}

api/builtins/LabelTransformer.go

@@ -17,7 +17,7 @@ type LabelTransformerPlugin struct {
 }
 
 func (p *LabelTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Labels = nil
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)

api/builtins/LegacyOrderTransformer.go

@@ -20,7 +20,7 @@ type LegacyOrderTransformerPlugin struct{}
 
 // Nothing needed for configuration.
 func (p *LegacyOrderTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
+	_ *resmap.PluginHelpers, _ []byte) (err error) {
 	return nil
 }
 

api/builtins/NamespaceTransformer.go

@@ -22,7 +22,7 @@ type NamespaceTransformerPlugin struct {
 }
 
 func (p *NamespaceTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Namespace = ""
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)
@@ -52,7 +52,7 @@ func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
 
 		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
 		if len(matches) != 1 {
-			return fmt.Errorf("namespace tranformation produces ID conflict: %#v", matches)
+			return fmt.Errorf("namespace tranformation produces ID conflict: %+v", matches)
 		}
 	}
 	return nil
@@ -75,7 +75,7 @@ func (p *NamespaceTransformerPlugin) applicableFieldSpecs(id resid.ResId) []type
 }
 
 func (p *NamespaceTransformerPlugin) changeNamespace(
-	referrer *resource.Resource) func(in interface{}) (interface{}, error) {
+	_ *resource.Resource) func(in interface{}) (interface{}, error) {
 	return func(in interface{}) (interface{}, error) {
 		switch in.(type) {
 		case string:

api/builtins/PrefixSuffixTransformer.go

@@ -33,7 +33,7 @@ var prefixSuffixFieldSpecsToSkip = []types.FieldSpec{
 }
 
 func (p *PrefixSuffixTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Prefix = ""
 	p.Suffix = ""
 	p.FieldSpecs = nil

api/builtins/ReplicaCountTransformer.go

@@ -22,8 +22,7 @@ type ReplicaCountTransformerPlugin struct {
 }
 
 func (p *ReplicaCountTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Replica = types.Replica{}
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)

api/filesys/file.go

@@ -6,34 +6,8 @@ package filesys
 import (
 	"io"
 	"os"
-	"time"
 )
 
-var _ os.FileInfo = &fileInfo{}
-
-// fileInfo implements os.FileInfo for a fileInMemory instance.
-type fileInfo struct {
-	*fileInMemory
-}
-
-// Name returns the name of the file
-func (fi *fileInfo) Name() string { return fi.name }
-
-// Size returns the size of the file
-func (fi *fileInfo) Size() int64 { return int64(len(fi.content)) }
-
-// Mode returns the file mode
-func (fi *fileInfo) Mode() os.FileMode { return 0777 }
-
-// ModTime returns the modification time
-func (fi *fileInfo) ModTime() time.Time { return time.Time{} }
-
-// IsDir returns if it is a directory
-func (fi *fileInfo) IsDir() bool { return fi.dir }
-
-// Sys should return underlying data source, but it now returns nil
-func (fi *fileInfo) Sys() interface{} { return nil }
-
 // File groups the basic os.File methods.
 type File interface {
 	io.ReadWriteCloser

api/filesys/fileinfo.go

@@ -0,0 +1,34 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"os"
+	"time"
+)
+
+var _ os.FileInfo = fileInfo{}
+
+// fileInfo implements os.FileInfo for a fileInMemory instance.
+type fileInfo struct {
+	node *fsNode
+}
+
+// Name returns the name of the file
+func (fi fileInfo) Name() string { return fi.node.Name() }
+
+// Size returns the size of the file
+func (fi fileInfo) Size() int64 { return fi.node.Size() }
+
+// Mode returns the file mode
+func (fi fileInfo) Mode() os.FileMode { return 0777 }
+
+// ModTime returns a bogus time
+func (fi fileInfo) ModTime() time.Time { return time.Time{} }
+
+// IsDir returns true if it is a directory
+func (fi fileInfo) IsDir() bool { return fi.node.isNodeADir() }
+
+// Sys should return underlying data source, but it now returns nil
+func (fi fileInfo) Sys() interface{} { return nil }

api/filesys/fileinmemory.go

@@ -1,56 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package filesys
-
-import (
-	"bytes"
-	"os"
-)
-
-var _ File = &fileInMemory{}
-
-// fileInMemory implements File in-memory for tests.
-type fileInMemory struct {
-	name    string
-	content []byte
-	dir     bool
-	open    bool
-}
-
-// makeDir makes a fake directory.
-func makeDir(name string) *fileInMemory {
-	return &fileInMemory{name: name, dir: true}
-}
-
-// Close marks the fake file closed.
-func (f *fileInMemory) Close() error {
-	f.open = false
-	return nil
-}
-
-// Read never fails, and doesn't mutate p.
-func (f *fileInMemory) Read(p []byte) (n int, err error) {
-	return len(p), nil
-}
-
-// Write saves the contents of the argument to memory.
-func (f *fileInMemory) Write(p []byte) (n int, err error) {
-	f.content = p
-	return len(p), nil
-}
-
-// ContentMatches returns true if v matches fake file's content.
-func (f *fileInMemory) ContentMatches(v []byte) bool {
-	return bytes.Equal(v, f.content)
-}
-
-// GetContent the content of a fake file.
-func (f *fileInMemory) GetContent() []byte {
-	return f.content
-}
-
-// Stat returns nil.
-func (f *fileInMemory) Stat() (os.FileInfo, error) {
-	return nil, nil
-}

api/filesys/filesystem.go

@@ -8,13 +8,20 @@ import (
 	"path/filepath"
 )
 
+const (
+	Separator = string(filepath.Separator)
+	SelfDir   = "."
+	ParentDir = ".."
+)
+
 // FileSystem groups basic os filesystem methods.
+// It's supposed be functional subset of https://golang.org/pkg/os
 type FileSystem interface {
 	// Create a file.
-	Create(name string) (File, error)
+	Create(path string) (File, error)
 	// MkDir makes a directory.
 	Mkdir(path string) error
-	// MkDir makes a directory path, creating intervening directories.
+	// MkDirAll makes a directory path, creating intervening directories.
 	MkdirAll(path string) error
 	// RemoveAll removes path and any children it contains.
 	RemoveAll(path string) error
@@ -30,11 +37,13 @@ type FileSystem interface {
 	CleanedAbs(path string) (ConfirmedDir, string, error)
 	// Exists is true if the path exists in the file system.
 	Exists(path string) bool
-	// Glob returns the list of matching files
+	// Glob returns the list of matching files,
+	// emulating https://golang.org/pkg/path/filepath/#Glob
 	Glob(pattern string) ([]string, error)
 	// ReadFile returns the contents of the file at the given path.
 	ReadFile(path string) ([]byte, error)
-	// WriteFile writes the data to a file at the given path.
+	// WriteFile writes the data to a file at the given path,
+	// overwriting anything that's already there.
 	WriteFile(path string, data []byte) error
 	// Walk walks the file system with the given WalkFunc.
 	Walk(path string, walkFn filepath.WalkFunc) error

api/filesys/fsinmemory.go

@@ -1,223 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package filesys
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-)
-
-var _ FileSystem = &fsInMemory{}
-
-// fsInMemory implements FileSystem using a in-memory filesystem
-// primarily for use in tests.
-type fsInMemory struct {
-	m map[string]*fileInMemory
-}
-
-// MakeFsInMemory returns an instance of fsInMemory with no files in it.
-func MakeFsInMemory() FileSystem {
-	result := &fsInMemory{m: map[string]*fileInMemory{}}
-	result.Mkdir(separator)
-	return result
-}
-
-const (
-	separator = string(filepath.Separator)
-	doubleSep = separator + separator
-)
-
-// Create assures a fake file appears in the in-memory file system.
-func (fs *fsInMemory) Create(name string) (File, error) {
-	f := &fileInMemory{}
-	f.open = true
-	fs.m[name] = f
-	return fs.m[name], nil
-}
-
-// Mkdir assures a fake directory appears in the in-memory file system.
-func (fs *fsInMemory) Mkdir(name string) error {
-	fs.m[name] = makeDir(name)
-	return nil
-}
-
-// MkdirAll delegates to Mkdir
-func (fs *fsInMemory) MkdirAll(name string) error {
-	return fs.Mkdir(name)
-}
-
-// RemoveAll presumably does rm -r on a path.
-// There's no error.
-func (fs *fsInMemory) RemoveAll(name string) error {
-	var toRemove []string
-	for k := range fs.m {
-		if strings.HasPrefix(k, name) {
-			toRemove = append(toRemove, k)
-		}
-	}
-	for _, k := range toRemove {
-		delete(fs.m, k)
-	}
-	return nil
-}
-
-// Open returns a fake file in the open state.
-func (fs *fsInMemory) Open(name string) (File, error) {
-	if _, found := fs.m[name]; !found {
-		return nil, fmt.Errorf("file %q cannot be opened", name)
-	}
-	return fs.m[name], nil
-}
-
-// CleanedAbs cannot fail.
-func (fs *fsInMemory) CleanedAbs(path string) (ConfirmedDir, string, error) {
-	if fs.IsDir(path) {
-		return ConfirmedDir(path), "", nil
-	}
-	d := filepath.Dir(path)
-	if d == path {
-		return ConfirmedDir(d), "", nil
-	}
-	return ConfirmedDir(d), filepath.Base(path), nil
-}
-
-// Exists returns true if file is known.
-func (fs *fsInMemory) Exists(name string) bool {
-	_, found := fs.m[name]
-	return found
-}
-
-// Glob returns the list of matching files
-func (fs *fsInMemory) Glob(pattern string) ([]string, error) {
-	var result []string
-	for p := range fs.m {
-		if fs.pathMatch(p, pattern) {
-			result = append(result, p)
-		}
-	}
-	sort.Strings(result)
-	return result, nil
-}
-
-// IsDir returns true if the file exists and is a directory.
-func (fs *fsInMemory) IsDir(name string) bool {
-	f, found := fs.m[name]
-	if found && f.dir {
-		return true
-	}
-	if !strings.HasSuffix(name, separator) {
-		name = name + separator
-	}
-	for k := range fs.m {
-		if strings.HasPrefix(k, name) {
-			return true
-		}
-	}
-	return false
-}
-
-// ReadFile always returns an empty bytes and error depending on content of m.
-func (fs *fsInMemory) ReadFile(name string) ([]byte, error) {
-	if ff, found := fs.m[name]; found {
-		return ff.content, nil
-	}
-	return nil, fmt.Errorf("cannot read file %q", name)
-}
-
-// WriteFile always succeeds and does nothing.
-func (fs *fsInMemory) WriteFile(name string, c []byte) error {
-	ff := &fileInMemory{}
-	ff.Write(c)
-	fs.m[name] = ff
-	return nil
-}
-
-// Walk implements filepath.Walk using the fake filesystem.
-func (fs *fsInMemory) Walk(path string, walkFn filepath.WalkFunc) error {
-	info, err := fs.lstat(path)
-	if err != nil {
-		err = walkFn(path, info, err)
-	} else {
-		err = fs.walk(path, info, walkFn)
-	}
-	if err == filepath.SkipDir {
-		return nil
-	}
-	return err
-}
-
-func (fs *fsInMemory) pathMatch(path, pattern string) bool {
-	match, _ := filepath.Match(pattern, path)
-	return match
-}
-
-func (fs *fsInMemory) lstat(path string) (*fileInfo, error) {
-	f, found := fs.m[path]
-	if !found {
-		return nil, os.ErrNotExist
-	}
-	return &fileInfo{f}, nil
-}
-
-func (fs *fsInMemory) join(elem ...string) string {
-	for i, e := range elem {
-		if e != "" {
-			return strings.Replace(
-				strings.Join(elem[i:], separator), doubleSep, separator, -1)
-		}
-	}
-	return ""
-}
-
-func (fs *fsInMemory) readDirNames(path string) []string {
-	var names []string
-	if !strings.HasSuffix(path, separator) {
-		path += separator
-	}
-	pathSegments := strings.Count(path, separator)
-	for name := range fs.m {
-		if name == path {
-			continue
-		}
-		if strings.Count(name, separator) > pathSegments {
-			continue
-		}
-		if strings.HasPrefix(name, path) {
-			names = append(names, filepath.Base(name))
-		}
-	}
-	sort.Strings(names)
-	return names
-}
-
-func (fs *fsInMemory) walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
-	if !info.IsDir() {
-		return walkFn(path, info, nil)
-	}
-
-	names := fs.readDirNames(path)
-	if err := walkFn(path, info, nil); err != nil {
-		return err
-	}
-	for _, name := range names {
-		filename := fs.join(path, name)
-		fileInfo, err := fs.lstat(filename)
-		if err != nil {
-			if err := walkFn(filename, fileInfo, os.ErrNotExist); err != nil && err != filepath.SkipDir {
-				return err
-			}
-		} else {
-			err = fs.walk(filename, fileInfo, walkFn)
-			if err != nil {
-				if !fileInfo.IsDir() || err != filepath.SkipDir {
-					return err
-				}
-			}
-		}
-	}
-	return nil
-}

api/filesys/fsinmemory_test.go

@@ -1,149 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package filesys_test
-
-import (
-	"bytes"
-	"reflect"
-	"testing"
-
-	. "sigs.k8s.io/kustomize/api/filesys"
-)
-
-func TestExists(t *testing.T) {
-	fSys := MakeFsInMemory()
-	if fSys.Exists("foo") {
-		t.Fatalf("expected no foo")
-	}
-	fSys.Mkdir("/")
-	if !fSys.IsDir("/") {
-		t.Fatalf("expected dir at /")
-	}
-}
-
-func TestIsDir(t *testing.T) {
-	fSys := MakeFsInMemory()
-	expectedName := "my-dir"
-	err := fSys.Mkdir(expectedName)
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	shouldExist(t, fSys, expectedName)
-	if !fSys.IsDir(expectedName) {
-		t.Fatalf(expectedName + " should be a dir")
-	}
-}
-
-func shouldExist(t *testing.T, fSys FileSystem, name string) {
-	if !fSys.Exists(name) {
-		t.Fatalf(name + " should exist")
-	}
-}
-
-func shouldNotExist(t *testing.T, fSys FileSystem, name string) {
-	if fSys.Exists(name) {
-		t.Fatalf(name + " should not exist")
-	}
-}
-
-func TestRemoveAll(t *testing.T) {
-	fSys := MakeFsInMemory()
-	fSys.WriteFile("/foo/project/file.yaml", []byte("Unused"))
-	fSys.WriteFile("/foo/project/subdir/file.yaml", []byte("Unused"))
-	fSys.WriteFile("/foo/apple/subdir/file.yaml", []byte("Unused"))
-	shouldExist(t, fSys, "/foo/project/file.yaml")
-	shouldExist(t, fSys, "/foo/project/subdir/file.yaml")
-	shouldExist(t, fSys, "/foo/apple/subdir/file.yaml")
-	err := fSys.RemoveAll("/foo/project")
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	shouldNotExist(t, fSys, "/foo/project/file.yaml")
-	shouldNotExist(t, fSys, "/foo/project/subdir/file.yaml")
-	shouldExist(t, fSys, "/foo/apple/subdir/file.yaml")
-}
-
-func TestIsDirDeeper(t *testing.T) {
-	fSys := MakeFsInMemory()
-	fSys.WriteFile("/foo/project/file.yaml", []byte("Unused"))
-	fSys.WriteFile("/foo/project/subdir/file.yaml", []byte("Unused"))
-	if !fSys.IsDir("/") {
-		t.Fatalf("/ should be a dir")
-	}
-	if !fSys.IsDir("/foo") {
-		t.Fatalf("/foo should be a dir")
-	}
-	if !fSys.IsDir("/foo/project") {
-		t.Fatalf("/foo/project should be a dir")
-	}
-	if fSys.IsDir("/fo") {
-		t.Fatalf("/fo should not be a dir")
-	}
-	if fSys.IsDir("/x") {
-		t.Fatalf("/x should not be a dir")
-	}
-}
-
-func TestCreate(t *testing.T) {
-	fSys := MakeFsInMemory()
-	f, err := fSys.Create("foo")
-	if f == nil {
-		t.Fatalf("expected file")
-	}
-	if err != nil {
-		t.Fatalf("unexpected error")
-	}
-	shouldExist(t, fSys, "foo")
-}
-
-func TestReadFile(t *testing.T) {
-	fSys := MakeFsInMemory()
-	f, err := fSys.Create("foo")
-	if f == nil {
-		t.Fatalf("expected file")
-	}
-	if err != nil {
-		t.Fatalf("unexpected error")
-	}
-	content, err := fSys.ReadFile("foo")
-	if len(content) != 0 {
-		t.Fatalf("expected no content")
-	}
-	if err != nil {
-		t.Fatalf("expected no error")
-	}
-}
-
-func TestWriteFile(t *testing.T) {
-	fSys := MakeFsInMemory()
-	c := []byte("heybuddy")
-	err := fSys.WriteFile("foo", c)
-	if err != nil {
-		t.Fatalf("expected no error")
-	}
-	content, err := fSys.ReadFile("foo")
-	if err != nil {
-		t.Fatalf("expected read to work: %v", err)
-	}
-	if !bytes.Equal(c, content) {
-		t.Fatalf("incorrect content: %v", content)
-	}
-}
-
-func TestGlob(t *testing.T) {
-	fSys := MakeFsInMemory()
-	fSys.Create("dir/foo")
-	fSys.Create("dir/bar")
-	files, err := fSys.Glob("dir/*")
-	if err != nil {
-		t.Fatalf("expected no error")
-	}
-	expected := []string{
-		"dir/bar",
-		"dir/foo",
-	}
-	if !reflect.DeepEqual(files, expected) {
-		t.Fatalf("incorrect files found by glob: %v", files)
-	}
-}

api/filesys/fsnode.go

@@ -0,0 +1,557 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var _ File = &fsNode{}
+var _ FileSystem = &fsNode{}
+
+// fsNode is either a file or a directory.
+type fsNode struct {
+	// What node owns me?
+	parent *fsNode
+
+	// Value to return as the Name() when the
+	// parent is nil.
+	nilParentName string
+
+	// A directory mapping names to nodes.
+	// If dir is nil, then self node is a file.
+	// If dir is non-nil, then self node is a directory,
+	// albeit possibly an empty directory.
+	dir map[string]*fsNode
+
+	// if this node is a file, this is the content.
+	content []byte
+
+	// if this node is a file, this tracks whether or
+	// not it is "open".
+	open bool
+}
+
+// MakeEmptyDirInMemory returns an empty directory.
+// The paths of nodes in this object will never
+// report a leading Separator, meaning they
+// aren't "absolute" in the sense defined by
+// https://golang.org/pkg/path/filepath/#IsAbs.
+func MakeEmptyDirInMemory() *fsNode {
+	return &fsNode{
+		dir: make(map[string]*fsNode),
+	}
+}
+
+// MakeFsInMemory returns an empty 'file system'.
+// The paths of nodes in this object will always
+// report a leading Separator, meaning they
+// are "absolute" in the sense defined by
+// https://golang.org/pkg/path/filepath/#IsAbs.
+// This is a relevant difference when using Walk,
+// Glob, Match, etc.
+func MakeFsInMemory() FileSystem {
+	return &fsNode{
+		nilParentName: Separator,
+		dir:           make(map[string]*fsNode),
+	}
+}
+
+// Name returns the name of the node.
+func (n *fsNode) Name() string {
+	if n.parent == nil {
+		// Unable to lookup name in parent.
+		return n.nilParentName
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir")
+	}
+	for key, value := range n.parent.dir {
+		if value == n {
+			return key
+		}
+	}
+	log.Fatal("unable to find fsNode name")
+	return ""
+}
+
+// Path returns the full path to the node.
+func (n *fsNode) Path() string {
+	if n.parent == nil {
+		return n.nilParentName
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir, structural error")
+	}
+	return filepath.Join(n.parent.Path(), n.Name())
+}
+
+// mySplit trims trailing separators from the directory
+// result of filepath.Split.
+func mySplit(s string) (string, string) {
+	dName, fName := filepath.Split(s)
+	return StripTrailingSeps(dName), fName
+}
+
+func (n *fsNode) addFile(name string, c []byte) (result *fsNode, err error) {
+	parent := n
+	dName, fileName := mySplit(name)
+	if dName != "" {
+		parent, err = parent.addDir(dName)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if !isLegalFileNameForCreation(fileName) {
+		return nil, fmt.Errorf(
+			"illegal name '%s' in file creation", fileName)
+	}
+	result, ok := parent.dir[fileName]
+	if ok {
+		// File already exists; overwrite it.
+		result.content = c
+		return result, nil
+	}
+	result = &fsNode{
+		content: c,
+		parent:  parent,
+	}
+	parent.dir[fileName] = result
+	return result, nil
+}
+
+// Create implements FileSystem.
+// Create makes an empty file.
+func (n *fsNode) Create(path string) (result File, err error) {
+	return n.AddFile(path, []byte{})
+}
+
+// WriteFile implements FileSystem.
+func (n *fsNode) WriteFile(path string, d []byte) error {
+	_, err := n.AddFile(path, d)
+	return err
+}
+
+// AddFile adds a file and any necessary containing
+// directories to the node.
+func (n *fsNode) AddFile(
+	name string, c []byte) (result *fsNode, err error) {
+	if n.dir == nil {
+		return nil, fmt.Errorf(
+			"cannot add a file to a non-directory '%s'", n.Name())
+	}
+	return n.addFile(cleanQueryPath(name), c)
+}
+
+func (n *fsNode) addDir(path string) (result *fsNode, err error) {
+	parent := n
+	dName, subDirName := mySplit(path)
+	if dName != "" {
+		parent, err = n.addDir(dName)
+		if err != nil {
+			return nil, err
+		}
+	}
+	switch subDirName {
+	case "", SelfDir:
+		return n, nil
+	case ParentDir:
+		if n.parent == nil {
+			return nil, fmt.Errorf(
+				"cannot add a directory above '%s'", n.Path())
+		}
+		return n.parent, nil
+	default:
+		if !isLegalFileNameForCreation(subDirName) {
+			return nil, fmt.Errorf(
+				"illegal name '%s' in directory creation", subDirName)
+		}
+		result, ok := parent.dir[subDirName]
+		if ok {
+			if result.isNodeADir() {
+				// it's already there.
+				return result, nil
+			}
+			return nil, fmt.Errorf(
+				"cannot make dir '%s'; a file of that name already exists in '%s'",
+				subDirName, parent.Name())
+		}
+		result = &fsNode{
+			dir:    make(map[string]*fsNode),
+			parent: parent,
+		}
+		parent.dir[subDirName] = result
+		return result, nil
+	}
+}
+
+// Mkdir implements FileSystem.
+// Mkdir creates a directory.
+func (n *fsNode) Mkdir(path string) error {
+	_, err := n.AddDir(path)
+	return err
+}
+
+// MkdirAll implements FileSystem.
+// MkdirAll creates a directory.
+func (n *fsNode) MkdirAll(path string) error {
+	_, err := n.AddDir(path)
+	return err
+}
+
+// AddDir adds a directory to the node, not complaining
+// if it is already there.
+func (n *fsNode) AddDir(path string) (result *fsNode, err error) {
+	if n.dir == nil {
+		return nil, fmt.Errorf(
+			"cannot add a directory to file node '%s'", n.Name())
+	}
+	return n.addDir(cleanQueryPath(path))
+}
+
+// CleanedAbs implements FileSystem.
+func (n *fsNode) CleanedAbs(path string) (ConfirmedDir, string, error) {
+	node, err := n.Find(path)
+	if err != nil {
+		return "", "", errors.Wrap(err, "unable to clean")
+	}
+	if node == nil {
+		return "", "", fmt.Errorf("'%s' doesn't exist", path)
+	}
+	if node.isNodeADir() {
+		return ConfirmedDir(node.Path()), "", nil
+	}
+	return ConfirmedDir(node.parent.Path()), node.Name(), nil
+}
+
+// Exists implements FileSystem.
+// Exists returns true if the path exists.
+func (n *fsNode) Exists(path string) bool {
+	if !n.isNodeADir() {
+		return n.Name() == path
+	}
+	result, err := n.Find(path)
+	if err != nil {
+		return false
+	}
+	return result != nil
+}
+
+func cleanQueryPath(path string) string {
+	// Always ignore leading separator?
+	// Remember that filepath.Clean returns "." if
+	// given an empty string argument.
+	return filepath.Clean(StripLeadingSeps(path))
+}
+
+// Find finds the given node, else nil if not found.
+// Return error on structural/argument errors.
+func (n *fsNode) Find(path string) (*fsNode, error) {
+	if !n.isNodeADir() {
+		return nil, fmt.Errorf("can only find inside a dir")
+	}
+	if path == "" {
+		// Special case; check *before* cleaning and *before*
+		// comparison to nilParentName.
+		return nil, nil
+	}
+	if (n.parent == nil && path == n.nilParentName) || path == SelfDir {
+		// Special case
+		return n, nil
+	}
+	return n.findIt(cleanQueryPath(path))
+}
+
+func (n *fsNode) findIt(path string) (result *fsNode, err error) {
+	parent := n
+	dName, item := mySplit(path)
+	if dName != "" {
+		parent, err = n.findIt(dName)
+		if err != nil {
+			return nil, err
+		}
+		if parent == nil {
+			// all done, target doesn't exist.
+			return nil, nil
+		}
+	}
+	if !parent.isNodeADir() {
+		return nil, fmt.Errorf("'%s' is not a directory", parent.Path())
+	}
+	return parent.dir[item], nil
+}
+
+// RemoveAll implements FileSystem.
+// RemoveAll removes an item and everything it contains.
+func (n *fsNode) RemoveAll(path string) error {
+	result, err := n.Find(path)
+	if err != nil {
+		return err
+	}
+	if result == nil {
+		return fmt.Errorf("cannot find '%s' to remove it", path)
+	}
+	return result.Remove()
+}
+
+// Remove drop the node, and everything it contains, from its parent.
+func (n *fsNode) Remove() error {
+	if n.parent == nil {
+		return fmt.Errorf("cannot remove a root node")
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir")
+	}
+	for key, value := range n.parent.dir {
+		if value == n {
+			delete(n.parent.dir, key)
+			return nil
+		}
+	}
+	log.Fatal("unable to find self in parent")
+	return nil
+}
+
+// isNodeADir returns true if the node is a directory.
+// Cannot collide with the poorly named "IsDir".
+func (n *fsNode) isNodeADir() bool {
+	return n.dir != nil
+}
+
+// IsDir implements FileSystem.
+// IsDir returns true if the argument resolves
+// to a directory rooted at the node.
+func (n *fsNode) IsDir(path string) bool {
+	result, err := n.Find(path)
+	if err != nil || result == nil {
+		return false
+	}
+	return result.isNodeADir()
+}
+
+// Size returns the size of the node.
+func (n *fsNode) Size() int64 {
+	if n.isNodeADir() {
+		return int64(len(n.dir))
+	}
+	return int64(len(n.content))
+}
+
+// Open implements FileSystem.
+// Open opens the node for reading (just marks it).
+func (n *fsNode) Open(path string) (File, error) {
+	result, err := n.Find(path)
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, fmt.Errorf("cannot find '%s' to open it", path)
+	}
+	result.open = true
+	return result, nil
+}
+
+// Close marks the node closed.
+func (n *fsNode) Close() error {
+	n.open = false
+	return nil
+}
+
+// ReadFile implements FileSystem.
+func (n *fsNode) ReadFile(path string) (c []byte, err error) {
+	result, err := n.Find(path)
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, fmt.Errorf("cannot find '%s' to read it", path)
+	}
+	c = make([]byte, len(result.content))
+	_, err = result.Read(c)
+	return c, err
+}
+
+// Read returns the content of the file node.
+func (n *fsNode) Read(d []byte) (c int, err error) {
+	if n.isNodeADir() {
+		return 0, fmt.Errorf(
+			"cannot read content from non-file '%s'", n.Path())
+	}
+	return copy(d, n.content), nil
+}
+
+// Write saves the contents of the argument to the file node.
+func (n *fsNode) Write(p []byte) (c int, err error) {
+	if n.isNodeADir() {
+		return 0, fmt.Errorf(
+			"cannot write content to non-file '%s'", n.Path())
+	}
+	n.content = make([]byte, len(p))
+	return copy(n.content, p), nil
+}
+
+// ContentMatches returns true if v matches fake file's content.
+func (n *fsNode) ContentMatches(v []byte) bool {
+	return bytes.Equal(v, n.content)
+}
+
+// GetContent the content of a fake file.
+func (n *fsNode) GetContent() []byte {
+	return n.content
+}
+
+// Stat returns an instance of FileInfo.
+func (n *fsNode) Stat() (os.FileInfo, error) {
+	return fileInfo{node: n}, nil
+}
+
+// Walk implements FileSystem.
+func (n *fsNode) Walk(path string, walkFn filepath.WalkFunc) error {
+	result, err := n.Find(path)
+	if err != nil {
+		return err
+	}
+	if result == nil {
+		return fmt.Errorf("cannot find '%s' to walk it", path)
+	}
+	return result.WalkMe(walkFn)
+}
+
+// Walk runs the given walkFn on each node.
+func (n *fsNode) WalkMe(walkFn filepath.WalkFunc) error {
+	fi, err := n.Stat()
+	// always visit self first
+	err = walkFn(n.Path(), fi, err)
+	if !n.isNodeADir() {
+		// it's a file, so nothing more to do
+		return err
+	}
+	// process self as a directory
+	if err == filepath.SkipDir {
+		return nil
+	}
+	// Walk is supposed to visit in lexical order.
+	for _, k := range n.sortedDirEntries() {
+		if err := n.dir[k].WalkMe(walkFn); err != nil {
+			if err == filepath.SkipDir {
+				// stop processing this directory
+				break
+			}
+			// bail out completely
+			return err
+		}
+	}
+	return nil
+}
+
+func (n *fsNode) sortedDirEntries() []string {
+	keys := make([]string, len(n.dir))
+	i := 0
+	for k := range n.dir {
+		keys[i] = k
+		i++
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// FileCount returns a count of files.
+// Directories, empty or otherwise, not counted.
+func (n *fsNode) FileCount() int {
+	count := 0
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			count++
+		}
+		return nil
+	})
+	return count
+}
+
+func (n *fsNode) DebugPrint() {
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("err '%v' at path %q\n", err, path)
+			return nil
+		}
+		if info.IsDir() {
+			if info.Size() == 0 {
+				fmt.Println("empty dir: " + path)
+			}
+		} else {
+			fmt.Println("     file: " + path)
+		}
+		return nil
+	})
+}
+
+var legalFileNamePattern = regexp.MustCompile("^[a-zA-Z0-9-_.]+$")
+
+// This rules enforced here should be simpler and tighter
+// than what's allowed on a real OS.
+// Should be fine for testing or in-memory purposes.
+func isLegalFileNameForCreation(n string) bool {
+	if n == "" || n == SelfDir || !legalFileNamePattern.MatchString(n) {
+		return false
+	}
+	return !strings.Contains(n, ParentDir)
+}
+
+// RegExpGlob returns a list of file paths matching the regexp.
+// Excludes directories.
+func (n *fsNode) RegExpGlob(pattern string) ([]string, error) {
+	var result []string
+	var expression = regexp.MustCompile(pattern)
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			if expression.MatchString(path) {
+				result = append(result, path)
+			}
+		}
+		return nil
+	})
+	sort.Strings(result)
+	return result, nil
+}
+
+// Glob implements FileSystem.
+// Glob returns the list of file paths matching
+// per filepath.Match semantics, i.e. unlike RegExpGlob,
+// Match("foo/a*") will not match sub-sub directories of foo.
+// This is how /bin/ls behaves.
+func (n *fsNode) Glob(pattern string) ([]string, error) {
+	var result []string
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			match, err := filepath.Match(pattern, path)
+			if err != nil {
+				return err
+			}
+			if match {
+				result = append(result, path)
+			}
+		}
+		return nil
+	})
+	sort.Strings(result)
+	return result, nil
+}

api/filesys/fsnode_test.go

@@ -0,0 +1,788 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+const content = `
+Lorem ipsum dolor sit amet,
+consectetur adipiscing elit,
+sed do eiusmod tempor incididunt
+ut labore et dolore magna aliqua.
+`
+const shortContent = "hi"
+
+var topCases = []pathCase{
+	{
+		what:   "dotdot",
+		arg:    ParentDir,
+		errStr: "illegal name '..' in file creation",
+	},
+	{
+		what:   "empty",
+		arg:    "",
+		name:   "",
+		errStr: "illegal name '.' in file creation",
+	},
+	{
+		what: "simple",
+		arg:  "bob",
+		name: "bob",
+		path: "bob",
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "bob"),
+	},
+	{
+		what: "longer yet",
+		arg:  filepath.Join("longer", "foo", "bar", "beans", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "foo", "bar", "beans", "bob"),
+	},
+	{
+		what: "tricky",
+		arg:  filepath.Join("bob", ParentDir, "sally"),
+		name: "sally",
+		path: "sally",
+	},
+	{
+		what: "trickier",
+		arg:  filepath.Join("bob", "sally", ParentDir, ParentDir, "jean"),
+		name: "jean",
+		path: "jean",
+	},
+}
+
+func TestMakeEmptyDirInMemory(t *testing.T) {
+	n := MakeEmptyDirInMemory()
+	if !n.isNodeADir() {
+		t.Fatalf("not a directory")
+	}
+	if n.Size() != 0 {
+		t.Fatalf("unexpected size %d", n.Size())
+	}
+	if n.Name() != "" {
+		t.Fatalf("unexpected name '%s'", n.Name())
+	}
+	if n.Path() != "" {
+		t.Fatalf("unexpected path '%s'", n.Path())
+	}
+	runBasicOperations(
+		t, "MakeEmptyDirInMemory", false, topCases, n)
+}
+
+func TestMakeFsInMemory(t *testing.T) {
+	runBasicOperations(
+		t, "MakeFsInMemory", true, topCases, MakeFsInMemory())
+}
+
+//nolint:gocyclo
+func runBasicOperations(
+	t *testing.T, tName string, isFSysRooted bool,
+	cases []pathCase, fSys FileSystem) {
+	buff := make([]byte, 500)
+	for _, c := range cases {
+		err := fSys.WriteFile(c.arg, []byte(content))
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error writing to  '%s'!", c.what, c.arg)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if !fSys.Exists(c.path) {
+			t.Fatalf("%s; expect existence of '%s'", c.what, c.path)
+		}
+		stuff, err := fSys.ReadFile(c.path)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(stuff) != content {
+			t.Fatalf("%s; unexpected content '%s'", c.what, stuff)
+		}
+		f, err := fSys.Open(c.arg)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		fi, err := f.Stat()
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if fi.Name() != c.name {
+			t.Fatalf("%s; expected name '%s', got '%s'", c.what, c.name, fi.Name())
+		}
+		count, err := f.Read(buff)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(buff[:count]) != content {
+			t.Fatalf("%s; unexpected buff '%s'", c.what, buff)
+		}
+		count, err = f.Write([]byte(shortContent))
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if count != len(shortContent) {
+			t.Fatalf("%s; unexpected count: %d", c.what, len(shortContent))
+		}
+		stuff, err = fSys.ReadFile(c.path)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(stuff) != shortContent {
+			t.Fatalf("%s; unexpected content '%s'", c.what, stuff)
+		}
+	}
+
+	var actualPaths []string
+	var err error
+	prefix := ""
+	{
+		root := SelfDir
+		if isFSysRooted {
+			root = Separator
+			prefix = Separator
+		}
+		err = fSys.Walk(root, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				fmt.Printf("err '%v' at path %q\n", err, path)
+				return nil
+			}
+			if !info.IsDir() {
+				actualPaths = append(actualPaths, path)
+			}
+			return nil
+		})
+	}
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	var expectedPaths []string
+	for _, c := range cases {
+		if c.errStr == "" {
+			expectedPaths = append(expectedPaths, prefix+c.path)
+		}
+	}
+	sort.Strings(expectedPaths)
+	assertEqualStringSlices(t, expectedPaths, actualPaths, tName)
+}
+
+type pathCase struct {
+	what   string
+	arg    string
+	name   string
+	path   string
+	errStr string
+}
+
+func TestAddDir(t *testing.T) {
+	cases := []pathCase{
+		{
+			what:   "dotdot",
+			arg:    ParentDir,
+			errStr: "cannot add a directory above ''",
+		},
+		{
+			what: "empty",
+			arg:  "",
+			name: "",
+			path: "",
+		},
+		{
+			what: "simple",
+			arg:  "bob",
+			name: "bob",
+			path: "bob",
+		},
+		{
+			what: "longer",
+			arg:  filepath.Join("longer", "bob"),
+			name: "bob",
+			path: filepath.Join("longer", "bob"),
+		},
+		{
+			what: "longer yet",
+			arg:  filepath.Join("longer", "foo", "bar", "beans", "bob"),
+			name: "bob",
+			path: filepath.Join("longer", "foo", "bar", "beans", "bob"),
+		},
+		{
+			what: "tricky",
+			arg:  filepath.Join("bob", ParentDir, "sally"),
+			name: "sally",
+			path: "sally",
+		},
+		{
+			what: "trickier",
+			arg:  filepath.Join("bob", "sally", ParentDir, ParentDir, "jean"),
+			name: "jean",
+			path: "jean",
+		},
+	}
+	for _, c := range cases {
+		n := MakeEmptyDirInMemory()
+		f, err := n.AddDir(c.arg)
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error!", c.what)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf(
+					"%s; expected error with '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		checkNode(t, c.what, f, c.name, 0, true, c.path)
+		checkOsStat(t, c.what, f, f.Name(), 0, true)
+	}
+}
+
+var bagOfCases = []pathCase{
+	{
+		what:   "empty",
+		arg:    "",
+		errStr: "illegal name '.' in file creation",
+	},
+	{
+		what: "simple",
+		arg:  "bob",
+		name: "bob",
+		path: "bob",
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "bob"),
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "sally"),
+		name: "sally",
+		path: filepath.Join("longer", "sally"),
+	},
+	{
+		what: "even longer",
+		arg:  filepath.Join("longer", "than", "the", "other", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "than", "the", "other", "bob"),
+	},
+	{
+		what: "even longer",
+		arg:  filepath.Join("even", "much", "longer", "than", "the", "other", "bob"),
+		name: "bob",
+		path: filepath.Join("even", "much", "longer", "than", "the", "other", "bob"),
+	},
+}
+
+func TestAddFile(t *testing.T) {
+	n := MakeEmptyDirInMemory()
+	if n.FileCount() != 0 {
+		t.Fatalf("expected no files, got %d", n.FileCount())
+	}
+	expectedFileCount := 0
+	for _, c := range bagOfCases {
+		f, err := n.AddFile(c.arg, []byte(content))
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error!", c.what)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error %v", c.what, err)
+		}
+		checkNode(t, c.what, f, c.name, len(content), false, c.path)
+		checkOsStat(t, c.what, f, f.Name(), len(content), false)
+
+		result, err := n.Find(c.arg)
+		if err != nil {
+			t.Fatalf("%s; unexpected find error %v", c.what, err)
+		}
+		if result != f {
+			t.Fatalf("%s; unexpected find result %v", c.what, result)
+		}
+
+		result, err = n.Find(filepath.Join("longer", "bogus"))
+		if err != nil {
+			t.Fatalf("%s; unexpected find error %v", c.what, err)
+		}
+		if result != nil {
+			t.Fatalf("%s; unexpected find result %v", c.what, result)
+		}
+
+		expectedFileCount++
+		fc := n.FileCount()
+		if fc != expectedFileCount {
+			t.Fatalf("expected file count %d, got %d",
+				expectedFileCount, fc)
+		}
+	}
+}
+
+func checkNode(
+	t *testing.T, what string, f *fsNode, name string,
+	size int, isDir bool, path string) {
+	if f.isNodeADir() != isDir {
+		t.Fatalf("%s; unexpected isNodeADir = %v", what, f.isNodeADir())
+	}
+	if f.Size() != int64(size) {
+		t.Fatalf("%s; unexpected size %d", what, f.Size())
+	}
+	if name != f.Name() {
+		t.Fatalf("%s; expected name '%s', got '%s'", what, name, f.Name())
+	}
+	if path != f.Path() {
+		t.Fatalf("%s; expected path '%s', got '%s'", what, path, f.Path())
+	}
+}
+
+func checkOsStat(
+	t *testing.T, what string, f File, name string,
+	size int, isDir bool) {
+	info, err := f.Stat()
+	if err != nil {
+		t.Fatalf("%s; unexpected stat error %v", what, err)
+	}
+	if info.IsDir() != isDir {
+		t.Fatalf("%s; unexpected info.isNodeADir = %v", what, info.IsDir())
+	}
+	if info.Size() != int64(size) {
+		t.Fatalf("%s; unexpected info.size %d", what, info.Size())
+	}
+	if info.Name() != name {
+		t.Fatalf("%s; expected name '%s', got info.Name '%s'", what, name, info.Name())
+	}
+}
+
+var bunchOfFiles = []struct {
+	path     string
+	addAsDir bool
+}{
+	{
+		path: filepath.Join("b", "e", "a", "c", "g"),
+	},
+	{
+		path: filepath.Join("z", "r", "a", "b", "g"),
+	},
+	{
+		path: filepath.Join("b", "q", "a", "c", "g"),
+	},
+	{
+		path:     filepath.Join("b", "a", "a", "m", "g"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("b", "w"),
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "m"),
+	},
+	{
+		path: filepath.Join("b", "d", "z"),
+	},
+	{
+		path: filepath.Join("b", "d", "y"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "n"),
+	},
+	{
+		path: filepath.Join("b", "d", "x"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "o"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "m"),
+	},
+	{
+		path:     filepath.Join("b", "d", "a", "c", "i"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("x"),
+	},
+	{
+		path: filepath.Join("y"),
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "i", "beans"),
+	},
+	{
+		path:     filepath.Join("b", "d", "a", "c", "r", "w"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "u"),
+	},
+}
+
+func makeLoadedFileTree(t *testing.T) *fsNode {
+	n := MakeEmptyDirInMemory()
+	var err error
+	expectedFileCount := 0
+	for _, item := range bunchOfFiles {
+		if item.addAsDir {
+			_, err = n.AddDir(item.path)
+		} else {
+			_, err = n.AddFile(item.path, []byte(content))
+			expectedFileCount++
+		}
+		if err != nil {
+			t.Fatalf("unexpected error %v", err)
+		}
+	}
+	fc := n.FileCount()
+	if fc != expectedFileCount {
+		t.Fatalf("expected file count %d, got %d",
+			expectedFileCount, fc)
+	}
+	return n
+}
+
+func TestWalkMe(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	var actualPaths []string
+	err := n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("err '%v' at path %q\n", err, path)
+			return nil
+		}
+		if info.IsDir() {
+			if info.Name() == "ignore" {
+				return filepath.SkipDir
+			}
+		} else {
+			actualPaths = append(actualPaths, path)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	var expectedPaths []string
+	for _, c := range bunchOfFiles {
+		if !c.addAsDir && !strings.Contains(c.path, "ignore") {
+			expectedPaths = append(expectedPaths, c.path)
+		}
+	}
+	sort.Strings(expectedPaths)
+	assertEqualStringSlices(t, expectedPaths, actualPaths, "testWalkMe")
+}
+
+func TestRemove(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	orgCount := n.FileCount()
+
+	// Remove the "ignore" directory and everything below it.
+	path := filepath.Join("b", "d", "ignore")
+	result, err := n.Find(path)
+	if err != nil {
+		t.Fatalf("%s; unexpected error %v", path, err)
+	}
+	if result == nil {
+		t.Fatalf("%s; expected to find '%s'", path, path)
+	}
+	if !result.isNodeADir() {
+		t.Fatalf("%s; expected to find a directory", path)
+	}
+	err = result.Remove()
+	if err != nil {
+		t.Fatalf("%s; unable to remove: %v", path, err)
+	}
+	result, err = n.Find(path)
+	if err != nil {
+		// Just because it's gone doesn't mean error.
+		t.Fatalf("%s; unexpected error %v", path, err)
+	}
+	if result != nil {
+		t.Fatalf("%s; should not have been able to find '%s'", path, path)
+	}
+
+	// There were three files below "ignore".
+	orgCount -= 3
+
+	// Now drop one more for a total of four dropped.
+	result, _ = n.Find(filepath.Join("y"))
+	err = result.Remove()
+	if err != nil {
+		t.Fatalf("%s; unable to remove: %v", path, err)
+	}
+	orgCount -= 1
+
+	fc := n.FileCount()
+	if fc != orgCount {
+		t.Fatalf("expected file count %d, got %d",
+			orgCount, fc)
+	}
+}
+
+func TestExists(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	path := filepath.Join("b", "d", "a")
+	if !n.Exists(path) {
+		t.Fatalf("expected existence at %s", path)
+	}
+	if !n.IsDir(path) {
+		t.Fatalf("expected directory at %s", path)
+	}
+}
+
+func TestRegExpGlob(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	expected := []string{
+		filepath.Join("b", "d", "a", "c", "i", "beans"),
+		filepath.Join("b", "d", "a", "c", "m"),
+		filepath.Join("b", "d", "a", "c", "u"),
+		filepath.Join("b", "d", "ignore", "c", "m"),
+		filepath.Join("b", "d", "ignore", "c", "n"),
+		filepath.Join("b", "d", "ignore", "c", "o"),
+		filepath.Join("b", "d", "x"),
+		filepath.Join("b", "d", "y"),
+		filepath.Join("b", "d", "z"),
+	}
+	paths, err := n.RegExpGlob("b/d/*")
+	if err != nil {
+		t.Fatalf("glob error: %v", err)
+	}
+	assertEqualStringSlices(t, expected, paths, "glob test")
+}
+
+func TestGlob(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	expected := []string{
+		filepath.Join("b", "d", "x"),
+		filepath.Join("b", "d", "y"),
+		filepath.Join("b", "d", "z"),
+	}
+	paths, err := n.Glob("b/d/*")
+	if err != nil {
+		t.Fatalf("glob error: %v", err)
+	}
+	assertEqualStringSlices(t, expected, paths, "glob test")
+}
+
+func assertEqualStringSlices(t *testing.T, expected, actual []string, message string) {
+	if len(expected) != len(actual) {
+		t.Fatalf(
+			"%s; unequal sizes; len(expected)=%d, len(actual)=%d\n%+v\n%+v\n",
+			message, len(expected), len(actual), expected, actual)
+	}
+	for i := range expected {
+		if expected[i] != actual[i] {
+			t.Fatalf(
+				"%s; unequal entries; expected=%s, actual=%s",
+				message, expected[i], actual[i])
+		}
+	}
+}
+
+func TestFind(t *testing.T) {
+	cases := []struct {
+		what       string
+		arg        string
+		expectDir  bool
+		expectFile bool
+		errStr     string
+	}{
+		{
+			what: "garbage",
+			arg:  "///1(*&SA",
+		},
+		{
+			what: "simple",
+			arg:  "bob",
+		},
+		{
+			what: "no directory",
+			arg:  filepath.Join("b", "rrrrrr"),
+		},
+		{
+			what:      "is a directory",
+			arg:       filepath.Join("b", "d", "ignore"),
+			expectDir: true,
+		},
+		{
+			what:       "longer, ending in file",
+			arg:        filepath.Join("b", "d", "x"),
+			expectFile: true,
+		},
+		{
+			what:       "moar longer, ending in file",
+			arg:        filepath.Join("b", "d", "a", "c", "u"),
+			expectFile: true,
+		},
+		{
+			what:      "directory",
+			arg:       filepath.Join("b"),
+			expectDir: true,
+		},
+		{
+			// Querying for the empty string could
+			// 1) be an error,
+			// 2) return no result (and no error) as with
+			//    any illegal and therefore non-existent
+			//    file name,
+			// 3) return the node itself, like running
+			//    'ls' with no argument.
+			// Going with option 2 (no result, no error),
+			// since at this low level it makes more sense
+			// if the results for the empty string query
+			// differ from the results for the "." query.
+			what: "empty name",
+			arg:  "",
+		},
+		{
+			what:      "self dir",
+			arg:       SelfDir,
+			expectDir: true,
+		},
+		{
+			what: "parent dir - doesn't exist",
+			arg:  ParentDir,
+		},
+		{
+			what: "many parents - doesn't exist",
+			arg:  filepath.Join(ParentDir, ParentDir, ParentDir),
+		},
+	}
+
+	n := makeLoadedFileTree(t)
+	for _, item := range cases {
+		result, err := n.Find(item.arg)
+		if item.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error", item.what)
+			}
+			if !strings.Contains(err.Error(), item.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					item.what, item.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", item.what, err)
+		}
+		if result == nil {
+			if item.expectDir {
+				t.Fatalf(
+					"%s; expected to find directory '%s'", item.what, item.arg)
+			}
+			if item.expectFile {
+				t.Fatalf(
+					"%s; expected to find file '%s'", item.what, item.arg)
+			}
+			continue
+		}
+		if item.expectDir {
+			if !result.isNodeADir() {
+				t.Fatalf(
+					"%s; expected '%s' to be a directory", item.what, item.arg)
+			}
+			continue
+		}
+		if item.expectFile {
+			if result.isNodeADir() {
+				t.Fatalf("%s; expected '%s' to be a file", item.what, item.arg)
+			}
+			continue
+		}
+		t.Fatalf(
+			"%s; expected nothing for '%s', but got '%s'",
+			item.what, item.arg, result.Path())
+	}
+}
+
+func TestCleanedAbs(t *testing.T) {
+	cases := []struct {
+		what   string
+		full   string
+		cDir   string
+		name   string
+		errStr string
+	}{
+		{
+			what:   "empty",
+			full:   "",
+			errStr: "doesn't exist",
+		},
+		{
+			what:   "simple",
+			full:   "bob",
+			errStr: "'bob' doesn't exist",
+		},
+		{
+			what:   "no directory",
+			full:   filepath.Join("b", "rrrrrr"),
+			errStr: "'b/rrrrrr' doesn't exist",
+		},
+		{
+			what: "longer, ending in file",
+			full: filepath.Join("b", "d", "x"),
+			cDir: filepath.Join("b", "d"),
+			name: "x",
+		},
+		{
+			what: "moar longer, ending in file",
+			full: filepath.Join("b", "d", "a", "c", "u"),
+			cDir: filepath.Join("b", "d", "a", "c"),
+			name: "u",
+		},
+		{
+			what: "directory",
+			full: filepath.Join("b", "d"),
+			cDir: filepath.Join("b", "d"),
+			name: "",
+		},
+	}
+
+	n := makeLoadedFileTree(t)
+	for _, item := range cases {
+		cDir, name, err := n.CleanedAbs(item.full)
+		if item.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error", item.what)
+			}
+			if !strings.Contains(err.Error(), item.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					item.what, item.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", item.what, err)
+		}
+		if cDir != ConfirmedDir(item.cDir) {
+			t.Fatalf("%s; expected cDir=%s, got '%s'", item.what, item.cDir, cDir)
+		}
+		if name != item.name {
+			t.Fatalf("%s; expected name=%s, got '%s'", item.what, item.name, name)
+		}
+	}
+}

api/filesys/rpath.go

@@ -1,12 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package filesys
-
-import "path/filepath"
-
-// RootedPath returns a rooted path, e.g. "/foo/bar" as
-// opposed to "foo/bar".
-func RootedPath(elem ...string) string {
-	return separator + filepath.Join(elem...)
-}

api/filesys/util.go

@@ -0,0 +1,30 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import "path/filepath"
+
+// RootedPath returns a rooted path, e.g. "/foo/bar" as
+// opposed to "foo/bar".
+func RootedPath(elem ...string) string {
+	return Separator + filepath.Join(elem...)
+}
+
+// StripTrailingSeps trims trailing filepath separators from input.
+func StripTrailingSeps(s string) string {
+	k := len(s)
+	for k > 0 && s[k-1] == filepath.Separator {
+		k--
+	}
+	return s[:k]
+}
+
+// StripLeadingSeps trims leading filepath separators from input.
+func StripLeadingSeps(s string) string {
+	k := 0
+	for k < len(s) && s[k] == filepath.Separator {
+		k++
+	}
+	return s[k:]
+}

api/filesys/util_test.go

@@ -0,0 +1,205 @@
+package filesys_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	. "sigs.k8s.io/kustomize/api/filesys"
+)
+
+// Confirm behavior of filepath.Match
+func TestFilePathMatch(t *testing.T) {
+	cases := []struct {
+		pattern  string
+		path     string
+		expected bool
+	}{
+		{
+			pattern:  "*e*",
+			path:     "hey",
+			expected: true,
+		},
+		{
+			pattern:  "*e*",
+			path:     "hay",
+			expected: false,
+		},
+		{
+			pattern:  "*e*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: false,
+		},
+		{
+			pattern:  "*/e/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "h/e/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/e/y",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/*/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/*/*",
+			path:     filepath.Join("h", "e", "y", "there"),
+			expected: false,
+		},
+		{
+			pattern:  "*/*/*/t*e",
+			path:     filepath.Join("h", "e", "y", "there"),
+			expected: true,
+		},
+	}
+	for _, item := range cases {
+		match, err := filepath.Match(item.pattern, item.path)
+		if err != nil {
+			t.Fatalf("unexpected err: %v", err)
+		}
+		if match != item.expected {
+			t.Fatalf("'%s' '%s' %v\n", item.pattern, item.path, match)
+		}
+	}
+}
+
+// Confirm behavior of filepath.Split
+func TestFilePathSplit(t *testing.T) {
+	cases := []struct {
+		full string
+		dir  string
+		file string
+	}{
+		{
+			full: "",
+			dir:  "",
+			file: "",
+		},
+		{
+			full: SelfDir,
+			dir:  "",
+			file: SelfDir,
+		},
+		{
+			full: "rabbit.jpg",
+			dir:  "",
+			file: "rabbit.jpg",
+		},
+		{
+			full: "/beans",
+			dir:  "/",
+			file: "beans",
+		},
+		{
+			full: "/home/foo/bar",
+			dir:  "/home/foo/",
+			file: "bar",
+		},
+		{
+			full: "/usr/local/",
+			dir:  "/usr/local/",
+			file: "",
+		},
+		{
+			full: "/usr//local//go",
+			dir:  "/usr//local//",
+			file: "go",
+		},
+	}
+	for _, p := range cases {
+		dir, file := filepath.Split(p.full)
+		if dir != p.dir || file != p.file {
+			t.Fatalf(
+				"in '%s',\ngot dir='%s' (expected '%s'),\n got file='%s' (expected %s).",
+				p.full, dir, p.dir, file, p.file)
+		}
+	}
+}
+
+func TestStripTrailingSeps(t *testing.T) {
+	cases := []struct {
+		full string
+		rem  string
+	}{
+		{
+			full: "foo",
+			rem:  "foo",
+		},
+		{
+			full: "",
+			rem:  "",
+		},
+		{
+			full: "foo/",
+			rem:  "foo",
+		},
+		{
+			full: "foo///bar///",
+			rem:  "foo///bar",
+		},
+		{
+			full: "/////",
+			rem:  "",
+		},
+		{
+			full: "/",
+			rem:  "",
+		},
+	}
+	for _, p := range cases {
+		dir := StripTrailingSeps(p.full)
+		if dir != p.rem {
+			t.Fatalf(
+				"in '%s', got dir='%s' (expected '%s')",
+				p.full, dir, p.rem)
+		}
+	}
+}
+
+func TestStripLeadingSeps(t *testing.T) {
+	cases := []struct {
+		full string
+		rem  string
+	}{
+		{
+			full: "foo",
+			rem:  "foo",
+		},
+		{
+			full: "",
+			rem:  "",
+		},
+		{
+			full: "/foo",
+			rem:  "foo",
+		},
+		{
+			full: "///foo///bar///",
+			rem:  "foo///bar///",
+		},
+		{
+			full: "/////",
+			rem:  "",
+		},
+		{
+			full: "/",
+			rem:  "",
+		},
+	}
+	for _, p := range cases {
+		dir := StripLeadingSeps(p.full)
+		if dir != p.rem {
+			t.Fatalf(
+				"in '%s', got dir='%s' (expected '%s')",
+				p.full, dir, p.rem)
+		}
+	}
+}

api/go.mod

@@ -6,12 +6,13 @@ require (
 	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/go-openapi/spec v0.19.4
 	github.com/golangci/golangci-lint v1.21.0
-	github.com/monopole/mdrip v1.0.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/pkg/errors v0.8.1
 	golang.org/x/tools v0.0.0-20191010075000-0337d82405ff
 	gopkg.in/yaml.v2 v2.2.4
+	k8s.io/api v0.17.0
+	k8s.io/apimachinery v0.17.0
+	k8s.io/client-go v0.17.0
 	k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a
-	sigs.k8s.io/kustomize/pluginator/v2 v2.0.0
-	sigs.k8s.io/kustomize/pseudo/k8s v0.1.0
 	sigs.k8s.io/yaml v1.1.0
 )

api/go.sum

@@ -1,14 +1,11 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -47,12 +44,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w=
-github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
@@ -112,13 +108,14 @@ github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b h1:ekuhfTjngPhisSjOJ0Q
 github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -136,18 +133,12 @@ github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgO
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.19.1 h1:g9xL8KW7UZDCkVlgHYJMA6F4Sj/sRVa0FoCeXI+Z3iM=
-github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps=
 github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI=
 github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
@@ -160,37 +151,33 @@ github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1
 github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
-github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0 h1:UykbtMB/w5No2LmE16gINgLj+r/vbziTgaoERQv6U+0=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a h1:YH0IojQwndMQdeRWdw1aPT8bkbiWaYR3WD+Zf5e09DU=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741 h1:OuuPl66BpF1q3OEkaPpp+VfzxrBBY62ATGdWqql/XX8=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
-github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -202,6 +189,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -229,12 +217,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86 h1:q6SrfsK4FojRnJ1j8+8OJzyq3g9Y1oSVyL6nYGJXXBk=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
 github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
 github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
@@ -253,8 +237,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.0 h1:RFDBa+tab6mW+gX4Ww2SZDc4kS6p01FwnLtgz64Il+I=
-github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -289,13 +271,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218 h1:O0yPHYL49quNL4Oj2wVq+zbGMu4dAM6iLoOQtm49TrQ=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
@@ -304,8 +281,6 @@ github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -338,8 +313,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -347,8 +320,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/ultraware/whitespace v0.0.3 h1:S5BCRRB5sttNy0bSOhbpw+0mb+cHiCmWfrvxpEzuUk0=
-github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
 github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs=
@@ -360,17 +331,22 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -381,21 +357,23 @@ golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -404,21 +382,22 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -427,24 +406,28 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs=
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -466,8 +449,15 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
+k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
+k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
+k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
+k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
@@ -475,20 +465,13 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4=
-sigs.k8s.io/kustomize/pluginator/v2 v2.0.0 h1:sES7e09G19Q0VjRp4ATSYKpTXoWaX8WMSHfw6u3G2Ok=
-sigs.k8s.io/kustomize/pluginator/v2 v2.0.0/go.mod h1:zrXhTv8BAKt0egmZX/8AtMOSFUSWM9YuoHvvqz8/eHE=
-sigs.k8s.io/kustomize/pseudo/k8s v0.0.0-20191108212413-1f86a0ca5d6c h1:t7fk+ljA3Ru4pro+/0RuOAZcODDhByL1fvIdyHLhjTY=
-sigs.k8s.io/kustomize/pseudo/k8s v0.0.0-20191108212413-1f86a0ca5d6c/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0 h1:otg4dLFc03c3gzl+2CV8GPGcd1kk8wjXwD+UhhcCn5I=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/internal/accumulator/loadconfigfromcrds.go

@@ -10,6 +10,7 @@ import (
 	"github.com/go-openapi/spec"
 	"github.com/pkg/errors"
 	"k8s.io/kube-openapi/pkg/common"
+	"sigs.k8s.io/kustomize/api/filesys"
 	"sigs.k8s.io/kustomize/api/ifc"
 	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
 	"sigs.k8s.io/kustomize/api/resid"
@@ -78,7 +79,7 @@ func makeConfigFromApiMap(m nameToApiMap) (*builtinconfig.TransformerConfig, err
 // openAPI definition once
 // "x-kubernetes-group-version-kind" is available in CRD
 func makeGvkFromTypeName(n string) resid.Gvk {
-	names := strings.Split(n, ".")
+	names := strings.Split(n, filesys.SelfDir)
 	kind := names[len(names)-1]
 	return resid.Gvk{Kind: kind}
 }

api/internal/accumulator/loadconfigfromcrds_test.go

@@ -7,9 +7,10 @@ import (
 	"reflect"
 	"testing"
 
-	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/loader"
+
 	. "sigs.k8s.io/kustomize/api/internal/accumulator"
-	"sigs.k8s.io/kustomize/api/internal/loadertest"
 	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/types"
@@ -41,7 +42,7 @@ More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#ty
 					"type": "string"
 				},
 				"metadata": {
-					"$ref": "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+					"$ref": "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 				},
 				"spec": {
 					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec"
@@ -54,7 +55,7 @@ More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#ty
 		"Dependencies": [
 			"github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec",
 			"github.com/example/pkg/apis/jingfang/v1beta1.BeeStatus",
-			"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+			"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 		]
 	},
 	"github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec": {
@@ -86,7 +87,7 @@ In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-con
 					"type": "string"
 				},
 				"metadata": {
-					"$ref": "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+					"$ref": "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 				},
 				"spec": {
 					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec"
@@ -99,7 +100,7 @@ In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-con
 		"Dependencies": [
 			"github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec",
 			"github.com/example/pkg/apis/jingfang/v1beta1.MyKindStatus",
-			"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+			"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 		]
 	},
 	"github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec": {
@@ -116,13 +117,13 @@ In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-con
 If it is not set we generate a secret dynamically",
 					"x-kubernetes-object-ref-api-version": "v1",
 					"x-kubernetes-object-ref-kind": "Secret",
-					"$ref": "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1.LocalObjectReference"
+					"$ref": "k8s.io/api/core/v1.LocalObjectReference"
 				}
 			}
 		},
 		"Dependencies": [
 			"github.com/example/pkg/apis/jingfang/v1beta1.Bee",
-			"sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1.LocalObjectReference"
+			"k8s.io/api/core/v1.LocalObjectReference"
 		]
 	},
 	"github.com/example/pkg/apis/jingfang/v1beta1.MyKindStatus": {
@@ -135,15 +136,6 @@ If it is not set we generate a secret dynamically",
 `
 )
 
-func makeLoader(t *testing.T) ifc.Loader {
-	ldr := loadertest.NewFakeLoader("/testpath")
-	err := ldr.AddFile("/testpath/crd.json", []byte(crdContent))
-	if err != nil {
-		t.Fatalf("Failed to setup fake ldr.")
-	}
-	return ldr
-}
-
 func TestLoadCRDs(t *testing.T) {
 	nbrs := []builtinconfig.NameBackReferences{
 		{
@@ -172,7 +164,14 @@ func TestLoadCRDs(t *testing.T) {
 		NameReference: nbrs,
 	}
 
-	actualTc, err := LoadConfigFromCRDs(makeLoader(t), []string{"crd.json"})
+	fSys := filesys.MakeFsInMemory()
+	fSys.WriteFile("/testpath/crd.json", []byte(crdContent))
+	ldr, err := loader.NewLoader(loader.RestrictionRootOnly, "/testpath", fSys)
+	if err != nil {
+		t.Fatalf("unexpected error:%v", err)
+	}
+
+	actualTc, err := LoadConfigFromCRDs(ldr, []string{"crd.json"})
 	if err != nil {
 		t.Fatalf("unexpected error:%v", err)
 	}

api/internal/crawl/cmd/crawler/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize//api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install -v ./cmd/crawler/crawler.go
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/crawler /
+ENTRYPOINT ["/crawler"]
+CMD []

api/internal/crawl/cmd/crawler/crawler.go

@@ -0,0 +1,207 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+const (
+	githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
+	redisCacheURL        = "REDIS_CACHE_URL"
+	redisKeyURL          = "REDIS_KEY_URL"
+	retryCount           = 3
+)
+
+type CrawlMode int
+const (
+	CrawlUnknown CrawlMode = iota
+	// Crawl all the kustomization files in all the repositories of a Github user
+	CrawlUser
+	// Crawl all the kustomization files in a Github repo
+	CrawlRepo
+	// Crawl all the documents in the index
+	CrawlIndex
+	// Crawl all the kustomization files on Github
+	CrawlGithub
+	// Crawl all the documents in the index and crawling all the kustomization files on Github
+	CrawlIndexAndGithub
+)
+
+func NewCrawlMode(s string) CrawlMode {
+	switch s {
+	case "github-user":
+		return CrawlUser
+	case "github-repo":
+		return CrawlRepo
+	case "":
+		return CrawlIndexAndGithub
+	case "index":
+		return CrawlIndex
+	case "github":
+		return CrawlGithub
+	default:
+		return CrawlUnknown
+	}
+}
+
+func Usage() {
+	fmt.Printf("Usage: %s [mode] [githubUser|githubRepo]\n", os.Args[0])
+	fmt.Printf("\tmode can be one of [github-user, github-repo, index, github]\n")
+	fmt.Printf("%s: crawl all the documents in the index and crawling all the kustomization files on Github\n", os.Args[0])
+	fmt.Printf("%s index: crawl all the documents in the index\n", os.Args[0])
+	fmt.Printf("%s gihub: crawl all the kustomization files on Github\n", os.Args[0])
+	fmt.Printf("%s github-user <github-user>: Crawl all the kustomization files in all the repositories of a Github user\n", os.Args[0])
+	fmt.Printf("\tFor example, %s github-user kubernetes-sigs\n", os.Args[0])
+	fmt.Printf("%s github-repo <github-repo>: Crawl all the kustomization files in a Github repo\n", os.Args[0])
+	fmt.Printf("\tFor example, %s github-repo kubernetes-sigs/kustomize\n", os.Args[0])
+}
+
+func main() {
+	githubToken := os.Getenv(githubAccessTokenVar)
+	if githubToken == "" {
+		fmt.Printf("Must set the variable '%s' to make github requests.\n",
+			githubAccessTokenVar)
+		return
+	}
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx)
+	if err != nil {
+		fmt.Printf("Could not create an index: %v\n", err)
+		return
+	}
+
+	cacheURL := os.Getenv(redisCacheURL)
+	cache, err := redis.DialURL(cacheURL)
+	clientCache := &http.Client{}
+	if err != nil {
+		fmt.Printf("Error: redis could not make a connection: %v\n", err)
+	} else {
+		clientCache = httpclient.NewClient(cache)
+	}
+
+	// docConverter takes in a plain document and processes it for the index.
+	docConverter := func(d *doc.Document) (crawler.CrawledDocument, error) {
+		kdoc := doc.KustomizationDocument{
+			Document: *d,
+		}
+
+		err := kdoc.ParseYAML()
+		return &kdoc, err
+	}
+
+	// Index updates the value in the index.
+	indexFunc := func(cdoc crawler.CrawledDocument, mode index.Mode) error {
+		switch d := cdoc.(type) {
+		case *doc.KustomizationDocument:
+			switch mode {
+			case index.Delete:
+				fmt.Println("Deleting: ", d)
+				return idx.Delete(d.ID())
+			default:
+				fmt.Println("Inserting: ", d)
+				return idx.Put(d.ID(), d)
+			}
+		default:
+			return fmt.Errorf("type %T not supported", d)
+		}
+	}
+
+	// seen tracks the IDs of all the documents in the index.
+	// This helps avoid indexing a given document multiple times.
+	seen := make(map[string]struct{})
+
+	var mode CrawlMode
+	if len(os.Args) == 1 {
+		mode = CrawlIndexAndGithub
+	} else {
+		mode = NewCrawlMode(os.Args[1])
+	}
+
+	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
+		if user != "" {
+			return 	github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.User(user)),
+			)
+		} else if repo != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.Repo(repo)),
+			)
+		} else {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization")),
+			)
+		}
+	}
+
+	seedDocs := make(crawler.CrawlSeed, 0)
+
+	// get all the documents in the index
+	getSeedDocsFunc := func() {
+		query := []byte(`{ "query":{ "match_all":{} } }`)
+		it := idx.IterateQuery(query, 10000, 60*time.Second)
+		for it.Next() {
+			for _, hit := range it.Value().Hits.Hits {
+				seedDocs = append(seedDocs, hit.Document.Copy())
+			}
+		}
+		if err := it.Err(); err != nil {
+			fmt.Printf("Error iterating: %v\n", err)
+		}
+	}
+
+	switch mode {
+	case CrawlIndexAndGithub:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlIndex:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+	case CrawlGithub:
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUser:
+		if len(os.Args) < 3 {
+			Usage()
+			log.Fatalf("Please specify a github user!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(os.Args[2], "")}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlRepo:
+		if len(os.Args) < 3 {
+			Usage()
+			log.Fatalf("Please specify a github repo!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", os.Args[2])}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUnknown:
+		Usage()
+		log.Fatalf("The crawler mode must be one of [github-user, github-repo, index, github]")
+	}
+}

api/internal/crawl/cmd/kustomize_stats/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install ./cmd/kustomize_stats
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/kustomize_stats /
+ENTRYPOINT ["/kustomize_stats"]

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -0,0 +1,150 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+	"sigs.k8s.io/kustomize/api/konfig"
+)
+
+// iterateArr adds each item in arr into countMap.
+func iterateArr(arr []string, countMap map[string]int) {
+	for _, item := range arr {
+		if _, ok := countMap[item]; !ok {
+			countMap[item] = 1
+		} else {
+			countMap[item]++
+		}
+	}
+
+}
+
+// isKustomizationFile determines whether a file path is a kustomization file
+func isKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
+		}
+	}
+	return false
+}
+
+// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValue(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool {return m[keys[i]] > m[keys[j]]})
+	return keys
+}
+
+func main() {
+	topKindsPtr := flag.Int(
+		"kinds", -1,
+		`the number of kubernetes object kinds to be listed according to their popularities.
+By default, all the kinds will be listed.
+If you only want to list the 10 most popular kinds, set the flag to 10.`)
+	topIdentifiersPtr := flag.Int(
+		"identifiers", -1,
+		`the number of identifiers to be listed according to their popularities.
+By default, all the identifiers will be listed.
+If you only want to list the 10 most popular identifiers, set the flag to 10.`)
+	topKustomizeFeaturesPtr := flag.Int(
+		"kustomize-features", -1,
+		`the number of kustomize features to be listed according to their popularities.
+By default, all the features will be listed.
+If you only want to list the 10 most popular features, set the flag to 10.`)
+	flag.Parse()
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx)
+	if err != nil {
+		log.Fatalf("Could not create an index: %v\n", err)
+	}
+
+	// count tracks the number of documents in the index
+	count := 0
+
+	// kustomizationFilecount tracks the number of kustomization files in the index
+	kustomizationFilecount := 0
+
+	kindsMap := make(map[string]int)
+	identifiersMap := make(map[string]int)
+	kustomizeIdentifiersMap := make(map[string]int)
+
+	// ids tracks the unique IDs of the documents in the index
+	ids := make(map[string]struct{})
+
+	// get all the documents in the index
+	query := []byte(`{ "query":{ "match_all":{} } }`)
+	it := idx.IterateQuery(query, 10000, 60*time.Second)
+	for it.Next() {
+		for _, hit := range it.Value().Hits.Hits {
+			// check whether there is any duplicate IDs in the index
+			if _, ok := ids[hit.ID]; !ok {
+				ids[hit.ID] = struct{}{}
+			} else {
+				fmt.Printf("Found duplicate ID (%s)\n", hit.ID)
+			}
+
+			count++
+			iterateArr(hit.Document.Kinds, kindsMap)
+			iterateArr(hit.Document.Identifiers, identifiersMap)
+
+			if isKustomizationFile(hit.Document.FilePath) {
+				kustomizationFilecount++
+				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
+			}
+		}
+	}
+	if err := it.Err(); err != nil {
+		fmt.Printf("Error iterating: %v\n", err)
+	}
+
+	sortedKindsMapKeys := SortMapKeyByValue(kindsMap)
+	sortedIdentifiersMapKeys := SortMapKeyByValue(identifiersMap)
+	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValue(kustomizeIdentifiersMap)
+
+	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
+There are %d documents in the kustomize index.
+%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
+	fmt.Printf("\n")
+
+	kindCount := 0
+	for _, key := range sortedKindsMapKeys {
+		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
+			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
+			kindCount++
+		}
+	}
+
+	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
+	identifierCount := 0
+	for _, key := range sortedIdentifiersMapKeys {
+		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
+			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
+			identifierCount++
+		}
+	}
+
+	fmt.Printf(`There are %d kustomization files in the kustomize index.
+%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
+	fmt.Printf("\n")
+	kustomizeFeatureCount := 0
+	for _, key := range sortedKustomizeIdentifiersMapKeys {
+		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
+			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
+			kustomizeFeatureCount++
+		}
+	}
+}

api/internal/crawl/cmd/log-parser/README.md

@@ -0,0 +1,8 @@
+This binary takes as its input a json file including GKE logs (which can be
+[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into 
+[Cloud Storage](https://cloud.google.com/storage/docs/)), 
+and extracts the `textPayload` field of each log entry.
+
+Here is an log entry example:
+
+{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"}

api/internal/crawl/cmd/log-parser/main.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatalf("The usage of the command is: \n\t%s <log-file.json>", os.Args[0])
+	}
+
+	file, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	closeFile := func(file *os.File) {
+		if err := file.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}
+	defer closeFile(file)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		var entry interface{}
+		if err := json.Unmarshal([]byte(line), &entry); err != nil {
+			log.Printf("failed to unmarshal a log entry: %s\n", line)
+		}
+
+		m := entry.(map[string]interface{})
+		if payload, ok := m["textPayload"]; ok {
+			fmt.Printf("%s", payload)
+		} else {
+			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}

api/internal/crawl/config/crawler/base/github_api_secret.txt

@@ -0,0 +1,2 @@
+<ADD YOUR GITHUB PERSONAL ACCESS TOKEN HERE WITHOUT A TRAILING NEWLINE>
+Run: printf "<your-token>" > github_api_secret.txt

api/internal/crawl/config/crawler/base/kustomization.yaml

@@ -5,7 +5,9 @@ configmapGenerator:
 - name: crawler-http-cache
   literals:
   - redis-cache-url="redis://redis-http-cache:6379"
-
+- name: redis-keystore
+  literals:
+  - keystore-url="redis://redis-docs-keystore:6379"
 
 secretGenerator:
 - name: github-access-token

api/internal/crawl/config/crawler/cronjob/cronjob.yaml

@@ -12,6 +12,7 @@ spec:
           containers:
           - name: crawler
             image: gcr.io/kustomize-search/crawler:latest
+            imagePullPolicy: Always
             env:
             - name: GITHUB_ACCESS_TOKEN
               valueFrom:

api/internal/crawl/config/crawler/job/README.md

@@ -0,0 +1,54 @@
+There are three ways of running the crawler job.
+
+# Crawling all the documents in the index and crawling all the kustomization files on Github
+
+This is the default setting of the crawler job. The `command` and `args` field
+of the container should be:
+
+```
+        command: ["/crawler"]
+        args: []
+```
+
+Or
+
+```
+        command: ["/crawler"]
+        args: [""]
+```
+
+# Crawling all the documents in the index
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["index"]
+```
+
+# Crawling all the kustomization files on Github
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["github"]
+```
+
+# Crawling all the kustomization files in a Github repo
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["github-repo", "kubernetes-sigs/kustomize"]
+```
+
+# Crawling all the kustomization files in all the repositories of a Github user
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["github-user", "kubernetes-sigs"]
+```

api/internal/crawl/config/crawler/job/job.yaml

@@ -8,7 +8,10 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: crawler
-        image: gcr.io/kustomize-search/crawler:latest
+        image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+        imagePullPolicy: Always
+        command: ["/crawler"]
+        args: ["github-repo", "kubernetes-sigs/kustomize"]
         env:
         - name: GITHUB_ACCESS_TOKEN
           valueFrom:

api/internal/crawl/config/crawler/kustomize_stats/job.yaml

@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kustomize-stats
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: kustomize-stats
+        image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
+        imagePullPolicy: Always
+        command: ["/kustomize_stats"]
+        args: ["--kinds=50", "--identifiers=50", "--kustomize-features=50"]
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url

api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml

api/internal/crawl/config/elastic/escluster.yaml

@@ -0,0 +1,43 @@
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESCluster
+metadata:
+  name: esbasic
+spec:
+  plugin:
+   pluginList:
+    - repository-gcs
+    - ingest-user-agent
+    - ingest-geoip
+  config:
+    env:
+      example: test
+  nodegroups:
+    - name: di
+      replicas: 2
+      data: true
+      ingest: true
+      config:
+        jvm:
+          - Djava.net.preferIPv4Stack=true
+          - Xms2g
+          - Xmx2g
+        es:
+          path.repo: '["/tmp/es_backup_basic"]'
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                es/nodegroup: di
+      resources:
+        requests:
+          memory: 3Gi
+        limits:
+          memory: 3Gi
+    - name: m
+      replicas: 2
+      master: true
+      config:
+        es:
+          path.repo: '["/tmp/es_backup_basic"]'

api/internal/crawl/config/elastic/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+spec:
+  selector:
+    custom-resource: v1alpha1.ESCluster
+    custom-resource-name: esbasic
+    custom-resource-namespace: default
+    es/data: "true"
+    using: escluster.Cluster
+  ports:
+  - protocol: "TCP"
+    port: 9200
+  type: LoadBalancer
+  loadBalancerIP: ""

api/internal/crawl/config/webapp/backend/deployment.yaml

@@ -17,14 +17,11 @@ spec:
       containers:
       - name: kustomize-search
         image: gcr.io/kustomize-search/backend:latest
+        imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /liveness
             port: backend-port
-        readinessProbe:
-          httpGet:
-            path: /readiness
-            port: backend-port
         ports:
         - name: backend-port
           containerPort: 8080

api/internal/crawl/config/webapp/frontend/deployment.yaml

@@ -17,6 +17,7 @@ spec:
       containers:
       - name: frontend
         image: gcr.io/kustomize-search/frontend:latest
+        imagePullPolicy: Always
         ports:
         - name: frontend-port
           containerPort: 80

api/internal/crawl/crawler/crawler.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"sync"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
 	_ "github.com/gomodule/redigo/redis"
 
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -40,142 +42,165 @@ type Crawler interface {
 type CrawledDocument interface {
 	ID() string
 	GetDocument() *doc.Document
+	// Get all the Documents directly referred in a Document.
 	GetResources() ([]*doc.Document, error)
 	WasCached() bool
 }
 
 type CrawlSeed []*doc.Document
 
-type IndexFunc func(CrawledDocument, Crawler) error
+type IndexFunc func(CrawledDocument, index.Mode) error
 type Converter func(*doc.Document) (CrawledDocument, error)
 
-// Cleaner, more efficient, and more extensible crawler implementation.
-// The seed must include the ids of each document in the index.
-func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
-	crawlers []Crawler, conv Converter, indx IndexFunc) {
-
-	seen := make(map[string]struct{})
-
-	logIfErr := func(err error) {
-		if err == nil {
-			return
-		}
-		logger.Println("error: ", err)
+func logIfErr(err error) {
+	if err == nil {
+		return
 	}
-
-	stack := make(CrawlSeed, 0)
-
-	findMatch := func(d *doc.Document) Crawler {
-		for _, crawl := range crawlers {
-			if crawl.Match(d) {
-				return crawl
-			}
-		}
-
-		return nil
-	}
-
-	addBranches := func(cdoc CrawledDocument, match Crawler) {
-		if _, ok := seen[cdoc.ID()]; ok {
-			return
-		}
-
-		seen[cdoc.ID()] = struct{}{}
-		// Insert into index
-		err := indx(cdoc, match)
-		logIfErr(err)
-		if err != nil {
-			return
-		}
-
-		deps, err := cdoc.GetResources()
-		logIfErr(err)
-		if err != nil {
-			return
-		}
-		for _, dep := range deps {
-			if _, ok := seen[dep.ID()]; ok {
-				continue
-			}
-			stack = append(stack, dep)
-		}
-	}
-
-	doCrawl := func(docsPtr *CrawlSeed) {
-		for len(*docsPtr) > 0 {
-			back := len(*docsPtr) - 1
-			next := (*docsPtr)[back]
-			*docsPtr = (*docsPtr)[:back]
-
-			match := findMatch(next)
-			if match == nil {
-				logIfErr(fmt.Errorf(
-					"%v could not match any crawler", next))
-				continue
-			}
-
-			err := match.FetchDocument(ctx, next)
-			logIfErr(err)
-			// If there was no change or there is an error, we don't have
-			// to branch out, since the dependencies are already in the
-			// index, or we cannot find the document.
-			if err != nil || next.WasCached() {
-				continue
-			}
-
-			cdoc, err := conv(next)
-			logIfErr(err)
-			if err != nil {
-				continue
-			}
-
-			addBranches(cdoc, match)
-		}
-	}
-	// Exploit seed to update bulk of corpus.
-	logger.Printf("updating %d documents from seed\n", len(seed))
-	doCrawl(&seed)
-	// Traverse any new links added while updating corpus.
-	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
-	doCrawl(&stack)
-
-	ch := make(chan CrawledDocument, 1<<10)
-	wg := sync.WaitGroup{}
-
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for cdoc := range ch {
-			if _, ok := seen[cdoc.ID()]; ok {
-				continue
-			}
-			match := findMatch(cdoc.GetDocument())
-			if match == nil {
-				logIfErr(fmt.Errorf(
-					"%v could not match any crawler", cdoc))
-				continue
-			}
-			addBranches(cdoc, match)
-		}
-	}()
-
-	// Exploration through APIs.
-	errs := CRunner(ctx, ch, crawlers)
-	if errs != nil {
-		for _, err := range errs {
-			logIfErr(err)
-		}
-	}
-	close(ch)
-	logger.Println("Processing the new documents from the crawlers' exploration.")
-	wg.Wait()
-	// Handle deps of newly discovered documents.
-	logger.Printf("crawling the %d new documents from the crawlers' exploration.",
-		len(stack))
-	doCrawl(&stack)
+	logger.Println("error: ", err)
 }
 
-// CRunner is a blocking function and only returns once all of the
+func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
+	for _, crawl := range crawlers {
+		if crawl.Match(d) {
+			return crawl
+		}
+	}
+	return nil
+}
+
+func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
+	seen map[string]struct{}, stack *CrawlSeed) {
+
+	seen[cdoc.ID()] = struct{}{}
+
+	// Insert into index
+	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
+		logger.Printf("Failed to insert or update %s %s: %v",
+			cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err)
+		return
+	}
+
+	deps, err := cdoc.GetResources()
+	if err != nil {
+		logger.Println(err)
+		return
+	}
+
+	for _, dep := range deps {
+		if _, ok := seen[dep.ID()]; ok {
+			continue
+		}
+		*stack = append(*stack, dep)
+	}
+}
+
+func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
+	seen map[string]struct{}, stack *CrawlSeed) {
+
+	UpdatedDocCount := 0
+	seenDocCount := 0
+	cachedDocCount := 0
+	findMatchErrCount := 0
+	FetchDocumentErrCount := 0
+	SetCreatedErrCount := 0
+	convErrCount := 0
+	deleteDocCount := 0
+
+	// During the execution of the for loop, more Documents may be added into (*docsPtr).
+	for len(*docsPtr) > 0 {
+		// get the last Document in (*docPtr), which will be crawled in this iteration.
+		tail := (*docsPtr)[len(*docsPtr)-1]
+
+		// remove the last Document in (*docPtr)
+		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
+
+		if _, ok := seen[tail.ID()]; ok {
+			seenDocCount++
+			continue
+		}
+
+		if tail.WasCached() {
+			logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath)
+			cachedDocCount++
+			continue
+		}
+
+		match := findMatch(tail, crawlers)
+		if match == nil {
+			logIfErr(fmt.Errorf("%v could not match any crawler", tail))
+			findMatchErrCount++
+			continue
+		}
+
+		logger.Println("Crawling ", tail.RepositoryURL, tail.FilePath)
+		if err := match.FetchDocument(ctx, tail); err != nil {
+			logger.Printf("FetchDocument failed on %s %s: %v",
+				tail.RepositoryURL, tail.FilePath, err)
+			FetchDocumentErrCount++
+			// delete the document from the index
+			cdoc := &doc.KustomizationDocument{
+				Document: *tail,
+			}
+			seen[cdoc.ID()] = struct{}{}
+			if err := indx(cdoc, index.Delete); err != nil {
+				logger.Printf("Failed to delete %s %s: %v",
+					cdoc.RepositoryURL, cdoc.FilePath, err)
+			}
+			deleteDocCount++
+			continue
+		}
+
+		if err := match.SetCreated(ctx, tail); err != nil {
+			logger.Printf("SetCreated failed on %s %s: %v",
+				tail.RepositoryURL, tail.FilePath, err)
+			SetCreatedErrCount++
+		}
+
+		cdoc, err := conv(tail)
+		// If conv returns an error, cdoc can still be added into the index so that
+		// cdoc.Document can be searched.
+		if err != nil {
+			logger.Printf("conv failed on %s %s: %v",
+				tail.RepositoryURL, tail.FilePath, err)
+			convErrCount++
+		}
+
+		UpdatedDocCount++
+		addBranches(cdoc, match, indx, seen, stack)
+	}
+	logger.Printf("Summary of doCrawl:\n")
+	logger.Printf("\t%d documents were updated\n", UpdatedDocCount)
+	logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount)
+	logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount)
+	logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount)
+	logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n",
+		FetchDocumentErrCount, deleteDocCount)
+	logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount)
+	logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount)
+}
+
+// CrawlFromSeed updates all the documents in seed, and crawls all the new
+// documents referred in the seed.
+func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
+	conv Converter, indx IndexFunc, seen map[string]struct{}) {
+
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// Exploit seed to update bulk of corpus.
+	logger.Printf("updating %d documents from seed\n", len(seed))
+	// each unique document in seed will be crawled once.
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack)
+
+	// Traverse any new documents added while updating corpus.
+	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
+	// While crawling each document in stack, the documents directly referred in the document
+	// will be added into stack.
+	// After this statement is done, stack will become empty.
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
+}
+
+// CrawlGithubRunner is a blocking function and only returns once all of the
 // crawlers are finished with execution.
 //
 // This function uses the output channel to forward kustomization documents
@@ -186,14 +211,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
 // index of the crawler that emitted the error. Although the errors themselves
 // can be nil, the array will always be exactly the size of the crawlers array.
 //
-// CRunner takes in a seed, which represents the documents stored in an
+// CrawlGithubRunner takes in a seed, which represents the documents stored in an
 // index somewhere. The document data is not required to be populated. If there
 // are many documents, this is preferable. The order of iteration over the seed
-// is not guaranteed, but the CRunner does guarantee that every element
+// is not guaranteed, but the CrawlGithub does guarantee that every element
 // from the seed will be processed before any other documents from the
 // crawlers.
-func CRunner(ctx context.Context,
-	output chan<- CrawledDocument, crawlers []Crawler) []error {
+func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
+	crawlers []Crawler) []error {
 
 	errs := make([]error, len(crawlers))
 	wg := sync.WaitGroup{}
@@ -234,3 +259,46 @@ func CRunner(ctx context.Context,
 	wg.Wait()
 	return errs
 }
+
+// CrawlGithub crawls all the kustomization files on Github.
+func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
+	indx IndexFunc, seen map[string]struct{}) {
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// ch is channel where all the crawlers sends the crawled documents to.
+	ch := make(chan CrawledDocument, 1<<10)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for cdoc := range ch {
+			if _, ok := seen[cdoc.ID()]; ok {
+				continue
+			}
+			match := findMatch(cdoc.GetDocument(), crawlers)
+			if match == nil {
+				logIfErr(fmt.Errorf(
+					"%v could not match any crawler", cdoc))
+				continue
+			}
+			addBranches(cdoc, match, indx, seen, &stack)
+		}
+	}()
+
+	logger.Println("processing the documents found from crawling github")
+	if errs := CrawlGithubRunner(ctx, ch, crawlers); errs != nil {
+		for _, err := range errs {
+			logIfErr(err)
+		}
+	}
+	close(ch)
+	wg.Wait()
+
+	// Handle deps of newly discovered documents.
+	logger.Printf("crawling the %d new documents referred by other documents",
+		len(stack))
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
+}

api/internal/crawl/crawler/crawler_test.go

@@ -11,8 +11,10 @@ import (
 	"testing"
 	"time"
 
-	"sigs.k8s.io/kustomize/api/konfig"
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/konfig"
 )
 
 const (
@@ -29,21 +31,22 @@ type testCrawler struct {
 }
 
 func (c testCrawler) Match(d *doc.Document) bool {
-	return d != nil && strings.HasPrefix(d.ID(), c.matchPrefix)
+	return d != nil
 }
 
-func (c testCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
+func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	if i, ok := c.lukp[d.ID()]; ok {
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
 	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
-		fmt.Println(d.ID(), "/", suffix)
-		i, ok := c.lukp[d.ID()+"/"+suffix]
+		savedFilePath := d.FilePath
+		d.FilePath += "/" + suffix
+		i, ok := c.lukp[d.ID()]
 		if !ok {
+			d.FilePath = savedFilePath
 			continue
 		}
-		d.FilePath += "/" + suffix
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
@@ -51,7 +54,7 @@ func (c testCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
 		d, c.matchPrefix)
 }
 
-func (c testCrawler) SetCreated(ctx context.Context, d *doc.Document) error {
+func (c testCrawler) SetCreated(_ context.Context, d *doc.Document) error {
 	d.CreationTime = &time.Time{}
 	return nil
 }
@@ -71,7 +74,7 @@ func newCrawler(matchPrefix string, err error,
 }
 
 // Crawl implements the Crawler interface for testing.
-func (c testCrawler) Crawl(ctx context.Context,
+func (c testCrawler) Crawl(_ context.Context,
 	output chan<- CrawledDocument) error {
 
 	for i, d := range c.docs {
@@ -106,8 +109,8 @@ func (s sortableDocs) Len() int {
 	return len(s)
 }
 
-func TestCrawlerRunner(t *testing.T) {
-	fmt.Println("testing CRunner")
+func TestCrawlGithubRunner(t *testing.T) {
+	fmt.Println("testing CrawlGithubRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -178,7 +181,7 @@ func TestCrawlerRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()
 
-			errs := CRunner(context.Background(),
+			errs := CrawlGithubRunner(context.Background(),
 				output, test.tc)
 
 			// Check that errors are returned as they should be.
@@ -302,29 +305,6 @@ resources:
 					RepositoryURL: kustomizeRepo,
 					FilePath:      "examples/seedcrawl2/job.yaml",
 				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/base/kustomization.yaml",
-					DocumentData: `
-resources:
-- ../app
-`,
-				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/kustomization.yaml",
-					DocumentData: `
-resources:
-- resource.yaml
-`,
-				}},
-				// Visited from crawling runner imported as resource.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/resource.yaml",
-				}},
 			},
 		},
 	}
@@ -338,10 +318,11 @@ resources:
 					Document: *d,
 				}, nil
 			},
-			func(d CrawledDocument, cr Crawler) error {
+			func(d CrawledDocument, mode index.Mode) error {
 				visited[d.ID()]++
 				return nil
 			},
+			make(map[string]struct{}),
 		)
 		if lv, lc := len(visited), len(tc.corpus); lv != lc {
 			t.Errorf("error: %d of %d documents visited.", lv, lc)

api/internal/crawl/crawler/github/crawler.go

@@ -16,11 +16,11 @@ import (
 	"strings"
 	"time"
 
-	"sigs.k8s.io/kustomize/api/internal/git"
-	"sigs.k8s.io/kustomize/api/konfig"
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
 	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
+	"sigs.k8s.io/kustomize/api/internal/git"
+	"sigs.k8s.io/kustomize/api/konfig"
 )
 
 var logger = log.New(os.Stdout, "Github Crawler: ",
@@ -34,11 +34,11 @@ type githubCrawler struct {
 
 type GhClient struct {
 	RequestConfig
-	retryCount uint64
-	client     *http.Client
+	retryCount  uint64
+	client      *http.Client
+	accessToken string
 }
 
-/*
 func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 	query Query) githubCrawler {
 
@@ -47,14 +47,13 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 			retryCount: retryCount,
 			client:     client,
 			RequestConfig: RequestConfig{
-				perPage:     githubMaxPageSize,
-				accessToken: accessToken,
+				perPage: githubMaxPageSize,
 			},
+			accessToken: accessToken,
 		},
 		query: query,
 	}
 }
-*/
 
 // Implements crawler.Crawler.
 func (gc githubCrawler) Crawl(
@@ -64,6 +63,7 @@ func (gc githubCrawler) Crawl(
 		RequestConfig: gc.client.RequestConfig,
 		client:        &http.Client{Timeout: gc.client.client.Timeout},
 		retryCount:    gc.client.retryCount,
+		accessToken:   gc.client.accessToken,
 	}
 
 	// Since Github returns a max of 1000 results per query, we can use
@@ -93,7 +93,21 @@ func (gc githubCrawler) Crawl(
 	return nil
 }
 
-func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
+// FetchDocument first tries to fetch the document with d.FilePath. If it fails,
+// it will try to add each string in konfig.RecognizedKustomizationFileNames() to
+// d.FilePath, and try to fetch the document again.
+func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
+	// set the default branch if it is empty
+	if d.DefaultBranch == "" {
+		url := gc.client.ReposRequest(d.RepositoryFullName())
+		defaultBranch, err := gc.client.GetDefaultBranch(url)
+		if err != nil {
+			logger.Printf(
+				"(error: %v) setting default_branch to master\n", err)
+			defaultBranch = "master"
+		}
+		d.DefaultBranch = defaultBranch
+	}
 	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
 	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
 	if err != nil {
@@ -104,9 +118,13 @@ func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) erro
 		"/" + repoSpec.Ref + "/" + repoSpec.Path
 
 	handle := func(resp *http.Response, err error, path string) error {
+		if resp == nil {
+			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
+				url, path, err)
+		}
 		if err == nil && resp.StatusCode == http.StatusOK {
 			d.IsSame = httpclient.FromCache(resp.Header)
-			defer resp.Body.Close()
+			defer CloseResponseBody(resp)
 			data, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				return err
@@ -115,26 +133,30 @@ func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) erro
 			d.FilePath = d.FilePath + path
 			return nil
 		}
+
 		return err
 	}
-	resp, err := gc.client.GetRawUserContent(url)
-	if err := handle(resp, err, ""); err == nil {
+	resp, errGetRawUserContent := gc.client.GetRawUserContent(url)
+	if err := handle(resp, errGetRawUserContent, ""); err == nil {
 		return nil
 	}
 
 	for _, file := range konfig.RecognizedKustomizationFileNames() {
-		resp, err = gc.client.GetRawUserContent(url + "/" + file)
-		err := handle(resp, err, "/"+file)
-		if err != nil {
-			continue
+		resp, errGetRawUserContent = gc.client.GetRawUserContent(url + "/" + file)
+		if err = handle(resp, errGetRawUserContent, "/"+file); err == nil {
+			return nil
 		}
 	}
-	return fmt.Errorf("file not found: %s", url)
+	return fmt.Errorf("file not found: %s, error: %v", url, err)
 }
 
-func (gc githubCrawler) SetCreated(ctx context.Context, d *doc.Document) error {
-	fs := GhFileSpec{}
-	fs.Repository.FullName = d.RepositoryURL + "/" + d.FilePath
+func (gc githubCrawler) SetCreated(_ context.Context, d *doc.Document) error {
+	fs := GhFileSpec{
+		Path: d.FilePath,
+		Repository: GitRepository{
+			FullName: d.RepositoryFullName(),
+		},
+	}
 	creationTime, err := gc.client.GetFileCreationTime(fs)
 	if err != nil {
 		return err
@@ -185,11 +207,13 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
 		for _, file := range page.Parsed.Items {
 			k, err := kustomizationResultAdapter(gcl, file)
 			if err != nil {
+				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
 				errorCnt++
-				continue
 			}
-			output <- k
+			if k != nil {
+				output <- k
+			}
 			totalCnt++
 		}
 
@@ -224,6 +248,17 @@ func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
 			RepositoryURL: k.Repository.URL,
 		},
 	}
+	creationTime, err := gcl.GetFileCreationTime(k)
+	if err != nil {
+		logger.Printf("GetFileCreationTime failed: %v", err)
+		return &d, err
+	}
+	d.CreationTime = &creationTime
+
+	if err := d.ParseYAML(); err != nil {
+		logger.Printf("ParseYAML failed: %v", err)
+		return &d, err
+	}
 
 	return &d, nil
 }
@@ -276,7 +311,10 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
 			k, url, err)
 	}
-	resp.Body.Close()
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}
 
 	type githubContentRawURL struct {
 		DownloadURL string `json:"download_url,omitempty"`
@@ -295,18 +333,24 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 			k, rawURL.DownloadURL, err)
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err = ioutil.ReadAll(resp.Body)
 	return data, err
 }
 
+func CloseResponseBody(resp *http.Response) {
+	if err := resp.Body.Close(); err != nil {
+		log.Printf("failed to close response body: %v", err)
+	}
+}
+
 func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
 			"'%s' could not get default_branch: %v", url, err)
 	}
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -358,7 +402,7 @@ func (gcl GhClient) GetFileCreationTime(
 		}
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return defaultTime, fmt.Errorf(
@@ -410,13 +454,15 @@ func (e multiError) Error() string {
 	return strings.Join(strs, "\n")
 }
 
+type GitRepository struct {
+	API      string `json:"url,omitempty"`
+	URL      string `json:"html_url,omitempty"`
+	FullName string `json:"full_name,omitempty"`
+}
+
 type GhFileSpec struct {
-	Path       string `json:"path,omitempty"`
-	Repository struct {
-		API      string `json:"url,omitempty"`
-		URL      string `json:"html_url,omitempty"`
-		FullName string `json:"full_name,omitempty"`
-	} `json:"repository,omitempty"`
+	Path       string        `json:"path,omitempty"`
+	Repository GitRepository `json:"repository,omitempty"`
 }
 
 type githubResponse struct {
@@ -479,7 +525,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	}
 
 	var data []byte
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
 	if requestInfo.Error != nil {
 		return requestInfo
@@ -513,7 +559,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 }
 
 // SearchGithubAPI performs a search query and handles rate limitting for
-// the 'code/search?' endpoint as well as timed retries in the case of abuse
+// the 'search/code?' endpoint as well as timed retries in the case of abuse
 // prevention.
 func (gcl GhClient) SearchGithubAPI(query string) (*http.Response, error) {
 	throttleSearchAPI()
@@ -534,19 +580,34 @@ func (gcl GhClient) GetRawUserContent(query string) (*http.Response, error) {
 	return gcl.getWithRetry(query)
 }
 
+func (gcl GhClient) Do(query string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", query, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("token %s", gcl.accessToken))
+
+	// gcl.client.Do: a non-2xx status code doesn't cause an error.
+	// See https://golang.org/pkg/net/http/#Client.Do for more info.
+	resp, err :=  gcl.client.Do(req)
+	if resp != nil && resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
+			query, resp.StatusCode)
+	}
+	return resp, err
+}
+
 func (gcl GhClient) getWithRetry(
 	query string) (resp *http.Response, err error) {
 
-	resp, err = gcl.client.Get(query)
+	resp, err = gcl.Do(query)
+
 	retryCount := gcl.retryCount
 
-	for err == nil &&
-		resp.StatusCode == http.StatusForbidden &&
-		retryCount > 0 {
-
+	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
 		retryTime := resp.Header.Get("Retry-After")
-		i, err := strconv.Atoi(retryTime)
-		if err != nil {
+		i, errAtoi := strconv.Atoi(retryTime)
+		if errAtoi != nil {
 			return resp, fmt.Errorf(
 				"query '%s' forbidden without 'Retry-After'", query)
 		}
@@ -556,7 +617,7 @@ func (gcl GhClient) getWithRetry(
 		logger.Printf("waiting %d seconds before retrying\n", i)
 		time.Sleep(time.Second * time.Duration(i))
 		retryCount--
-		resp, err = gcl.client.Get(query)
+		resp, err = gcl.Do(query)
 	}
 
 	if err != nil {

api/internal/crawl/crawler/github/queries.go

@@ -7,10 +7,11 @@ import (
 )
 
 const (
-	perPageArg     = "per_page"
-	accessTokenArg = "access_token"
+	perPageArg = "per_page"
 )
 
+const githubMaxPageSize = 100
+
 // Implementation detail, not important to external API.
 type queryField struct {
 	name  string
@@ -89,6 +90,17 @@ func Path(p string) queryField {
 	return queryField{name: "path", value: p}
 }
 
+// Repo takes a repository (i.e., kubernetes-sigs/kustomize) and formats
+// it according to the Github API.
+func Repo(r string) queryField {
+	return queryField{name: "repo", value: r}
+}
+
+// Path takes a github username and formats it according to the Github API.
+func User(u string) queryField {
+	return queryField{name: "user", value: u}
+}
+
 // RequestConfig stores common variables that must be present for the queries.
 // - CodeSearchRequests: ask Github to check the code indices given a query.
 // - ContentsRequests: ask Github where to download a resource given a repo and a
@@ -96,15 +108,7 @@ func Path(p string) queryField {
 // - CommitsRequests: asks Github to list commits made one a file. Useful to
 // determine the date of a file.
 type RequestConfig struct {
-	perPage     uint64
-	accessToken string
-}
-
-func NewRequestConfig(perPage uint64, accessToken string) RequestConfig {
-	return RequestConfig{
-		perPage:     perPage,
-		accessToken: accessToken,
-	}
+	perPage uint64
 }
 
 // CodeSearchRequestWith given a list of query parameters that specify the
@@ -112,9 +116,11 @@ func NewRequestConfig(perPage uint64, accessToken string) RequestConfig {
 // the URL method to get the string value of the URL. See request.CopyWith, to
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
-	req := rc.makeRequest("search/code", query)
-	req.vals.Set("sort", "indexed")
-	req.vals.Set("order", "desc")
+	vals := url.Values{
+		"sort": []string{"indexed"},
+		"order": []string{"desc"},
+	}
+	req := rc.makeRequest("search/code", query, vals)
 	return req
 }
 
@@ -122,26 +128,25 @@ func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 // query for the Github API to find the dowload information of this filepath.
 func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 func (rc RequestConfig) ReposRequest(fullRepoName string) string {
 	uri := fmt.Sprintf("repos/%s", fullRepoName)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 // CommitsRequest given the repo name, and a filepath returns a formatted query
 // for the Github API to find the commits that affect this file.
 func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
-	return rc.makeRequest(uri, Query{Path(path)}).URL()
+	vals := url.Values{
+		"path": []string{path},
+	}
+	return rc.makeRequest(uri, Query{}, vals).URL()
 }
 
-func (rc RequestConfig) makeRequest(path string, query Query) request {
-	vals := url.Values{}
-	if rc.accessToken != "" {
-		vals.Set(accessTokenArg, rc.accessToken)
-	}
+func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
 	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
 
 	return request{
@@ -183,7 +188,7 @@ func (r request) URL() string {
 	if encoded == "" && query != "" {
 		sep = "?"
 	}
-	r.url.RawQuery = encoded + sep + query
+	r.url.RawQuery = query + sep + encoded
 	return r.url.String()
 }
 

api/internal/crawl/crawler/github/queries_test.go

@@ -53,8 +53,11 @@ func TestQueryType(t *testing.T) {
 				Filename("kustomization.yaml"),
 				Keyword("keyword1"),
 				Keyword("keyword2"),
+				Repo("user1/repo1"),
+				User("user1"),
 			),
-			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2",
+			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2+" +
+				"repo:user1/repo1+user:user1",
 		},
 	}
 
@@ -68,8 +71,7 @@ func TestQueryType(t *testing.T) {
 
 func TestGithubSearchQuery(t *testing.T) {
 	const (
-		accessToken = "random_token"
-		perPage     = 100
+		perPage = 100
 	)
 
 	testCases := []struct {
@@ -83,8 +85,7 @@ func TestGithubSearchQuery(t *testing.T) {
 	}{
 		{
 			rc: RequestConfig{
-				perPage:     perPage,
-				accessToken: accessToken,
+				perPage: perPage,
 			},
 			codeQuery: Query{
 				Filename("kustomization.yaml"),
@@ -94,13 +95,33 @@ func TestGithubSearchQuery(t *testing.T) {
 			path:         "examples/helloWorld/kustomization.yaml",
 
 			expectedCodeQuery: "https://api.github.com/search/code?" +
-				"access_token=random_token&order=desc&per_page=100&sort=indexed&q=filename:kustomization.yaml+size:64..128",
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
 
 			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
-				"examples/helloWorld/kustomization.yaml?access_token=random_token&per_page=100",
+				"examples/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"access_token=random_token&per_page=100&q=path:examples/helloWorld/kustomization.yaml",
+				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+		},
+		{
+			rc: RequestConfig{
+				perPage: perPage,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples 1/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples%201/helloWorld/kustomization.yaml?per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
 		},
 	}
 

api/internal/crawl/crawler/github/split_search_ranges.go

@@ -243,7 +243,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	logger.Println("total files: ", totalFiles)
+	logger.Println("total kustomization files: ", totalFiles)
 
 	if githubMaxResultsPerQuery >= totalFiles {
 		return []string{

api/internal/crawl/doc/doc.go

@@ -2,6 +2,7 @@ package doc
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
@@ -43,6 +44,12 @@ type KustomizationDocument struct {
 
 type set map[string]struct{}
 
+func (doc *KustomizationDocument) String() string {
+	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
+		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
+}
+
 // Implements the CrawlerDocument interface.
 func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 	isResource := true
@@ -71,6 +78,9 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 
 	res := make([]*Document, 0, len(k.Resources))
 	for _, r := range k.Resources {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
 		next, err := doc.Document.FromRelativePath(r)
 		if err != nil {
 			fmt.Printf("GetResources error: %v\n", err)
@@ -110,6 +120,8 @@ func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error)
 	return configs, nil
 }
 
+// ParseYAML parses doc.Document and sets the following fields of doc:
+// Kinds, Values, Identifiers.
 func (doc *KustomizationDocument) ParseYAML() error {
 	doc.Identifiers = make([]string, 0)
 	doc.Values = make([]string, 0)
@@ -117,6 +129,8 @@ func (doc *KustomizationDocument) ParseYAML() error {
 
 	identifierSet := make(set)
 	valueSet := make(set)
+	kindSet := make(set)
+
 	getKind := func(m map[string]interface{}) string {
 		const defaultStr = "Kustomization"
 		kind, ok := m["kind"]
@@ -135,10 +149,14 @@ func (doc *KustomizationDocument) ParseYAML() error {
 	}
 
 	for _, contents := range ks {
-		doc.Kinds = append(doc.Kinds, getKind(contents))
+		kindSet[getKind(contents)] = struct{}{}
 		createFlatStructure(identifierSet, valueSet, contents)
 	}
 
+	for val := range kindSet {
+		doc.Kinds = append(doc.Kinds, val)
+	}
+
 	for val := range valueSet {
 		doc.Values = append(doc.Values, val)
 	}
@@ -147,6 +165,13 @@ func (doc *KustomizationDocument) ParseYAML() error {
 		doc.Identifiers = append(doc.Identifiers, key)
 	}
 
+	// Without sorting these fields, every time when the string order in these fields changes,
+	// the document in the index will be updated.
+	// Sorting these fields are necessary to avoid a document being updated unnecessarily.
+	sort.Strings(doc.Kinds)
+	sort.Strings(doc.Values)
+	sort.Strings(doc.Identifiers)
+
 	return nil
 }
 

api/internal/crawl/doc/doc_test.go

@@ -121,6 +121,38 @@ metadata:
 kind: Custom
 metadata:
   name: app-crd
+`,
+		},
+		{
+			identifiers: []string{
+				"kind",
+				"metadata",
+				"metadata:name",
+			},
+			values: []string{
+				"kind=Deployment",
+				"kind=Service",
+				"metadata:name=app1",
+				"metadata:name=app2",
+			},
+			kinds: []string{
+				"Deployment",
+				"Service",
+			},
+			filepath: "resources.yaml",
+			yaml: `
+---
+kind: Deployment
+metadata:
+  name: app1
+---
+kind: Deployment
+metadata:
+  name: app2
+---
+kind: Service
+metadata:
+  name: app1
 `,
 		},
 	}

api/internal/crawl/doc/docname.go

@@ -1,7 +1,10 @@
 package doc
 
 import (
+	"crypto/sha256"
+	"fmt"
 	"path"
+	"strings"
 	"time"
 
 	"sigs.k8s.io/kustomize/api/internal/git"
@@ -21,6 +24,17 @@ func (doc *Document) GetDocument() *Document {
 	return doc
 }
 
+func (doc *Document) Copy() *Document {
+	return &Document{
+		RepositoryURL: doc.RepositoryURL,
+		FilePath:      doc.FilePath,
+		DefaultBranch: doc.DefaultBranch,
+		DocumentData:  doc.DocumentData,
+		CreationTime:  doc.CreationTime,
+		IsSame:        doc.IsSame,
+	}
+}
+
 // Implements the CrawlerDocument interface.
 func (doc *Document) WasCached() bool {
 	return doc.IsSame
@@ -53,6 +67,28 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 }
 
 func (doc *Document) ID() string {
-	return doc.RepositoryURL + "/" +
-		doc.DefaultBranch + "/" + doc.FilePath
+	sum := sha256.Sum256([]byte(strings.Join(
+		[]string{
+			doc.RepositoryURL,
+			doc.DefaultBranch,
+			doc.FilePath,
+		},
+		"---|---")))
+	return fmt.Sprintf("%x", sum)
+}
+
+func (doc *Document) RepositoryFullName() string {
+	url := strings.TrimRight(doc.RepositoryURL, "/")
+
+	gitPrefix := "git@github.com:"
+	if strings.HasPrefix(url, gitPrefix) {
+		url = url[len(gitPrefix):]
+	}
+
+	sections := strings.Split(url, "/")
+	l := len(sections)
+	if l < 2 {
+		return url
+	}
+	return path.Join(sections[l-2], sections[l-1])
 }

api/internal/crawl/doc/docname_test.go

@@ -62,3 +62,50 @@ func TestFromRelativePath(t *testing.T) {
 		}
 	}
 }
+
+func TestDocument_RepositoryFullName(t *testing.T) {
+	testCases := []struct {
+		doc Document
+		expectedRepositoryFullName string
+	}{
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com/user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com//user/repo////",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "repo/",
+			},
+			expectedRepositoryFullName: "repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "",
+			},
+			expectedRepositoryFullName: "",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "git@github.com:user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedRepositoryFullName := tc.doc.RepositoryFullName()
+		if returnedRepositoryFullName != tc.expectedRepositoryFullName {
+			t.Errorf("RepositoryFullName expected %s, got %s",
+				tc.expectedRepositoryFullName,
+				returnedRepositoryFullName)
+		}
+	}
+}

api/internal/crawl/go.mod

@@ -1,13 +1,13 @@
 module sigs.k8s.io/kustomize/api/internal/crawl
 
-go 1.13
+go 1.12
 
 require (
-	github.com/elastic/go-elasticsearch/v6 v6.8.2
+	github.com/elastic/go-elasticsearch/v6 v6.8.5
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/gorilla/mux v1.7.3
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/rs/cors v1.7.0
-	sigs.k8s.io/kustomize/api v0.2.0
+	sigs.k8s.io/kustomize/api v0.3.1
 	sigs.k8s.io/yaml v1.1.0
 )

api/internal/crawl/go.sum

@@ -1,14 +1,11 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -25,6 +22,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -41,13 +39,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/elastic/go-elasticsearch/v6 v6.8.2 h1:rp5DGrd63V5c6nHLjF6QEXUpZSvs0+QM3ld7m9VhV2g=
-github.com/elastic/go-elasticsearch/v6 v6.8.2/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/elastic/go-elasticsearch/v6 v6.8.5 h1:U2HtkBseC1FNBmDr0TR2tKltL6FxoY+niDAlj5M8TK8=
+github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -87,14 +84,16 @@ github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Il
 github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
 github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -106,10 +105,10 @@ github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvL
 github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
 github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
@@ -117,34 +116,35 @@ github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunE
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
 github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
-github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -154,6 +154,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -177,8 +178,8 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -192,7 +193,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -225,18 +225,15 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -261,12 +258,13 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
@@ -274,16 +272,22 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -294,19 +298,23 @@ golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -315,19 +323,22 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -336,20 +347,26 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -371,7 +388,14 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
+k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
+k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
+k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
+k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
@@ -379,14 +403,12 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.2.0 h1:e++6JpysnnlUbHmFrv6jvfF5rFlgQ103bS1DO7r5bWA=
-sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0 h1:otg4dLFc03c3gzl+2CV8GPGcd1kk8wjXwD+UhhcCn5I=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
+sigs.k8s.io/kustomize/api v0.3.1 h1:oqMIXvS6tFEUVuKIRUKDa05eC4Hh+cb9JYg8Zhp2d24=
+sigs.k8s.io/kustomize/api v0.3.1/go.mod h1:A+ATnlHqzictQfQC1q3KB/T6MSr0UWQsrrLxMWkge2E=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/internal/crawl/index/elasticsearch.go

@@ -6,13 +6,46 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"time"
 
 	es "github.com/elastic/go-elasticsearch/v6"
 	"github.com/elastic/go-elasticsearch/v6/esapi"
 )
 
+const IndexConfig = `
+{
+	"mappings": {
+		"_doc": {
+			"properties": {
+				"repositoryUrl": {
+					"type": "keyword"
+				},
+				"filePath": {
+					"type": "keyword"
+				},
+				"defaultBranch": {
+					"type": "keyword"
+				},
+				"document": {
+					"type": "text"
+				},
+				"creationTime": {
+					"type": "date"
+				},
+				"kinds": {
+					"type": "text"
+				},
+				"identifiers": {
+					"type": "text"
+				},
+				"values": {
+					"type": "text"
+				}
+			}
+		}
+	}
+}`
+
 // TODO(damienr74) Split index into reader and writer?
 type index struct {
 	ctx    context.Context
@@ -118,21 +151,19 @@ func (idx *index) UpdateSetting(settings []byte) error {
 		res, err, ignoreResponseBody)
 }
 
-// Create an index providing both the mappings and the settings.
-func (idx *index) CreateIndex(mappings []byte, settings []byte) error {
-	request := byteJoin(`{ "mappings":`, mappings, `, "settings":`, settings, `}`)
+// Create an index providing the config for both the mappings and the settings.
+func (idx *index) CreateIndex(config []byte) error {
 	op := idx.client.Indices.Create
 	res, err := op(
 		idx.name,
-		op.WithBody(bytes.NewReader(request)),
+		op.WithBody(bytes.NewReader(config)),
 		op.WithContext(idx.ctx),
 		op.WithHuman(),
 		op.WithPretty(),
-		op.WithIncludeTypeName(true),
 	)
 
 	return idx.responseErrorOrNil(
-		fmt.Sprintf("could not create index with config '%s'", request),
+		fmt.Sprintf("could not create index with config '%s'", config),
 		res, err, ignoreResponseBody)
 }
 
@@ -147,44 +178,47 @@ func (idx *index) DeleteIndex() error {
 }
 
 // Insert or update the document by ID.
-func (idx *index) Put(uniqueID string, doc interface{}) (string, error) {
-	body, err := json.Marshal(doc)
+func (idx *index) Put(uniqueID string, doc interface{}) error {
+	exists, err := idx.Exists(uniqueID)
 	if err != nil {
-		return "", err
+		return err
 	}
 
-	req := esapi.IndexRequest{
-		Index:      idx.name,
-		Body:       bytes.NewReader(body),
-		DocumentID: uniqueID,
-	}
-	res, err := req.Do(idx.ctx, idx.client)
-
-	var id string
-	readId := func(reader io.Reader) error {
-		type InsertResult struct {
-			ID string `json:"_id,omitempty"`
+	if exists {
+		docBytes, err := json.Marshal(doc)
+		if err != nil {
+			return err
 		}
-		var ir InsertResult
-		data, err := ioutil.ReadAll(reader)
+		body := byteJoin(`{"doc":`, docBytes, `}`)
+
+		// For a document with a given id, every call of IndexRequest.Do will increase the version of a document.
+		// To avoid increasing the document version unnecessarily, use UpdateRequest here.
+		req := esapi.UpdateRequest{
+			Index:      idx.name,
+			Body:       bytes.NewReader(body),
+			DocumentID: uniqueID,
+		}
+		res, err := req.Do(idx.ctx, idx.client)
+
+		err = idx.responseErrorOrNil("could not update document",
+			res, err, ignoreResponseBody)
+	} else {
+		body, err := json.Marshal(doc)
 		if err != nil {
 			return err
 		}
 
-		err = json.Unmarshal(data, &ir)
-		if err != nil {
-			return err
+		req := esapi.IndexRequest{
+			Index:      idx.name,
+			Body:       bytes.NewReader(body),
+			DocumentID: uniqueID,
 		}
-		id = ir.ID
+		res, err := req.Do(idx.ctx, idx.client)
 
-		return nil
+		err = idx.responseErrorOrNil("could not insert document",
+			res, err, ignoreResponseBody)
 	}
-
-	// populates the id field.
-	err = idx.responseErrorOrNil("could not insert document",
-		res, err, readId)
-
-	return id, err
+	return err
 }
 
 type scrollUpdater func(string, readerFunc) error
@@ -264,3 +298,24 @@ func (idx *index) Delete(id string) error {
 		fmt.Sprintf("could not delete id(%s) from index(%s)", id, idx.name),
 		res, err, ignoreResponseBody)
 }
+
+// Check whether a given document id is in the index
+func (idx *index) Exists(id string) (bool, error) {
+	op := idx.client.Exists
+	res, err := op(
+		idx.name,
+		id,
+		op.WithContext(idx.ctx),
+		op.WithPretty(),
+	)
+
+	if res != nil && !res.IsError() {
+		return true, nil
+	} else if res != nil && res.StatusCode == 404 {
+		return false, nil
+	} else {
+		return false, idx.responseErrorOrNil(
+			fmt.Sprintf("could not check the existence of id(%s) from index(%s)", id, idx.name),
+			res, err, ignoreResponseBody)
+	}
+}

api/internal/crawl/index/kustomize.go

@@ -16,6 +16,12 @@ const (
 	AggregationKeyword = "aggs"
 )
 
+type Mode int
+const (
+	InsertOrUpdate = iota
+	Delete
+)
+
 // Redefinition of Hits structure. Must match the json string of
 // KustomizeResult.Hits.Hits. Declared as a convenience for iteration.
 type KustomizeHits []struct {
@@ -96,6 +102,24 @@ func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	indicesExistsOp := idx.client.Indices.Exists
+	resp, err := indicesExistsOp([]string{"kustomize"},
+		indicesExistsOp.WithContext(idx.ctx),
+		indicesExistsOp.WithPretty())
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode == 200 {
+		fmt.Printf("The kustomize index already exists\n")
+	} else {
+		fmt.Printf("Creating the kustomize index\n")
+		if err := idx.CreateIndex([]byte(IndexConfig)); err != nil {
+			return nil, err
+		}
+	}
+
 	return &KustomizeIndex{idx}, nil
 }
 
@@ -275,12 +299,13 @@ func (ki *KustomizeIndex) IterateQuery(query []byte, batchSize int,
 }
 
 // type specific Put for inserting structured kustomization documents.
-func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) (string, error) {
-	id, err := ki.index.Put(id, doc)
-	if err != nil {
-		return id, fmt.Errorf("could not insert in elastic: %v", err)
-	}
-	return id, nil
+func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) error {
+	return ki.index.Put(id, doc)
+}
+
+// Delete a document with a given id from the kustomize index.
+func (ki *KustomizeIndex) Delete(id string) error {
+	return ki.index.Delete(id)
 }
 
 // Kustomize search options: What metrics should be returned? Kind Aggregation,

api/internal/crawl/search_cmds/creationTime.md

@@ -0,0 +1,180 @@
+Find out the largest value of the `creationTime` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "max_creationTime" : { "max" : { "field" : "creationTime" } }
+    }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "min_creationTime" : { "min" : { "field" : "creationTime" } }
+    }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all the kustomization files:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+        { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize resource files:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }
+      }
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Query all the documents whose `creationTime` <= `2016-07-29T17:38:26.000Z`:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "range": {
+      "creationTime": {
+       "lte": "2016-07-29T17:38:26.000Z"
+      }
+    }
+  }
+}
+'
+```
+
+Query all the documents whose `creationTime` falls within the specific range:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "range": {
+      "creationTime": {
+       "gte": "2016-07-29T17:38:26.000Z",
+       "lte": "2016-08-29T17:38:26.000Z"
+      }
+    }
+  }
+}
+'
+```
+
+Aggregate how many new kustomization files were added into Github each month:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize resource files were added into Github each month:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomization files were added into Github each year:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize resource files were added into Github each year:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/defaultBranch.md

@@ -0,0 +1,32 @@
+Count distinct values of the `defaultBranch` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "defaultBranch_count" : {
+            "cardinality" : {
+                "field" : "defaultBranch",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the github branches where kustomization files and kustomize resource files live, 
+and how many kustomization files and kustomize resource files live in each branch:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "defaultBranch" : {
+            "terms" : {
+              "field" : "defaultBranch",
+              "size": 41
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/fieldExistence.md

@@ -0,0 +1,55 @@
+Count the documents whose `document` field is empty (The reason why the `document` field
+of a document is empty is because of empty documents):
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "document"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Find all the documents having the `creationTime` field set:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "creationTime"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `creationTime` field is not set:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "creationTime"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+The following fields of a document in the kustomize index are always non-empty:
+`repositoryUrl`, `filePath`, `defaultBranch`.
+
+The following fields of a document in the kustomize index may be empty:
+`kinds`, `identifiers`, `values`.

api/internal/crawl/search_cmds/keyword_search.md

@@ -0,0 +1,66 @@
+Count the documents in the index whose `repositoryUrl` field starts with
+`https://github.com/`:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Count the documents in the index whose `repositoryUrl` field does not start with
+`https://github.com/`:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": [
+       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents matching the given `repositoryUrl` and `filePath`, and return 
+a version for each search hit:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "version": true,
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "repositoryUrl": "git@github.com:talos-systems/talos-controller-manager" }},
+       { "regexp": { "filePath": "hack/config.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents whose filePath ends with one of these following three filenames:
+`kustomization.yaml`, `kustomization.yml`, `kustomization`:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+      ]
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/misc.md

@@ -0,0 +1,19 @@
+Check the health status of an ElasticSearch cluster:
+```
+curl -X GET "${ElasticSearchURL}:9200/_cat/health?v&pretty"
+```
+
+Check the indices in an ElasticSearch cluster:
+```
+curl "${ElasticSearchURL}:9200/_cat/indices?v"
+```
+
+Get the mapping of the index:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_mapping?pretty"
+```
+
+Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**):
+```
+curl -X DELETE "${ElasticSearchURL}:9200/kustomize?pretty"
+```

api/internal/crawl/search_cmds/repositoryUrl.md

@@ -0,0 +1,125 @@
+Count distinct values of the `repositoryUrl` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count how many Github repositories include kustomization files:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+             ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count how many Github repositories include kustomize resource files:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }
+            }
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the github repositories including kustomization files and kustomize resource files,
+and how many kustomization files and kustomize resource files each github repository includes
+(the github repository including the most kustomization files is listed first):
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 2082
+            }
+        }
+    }
+}
+'
+```
+
+List the top 20 Github repositories including the most amount of kustomization files:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+             ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+List the top 20 Github repositories including the most amount of kustomize resource files:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }
+            }
+        }
+    },
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 20
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/text_search.md

@@ -0,0 +1,148 @@
+Search for all the kustomize resource files including a Deployment object:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files including a Deployment object, but only
+including the `kinds` field in the result:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "_source": {
+    "includes": ["kinds"]
+  },
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files including both a Deployment object and 
+a Service object:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment Service",
+        "operator" : "and"
+      }
+    }
+  }
+}
+'
+```
+
+Count the number of documents including Deployment and the number of documents 
+including Service:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 0,
+  "aggs" : {
+    "messages" : {
+      "filters" : {
+        "filters" : {
+          "Deployment" : { "match" : { "kinds" : "Deployment"   }},
+          "Service" : { "match" : { "kinds" : "Service" }}
+        }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files involving CRDs:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "crds"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files defining configMapGenerator:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "configMapGenerator"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents having a `kind` field:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "match" : { "identifiers" : { "query" : "kind" }}}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kuostmization files having a `kind` field:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)" }},
+       { "match" : { "identifiers" : { "query" : "kind" }}}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files defining the `generatorOptions:disableNameSuffixHash` feature:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "generatorOptions:disableNameSuffixHash"
+      }
+    }
+  }
+}
+'
+```

api/internal/crawl/ui/package.json

@@ -11,43 +11,45 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "~8.1.0",
-    "@angular/cdk": "~8.0.2",
-    "@angular/common": "~8.1.0",
-    "@angular/compiler": "~8.1.0",
-    "@angular/core": "~8.1.0",
-    "@angular/forms": "~8.1.0",
+    "@angular/animations": "^8.2.14",
+    "@angular/cdk": "~8.2.3",
+    "@angular/common": "^8.2.14",
+    "@angular/compiler": "^8.2.14",
+    "@angular/core": "^8.2.14",
+    "@angular/forms": "^8.2.14",
     "@angular/http": "^7.2.15",
-    "@angular/material": "^8.0.2",
-    "@angular/platform-browser": "~8.1.0",
-    "@angular/platform-browser-dynamic": "~8.1.0",
-    "@angular/router": "~8.1.0",
+    "@angular/material": "^8.2.3",
+    "@angular/platform-browser": "^8.2.14",
+    "@angular/platform-browser-dynamic": "^8.2.14",
+    "@angular/router": "^8.2.14",
     "angular-google-charts": "^0.1.6",
-    "chart.js": "^2.8.0",
+    "chart.js": "^2.9.3",
+    "core-js": "^3.5.0",
     "hammerjs": "^2.0.8",
-    "rxjs": "~6.4.0",
-    "tslib": "^1.9.0",
-    "zone.js": "~0.9.1"
+    "rxjs": "~6.5.3",
+    "serialize-javascript": "^2.1.2",
+    "tslib": "^1.10.0",
+    "zone.js": "~0.10.2"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "~0.801.0",
-    "@angular/cli": "^8.1.2",
-    "@angular/compiler-cli": "~8.1.0",
-    "@angular/language-service": "~8.1.0",
-    "@types/jasmine": "~3.3.8",
-    "@types/jasminewd2": "~2.0.3",
-    "@types/node": "~8.9.4",
-    "codelyzer": "^5.0.0",
-    "jasmine-core": "~3.4.0",
+    "@angular-devkit/build-angular": "^0.803.20",
+    "@angular/cli": "^8.3.20",
+    "@angular/compiler-cli": "^8.2.14",
+    "@angular/language-service": "^8.2.14",
+    "@types/jasmine": "^3.5.0",
+    "@types/jasminewd2": "^2.0.8",
+    "@types/node": "~12.12.17",
+    "codelyzer": "^5.2.0",
+    "jasmine-core": "~3.5.0",
     "jasmine-spec-reporter": "~4.2.1",
-    "karma": "~4.1.0",
-    "karma-chrome-launcher": "~2.2.0",
-    "karma-coverage-istanbul-reporter": "~2.0.1",
+    "karma": "~4.4.1",
+    "karma-chrome-launcher": "~3.1.0",
+    "karma-coverage-istanbul-reporter": "^2.1.1",
     "karma-jasmine": "~2.0.1",
-    "karma-jasmine-html-reporter": "^1.4.0",
-    "protractor": "~5.4.0",
-    "ts-node": "~7.0.0",
-    "tslint": "~5.15.0",
-    "typescript": "~3.4.3"
+    "karma-jasmine-html-reporter": "^1.4.2",
+    "protractor": "~5.4.2",
+    "ts-node": "~8.5.4",
+    "tslint": "~5.20.1",
+    "typescript": "~3.7.3"
   }
 }

api/internal/git/cloner.go

@@ -74,8 +74,28 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 	cmd.Dir = repoSpec.Dir.String()
 	err = cmd.Run()
 	if err != nil {
-		log.Printf("Error performing git fetch: %s", out.String())
-		return errors.Wrapf(err, "trouble fetching %s", repoSpec.Ref)
+		cmd = exec.Command(
+			gitProgram,
+			"pull",
+			"origin",
+			"master")
+		var out bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Dir = repoSpec.Dir.String()
+		err := cmd.Run()
+		if err != nil {
+			return errors.Wrapf(err, "trouble pulling %s", repoSpec.OrgRepo)
+		}
+		if repoSpec.Ref == "" {
+			repoSpec.Ref = "master"
+		}
+		cmd = exec.Command(gitProgram, "checkout", repoSpec.Ref)
+		cmd.Dir = repoSpec.Dir.String()
+		err = cmd.Run()
+		if err != nil {
+			return errors.Wrapf(
+				err, "trouble checking out href %s", repoSpec.Ref)
+		}
 	}
 
 	cmd = exec.Command(

api/internal/k8sdeps/configmapandsecret/configmapfactory.go

@@ -9,8 +9,8 @@ import (
 	"unicode/utf8"
 
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/kustomize/api/types"
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
 )
 
 func makeFreshConfigMap(

api/internal/k8sdeps/configmapandsecret/configmapfactory_test.go

@@ -8,13 +8,13 @@ import (
 	"reflect"
 	"testing"
 
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/kustomize/api/filesys"
 	"sigs.k8s.io/kustomize/api/kv"
 	"sigs.k8s.io/kustomize/api/loader"
 	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
 	"sigs.k8s.io/kustomize/api/types"
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
-	metav1 "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1"
 )
 
 func makeEnvConfigMap(name string) *corev1.ConfigMap {

api/internal/k8sdeps/configmapandsecret/secretfactory.go

@@ -6,8 +6,8 @@ package configmapandsecret
 import (
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/kustomize/api/types"
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
 )
 
 func makeFreshSecret(

api/internal/k8sdeps/configmapandsecret/secretfactory_test.go

@@ -7,13 +7,13 @@ import (
 	"reflect"
 	"testing"
 
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/kustomize/api/filesys"
 	"sigs.k8s.io/kustomize/api/kv"
 	"sigs.k8s.io/kustomize/api/loader"
 	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
 	"sigs.k8s.io/kustomize/api/types"
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
-	metav1 "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1"
 )
 
 func makeEnvSecret(name string) *corev1.Secret {

api/internal/k8sdeps/transformer/patch/conflictdetector.go

@@ -8,14 +8,14 @@ import (
 	"fmt"
 
 	jsonpatch "github.com/evanphx/json-patch"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/mergepatch"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	"k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/runtime"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/util/mergepatch"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/util/strategicpatch"
-	"sigs.k8s.io/kustomize/pseudo/k8s/client-go/kubernetes/scheme"
 )
 
 type conflictDetector interface {

api/internal/loadertest/fakeloader.go

@@ -1,76 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-// Package loadertest holds a fake for the Loader interface.
-package loadertest
-
-import (
-	"log"
-
-	"sigs.k8s.io/kustomize/api/filesys"
-	"sigs.k8s.io/kustomize/api/ifc"
-	"sigs.k8s.io/kustomize/api/loader"
-)
-
-// FakeLoader encapsulates the delegate Loader and the fake file system.
-type FakeLoader struct {
-	fs       filesys.FileSystem
-	delegate ifc.Loader
-}
-
-// NewFakeLoader returns a Loader that uses a fake filesystem.
-// The loader will be restricted to root only.
-// The initialDir argument should be an absolute file path.
-func NewFakeLoader(initialDir string) FakeLoader {
-	return NewFakeLoaderWithRestrictor(
-		loader.RestrictionRootOnly, initialDir)
-}
-
-// NewFakeLoaderWithRestrictor returns a Loader that
-// uses a fake filesystem.
-// The initialDir argument should be an absolute file path.
-func NewFakeLoaderWithRestrictor(
-	lr loader.LoadRestrictorFunc, initialDir string) FakeLoader {
-	// Create fake filesystem and inject it into initial Loader.
-	fSys := filesys.MakeFsInMemory()
-	fSys.Mkdir(initialDir)
-	ldr, err := loader.NewLoader(lr, initialDir, fSys)
-	if err != nil {
-		log.Fatalf("Unable to make loader: %v", err)
-	}
-	return FakeLoader{fs: fSys, delegate: ldr}
-}
-
-// AddFile adds a fake file to the file system.
-func (f FakeLoader) AddFile(fullFilePath string, content []byte) error {
-	return f.fs.WriteFile(fullFilePath, content)
-}
-
-// AddDirectory adds a fake directory to the file system.
-func (f FakeLoader) AddDirectory(fullDirPath string) error {
-	return f.fs.Mkdir(fullDirPath)
-}
-
-// Root delegates.
-func (f FakeLoader) Root() string {
-	return f.delegate.Root()
-}
-
-// New creates a new loader from a new root.
-func (f FakeLoader) New(newRoot string) (ifc.Loader, error) {
-	l, err := f.delegate.New(newRoot)
-	if err != nil {
-		return nil, err
-	}
-	return FakeLoader{fs: f.fs, delegate: l}, nil
-}
-
-// Load delegates.
-func (f FakeLoader) Load(location string) ([]byte, error) {
-	return f.delegate.Load(location)
-}
-
-// Cleanup delegates.
-func (f FakeLoader) Cleanup() error {
-	return f.delegate.Cleanup()
-}

api/internal/plugins/builtinconfig/loaddefaultconfig_test.go

@@ -7,21 +7,30 @@ import (
 	"reflect"
 	"testing"
 
-	"sigs.k8s.io/kustomize/api/internal/loadertest"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/loader"
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/types"
 )
 
 func TestLoadDefaultConfigsFromFiles(t *testing.T) {
-	ldr := loadertest.NewFakeLoader("/app")
-	ldr.AddFile("/app/config.yaml", []byte(`
+	fSys := filesys.MakeFsInMemory()
+	err := fSys.WriteFile("config.yaml", []byte(`
 namePrefix:
 - path: nameprefix/path
   kind: SomeKind
 `))
-	tcfg, err := loadDefaultConfig(ldr, []string{"/app/config.yaml"})
 	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
+		t.Fatal(err)
+	}
+	ldr, err := loader.NewLoader(
+		loader.RestrictionRootOnly, filesys.Separator, fSys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tCfg, err := loadDefaultConfig(ldr, []string{"config.yaml"})
+	if err != nil {
+		t.Fatal(err)
 	}
 	expected := &TransformerConfig{
 		NamePrefix: []types.FieldSpec{
@@ -31,7 +40,7 @@ namePrefix:
 			},
 		},
 	}
-	if !reflect.DeepEqual(tcfg, expected) {
-		t.Fatalf("expected %v\n but go6t %v\n", expected, tcfg)
+	if !reflect.DeepEqual(tCfg, expected) {
+		t.Fatalf("expected %v\n but go6t %v\n", expected, tCfg)
 	}
 }

api/internal/plugins/compiler/compiler.go

@@ -37,6 +37,16 @@ func DeterminePluginSrcRoot(fSys filesys.FileSystem) (string, error) {
 		"source directory", fSys, []konfig.NotedFunc{
 			{
 				Note: "relative to unit test",
+				F: func() string {
+					return filepath.Clean(
+						filepath.Join(
+							os.Getenv("PWD"),
+							"..", "..",
+							konfig.RelPluginHome))
+				},
+			},
+			{
+				Note: "relative to unit test (internal pkg)",
 				F: func() string {
 					return filepath.Clean(
 						filepath.Join(

api/internal/plugins/execplugin/execplugin.go

@@ -12,6 +12,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/google/shlex"
+
 	"github.com/pkg/errors"
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/resmap"
@@ -86,7 +88,7 @@ func (p *ExecPlugin) processOptionalArgsFields() error {
 	var c argsConfig
 	yaml.Unmarshal(p.cfg, &c)
 	if c.ArgsOneLiner != "" {
-		p.args = strings.Split(c.ArgsOneLiner, " ")
+		p.args, _ = shlex.Split(c.ArgsOneLiner)
 	}
 	if c.ArgsFromFile != "" {
 		content, err := p.h.Loader().Load(c.ArgsFromFile)

api/internal/plugins/execplugin/execplugin_test.go

@@ -8,11 +8,12 @@ import (
 	"strings"
 	"testing"
 
-	"sigs.k8s.io/kustomize/api/internal/loadertest"
+	"sigs.k8s.io/kustomize/api/filesys"
 	. "sigs.k8s.io/kustomize/api/internal/plugins/execplugin"
-	"sigs.k8s.io/kustomize/api/internal/plugins/loader"
+	pLdr "sigs.k8s.io/kustomize/api/internal/plugins/loader"
 	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
 	"sigs.k8s.io/kustomize/api/konfig"
+	fLdr "sigs.k8s.io/kustomize/api/loader"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
 	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
@@ -20,11 +21,20 @@ import (
 )
 
 func TestExecPluginConfig(t *testing.T) {
-	path := "/app"
+	fSys := filesys.MakeFsInMemory()
+	fSys.WriteFile("sed-input.txt", []byte(`
+s/$FOO/foo/g
+s/$BAR/bar baz/g
+ \ \ \ 
+`))
+	ldr, err := fLdr.NewLoader(
+		fLdr.RestrictionRootOnly, filesys.Separator, fSys)
+	if err != nil {
+		t.Fatal(err)
+	}
 	rf := resmap.NewFactory(
 		resource.NewFactory(
 			kunstruct.NewKunstructuredFactoryImpl()), nil)
-	ldr := loadertest.NewFakeLoader(path)
 	v := valtest_test.MakeFakeValidator()
 	pluginConfig := rf.RF().FromMap(
 		map[string]interface{}{
@@ -33,18 +43,12 @@ func TestExecPluginConfig(t *testing.T) {
 			"metadata": map[string]interface{}{
 				"name": "some-random-name",
 			},
-			"argsOneLiner": "one two",
+			"argsOneLiner": "one two 'foo bar'",
 			"argsFromFile": "sed-input.txt",
 		})
 
-	ldr.AddFile("/app/sed-input.txt", []byte(`
-s/$FOO/foo/g
-s/$BAR/bar/g
- \ \ \ 
-`))
-
 	p := NewExecPlugin(
-		loader.AbsolutePluginPath(
+		pLdr.AbsolutePluginPath(
 			konfig.DisabledPluginConfig(),
 			pluginConfig.OrgId()))
 	// Not checking to see if the plugin is executable,
@@ -66,7 +70,7 @@ s/$BAR/bar/g
 
 	expected = `apiVersion: someteam.example.com/v1
 argsFromFile: sed-input.txt
-argsOneLiner: one two
+argsOneLiner: one two 'foo bar'
 kind: SedTransformer
 metadata:
   name: some-random-name
@@ -75,15 +79,16 @@ metadata:
 		t.Fatalf("expected cfg '%s', got '%s'", expected, string(p.Cfg()))
 
 	}
-	if len(p.Args()) != 5 {
-		t.Fatalf("unexpected arg len %d, %v", len(p.Args()), p.Args())
+	if len(p.Args()) != 6 {
+		t.Fatalf("unexpected arg len %d, %#v", len(p.Args()), p.Args())
 	}
 	if p.Args()[0] != "one" ||
 		p.Args()[1] != "two" ||
-		p.Args()[2] != "s/$FOO/foo/g" ||
-		p.Args()[3] != "s/$BAR/bar/g" ||
-		p.Args()[4] != "\\ \\ \\ " {
-		t.Fatalf("unexpected arg array: %v", p.Args())
+		p.Args()[2] != "foo bar" ||
+		p.Args()[3] != "s/$FOO/foo/g" ||
+		p.Args()[4] != "s/$BAR/bar baz/g" ||
+		p.Args()[5] != "\\ \\ \\ " {
+		t.Fatalf("unexpected arg array: %#v", p.Args())
 	}
 }
 

api/internal/plugins/loader/loader_test.go

@@ -6,10 +6,11 @@ package loader_test
 import (
 	"testing"
 
-	"sigs.k8s.io/kustomize/api/internal/loadertest"
+	"sigs.k8s.io/kustomize/api/filesys"
 	. "sigs.k8s.io/kustomize/api/internal/plugins/loader"
 	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
 	"sigs.k8s.io/kustomize/api/konfig"
+	"sigs.k8s.io/kustomize/api/loader"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
 	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
@@ -45,36 +46,33 @@ port: "12345"
 )
 
 func TestLoader(t *testing.T) {
-	tc := kusttest_test.NewPluginTestEnv(t).Set()
-	defer tc.Reset()
-
-	tc.BuildGoPlugin(
-		"builtin", "", "SecretGenerator")
-	tc.BuildGoPlugin(
-		"someteam.example.com", "v1", "SomeServiceGenerator")
-
+	th := kusttest_test.MakeEnhancedHarness(t).
+		BuildGoPlugin("builtin", "", "SecretGenerator").
+		BuildGoPlugin("someteam.example.com", "v1", "SomeServiceGenerator")
+	defer th.Reset()
 	rmF := resmap.NewFactory(resource.NewFactory(
 		kunstruct.NewKunstructuredFactoryImpl()), nil)
-
-	ldr := loadertest.NewFakeLoader("/foo")
-
+	fLdr, err := loader.NewLoader(
+		loader.RestrictionRootOnly,
+		filesys.Separator, filesys.MakeFsInMemory())
+	if err != nil {
+		t.Fatal(err)
+	}
 	c, err := konfig.EnabledPluginConfig()
 	if err != nil {
 		t.Fatal(err)
 	}
-
 	pLdr := NewLoader(c, rmF)
 	if pLdr == nil {
 		t.Fatal("expect non-nil loader")
 	}
-
 	m, err := rmF.NewResMapFromBytes([]byte(
 		someServiceGenerator + "---\n" + secretGenerator))
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	_, err = pLdr.LoadGenerators(ldr, valtest_test.MakeFakeValidator(), m)
+	_, err = pLdr.LoadGenerators(
+		fLdr, valtest_test.MakeFakeValidator(), m)
 	if err != nil {
 		t.Fatal(err)
 	}

api/internal/target/accumulation_test.go

@@ -1,60 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package target_test
-
-import (
-	"testing"
-
-	. "sigs.k8s.io/kustomize/api/internal/target"
-	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
-)
-
-func TestTargetMustHaveKustomizationFile(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app")
-	th.WriteF("/app/service.yaml", `
-apiVersion: v1
-kind: Service
-metadata:
-  name: aService
-`)
-	th.WriteF("/app/deeper/service.yaml", `
-apiVersion: v1
-kind: Service
-metadata:
-  name: anotherService
-`)
-	_, err := th.MakeKustTargetOrErr()
-	if err == nil {
-		t.Fatalf("expected an error")
-	}
-	if !IsMissingKustomizationFileError(err) {
-		t.Fatalf("unexpected error: %q", err)
-	}
-}
-
-func TestResourceDirectoryMustHaveKustomizationFile(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app")
-	th.WriteK("/app", `
-resources:
-- base
-`)
-	th.WriteF("/app/base/service.yaml", `
-apiVersion: v1
-kind: Service
-metadata:
-  name: myService
-spec:
-  selector:
-    backend: bungie
-  ports:
-    - port: 7002
-`)
-	_, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err == nil {
-		t.Fatalf("expected an error")
-	}
-	if !IsMissingKustomizationFileError(err) {
-		t.Fatalf("unexpected error: %q", err)
-	}
-}

api/internal/target/errmissingkustomization.go

@@ -0,0 +1,48 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package target
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	"sigs.k8s.io/kustomize/api/konfig"
+)
+
+type errMissingKustomization struct {
+	path string
+}
+
+func (e *errMissingKustomization) Error() string {
+	return fmt.Sprintf(
+		"unable to find one of %v in directory '%s'",
+		commaOr(quoted(konfig.RecognizedKustomizationFileNames())),
+		e.path)
+}
+
+func IsMissingKustomizationFileError(err error) bool {
+	_, ok := err.(*errMissingKustomization)
+	if ok {
+		return true
+	}
+	_, ok = errors.Cause(err).(*errMissingKustomization)
+	return ok
+}
+
+func NewErrMissingKustomization(p string) *errMissingKustomization {
+	return &errMissingKustomization{path: p}
+}
+
+func quoted(l []string) []string {
+	r := make([]string, len(l))
+	for i, v := range l {
+		r[i] = "'" + v + "'"
+	}
+	return r
+}
+
+func commaOr(q []string) string {
+	return strings.Join(q[:len(q)-1], ", ") + " or " + q[len(q)-1]
+}

api/internal/target/kusttarget.go

@@ -1,8 +1,6 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-// Package target implements state for the set of all
-// resources to customize.
 package target
 
 import (
@@ -36,50 +34,43 @@ type KustTarget struct {
 	pLdr          *loader.Loader
 }
 
-// NewKustTarget returns a new instance of KustTarget primed with a Loader.
+// NewKustTarget returns a new instance of KustTarget.
 func NewKustTarget(
 	ldr ifc.Loader,
 	validator ifc.Validator,
 	rFactory *resmap.Factory,
 	tFactory resmap.PatchFactory,
-	pLdr *loader.Loader) (*KustTarget, error) {
-	content, err := loadKustFile(ldr)
+	pLdr *loader.Loader) *KustTarget {
+	return &KustTarget{
+		ldr:       ldr,
+		validator: validator,
+		rFactory:  rFactory,
+		tFactory:  tFactory,
+		pLdr:      pLdr,
+	}
+}
+
+// Load attempts to load the target's kustomization file.
+func (kt *KustTarget) Load() error {
+	content, err := loadKustFile(kt.ldr)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	content = types.FixKustomizationPreUnmarshalling(content)
 	var k types.Kustomization
 	err = unmarshal(content, &k)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	k.FixKustomizationPostUnmarshalling()
 	errs := k.EnforceFields()
 	if len(errs) > 0 {
-		return nil, fmt.Errorf(
+		return fmt.Errorf(
 			"Failed to read kustomization file under %s:\n"+
-				strings.Join(errs, "\n"), ldr.Root())
+				strings.Join(errs, "\n"), kt.ldr.Root())
 	}
-	return &KustTarget{
-		kustomization: &k,
-		ldr:           ldr,
-		validator:     validator,
-		rFactory:      rFactory,
-		tFactory:      tFactory,
-		pLdr:          pLdr,
-	}, nil
-}
-
-func quoted(l []string) []string {
-	r := make([]string, len(l))
-	for i, v := range l {
-		r[i] = "'" + v + "'"
-	}
-	return r
-}
-
-func commaOr(q []string) string {
-	return strings.Join(q[:len(q)-1], ", ") + " or " + q[len(q)-1]
+	kt.kustomization = &k
+	return nil
 }
 
 func loadKustFile(ldr ifc.Loader) ([]byte, error) {
@@ -103,30 +94,6 @@ func loadKustFile(ldr ifc.Loader) ([]byte, error) {
 	}
 }
 
-type errMissingKustomization struct {
-	path string
-}
-
-func (e *errMissingKustomization) Error() string {
-	return fmt.Sprintf(
-		"unable to find one of %v in directory '%s'",
-		commaOr(quoted(konfig.RecognizedKustomizationFileNames())),
-		e.path)
-}
-
-func NewErrMissingKustomization(p string) *errMissingKustomization {
-	return &errMissingKustomization{path: p}
-}
-
-func IsMissingKustomizationFileError(err error) bool {
-	_, ok := err.(*errMissingKustomization)
-	if ok {
-		return true
-	}
-	_, ok = errors.Cause(err).(*errMissingKustomization)
-	return ok
-}
-
 func unmarshal(y []byte, o interface{}) error {
 	j, err := yaml.YAMLToJSON(y)
 	if err != nil {
@@ -137,8 +104,8 @@ func unmarshal(y []byte, o interface{}) error {
 	return dec.Decode(o)
 }
 
-// MakeCustomizedResMap creates a ResMap per kustomization instructions.
-// The Resources in the returned ResMap are fully customized.
+// MakeCustomizedResMap creates a fully customized ResMap
+// per the instructions contained in its kustomiztion instance.
 func (kt *KustTarget) MakeCustomizedResMap() (resmap.ResMap, error) {
 	return kt.makeCustomizedResMap(types.GarbageIgnore)
 }
@@ -356,8 +323,9 @@ func (kt *KustTarget) accumulateResources(
 func (kt *KustTarget) accumulateDirectory(
 	ra *accumulator.ResAccumulator, ldr ifc.Loader) error {
 	defer ldr.Cleanup()
-	subKt, err := NewKustTarget(
+	subKt := NewKustTarget(
 		ldr, kt.validator, kt.rFactory, kt.tFactory, kt.pLdr)
+	err := subKt.Load()
 	if err != nil {
 		return errors.Wrapf(
 			err, "couldn't make target for path '%s'", ldr.Root())

api/internal/target/kusttarget_test.go

@@ -5,23 +5,23 @@ package target_test
 
 import (
 	"encoding/base64"
-	"fmt"
-	"strings"
 	"testing"
 
+	"sigs.k8s.io/kustomize/api/filesys"
 	"sigs.k8s.io/kustomize/api/ifc"
-	"sigs.k8s.io/kustomize/api/internal/loadertest"
-	. "sigs.k8s.io/kustomize/api/internal/target"
-	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
 	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
-	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
-	"sigs.k8s.io/kustomize/api/types"
 )
 
-const (
-	kustomizationContent = `
+// KustTarget is primarily tested in the krusty package with
+// high level tests.
+
+func TestMakeCustomizedResMap(t *testing.T) {
+	fSys := filesys.MakeFsInMemory()
+	th := kusttest_test.MakeHarnessWithFs(t, fSys)
+	th.WriteK("/whatever", `
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namePrefix: foo-
@@ -54,33 +54,27 @@ patchesJson6902:
     kind: Deployment
     name: dply1
   path: jsonpatch.json
-`
-	deploymentContent = `
+`)
+	th.WriteF("/whatever/deployment.yaml", `
 apiVersion: apps/v1
 metadata:
   name: dply1
 kind: Deployment
-`
-	namespaceContent = `
+`)
+	th.WriteF("/whatever/namespace.yaml", `
 apiVersion: v1
 kind: Namespace
 metadata:
   name: ns1
-`
-	jsonpatchContent = `[
+`)
+	th.WriteF("/whatever/jsonpatch.json", `[
     {"op": "add", "path": "/spec/replica", "value": "3"}
-]`
-)
+]`)
 
-func TestResources(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/whatever")
-	th.WriteK("/whatever/", kustomizationContent)
-	th.WriteF("/whatever/deployment.yaml", deploymentContent)
-	th.WriteF("/whatever/namespace.yaml", namespaceContent)
-	th.WriteF("/whatever/jsonpatch.json", jsonpatchContent)
+	resFactory := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())
 
 	resources := []*resource.Resource{
-		th.RF().FromMapWithName("dply1", map[string]interface{}{
+		resFactory.FromMapWithName("dply1", map[string]interface{}{
 			"apiVersion": "apps/v1",
 			"kind":       "Deployment",
 			"metadata": map[string]interface{}{
@@ -112,7 +106,7 @@ func TestResources(t *testing.T) {
 				},
 			},
 		}),
-		th.RF().FromMapWithName("ns1", map[string]interface{}{
+		resFactory.FromMapWithName("ns1", map[string]interface{}{
 			"apiVersion": "v1",
 			"kind":       "Namespace",
 			"metadata": map[string]interface{}{
@@ -125,7 +119,7 @@ func TestResources(t *testing.T) {
 				},
 			},
 		}),
-		th.RF().FromMapWithName("literalConfigMap",
+		resFactory.FromMapWithName("literalConfigMap",
 			map[string]interface{}{
 				"apiVersion": "v1",
 				"kind":       "ConfigMap",
@@ -144,7 +138,7 @@ func TestResources(t *testing.T) {
 					"DB_PASSWORD": "somepw",
 				},
 			}),
-		th.RF().FromMapWithName("secret",
+		resFactory.FromMapWithName("secret",
 			map[string]interface{}{
 				"apiVersion": "v1",
 				"kind":       "Secret",
@@ -173,7 +167,8 @@ func TestResources(t *testing.T) {
 		}
 	}
 
-	actual, err := th.MakeKustTarget().MakeCustomizedResMap()
+	actual, err := makeKustTargetWithRf(
+		t, fSys, "/whatever", resFactory).MakeCustomizedResMap()
 	if err != nil {
 		t.Fatalf("unexpected Resources error %v", err)
 	}
@@ -182,265 +177,3 @@ func TestResources(t *testing.T) {
 		t.Fatalf("unexpected inequality: %v", err)
 	}
 }
-
-func TestKustomizationNotFound(t *testing.T) {
-	_, err := NewKustTarget(
-		loadertest.NewFakeLoader("/foo"),
-		valtest_test.MakeFakeValidator(), nil, nil, nil)
-	if err == nil {
-		t.Fatalf("expected an error")
-	}
-	if err.Error() !=
-		`unable to find one of 'kustomization.yaml', 'kustomization.yml' or 'Kustomization' in directory '/foo'` {
-		t.Fatalf("unexpected error: %q", err)
-	}
-}
-
-func TestResourceNotFound(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/whatever")
-	th.WriteK("/whatever", kustomizationContent)
-	_, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err == nil {
-		t.Fatalf("Didn't get the expected error for an unknown resource")
-	}
-	if !strings.Contains(err.Error(), `cannot read file`) {
-		t.Fatalf("unexpected error: %q", err)
-	}
-}
-
-func findSecret(m resmap.ResMap) *resource.Resource {
-	for _, r := range m.Resources() {
-		if r.OrgId().Kind == "Secret" {
-			return r
-		}
-	}
-	return nil
-}
-
-func TestDisableNameSuffixHash(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/whatever")
-	th.WriteK("/whatever/", kustomizationContent)
-	th.WriteF("/whatever/deployment.yaml", deploymentContent)
-	th.WriteF("/whatever/namespace.yaml", namespaceContent)
-	th.WriteF("/whatever/jsonpatch.json", jsonpatchContent)
-
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("unexpected Resources error %v", err)
-	}
-	secret := findSecret(m)
-	if secret == nil {
-		t.Errorf("Expected to find a Secret")
-	}
-	if secret.GetName() != "foo-secret-bar-9btc7bt4kb" {
-		t.Errorf("unexpected secret resource name: %s", secret.GetName())
-	}
-
-	th.WriteK("/whatever/",
-		strings.Replace(kustomizationContent,
-			"disableNameSuffixHash: false",
-			"disableNameSuffixHash: true", -1))
-	m, err = th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("unexpected Resources error %v", err)
-	}
-	secret = findSecret(m)
-	if secret == nil {
-		t.Errorf("Expected to find a Secret")
-	}
-	if secret.GetName() != "foo-secret-bar" { // No hash at end.
-		t.Errorf("unexpected secret resource name: %s", secret.GetName())
-	}
-}
-
-func TestIssue596AllowDirectoriesThatAreSubstringsOfEachOther(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/overlays/aws-sandbox2.us-east-1")
-	th.WriteK("/app/base", "")
-	th.WriteK("/app/overlays/aws", `
-resources:
-- ../../base
-`)
-	th.WriteK("/app/overlays/aws-nonprod", `
-resources:
-- ../aws
-`)
-	th.WriteK("/app/overlays/aws-sandbox2.us-east-1", `
-resources:
-- ../aws-nonprod
-`)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
-	th.AssertActualEqualsExpected(m, "")
-}
-
-// To simplify tests, these vars specified in alphabetical order.
-var someVars = []types.Var{
-	{
-		Name: "AWARD",
-		ObjRef: types.Target{
-			APIVersion: "v7",
-			Gvk:        resid.Gvk{Kind: "Service"},
-			Name:       "nobelPrize"},
-		FieldRef: types.FieldSelector{FieldPath: "some.arbitrary.path"},
-	},
-	{
-		Name: "BIRD",
-		ObjRef: types.Target{
-			APIVersion: "v300",
-			Gvk:        resid.Gvk{Kind: "Service"},
-			Name:       "heron"},
-		FieldRef: types.FieldSelector{FieldPath: "metadata.name"},
-	},
-	{
-		Name: "FRUIT",
-		ObjRef: types.Target{
-			Gvk:  resid.Gvk{Kind: "Service"},
-			Name: "apple"},
-		FieldRef: types.FieldSelector{FieldPath: "metadata.name"},
-	},
-	{
-		Name: "VEGETABLE",
-		ObjRef: types.Target{
-			Gvk:  resid.Gvk{Kind: "Leafy"},
-			Name: "kale"},
-		FieldRef: types.FieldSelector{FieldPath: "metadata.name"},
-	},
-}
-
-func TestGetAllVarsSimple(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app")
-	th.WriteK("/app", `
-vars:
-  - name: AWARD
-    objref:
-      kind: Service
-      name: nobelPrize
-      apiVersion: v7
-    fieldref:
-      fieldpath: some.arbitrary.path
-  - name: BIRD
-    objref:
-      kind: Service
-      name: heron
-      apiVersion: v300
-`)
-	ra, err := th.MakeKustTarget().AccumulateTarget()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
-	vars := ra.Vars()
-	if len(vars) != 2 {
-		t.Fatalf("unexpected size %d", len(vars))
-	}
-	for i := range vars[:2] {
-		// By using Var.DeepEqual, we are protecting the code
-		// from a potential invocation of vars[i].ObjRef.GVK()
-		// during AccumulateTarget
-		if !vars[i].DeepEqual(someVars[i]) {
-			t.Fatalf("unexpected var[%d]:\n  %v\n  %v", i, vars[i], someVars[i])
-		}
-	}
-}
-
-func TestGetAllVarsNested(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/overlays/o2")
-	th.WriteK("/app/base", `
-vars:
-  - name: AWARD
-    objref:
-      kind: Service
-      name: nobelPrize
-      apiVersion: v7
-    fieldref:
-      fieldpath: some.arbitrary.path
-  - name: BIRD
-    objref:
-      kind: Service
-      name: heron
-      apiVersion: v300
-`)
-	th.WriteK("/app/overlays/o1", `
-vars:
-  - name: FRUIT
-    objref:
-      kind: Service
-      name: apple
-resources:
-- ../../base
-`)
-	th.WriteK("/app/overlays/o2", `
-vars:
-  - name: VEGETABLE
-    objref:
-      kind: Leafy
-      name: kale
-resources:
-- ../o1
-`)
-	ra, err := th.MakeKustTarget().AccumulateTarget()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
-	vars := ra.Vars()
-	if len(vars) != 4 {
-		for i, v := range vars {
-			fmt.Printf("%v: %v\n", i, v)
-		}
-		t.Fatalf("expected 4 vars, got %d", len(vars))
-	}
-	for i := range vars {
-		// By using Var.DeepEqual, we are protecting the code
-		// from a potential invocation of vars[i].ObjRef.GVK()
-		// during AccumulateTarget
-		if !vars[i].DeepEqual(someVars[i]) {
-			t.Fatalf("unexpected var[%d]:\n  %v\n  %v", i, vars[i], someVars[i])
-		}
-	}
-}
-
-func TestVarCollisionsForbidden(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/overlays/o2")
-	th.WriteK("/app/base", `
-vars:
-  - name: AWARD
-    objref:
-      kind: Service
-      name: nobelPrize
-      apiVersion: v7
-    fieldref:
-      fieldpath: some.arbitrary.path
-  - name: BIRD
-    objref:
-      kind: Service
-      name: heron
-      apiVersion: v300
-`)
-	th.WriteK("/app/overlays/o1", `
-vars:
-  - name: AWARD
-    objref:
-      kind: Service
-      name: academy
-resources:
-- ../../base
-`)
-	th.WriteK("/app/overlays/o2", `
-vars:
-  - name: VEGETABLE
-    objref:
-      kind: Leafy
-      name: kale
-resources:
-- ../o1
-`)
-	_, err := th.MakeKustTarget().AccumulateTarget()
-	if err == nil {
-		t.Fatalf("expected var collision")
-	}
-	if !strings.Contains(err.Error(),
-		"var 'AWARD' already encountered") {
-		t.Fatalf("unexpected error: %v", err)
-	}
-}

api/internal/target/maker_test.go

@@ -0,0 +1,51 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package target_test
+
+import (
+	"testing"
+
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/internal/k8sdeps/transformer"
+	pLdr "sigs.k8s.io/kustomize/api/internal/plugins/loader"
+	"sigs.k8s.io/kustomize/api/internal/target"
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/api/konfig"
+	fLdr "sigs.k8s.io/kustomize/api/loader"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
+)
+
+func makeKustTarget(
+	t *testing.T,
+	fSys filesys.FileSystem,
+	root string) *target.KustTarget {
+	return makeKustTargetWithRf(
+		t, fSys, root,
+		resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()))
+}
+
+func makeKustTargetWithRf(
+	t *testing.T,
+	fSys filesys.FileSystem,
+	root string,
+	resFact *resource.Factory) *target.KustTarget {
+	rf := resmap.NewFactory(resFact, transformer.NewFactoryImpl())
+	pc := konfig.DisabledPluginConfig()
+	ldr, err := fLdr.NewLoader(fLdr.RestrictionRootOnly, root, fSys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	kt := target.NewKustTarget(
+		ldr,
+		valtest_test.MakeFakeValidator(),
+		rf,
+		transformer.NewFactoryImpl(),
+		pLdr.NewLoader(pc, rf))
+	if err = kt.Load(); err != nil {
+		t.Fatalf("Unexpected construction error %v", err)
+	}
+	return kt
+}

api/internal/target/plugindir_test.go

@@ -1,93 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package target_test
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"sigs.k8s.io/kustomize/api/filesys"
-	"sigs.k8s.io/kustomize/api/internal/k8sdeps/transformer"
-	pLdr "sigs.k8s.io/kustomize/api/internal/plugins/loader"
-	. "sigs.k8s.io/kustomize/api/internal/target"
-	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
-	"sigs.k8s.io/kustomize/api/konfig"
-	fLdr "sigs.k8s.io/kustomize/api/loader"
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/resource"
-	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
-	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
-)
-
-func TestPluginDir(t *testing.T) {
-	tc := kusttest_test.NewPluginTestEnv(t).Set()
-	defer tc.Reset()
-
-	tc.BuildExecPlugin(
-		"someteam.example.com", "v1", "PrintWorkDir")
-
-	base, err := os.Getwd()
-	if err != nil {
-		t.Fatalf("err %v", err)
-	}
-	dir, err := ioutil.TempDir(base, "kustomize-")
-	if err != nil {
-		t.Fatalf("err %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	fSys := filesys.MakeFsOnDisk()
-	err = fSys.WriteFile(filepath.Join(dir, "kustomization.yaml"), []byte(`
-generators:
-- config.yaml
-`))
-	if err != nil {
-		t.Fatalf("err %v", err)
-	}
-	err = fSys.WriteFile(filepath.Join(dir, "config.yaml"), []byte(`
-apiVersion: someteam.example.com/v1
-kind: PrintWorkDir
-metadata:
-  name: some-random-name
-`))
-	if err != nil {
-		t.Fatalf("err %v", err)
-	}
-
-	ldr, err := fLdr.NewLoader(
-		fLdr.RestrictionRootOnly, dir, fSys)
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
-	rf := resmap.NewFactory(resource.NewFactory(
-		kunstruct.NewKunstructuredFactoryImpl()), nil)
-
-	c, err := konfig.EnabledPluginConfig()
-	if err != nil {
-		t.Fatal(err)
-	}
-	pl := pLdr.NewLoader(c, rf)
-	tg, err := NewKustTarget(
-		ldr, valtest_test.MakeFakeValidator(), rf, transformer.NewFactoryImpl(), pl)
-	if err != nil {
-		t.Fatalf("err %v", err)
-	}
-
-	m, err := tg.MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
-
-	th := kusttest_test.NewKustTestHarness(t, ".")
-	th.AssertActualEqualsExpected(m, `
-apiVersion: v1
-kind: WorkDir
-metadata:
-  name: `+dir+`
-spec:
-  path: `+dir+`
-`)
-}

api/internal/target/vars_test.go

@@ -0,0 +1,188 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package target_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+// To simplify tests, these vars specified in alphabetical order.
+var someVars = []types.Var{
+	{
+		Name: "AWARD",
+		ObjRef: types.Target{
+			APIVersion: "v7",
+			Gvk:        resid.Gvk{Kind: "Service"},
+			Name:       "nobelPrize"},
+		FieldRef: types.FieldSelector{FieldPath: "some.arbitrary.path"},
+	},
+	{
+		Name: "BIRD",
+		ObjRef: types.Target{
+			APIVersion: "v300",
+			Gvk:        resid.Gvk{Kind: "Service"},
+			Name:       "heron"},
+		FieldRef: types.FieldSelector{FieldPath: "metadata.name"},
+	},
+	{
+		Name: "FRUIT",
+		ObjRef: types.Target{
+			Gvk:  resid.Gvk{Kind: "Service"},
+			Name: "apple"},
+		FieldRef: types.FieldSelector{FieldPath: "metadata.name"},
+	},
+	{
+		Name: "VEGETABLE",
+		ObjRef: types.Target{
+			Gvk:  resid.Gvk{Kind: "Leafy"},
+			Name: "kale"},
+		FieldRef: types.FieldSelector{FieldPath: "metadata.name"},
+	},
+}
+
+func TestGetAllVarsSimple(t *testing.T) {
+	th := kusttest_test.MakeHarness(t)
+	th.WriteK("/app", `
+vars:
+  - name: AWARD
+    objref:
+      kind: Service
+      name: nobelPrize
+      apiVersion: v7
+    fieldref:
+      fieldpath: some.arbitrary.path
+  - name: BIRD
+    objref:
+      kind: Service
+      name: heron
+      apiVersion: v300
+`)
+	ra, err := makeKustTarget(
+		t, th.GetFSys(), "/app").AccumulateTarget()
+	if err != nil {
+		t.Fatalf("Err: %v", err)
+	}
+	vars := ra.Vars()
+	if len(vars) != 2 {
+		t.Fatalf("unexpected size %d", len(vars))
+	}
+	for i := range vars[:2] {
+		// By using Var.DeepEqual, we are protecting the code
+		// from a potential invocation of vars[i].ObjRef.GVK()
+		// during accumulateTarget
+		if !vars[i].DeepEqual(someVars[i]) {
+			t.Fatalf("unexpected var[%d]:\n  %v\n  %v", i, vars[i], someVars[i])
+		}
+	}
+}
+
+func TestGetAllVarsNested(t *testing.T) {
+	th := kusttest_test.MakeHarness(t)
+	th.WriteK("/app/base", `
+vars:
+  - name: AWARD
+    objref:
+      kind: Service
+      name: nobelPrize
+      apiVersion: v7
+    fieldref:
+      fieldpath: some.arbitrary.path
+  - name: BIRD
+    objref:
+      kind: Service
+      name: heron
+      apiVersion: v300
+`)
+	th.WriteK("/app/overlays/o1", `
+vars:
+  - name: FRUIT
+    objref:
+      kind: Service
+      name: apple
+resources:
+- ../../base
+`)
+	th.WriteK("/app/overlays/o2", `
+vars:
+  - name: VEGETABLE
+    objref:
+      kind: Leafy
+      name: kale
+resources:
+- ../o1
+`)
+
+	ra, err := makeKustTarget(
+		t, th.GetFSys(), "/app/overlays/o2").AccumulateTarget()
+	if err != nil {
+		t.Fatalf("Err: %v", err)
+	}
+	vars := ra.Vars()
+	if len(vars) != 4 {
+		for i, v := range vars {
+			fmt.Printf("%v: %v\n", i, v)
+		}
+		t.Fatalf("expected 4 vars, got %d", len(vars))
+	}
+	for i := range vars {
+		// By using Var.DeepEqual, we are protecting the code
+		// from a potential invocation of vars[i].ObjRef.GVK()
+		// during accumulateTarget
+		if !vars[i].DeepEqual(someVars[i]) {
+			t.Fatalf("unexpected var[%d]:\n  %v\n  %v", i, vars[i], someVars[i])
+		}
+	}
+}
+
+func TestVarCollisionsForbidden(t *testing.T) {
+	th := kusttest_test.MakeHarness(t)
+	th.WriteK("/app/base", `
+vars:
+  - name: AWARD
+    objref:
+      kind: Service
+      name: nobelPrize
+      apiVersion: v7
+    fieldref:
+      fieldpath: some.arbitrary.path
+  - name: BIRD
+    objref:
+      kind: Service
+      name: heron
+      apiVersion: v300
+`)
+	th.WriteK("/app/overlays/o1", `
+vars:
+  - name: AWARD
+    objref:
+      kind: Service
+      name: academy
+resources:
+- ../../base
+`)
+	th.WriteK("/app/overlays/o2", `
+vars:
+  - name: VEGETABLE
+    objref:
+      kind: Leafy
+      name: kale
+resources:
+- ../o1
+`)
+	_, err := makeKustTarget(
+		t, th.GetFSys(), "/app/overlays/o2").AccumulateTarget()
+	if err == nil {
+		t.Fatalf("expected var collision")
+	}
+	if !strings.Contains(err.Error(),
+		"var 'AWARD' already encountered") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}

api/internal/tools/tools.go

@@ -3,7 +3,10 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-// This file exists to trigger installs of the given tools.
+// This file exists to automatically trigger installs
+// of the given tools, and is the offical 'unofficial'
+// way to declare a dependence on a Go binary until
+// some better technique comes along.
 
 package tools
 
@@ -12,8 +15,11 @@ import (
 	_ "golang.org/x/tools/cmd/stringer"
 	// for lint checks
 	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
-	// for integration tests driven by the examples
-	_ "github.com/monopole/mdrip"
-	// for generating code for builtin plugins
-	_ "sigs.k8s.io/kustomize/pluginator/v2"
+	// REMOVED pluginator from this process, and leaving
+	// this note to discourage its reintroduction,
+	// because pluginator depends on the api, forcing
+	// major version increments in pluginator with each
+	// api release to allow this trick to work and not
+	// introduce cycles.
+	// _ "sigs.k8s.io/kustomize/pluginator/v2"
 )

api/k8sdeps/doc.go

@@ -21,18 +21,18 @@
 //       tree/master/pkg/util/yaml
 //
 // It vendors that package into
-//   sigs.k8s.io/kustomize/vendor/sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/
+//   sigs.k8s.io/kustomize/vendor/k8s.io/apimachinery/
 //
 // Whereas kubectl-kustomize would have to depend on the "staging"
 // version of this code, located at
 //
 //   https://github.com/kubernetes/kubernetes/
-//       blob/master/staging/src/sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/util/yaml
+//       blob/master/staging/src/k8s.io/apimachinery/pkg/util/yaml
 //
 // which is "vendored" via symlinks:
-//   k8s.io/kubernetes/vendor/sigs.k8s.io/kustomize/pseudo/k8s/apimachinery
+//   k8s.io/kubernetes/vendor/k8s.io/apimachinery
 // is a symlink to
-//   ../../staging/src/sigs.k8s.io/kustomize/pseudo/k8s/apimachinery
+//   ../../staging/src/k8s.io/apimachinery
 //
 // The staging version is the canonical, under-development
 // version of the code that kubectl depends on, whereas the packages

api/k8sdeps/kunstruct/factory.go

@@ -10,11 +10,11 @@ import (
 	"strconv"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/kustomize/api/ifc"
 	"sigs.k8s.io/kustomize/api/internal/k8sdeps/configmapandsecret"
 	"sigs.k8s.io/kustomize/api/types"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1/unstructured"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/util/yaml"
 )
 
 // KunstructuredFactoryImpl hides construction using apimachinery types.

api/k8sdeps/kunstruct/hasher.go

@@ -7,10 +7,10 @@ import (
 	"encoding/json"
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sigs.k8s.io/kustomize/api/hasher"
 	"sigs.k8s.io/kustomize/api/ifc"
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
 // kustHash computes a hash of an unstructured object.

api/k8sdeps/kunstruct/hasher_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 )
 
 func TestConfigMapHash(t *testing.T) {

api/k8sdeps/kunstruct/unstructadapter.go

@@ -9,15 +9,15 @@ import (
 	"fmt"
 
 	jsonpatch "github.com/evanphx/json-patch"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	"k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/kustomize/api/ifc"
 	"sigs.k8s.io/kustomize/api/resid"
-	metav1 "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1/unstructured"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/labels"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/runtime"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/util/strategicpatch"
-	"sigs.k8s.io/kustomize/pseudo/k8s/client-go/kubernetes/scheme"
 )
 
 var _ ifc.Kunstructured = &UnstructAdapter{}

api/k8sdeps/validator/validators.go

@@ -10,10 +10,10 @@ import (
 	"fmt"
 	"strings"
 
-	apivalidation "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/api/validation"
-	v1validation "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1/validation"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/util/validation"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/util/validation/field"
+	apivalidation "k8s.io/apimachinery/pkg/api/validation"
+	v1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
 // KustValidator validates Labels and annotations by apimachinery

api/konfig/builtinpluginconsts/varreference.go

@@ -190,5 +190,7 @@ varReference:
   kind: StatefulSet
 
 - path: metadata/labels
+
+- path: metadata/annotations
 `
 )

api/konfig/plugins.go

@@ -139,7 +139,7 @@ func CurrentWorkingDir() string {
 	if len(pwd) > 0 {
 		return pwd
 	}
-	return "."
+	return filesys.SelfDir
 }
 
 func pwdEnv() string {

api/krusty/accumulation_test.go

@@ -0,0 +1,95 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package krusty_test
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	. "sigs.k8s.io/kustomize/api/internal/target"
+	"sigs.k8s.io/kustomize/api/konfig"
+	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
+)
+
+func TestTargetMustHaveKustomizationFile(t *testing.T) {
+	th := kusttest_test.MakeHarness(t)
+	th.WriteF("/app/service.yaml", `
+apiVersion: v1
+kind: Service
+metadata:
+  name: aService
+`)
+	th.WriteF("/app/deeper/service.yaml", `
+apiVersion: v1
+kind: Service
+metadata:
+  name: anotherService
+`)
+	err := th.RunWithErr("/app", th.MakeDefaultOptions())
+	if err == nil {
+		t.Fatalf("expected an error")
+	}
+	if !IsMissingKustomizationFileError(err) {
+		t.Fatalf("unexpected error: %q", err)
+	}
+}
+
+func TestTargetMustHaveOnlyOneKustomizationFile(t *testing.T) {
+	th := kusttest_test.MakeHarness(t)
+	for _, n := range konfig.RecognizedKustomizationFileNames() {
+		th.WriteF(filepath.Join("/app", n), `
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+`)
+	}
+	err := th.RunWithErr("/app", th.MakeDefaultOptions())
+	if err == nil {
+		t.Fatalf("expected an error")
+	}
+	if !strings.Contains(err.Error(), "Found multiple kustomization files under: /app") {
+		t.Fatalf("unexpected error: %q", err)
+	}
+}
+
+func TestBaseMustHaveKustomizationFile(t *testing.T) {
+	th := kusttest_test.MakeHarness(t)
+	th.WriteK("/app", `
+resources:
+- base
+`)
+	th.WriteF("/app/base/service.yaml", `
+apiVersion: v1
+kind: Service
+metadata:
+  name: myService
+spec:
+  selector:
+    backend: bungie
+  ports:
+    - port: 7002
+`)
+	err := th.RunWithErr("/app", th.MakeDefaultOptions())
+	if err == nil {
+		t.Fatalf("expected an error")
+	}
+	if !IsMissingKustomizationFileError(err) {
+		t.Fatalf("unexpected error: %q", err)
+	}
+}
+
+func TestResourceNotFound(t *testing.T) {
+	th := kusttest_test.MakeHarness(t)
+	th.WriteK("/app", `
+resources:
+- deployment.yaml
+`)
+	err := th.RunWithErr("/app", th.MakeDefaultOptions())
+	if err == nil {
+		t.Fatalf("expected an error")
+	}
+	if !strings.Contains(err.Error(), "'/app/deployment.yaml' doesn't exist") {
+		t.Fatalf("unexpected error: %q", err)
+	}
+}

api/krusty/baseandoverlaymedium_test.go

@@ -1,7 +1,7 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package target_test
+package krusty_test
 
 import (
 	"testing"
@@ -9,7 +9,7 @@ import (
 	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
 )
 
-func writeMediumBase(th *kusttest_test.KustTestHarness) {
+func writeMediumBase(th kusttest_test.Harness) {
 	th.WriteK("/app/base", `
 namePrefix: baseprefix-
 commonLabels:
@@ -59,12 +59,9 @@ spec:
 }
 
 func TestMediumBase(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/base")
+	th := kusttest_test.MakeHarness(t)
 	writeMediumBase(th)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app/base", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, `
 apiVersion: apps/v1
 kind: Deployment
@@ -116,7 +113,7 @@ spec:
 }
 
 func TestMediumOverlay(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/overlay")
+	th := kusttest_test.MakeHarness(t)
 	writeMediumBase(th)
 	th.WriteK("/app/overlay", `
 namePrefix: test-infra-
@@ -156,9 +153,9 @@ FRUIT=banana
 LEGUME=chickpea
 `)
 	th.WriteF("/app/overlay/configmap/dummy.txt",
-		`Lorem ipsum dolor sit amet, consectetur
-adipiscing elit, sed do eiusmod tempor
-incididunt ut labore et dolore magna aliqua. 
+		`Lorem ipsum dolor sit amet, consectetur  
+adipiscing elit, sed do eiusmod tempor		
+incididunt ut labore et dolore magna aliqua.   
 `)
 	th.WriteF("/app/overlay/deployment/deployment.yaml", `
 apiVersion: apps/v1
@@ -193,10 +190,7 @@ spec:
           name: app-env
         name: app-env
 `)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app/overlay", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, `
 apiVersion: apps/v1
 kind: Deployment
@@ -298,8 +292,10 @@ metadata:
 ---
 apiVersion: v1
 data:
-  nonsense: "Lorem ipsum dolor sit amet, consectetur\nadipiscing elit, sed do eiusmod
-    tempor\nincididunt ut labore et dolore magna aliqua. \n"
+  nonsense: |
+    Lorem ipsum dolor sit amet, consectetur
+    adipiscing elit, sed do eiusmod tempor
+    incididunt ut labore et dolore magna aliqua.
 kind: ConfigMap
 metadata:
   annotations:
@@ -308,6 +304,6 @@ metadata:
     app: mungebot
     org: kubernetes
     repo: test-infra
-  name: test-infra-app-config-f462h769f9
+  name: test-infra-app-config-4mt28b5bg2
 `)
 }

api/krusty/baseandoverlaysmall_test.go

@@ -1,19 +1,19 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package target_test
+package krusty_test
 
 import (
 	"strings"
 	"testing"
 
-	"sigs.k8s.io/kustomize/api/konfig"
-	"sigs.k8s.io/kustomize/api/loader"
+	. "sigs.k8s.io/kustomize/api/krusty"
 	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
 func TestOrderPreserved(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/prod")
+	th := kusttest_test.MakeHarness(t)
 	th.WriteK("/app/base", `
 namePrefix: b-
 resources:
@@ -65,11 +65,7 @@ kind: Namespace
 metadata:
   name: myNs2
 `)
-
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app/prod", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, `
 apiVersion: v1
 kind: Namespace
@@ -104,7 +100,7 @@ metadata:
 }
 
 func TestBaseInResourceList(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/prod")
+	th := kusttest_test.MakeHarness(t)
 	th.WriteK("/app/prod", `
 namePrefix: b-
 resources:
@@ -124,10 +120,7 @@ spec:
   selector:
     backend: bungie
 `)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app/prod", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, `
 apiVersion: v1
 kind: Service
@@ -139,7 +132,7 @@ spec:
 `)
 }
 
-func writeSmallBase(th *kusttest_test.KustTestHarness) {
+func writeSmallBase(th kusttest_test.Harness) {
 	th.WriteK("/app/base", `
 namePrefix: a-
 commonLabels:
@@ -177,12 +170,9 @@ spec:
 }
 
 func TestSmallBase(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/base")
+	th := kusttest_test.MakeHarness(t)
 	writeSmallBase(th)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app/base", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, `
 apiVersion: apps/v1
 kind: Deployment
@@ -220,7 +210,7 @@ spec:
 }
 
 func TestSmallOverlay(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/overlay")
+	th := kusttest_test.MakeHarness(t)
 	writeSmallBase(th)
 	th.WriteK("/app/overlay", `
 namePrefix: b-
@@ -251,10 +241,7 @@ metadata:
 spec:
   replicas: 1000
 `)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app/overlay", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, `
 apiVersion: apps/v1
 kind: Deployment
@@ -298,9 +285,7 @@ spec:
 }
 
 func TestSharedPatchDisAllowed(t *testing.T) {
-	th := kusttest_test.NewKustTestHarnessFull(
-		t, "/app/overlay",
-		loader.RestrictionRootOnly, konfig.DisabledPluginConfig())
+	th := kusttest_test.MakeHarness(t)
 	writeSmallBase(th)
 	th.WriteK("/app/overlay", `
 commonLabels:
@@ -318,10 +303,11 @@ metadata:
 spec:
   replicas: 1000
 `)
-	_, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err == nil {
-		t.Fatalf("expected error")
-	}
+	err := th.RunWithErr("/app/overlay", func() Options {
+		o := th.MakeDefaultOptions()
+		o.LoadRestrictions = types.LoadRestrictionsRootOnly
+		return o
+	}())
 	if !strings.Contains(
 		err.Error(),
 		"security; file '/app/shared/deployment-patch.yaml' is not in or below '/app/overlay'") {
@@ -330,9 +316,7 @@ spec:
 }
 
 func TestSharedPatchAllowed(t *testing.T) {
-	th := kusttest_test.NewKustTestHarnessFull(
-		t, "/app/overlay",
-		loader.RestrictionNone, konfig.DisabledPluginConfig())
+	th := kusttest_test.MakeHarness(t)
 	writeSmallBase(th)
 	th.WriteK("/app/overlay", `
 commonLabels:
@@ -350,10 +334,11 @@ metadata:
 spec:
   replicas: 1000
 `)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app/overlay", func() Options {
+		o := th.MakeDefaultOptions()
+		o.LoadRestrictions = types.LoadRestrictionsNone
+		return o
+	}())
 	th.AssertActualEqualsExpected(m, `
 apiVersion: apps/v1
 kind: Deployment
@@ -397,7 +382,7 @@ spec:
 }
 
 func TestSmallOverlayJSONPatch(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/overlay")
+	th := kusttest_test.MakeHarness(t)
 	writeSmallBase(th)
 	th.WriteK("/app/overlay", `
 resources:
@@ -415,10 +400,7 @@ patchesJson6902:
   path: /spec/selector/backend
   value: beagle
 `)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app/overlay", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, `
 apiVersion: apps/v1
 kind: Deployment

api/krusty/basereusenameprefix_test.go

@@ -1,7 +1,7 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package target_test
+package krusty_test
 
 import (
 	"testing"
@@ -40,7 +40,7 @@ import (
 // ├── kustomization.yaml
 
 func TestBaseReuseNameConflict(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app")
+	th := kusttest_test.MakeHarness(t)
 	th.WriteK("/app/component1/base", `
 resources:
   - ../../shared
@@ -117,10 +117,7 @@ resources:
   - component2/overlay
 `)
 
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, `
 apiVersion: v1
 kind: PersistentVolumeClaim

api/krusty/chartinflatorplugin_test.go

@@ -5,13 +5,13 @@
 
 // Disabled on travis, because don't want to install helm on travis.
 
-package target_test
+package krusty_test
 
 import (
 	"regexp"
 	"testing"
 
-	"sigs.k8s.io/kustomize/api/testutils/kusttest"
+	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
 )
 
 // This is an example of using a helm chart as a base,
@@ -28,13 +28,10 @@ import (
 // TODO: Download and inflate the chart, and check that
 // in for the test.
 func TestChartInflatorPlugin(t *testing.T) {
-	tc := kusttest_test.NewPluginTestEnv(t).Set()
-	defer tc.Reset()
+	th := kusttest_test.MakeEnhancedHarness(t).
+		PrepExecPlugin("someteam.example.com", "v1", "ChartInflator")
+	defer th.Reset()
 
-	tc.BuildExecPlugin(
-		"someteam.example.com", "v1", "ChartInflator")
-
-	th := kusttest_test.NewKustTestHarnessAllowPlugins(t, "/app")
 	th.WriteK("/app", `
 generators:
 - chartInflator.yaml
@@ -49,10 +46,7 @@ metadata:
 chartName: minecraft
 `)
 
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Err: %v", err)
-	}
+	m := th.Run("/app", th.MakeOptionsPluginsEnabled())
 	chartName := regexp.MustCompile("chart: minecraft-[0-9.]+")
 	th.AssertActualEqualsExpectedWithTweak(m,
 		func(x []byte) []byte {

api/krusty/complexcomposition_test.go

@@ -1,13 +1,15 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package target_test
+package krusty_test
 
 import (
 	"strings"
 	"testing"
 
+	. "sigs.k8s.io/kustomize/api/krusty"
 	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
 const httpsService = `
@@ -24,7 +26,7 @@ spec:
     app: my-app
 `
 
-func writeStatefulSetBase(th *kusttest_test.KustTestHarness) {
+func writeStatefulSetBase(th kusttest_test.Harness) {
 	th.WriteK("/app/base", `
 resources:
 - statefulset.yaml
@@ -53,7 +55,7 @@ spec:
 `)
 }
 
-func writeHTTPSOverlay(th *kusttest_test.KustTestHarness) {
+func writeHTTPSOverlay(th kusttest_test.Harness) {
 	th.WriteK("/app/https", `
 resources:
 - ../base
@@ -72,7 +74,7 @@ spec:
 `)
 }
 
-func writeHTTPSTransformerRaw(th *kusttest_test.KustTestHarness) {
+func writeHTTPSTransformerRaw(th kusttest_test.Harness) {
 	th.WriteF("/app/https/service/https-svc.yaml", httpsService)
 	th.WriteF("/app/https/transformer/transformer.yaml", `
 apiVersion: builtin
@@ -94,7 +96,7 @@ patch: |-
 `)
 }
 
-func writeHTTPSTransformerBase(th *kusttest_test.KustTestHarness) {
+func writeHTTPSTransformerBase(th kusttest_test.Harness) {
 	th.WriteK("/app/https/service", `
 resources:
 - https-svc.yaml
@@ -106,7 +108,7 @@ resources:
 	writeHTTPSTransformerRaw(th)
 }
 
-func writeConfigFromEnvOverlay(th *kusttest_test.KustTestHarness) {
+func writeConfigFromEnvOverlay(th kusttest_test.Harness) {
 	th.WriteK("/app/config", `
 resources:
 - ../base
@@ -135,7 +137,7 @@ spec:
 `)
 }
 
-func writeConfigFromEnvTransformerRaw(th *kusttest_test.KustTestHarness) {
+func writeConfigFromEnvTransformerRaw(th kusttest_test.Harness) {
 	th.WriteF("/app/config/map/generator.yaml", `
 apiVersion: builtin
 kind: ConfigMapGenerator
@@ -170,7 +172,7 @@ patch: |-
               name: my-config
 `)
 }
-func writeConfigFromEnvTransformerBase(th *kusttest_test.KustTestHarness) {
+func writeConfigFromEnvTransformerBase(th kusttest_test.Harness) {
 	th.WriteK("/app/config/map", `
 resources:
 - generator.yaml
@@ -182,7 +184,7 @@ resources:
 	writeConfigFromEnvTransformerRaw(th)
 }
 
-func writeTolerationsOverlay(th *kusttest_test.KustTestHarness) {
+func writeTolerationsOverlay(th kusttest_test.Harness) {
 	th.WriteK("/app/tolerations", `
 resources:
 - ../base
@@ -204,7 +206,7 @@ spec:
 `)
 }
 
-func writeTolerationsTransformerRaw(th *kusttest_test.KustTestHarness) {
+func writeTolerationsTransformerRaw(th kusttest_test.Harness) {
 	th.WriteF("/app/tolerations/transformer.yaml", `
 apiVersion: builtin
 kind: PatchTransformer
@@ -230,7 +232,7 @@ patch: |-
 `)
 }
 
-func writeTolerationsTransformerBase(th *kusttest_test.KustTestHarness) {
+func writeTolerationsTransformerBase(th kusttest_test.Harness) {
 	th.WriteK("/app/tolerations", `
 resources:
 - transformer.yaml
@@ -238,7 +240,7 @@ resources:
 	writeTolerationsTransformerRaw(th)
 }
 
-func writeStorageOverlay(th *kusttest_test.KustTestHarness) {
+func writeStorageOverlay(th kusttest_test.Harness) {
 	th.WriteK("/app/storage", `
 resources:
 - ../base
@@ -255,7 +257,7 @@ patchesJson6902:
 `)
 }
 
-func writeStorageTransformerRaw(th *kusttest_test.KustTestHarness) {
+func writeStorageTransformerRaw(th kusttest_test.Harness) {
 	th.WriteF("/app/storage/transformer.yaml", `
 apiVersion: builtin
 kind: PatchTransformer
@@ -271,7 +273,7 @@ patch: |-
 `)
 }
 
-func writeStorageTransformerBase(th *kusttest_test.KustTestHarness) {
+func writeStorageTransformerBase(th kusttest_test.Harness) {
 	th.WriteK("/app/storage", `
 resources:
 - transformer.yaml
@@ -279,14 +281,14 @@ resources:
 	writeStorageTransformerRaw(th)
 }
 
-func writePatchingOverlays(th *kusttest_test.KustTestHarness) {
+func writePatchingOverlays(th kusttest_test.Harness) {
 	writeStorageOverlay(th)
 	writeConfigFromEnvOverlay(th)
 	writeTolerationsOverlay(th)
 	writeHTTPSOverlay(th)
 }
 
-func writePatchingTransformersRaw(th *kusttest_test.KustTestHarness) {
+func writePatchingTransformersRaw(th kusttest_test.Harness) {
 	writeStorageTransformerRaw(th)
 	writeConfigFromEnvTransformerRaw(th)
 	writeTolerationsTransformerRaw(th)
@@ -309,7 +311,7 @@ func writePatchingTransformersRaw(th *kusttest_test.KustTestHarness) {
 // must be self-contained, i.e. the config may not have fields that
 // refer to local files, since those files won't be present when
 // the plugin is instantiated and used.
-func writePatchingTransformerBases(th *kusttest_test.KustTestHarness) {
+func writePatchingTransformerBases(th kusttest_test.Harness) {
 	writeStorageTransformerBase(th)
 	writeConfigFromEnvTransformerBase(th)
 	writeTolerationsTransformerBase(th)
@@ -352,7 +354,7 @@ func writePatchingTransformerBases(th *kusttest_test.KustTestHarness) {
 //   - prod: Combines the config, tolerations and https intermediate overlays.
 
 func TestComplexComposition_Dev_Failure(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/dev")
+	th := kusttest_test.MakeHarness(t)
 	writeStatefulSetBase(th)
 	writePatchingOverlays(th)
 	th.WriteK("/app/dev", `
@@ -360,7 +362,7 @@ resources:
 - ../storage
 - ../config
 `)
-	_, err := th.MakeKustTarget().MakeCustomizedResMap()
+	err := th.RunWithErr("/app/dev", th.MakeDefaultOptions())
 	if err == nil {
 		t.Fatalf("Expected resource accumulation error")
 	}
@@ -404,7 +406,7 @@ metadata:
 `
 
 func TestComplexComposition_Dev_SuccessWithRawTransformers(t *testing.T) {
-	th := kusttest_test.NewKustTestHarnessNoLoadRestrictor(t, "/app/dev")
+	th := kusttest_test.MakeHarness(t)
 	writeStatefulSetBase(th)
 	writePatchingTransformersRaw(th)
 	th.WriteK("/app/dev", `
@@ -416,15 +418,16 @@ transformers:
 - ../config/transformer/transformer.yaml
 - ../storage/transformer.yaml
 `)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Unexpected err: %v", err)
-	}
+	m := th.Run("/app/dev", func() Options {
+		o := th.MakeDefaultOptions()
+		o.LoadRestrictions = types.LoadRestrictionsNone
+		return o
+	}())
 	th.AssertActualEqualsExpected(m, devDesiredResult)
 }
 
 func TestComplexComposition_Dev_SuccessWithBaseTransformers(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/dev")
+	th := kusttest_test.MakeHarness(t)
 	writeStatefulSetBase(th)
 	writePatchingTransformerBases(th)
 	th.WriteK("/app/dev", `
@@ -436,15 +439,12 @@ transformers:
 - ../config/transformer
 - ../storage
 `)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Unexpected err: %v", err)
-	}
+	m := th.Run("/app/dev", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, devDesiredResult)
 }
 
 func TestComplexComposition_Prod_Failure(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/prod")
+	th := kusttest_test.MakeHarness(t)
 	writeStatefulSetBase(th)
 	writePatchingOverlays(th)
 	th.WriteK("/app/prod", `
@@ -453,7 +453,7 @@ resources:
 - ../tolerations
 - ../https
 `)
-	_, err := th.MakeKustTarget().MakeCustomizedResMap()
+	err := th.RunWithErr("/app/prod", th.MakeDefaultOptions())
 	if err == nil {
 		t.Fatalf("Expected resource accumulation error")
 	}
@@ -513,7 +513,7 @@ metadata:
 `
 
 func TestComplexComposition_Prod_SuccessWithRawTransformers(t *testing.T) {
-	th := kusttest_test.NewKustTestHarnessNoLoadRestrictor(t, "/app/prod")
+	th := kusttest_test.MakeHarness(t)
 	writeStatefulSetBase(th)
 	writePatchingTransformersRaw(th)
 	th.WriteK("/app/prod", `
@@ -527,15 +527,16 @@ transformers:
 - ../https/transformer/transformer.yaml
 - ../tolerations/transformer.yaml
 `)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Unexpected err: %v", err)
-	}
+	m := th.Run("/app/prod", func() Options {
+		o := th.MakeDefaultOptions()
+		o.LoadRestrictions = types.LoadRestrictionsNone
+		return o
+	}())
 	th.AssertActualEqualsExpected(m, prodDesiredResult)
 }
 
 func TestComplexComposition_Prod_SuccessWithBaseTransformers(t *testing.T) {
-	th := kusttest_test.NewKustTestHarness(t, "/app/prod")
+	th := kusttest_test.MakeHarness(t)
 	writeStatefulSetBase(th)
 	writePatchingTransformerBases(th)
 	th.WriteK("/app/prod", `
@@ -549,9 +550,6 @@ transformers:
 - ../https/transformer
 - ../tolerations
 `)
-	m, err := th.MakeKustTarget().MakeCustomizedResMap()
-	if err != nil {
-		t.Fatalf("Unexpected err: %v", err)
-	}
+	m := th.Run("/app/prod", th.MakeDefaultOptions())
 	th.AssertActualEqualsExpected(m, prodDesiredResult)
 }