Add lint-kustomize to prow-presubmit-check

To do tool installs.

.gitignore

@@ -5,6 +5,9 @@
 *.so
 *.dylib
 
+.idea
+*.iml
+
 # Test binary, build with `go test -c`
 *.test
 

.golangci-kustomize.yml

@@ -0,0 +1,50 @@
+run:
+  deadline: 5m
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - depguard
+    # - dogsled
+    - dupl
+    # - errcheck
+    # - funlen
+    # - gochecknoinits
+    - goconst
+    # - gocritic
+    - gocyclo
+    - gofmt
+    - goimports
+    - golint
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - interfacer
+    - lll
+    - misspell
+    - nakedret
+    # - scopelint
+    - staticcheck
+    - structcheck
+    # stylecheck demands that acronyms not be treated as words
+    # in camelCase, so JsonOp become JSONOp, etc.  Yuck.
+    # - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    # - whitespace
+
+linters-settings:
+  dupl:
+    threshold: 400
+  lll:
+    line-length: 170
+  gocyclo:
+    min-complexity: 15
+  golint:
+    min-confidence: 0.85

.golangci.yml

@@ -1,30 +0,0 @@
-run:
-  deadline: 5m
-
-linters:
-  disable-all: true
-  enable:
-    - dupl
-    - goconst
-    - gocyclo
-    - gofmt
-    - golint
-    - govet
-    - ineffassign
-    - interfacer
-    - lll
-    - misspell
-    - nakedret
-    - structcheck
-    - unparam
-    - varcheck
-
-linters-settings:
-  dupl:
-    threshold: 400 
-  lll:   
-    line-length: 170
-  gocyclo:
-    min-complexity: 15
-  golint:
-    min-confidence: 0.85

.travis.yml

@@ -1,7 +1,10 @@
 os:
   - linux
   - osx
-# TODO: Uncomment when tests running on Windows.
+# TODO: Speed up the slowness of the osx travis runs
+# Maybe cache brew installs?
+#
+# TODO: Uncomment when some gets the tests running on Windows.
 #  - windows
 
 addons:
@@ -20,7 +23,7 @@ git:
 language: go
 
 go:
-  - "1.12"
+  - "1.13"
 
 go_import_path: sigs.k8s.io/kustomize
 
@@ -31,7 +34,9 @@ before_install:
 install: true
 
 script:
-  - ./travis/pre-commit.sh
+  - make verify-kustomize
+  - ./travis/kyaml-pre-commit.sh
+  - ./travis/check-go-mod.sh
 
 # TBD. Suppressing for now.
 notifications:

CONTRIBUTING.md

@@ -6,11 +6,15 @@ _As contributors and maintainers of this project, and in the interest of fosteri
 
 ## Getting Started
 
+Dev guides:
+
+- [Mac](docs/macDevGuide.md)
+
 We have full documentation on how to get started contributing here:
 
 - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests
 - [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing)
-- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet.md) - Common resources for existing developers
+- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet/README.md) - Common resources for existing developers
 
 ## Mentorship
 

Makefile

@@ -1,37 +1,285 @@
-BIN_NAME=kustomize
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Makefile for kustomize CLI and API.
 
-COVER_FILE=coverage.out
+MYGOBIN := $(shell go env GOPATH)/bin
+SHELL := /bin/bash
+export PATH := $(MYGOBIN):$(PATH)
 
-export GO111MODULE=on
+.PHONY: all
+all: verify-kustomize
 
-all: test build
+.PHONY: verify-kustomize
+verify-kustomize: \
+	lint-kustomize \
+	test-unit-kustomize-all \
+	test-examples-kustomize-against-HEAD \
+	test-examples-kustomize-against-latest
 
-test: generate-code test-lint test-go
+# The following target referenced by a file in
+# https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/kustomize
+.PHONY: prow-presubmit-check
+prow-presubmit-check: \
+	lint-kustomize \
+	test-unit-kustomize-all
 
-test-go:
-	go test -v ./...
+.PHONY: verify-kustomize-e2e
+verify-kustomize-e2e: test-examples-e2e-kustomize
 
-test-lint:
-	golangci-lint run ./...
+# Other builds in this repo might want a different linter version.
+# Without one Makefile to rule them all, the different makes
+# cannot assume that golanci-lint is at the version they want
+# since everything uses the same implicit GOPATH.
+# This installs in a temp dir to avoid overwriting someone else's
+# linter, then installs in MYGOBIN with a new name.
+# Version pinned by api/go.mod
+$(MYGOBIN)/golangci-lint-kustomize:
+	( \
+		set -e; \
+		export GOBIN=$$(mktemp -d) \
+		cd api; \
+		GO111MODULE=on go install github.com/golangci/golangci-lint/cmd/golangci-lint; \
+		mv $$GOBIN/golangci-lint $(MYGOBIN)/golangci-lint-kustomize \
+	)
 
-generate-code:
-	./plugin/generateBuiltins.sh $(GOPATH)
+# Version pinned by api/go.mod
+$(MYGOBIN)/mdrip:
+	cd api; \
+	go install github.com/monopole/mdrip
 
-build:
-	go build -o $(BIN_NAME) cmd/kustomize/main.go
+# Version pinned by api/go.mod
+$(MYGOBIN)/stringer:
+	cd api; \
+	go install golang.org/x/tools/cmd/stringer
 
-install:
-	go install $(PWD)/cmd/kustomize
+# Version pinned by api/go.mod
+$(MYGOBIN)/goimports:
+	cd api; \
+	go install golang.org/x/tools/cmd/goimports
 
-cover:
-	# The plugin directory eludes coverage, and is therefore omitted
-	go test ./pkg/... ./k8sdeps/... ./internal/... -coverprofile=$(COVER_FILE) && \
-	go tool cover -html=$(COVER_FILE)
+# Install resource from whatever is checked out.
+$(MYGOBIN)/resource:
+	cd cmd/resource; \
+	go install .
 
+# To pin pluginator, use this recipe instead:
+#   cd api;
+#   go install sigs.k8s.io/kustomize/pluginator/v2
+$(MYGOBIN)/pluginator:
+	cd pluginator; \
+	go install .
 
-clean:
-	go clean
-	rm -f $(BIN_NAME)
-	rm -f $(COVER_FILE)
+# Install kustomize from whatever is checked out.
+$(MYGOBIN)/kustomize:
+	cd kustomize; \
+	go install .
 
-.PHONY: test build install clean generate-code test-go test-lint cover
+.PHONY: install-tools
+install-tools: \
+	$(MYGOBIN)/goimports \
+	$(MYGOBIN)/golangci-lint-kustomize \
+	$(MYGOBIN)/mdrip \
+	$(MYGOBIN)/pluginator \
+	$(MYGOBIN)/stringer
+
+### Begin kustomize plugin rules.
+#
+# The rules to deal with builtin plugins are a bit
+# complicated because
+#
+# - Every builtin plugin is a Go plugin -
+#   meaning it gets its own module directory
+#   (outside of the api module) with Go
+#   code in a 'main' package per Go plugin rules.
+# - kustomize locates plugins using the
+#   'apiVersion' and 'kind' fields from the
+#   plugin config file.
+# - k8s wants CamelCase in 'kind' fields.
+# - The module name (the last name in the path)
+#   must be the lowercased 'kind' of the
+#   plugin because Go and related tools
+#   demand lowercase in import paths, but
+#   allow CamelCase in file names.
+# - the generated code must live in the api
+#   module (it's linked into the api).
+
+# Where all generated builtin plugin code should go.
+pGen=api/builtins
+# Where the builtin Go plugin modules live.
+pSrc=plugin/builtin
+
+_builtinplugins = \
+	AnnotationsTransformer.go \
+	ConfigMapGenerator.go \
+	HashTransformer.go \
+	ImageTagTransformer.go \
+	InventoryTransformer.go \
+	LabelTransformer.go \
+	LegacyOrderTransformer.go \
+	NamespaceTransformer.go \
+	PatchJson6902Transformer.go \
+	PatchStrategicMergeTransformer.go \
+	PatchTransformer.go \
+	PrefixSuffixTransformer.go \
+	ReplicaCountTransformer.go \
+	SecretGenerator.go
+
+# Maintaining this explicit list of generated files, and
+# adding it as a dependency to a few targets, to assure
+# they get recreated if deleted.  The rules below on how
+# to make them don't, by themselves, assure they will be
+# recreated if deleted.
+builtinplugins = $(patsubst %,$(pGen)/%,$(_builtinplugins))
+
+# These rules are verbose, but assure that if a source file
+# is modified, the corresponding generated file, and only
+# that file, will be recreated.
+$(pGen)/AnnotationsTransformer.go: $(pSrc)/annotationstransformer/AnnotationsTransformer.go
+$(pGen)/ConfigMapGenerator.go: $(pSrc)/configmapgenerator/ConfigMapGenerator.go
+$(pGen)/HashTransformer.go: $(pSrc)/hashtransformer/HashTransformer.go
+$(pGen)/ImageTagTransformer.go: $(pSrc)/imagetagtransformer/ImageTagTransformer.go
+$(pGen)/InventoryTransformer.go: $(pSrc)/inventorytransformer/InventoryTransformer.go
+$(pGen)/LabelTransformer.go: $(pSrc)/labeltransformer/LabelTransformer.go
+$(pGen)/LegacyOrderTransformer.go: $(pSrc)/legacyordertransformer/LegacyOrderTransformer.go
+$(pGen)/NamespaceTransformer.go: $(pSrc)/namespacetransformer/NamespaceTransformer.go
+$(pGen)/PatchJson6902Transformer.go: $(pSrc)/patchjson6902transformer/PatchJson6902Transformer.go
+$(pGen)/PatchStrategicMergeTransformer.go: $(pSrc)/patchstrategicmergetransformer/PatchStrategicMergeTransformer.go
+$(pGen)/PatchTransformer.go: $(pSrc)/patchtransformer/PatchTransformer.go
+$(pGen)/PrefixSuffixTransformer.go: $(pSrc)/prefixsuffixtransformer/PrefixSuffixTransformer.go
+$(pGen)/ReplicaCountTransformer.go: $(pSrc)/replicacounttransformer/ReplicaCountTransformer.go
+$(pGen)/SecretGenerator.go: $(pSrc)/secretgenerator/SecretGenerator.go
+
+# The (verbose but portable) Makefile way to convert to lowercase.
+toLowerCase = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+
+$(pGen)/%.go: $(MYGOBIN)/pluginator
+	@echo "generating $*"
+	( \
+		set -e; \
+		cd $(pSrc)/$(call toLowerCase,$*); \
+		go generate .; \
+		cd ../../../$(pGen); \
+		$(MYGOBIN)/goimports -w $*.go \
+	)
+
+# Target is for debugging.
+.PHONY: generate-kustomize-builtin-plugins
+generate-kustomize-builtin-plugins: $(builtinplugins)
+
+.PHONY: kustomize-external-go-plugin-build
+kustomize-external-go-plugin-build:
+	./hack/buildExternalGoPlugins.sh ./plugin
+
+.PHONY: kustomize-external-go-plugin-clean
+kustomize-external-go-plugin-clean:
+	./hack/buildExternalGoPlugins.sh ./plugin clean
+
+### End kustomize plugin rules.
+
+.PHONY: lint-kustomize
+lint-kustomize: install-tools $(builtinplugins)
+	cd api; \
+	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
+	cd kustomize; \
+	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
+	cd pluginator; \
+	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
+
+.PHONY: test-unit-kustomize-api
+test-unit-kustomize-api: $(builtinplugins)
+	cd api; go test ./...
+
+.PHONY: test-unit-kustomize-plugins
+test-unit-kustomize-plugins:
+	./hack/testUnitKustomizePlugins.sh
+
+.PHONY: test-unit-kustomize-cli
+test-unit-kustomize-cli:
+	cd kustomize; go test ./...
+
+.PHONY: test-unit-kustomize-all
+test-unit-kustomize-all: \
+	test-unit-kustomize-api \
+	test-unit-kustomize-cli \
+	test-unit-kustomize-plugins
+
+.PHONY:
+test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
+	./hack/testExamplesAgainstKustomize.sh HEAD
+
+.PHONY:
+test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from ."; \
+		cd kustomize; go install .; cd ..; \
+		./hack/testExamplesE2EAgainstKustomize.sh .; \
+	)
+
+.PHONY:
+test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from latest."; \
+		GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v3; \
+		./hack/testExamplesAgainstKustomize.sh latest; \
+		echo "Reinstalling kustomize from HEAD."; \
+		cd kustomize; go install .; \
+	)
+
+# linux only.
+# This is for testing an example plugin that
+# uses kubeval for validation.
+# Don't want to add a hard dependence in go.mod file
+# to github.com/instrumenta/kubeval.
+# Instead, download the binary.
+$(MYGOBIN)/kubeval:
+	( \
+		set -e; \
+		d=$(shell mktemp -d); cd $$d; \
+		wget https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz; \
+		tar xf kubeval-linux-amd64.tar.gz; \
+		mv kubeval $(MYGOBIN); \
+		rm -rf $$d; \
+	)
+
+# linux only.
+# This is for testing an example plugin that
+# uses helm to inflate a chart for subsequent kustomization.
+# Don't want to add a hard dependence in go.mod file
+# to helm.
+# Instead, download the binary.
+$(MYGOBIN)/helm:
+	( \
+		set -e; \
+		d=$(shell mktemp -d); cd $$d; \
+		wget https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz; \
+		tar -xvzf helm-v2.13.1-linux-amd64.tar.gz; \
+		mv linux-amd64/helm $(MYGOBIN); \
+		rm -rf $$d \
+	)
+
+$(MYGOBIN)/kind:
+	( \
+        set -e; \
+        d=$(shell mktemp -d); cd $$d; \
+        wget -O ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(shell uname)-amd64; \
+        chmod +x ./kind; \
+        mv ./kind $(MYGOBIN); \
+        rm -rf $$d; \
+	)
+
+.PHONY: clean
+clean: kustomize-external-go-plugin-clean
+	go clean --cache
+	rm -f $(builtinplugins)
+	rm -f $(MYGOBIN)/pluginator
+	rm -f $(MYGOBIN)/kustomize
+	rm -f $(MYGOBIN)/golangci-lint-kustomize
+
+.PHONY: nuke
+nuke: clean
+	sudo rm -rf $(shell go env GOPATH)/pkg/mod/sigs.k8s.io

README.md

@@ -28,9 +28,9 @@ Since [v1.14][kubectl announcement] the kustomize build system has been included
 
 | kubectl version | kustomize version |
 |---------|--------|
-| v1.16.0 | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
-| v1.15.x | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
-| v1.14.x | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
+| v1.16.0 | [v2.0.3](/../../tree/v2.0.3) |
+| v1.15.x | [v2.0.3](/../../tree/v2.0.3) |
+| v1.14.x | [v2.0.3](/../../tree/v2.0.3) |
 
 For examples and guides for using the kubectl integration please see the [kubectl book] or the [kubernetes documentation].
 
@@ -167,7 +167,7 @@ is governed by the [Kubernetes Code of Conduct].
 [eschewed feature list]: docs/eschewedFeatures.md
 [imageBase]: docs/images/base.jpg
 [imageOverlay]: docs/images/overlay.jpg
-[kind/feature]: https://github.com/kubernetes-sigs/kustomize/labels/kind%2Ffeature
+[kind/feature]: /../../labels/kind%2Ffeature
 [kubectl announcement]: https://kubernetes.io/blog/2019/03/25/kubernetes-1-14-release-announcement
 [kubectl book]: https://kubectl.docs.kubernetes.io/pages/app_customization/introduction.html
 [kubernetes documentation]: https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/
@@ -175,12 +175,12 @@ is governed by the [Kubernetes Code of Conduct].
 [kustomization]: docs/glossary.md#kustomization
 [overlay]: docs/glossary.md#overlay
 [overlays]: docs/glossary.md#overlay
-[release page]: https://github.com/kubernetes-sigs/kustomize/releases
+[release page]: /../../releases
 [resource]: docs/glossary.md#resource
 [resources]: docs/glossary.md#resource
 [sig-cli]: https://github.com/kubernetes/community/blob/master/sig-cli/README.md
 [variant]: docs/glossary.md#variant
 [variants]: docs/glossary.md#variant
-[v2.0.3]: https://github.com/kubernetes-sigs/kustomize/releases/tag/v2.0.3
-[v2.1.0]: https://github.com/kubernetes-sigs/kustomize/releases/tag/v2.1.0
+[v2.0.3]: /../../releases/tag/v2.0.3
+[v2.1.0]: /../../releases/tag/v2.1.0
 [workflows]: docs/workflows.md

api/builtins/AnnotationsTransformer.go

@@ -1,29 +1,30 @@
 // Code generated by pluginator on AnnotationsTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
 // Add the given annotations to the given field specifications.
 type AnnotationsTransformerPlugin struct {
-	Annotations map[string]string  `json:"annotations,omitempty" yaml:"annotations,omitempty"`
-	FieldSpecs  []config.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
+	FieldSpecs  []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
 }
 
 func (p *AnnotationsTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Annotations = nil
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)
 }
 
 func (p *AnnotationsTransformerPlugin) Transform(m resmap.ResMap) error {
-	t, err := transformers.NewMapTransformer(
+	t, err := transform.NewMapTransformer(
 		p.FieldSpecs,
 		p.Annotations,
 	)

api/builtins/ConfigMapGenerator.go

@@ -1,23 +1,24 @@
 // Code generated by pluginator on ConfigMapGenerator; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/kv"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
 type ConfigMapGeneratorPlugin struct {
-	ldr              ifc.Loader
-	rf               *resmap.Factory
+	h                *resmap.PluginHelpers
 	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
 	types.GeneratorOptions
 	types.ConfigMapArgs
 }
 
 func (p *ConfigMapGeneratorPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, config []byte) (err error) {
+	h *resmap.PluginHelpers, config []byte) (err error) {
 	p.GeneratorOptions = types.GeneratorOptions{}
 	p.ConfigMapArgs = types.ConfigMapArgs{}
 	err = yaml.Unmarshal(config, p)
@@ -27,13 +28,14 @@ func (p *ConfigMapGeneratorPlugin) Config(
 	if p.ConfigMapArgs.Namespace == "" {
 		p.ConfigMapArgs.Namespace = p.Namespace
 	}
-	p.ldr = ldr
-	p.rf = rf
+	p.h = h
 	return
 }
 
 func (p *ConfigMapGeneratorPlugin) Generate() (resmap.ResMap, error) {
-	return p.rf.FromConfigMapArgs(p.ldr, &p.GeneratorOptions, p.ConfigMapArgs)
+	return p.h.ResmapFactory().FromConfigMapArgs(
+		kv.NewLoader(p.h.Loader(), p.h.Validator()),
+		&p.GeneratorOptions, p.ConfigMapArgs)
 }
 
 func NewConfigMapGeneratorPlugin() resmap.GeneratorPlugin {

api/builtins/HashTransformer.go

@@ -1,11 +1,13 @@
 // Code generated by pluginator on HashTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"fmt"
 
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/resmap"
 )
 
 type HashTransformerPlugin struct {
@@ -13,8 +15,8 @@ type HashTransformerPlugin struct {
 }
 
 func (p *HashTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, config []byte) (err error) {
-	p.hasher = rf.RF().Hasher()
+	h *resmap.PluginHelpers, _ []byte) (err error) {
+	p.hasher = h.ResmapFactory().RF().Hasher()
 	return nil
 }
 

api/builtins/ImageTagTransformer.go

@@ -1,29 +1,30 @@
 // Code generated by pluginator on ImageTagTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"fmt"
 	"regexp"
 	"strings"
 
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/image"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+
+	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/yaml"
 )
 
 // Find matching image declarations and replace
 // the name, tag and/or digest.
 type ImageTagTransformerPlugin struct {
-	ImageTag   image.Image        `json:"imageTag,omitempty" yaml:"imageTag,omitempty"`
-	FieldSpecs []config.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+	ImageTag   types.Image       `json:"imageTag,omitempty" yaml:"imageTag,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
 }
 
 func (p *ImageTagTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
-	p.ImageTag = image.Image{}
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.ImageTag = types.Image{}
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)
 }
@@ -34,7 +35,7 @@ func (p *ImageTagTransformerPlugin) Transform(m resmap.ResMap) error {
 			if !r.OrgId().IsSelected(&path.Gvk) {
 				continue
 			}
-			err := transformers.MutateField(
+			err := transform.MutateField(
 				r.Map(), path.PathSlice(), false, p.mutateImage)
 			if err != nil {
 				return err
@@ -81,7 +82,7 @@ func (p *ImageTagTransformerPlugin) findAndReplaceImage(obj map[string]interface
 	updated := false
 	for _, path := range paths {
 		containers, found := obj[path]
-		if found {
+		if found && containers != nil {
 			if _, err := p.updateContainers(containers); err != nil {
 				return err
 			}

api/builtins/InventoryTransformer.go

@@ -1,31 +1,30 @@
 // Code generated by pluginator on InventoryTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"fmt"
 
-	"sigs.k8s.io/kustomize/v3/pkg/resource"
-
-	"sigs.k8s.io/kustomize/v3/pkg/hasher"
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/inventory"
-	"sigs.k8s.io/kustomize/v3/pkg/resid"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/hasher"
+	"sigs.k8s.io/kustomize/api/inventory"
+	"sigs.k8s.io/kustomize/api/kv"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
 type InventoryTransformerPlugin struct {
-	ldr              ifc.Loader
-	rf               *resmap.Factory
+	h                *resmap.PluginHelpers
 	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
 	Policy           string `json:"policy,omitempty" yaml:"policy,omitempty"`
 }
 
 func (p *InventoryTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
-	p.ldr = ldr
-	p.rf = rf
+	h *resmap.PluginHelpers, c []byte) (err error) {
+	p.h = h
 	err = yaml.Unmarshal(c, p)
 	if err != nil {
 		return err
@@ -74,7 +73,8 @@ func (p *InventoryTransformerPlugin) Transform(m resmap.ResMap) error {
 		return err
 	}
 
-	cm, err := p.rf.RF().MakeConfigMap(p.ldr, opts, &args)
+	cm, err := p.h.ResmapFactory().RF().MakeConfigMap(
+		kv.NewLoader(p.h.Loader(), p.h.Validator()), opts, &args)
 	if err != nil {
 		return err
 	}

api/builtins/LabelTransformer.go

@@ -1,29 +1,30 @@
 // Code generated by pluginator on LabelTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
 // Add the given labels to the given field specifications.
 type LabelTransformerPlugin struct {
-	Labels     map[string]string  `json:"labels,omitempty" yaml:"labels,omitempty"`
-	FieldSpecs []config.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+	Labels     map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
 }
 
 func (p *LabelTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Labels = nil
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)
 }
 
 func (p *LabelTransformerPlugin) Transform(m resmap.ResMap) error {
-	t, err := transformers.NewMapTransformer(
+	t, err := transform.NewMapTransformer(
 		p.FieldSpecs,
 		p.Labels,
 	)

api/builtins/LegacyOrderTransformer.go

@@ -1,13 +1,14 @@
 // Code generated by pluginator on LegacyOrderTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"sort"
 
 	"github.com/pkg/errors"
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/resource"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
 )
 
 // Sort the resources using an ordering defined in the Gvk class.
@@ -19,7 +20,7 @@ type LegacyOrderTransformerPlugin struct{}
 
 // Nothing needed for configuration.
 func (p *LegacyOrderTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
+	_ *resmap.PluginHelpers, _ []byte) (err error) {
 	return nil
 }
 

api/builtins/NamespaceTransformer.go

@@ -1,27 +1,28 @@
 // Code generated by pluginator on NamespaceTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"fmt"
 
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resid"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/resource"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/transform"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
 // Change or set the namespace of non-cluster level resources.
 type NamespaceTransformerPlugin struct {
 	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
-	FieldSpecs       []config.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+	FieldSpecs       []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
 }
 
 func (p *NamespaceTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Namespace = ""
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)
@@ -41,7 +42,7 @@ func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
 		applicableFs := p.applicableFieldSpecs(id)
 
 		for _, fs := range applicableFs {
-			err := transformers.MutateField(
+			err := transform.MutateField(
 				r.Map(), fs.PathSlice(), fs.CreateIfNotPresent,
 				p.changeNamespace(r))
 			if err != nil {
@@ -51,7 +52,7 @@ func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
 
 		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
 		if len(matches) != 1 {
-			return fmt.Errorf("namespace tranformation produces ID conflict: %#v", matches)
+			return fmt.Errorf("namespace tranformation produces ID conflict: %+v", matches)
 		}
 	}
 	return nil
@@ -63,8 +64,8 @@ const metaNamespace = "metadata/namespace"
 // all objects have it, even "ClusterKind" objects
 // that don't exist in a namespace (the Namespace
 // object itself doesn't live in a namespace).
-func (p *NamespaceTransformerPlugin) applicableFieldSpecs(id resid.ResId) []config.FieldSpec {
-	var res []config.FieldSpec
+func (p *NamespaceTransformerPlugin) applicableFieldSpecs(id resid.ResId) []types.FieldSpec {
+	var res []types.FieldSpec
 	for _, fs := range p.FieldSpecs {
 		if id.IsSelected(&fs.Gvk) && (fs.Path != metaNamespace || (fs.Path == metaNamespace && id.IsNamespaceableKind())) {
 			res = append(res, fs)
@@ -74,7 +75,7 @@ func (p *NamespaceTransformerPlugin) applicableFieldSpecs(id resid.ResId) []conf
 }
 
 func (p *NamespaceTransformerPlugin) changeNamespace(
-	referrer *resource.Resource) func(in interface{}) (interface{}, error) {
+	_ *resource.Resource) func(in interface{}) (interface{}, error) {
 	return func(in interface{}) (interface{}, error) {
 		switch in.(type) {
 		case string:

api/builtins/PatchJson6902Transformer.go

@@ -1,16 +1,17 @@
 // Code generated by pluginator on PatchJson6902Transformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"fmt"
 
 	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/pkg/errors"
-	"sigs.k8s.io/kustomize/v3/pkg/gvk"
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resid"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
@@ -23,8 +24,8 @@ type PatchJson6902TransformerPlugin struct {
 }
 
 func (p *PatchJson6902TransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
-	p.ldr = ldr
+	h *resmap.PluginHelpers, c []byte) (err error) {
+	p.ldr = h.Loader()
 	err = yaml.Unmarshal(c, p)
 	if err != nil {
 		return err
@@ -70,7 +71,7 @@ func (p *PatchJson6902TransformerPlugin) Config(
 
 func (p *PatchJson6902TransformerPlugin) Transform(m resmap.ResMap) error {
 	id := resid.NewResIdWithNamespace(
-		gvk.Gvk{
+		resid.Gvk{
 			Group:   p.Target.Group,
 			Version: p.Target.Version,
 			Kind:    p.Target.Kind,

api/builtins/PatchStrategicMergeTransformer.go

@@ -1,28 +1,27 @@
 // Code generated by pluginator on PatchStrategicMergeTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"fmt"
 
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/resource"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
 type PatchStrategicMergeTransformerPlugin struct {
-	ldr           ifc.Loader
-	rf            *resmap.Factory
+	h             *resmap.PluginHelpers
 	loadedPatches []*resource.Resource
 	Paths         []types.PatchStrategicMerge `json:"paths,omitempty" yaml:"paths,omitempty"`
 	Patches       string                      `json:"patches,omitempty" yaml:"patches,omitempty"`
 }
 
 func (p *PatchStrategicMergeTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
-	p.ldr = ldr
-	p.rf = rf
+	h *resmap.PluginHelpers, c []byte) (err error) {
+	p.h = h
 	err = yaml.Unmarshal(c, p)
 	if err != nil {
 		return err
@@ -32,12 +31,13 @@ func (p *PatchStrategicMergeTransformerPlugin) Config(
 	}
 	if len(p.Paths) != 0 {
 		for _, onePath := range p.Paths {
-			res, err := p.rf.RF().SliceFromBytes([]byte(onePath))
+			res, err := p.h.ResmapFactory().RF().SliceFromBytes([]byte(onePath))
 			if err == nil {
 				p.loadedPatches = append(p.loadedPatches, res...)
 				continue
 			}
-			res, err = p.rf.RF().SliceFromPatches(ldr, []types.PatchStrategicMerge{onePath})
+			res, err = p.h.ResmapFactory().RF().SliceFromPatches(
+				p.h.Loader(), []types.PatchStrategicMerge{onePath})
 			if err != nil {
 				return err
 			}
@@ -45,7 +45,7 @@ func (p *PatchStrategicMergeTransformerPlugin) Config(
 		}
 	}
 	if p.Patches != "" {
-		res, err := p.rf.RF().SliceFromBytes([]byte(p.Patches))
+		res, err := p.h.ResmapFactory().RF().SliceFromBytes([]byte(p.Patches))
 		if err != nil {
 			return err
 		}
@@ -60,7 +60,7 @@ func (p *PatchStrategicMergeTransformerPlugin) Config(
 }
 
 func (p *PatchStrategicMergeTransformerPlugin) Transform(m resmap.ResMap) error {
-	patches, err := p.rf.MergePatches(p.loadedPatches)
+	patches, err := p.h.ResmapFactory().MergePatches(p.loadedPatches)
 	if err != nil {
 		return err
 	}

api/builtins/PatchTransformer.go

@@ -1,21 +1,20 @@
 // Code generated by pluginator on PatchTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"fmt"
 
-	"github.com/evanphx/json-patch"
+	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/pkg/errors"
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/resource"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
 type PatchTransformerPlugin struct {
-	ldr          ifc.Loader
-	rf           *resmap.Factory
 	loadedPatch  *resource.Resource
 	decodedPatch jsonpatch.Patch
 	Path         string          `json:"path,omitempty" yaml:"path,omitempty"`
@@ -24,9 +23,7 @@ type PatchTransformerPlugin struct {
 }
 
 func (p *PatchTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
-	p.ldr = ldr
-	p.rf = rf
+	h *resmap.PluginHelpers, c []byte) (err error) {
 	err = yaml.Unmarshal(c, p)
 	if err != nil {
 		return err
@@ -43,7 +40,7 @@ func (p *PatchTransformerPlugin) Config(
 	}
 	var in []byte
 	if p.Path != "" {
-		in, err = ldr.Load(p.Path)
+		in, err = h.Loader().Load(p.Path)
 		if err != nil {
 			return
 		}
@@ -52,7 +49,7 @@ func (p *PatchTransformerPlugin) Config(
 		in = []byte(p.Patch)
 	}
 
-	patchSM, errSM := p.rf.RF().FromBytes(in)
+	patchSM, errSM := h.ResmapFactory().RF().FromBytes(in)
 	patchJson, errJson := jsonPatchFromBytes(in)
 	if errSM != nil && errJson != nil {
 		err = fmt.Errorf(

api/builtins/PrefixSuffixTransformer.go

@@ -1,38 +1,39 @@
 // Code generated by pluginator on PrefixSuffixTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"errors"
 	"fmt"
 
-	"sigs.k8s.io/kustomize/v3/pkg/gvk"
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resid"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/yaml"
 )
 
 // Add the given prefix and suffix to the field.
 type PrefixSuffixTransformerPlugin struct {
-	Prefix     string             `json:"prefix,omitempty" yaml:"prefix,omitempty"`
-	Suffix     string             `json:"suffix,omitempty" yaml:"suffix,omitempty"`
-	FieldSpecs []config.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+	Prefix     string            `json:"prefix,omitempty" yaml:"prefix,omitempty"`
+	Suffix     string            `json:"suffix,omitempty" yaml:"suffix,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
 }
 
 // Not placed in a file yet due to lack of demand.
-var prefixSuffixFieldSpecsToSkip = []config.FieldSpec{
+var prefixSuffixFieldSpecsToSkip = []types.FieldSpec{
 	{
-		Gvk: gvk.Gvk{Kind: "CustomResourceDefinition"},
+		Gvk: resid.Gvk{Kind: "CustomResourceDefinition"},
 	},
 	{
-		Gvk: gvk.Gvk{Group: "apiregistration.k8s.io", Kind: "APIService"},
+		Gvk: resid.Gvk{Group: "apiregistration.k8s.io", Kind: "APIService"},
 	},
 }
 
 func (p *PrefixSuffixTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Prefix = ""
 	p.Suffix = ""
 	p.FieldSpecs = nil
@@ -81,7 +82,7 @@ func (p *PrefixSuffixTransformerPlugin) Transform(m resmap.ResMap) error {
 			// the addPrefixSuffix method will not
 			// change the name if both the prefix and suffix
 			// are empty.
-			err := transformers.MutateField(
+			err := transform.MutateField(
 				r.Map(),
 				path.PathSlice(),
 				path.CreateIfNotPresent,
@@ -94,7 +95,7 @@ func (p *PrefixSuffixTransformerPlugin) Transform(m resmap.ResMap) error {
 	return nil
 }
 
-func smellsLikeANameChange(fs *config.FieldSpec) bool {
+func smellsLikeANameChange(fs *types.FieldSpec) bool {
 	return fs.Path == "metadata/name"
 }
 

api/builtins/ReplicaCountTransformer.go

@@ -1,35 +1,34 @@
 // Code generated by pluginator on ReplicaCountTransformer; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
 	"fmt"
 
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resid"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/transform"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
 // Find matching replicas declarations and replace the count.
 // Eases the kustomization configuration of replica changes.
 type ReplicaCountTransformerPlugin struct {
-	Replica    types.Replica      `json:"replica,omitempty" yaml:"replica,omitempty"`
-	FieldSpecs []config.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+	Replica    types.Replica     `json:"replica,omitempty" yaml:"replica,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
 }
 
 func (p *ReplicaCountTransformerPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, c []byte) (err error) {
-
+	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Replica = types.Replica{}
 	p.FieldSpecs = nil
 	return yaml.Unmarshal(c, p)
 }
 
 func (p *ReplicaCountTransformerPlugin) Transform(m resmap.ResMap) error {
-
 	found := false
 	for i, replicaSpec := range p.FieldSpecs {
 		matcher := p.createMatcher(i)
@@ -38,7 +37,7 @@ func (p *ReplicaCountTransformerPlugin) Transform(m resmap.ResMap) error {
 
 		for _, res := range append(matchOriginal, matchCurrent...) {
 			found = true
-			err := transformers.MutateField(
+			err := transform.MutateField(
 				res.Map(), replicaSpec.PathSlice(),
 				replicaSpec.CreateIfNotPresent, p.addReplicas)
 			if err != nil {

api/builtins/SecretGenerator.go

@@ -1,23 +1,23 @@
 // Code generated by pluginator on SecretGenerator; DO NOT EDIT.
-package builtin
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
 
 import (
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/kv"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
 type SecretGeneratorPlugin struct {
-	ldr              ifc.Loader
-	rf               *resmap.Factory
+	h                *resmap.PluginHelpers
 	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
 	types.GeneratorOptions
 	types.SecretArgs
 }
 
-func (p *SecretGeneratorPlugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, config []byte) (err error) {
+func (p *SecretGeneratorPlugin) Config(h *resmap.PluginHelpers, config []byte) (err error) {
 	p.GeneratorOptions = types.GeneratorOptions{}
 	p.SecretArgs = types.SecretArgs{}
 	err = yaml.Unmarshal(config, p)
@@ -27,13 +27,14 @@ func (p *SecretGeneratorPlugin) Config(
 	if p.SecretArgs.Namespace == "" {
 		p.SecretArgs.Namespace = p.Namespace
 	}
-	p.ldr = ldr
-	p.rf = rf
+	p.h = h
 	return
 }
 
 func (p *SecretGeneratorPlugin) Generate() (resmap.ResMap, error) {
-	return p.rf.FromSecretArgs(p.ldr, &p.GeneratorOptions, p.SecretArgs)
+	return p.h.ResmapFactory().FromSecretArgs(
+		kv.NewLoader(p.h.Loader(), p.h.Validator()),
+		&p.GeneratorOptions, p.SecretArgs)
 }
 
 func NewSecretGeneratorPlugin() resmap.GeneratorPlugin {

api/builtins/doc.go

@@ -0,0 +1,8 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package builtins holds code generated from the builtin plugins.
+// The "builtin" plugins are written as normal plugins and can
+// be used as such, but they are also used to generate the code
+// in this package so they can be statically linked to client code.
+package builtins

api/filesys/confirmeddir.go

@@ -1,7 +1,7 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package fs
+package filesys
 
 import (
 	"io/ioutil"

api/filesys/confirmeddir_test.go

@@ -1,11 +1,13 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package fs
+package filesys_test
 
 import (
 	"path/filepath"
 	"testing"
+
+	. "sigs.k8s.io/kustomize/api/filesys"
 )
 
 func TestJoin(t *testing.T) {
@@ -27,7 +29,8 @@ func TestJoin(t *testing.T) {
 }
 
 func TestHasPrefix_Slash(t *testing.T) {
-	d, f, err := MakeFsInMemory().CleanedAbs("/")
+	fSys := MakeFsInMemory()
+	d, f, err := fSys.CleanedAbs("/")
 	if err != nil {
 		t.Fatalf("unexpected err: %v", err)
 	}

api/filesys/file.go

@@ -0,0 +1,15 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"io"
+	"os"
+)
+
+// File groups the basic os.File methods.
+type File interface {
+	io.ReadWriteCloser
+	Stat() (os.FileInfo, error)
+}

api/filesys/fileinfo.go

@@ -0,0 +1,34 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"os"
+	"time"
+)
+
+var _ os.FileInfo = fileInfo{}
+
+// fileInfo implements os.FileInfo for a fileInMemory instance.
+type fileInfo struct {
+	node *fsNode
+}
+
+// Name returns the name of the file
+func (fi fileInfo) Name() string { return fi.node.Name() }
+
+// Size returns the size of the file
+func (fi fileInfo) Size() int64 { return fi.node.Size() }
+
+// Mode returns the file mode
+func (fi fileInfo) Mode() os.FileMode { return 0777 }
+
+// ModTime returns a bogus time
+func (fi fileInfo) ModTime() time.Time { return time.Time{} }
+
+// IsDir returns true if it is a directory
+func (fi fileInfo) IsDir() bool { return fi.node.isNodeADir() }
+
+// Sys should return underlying data source, but it now returns nil
+func (fi fileInfo) Sys() interface{} { return nil }

api/filesys/fileondisk.go

@@ -1,7 +1,7 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package fs
+package filesys
 
 import (
 	"os"

api/filesys/filesystem.go

@@ -1,22 +1,27 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-// Package fs provides a file system abstraction layer.
-package fs
+// Package filesys provides a file system abstraction layer.
+package filesys
 
 import (
-	"io"
-	"os"
 	"path/filepath"
 )
 
+const (
+	Separator = string(filepath.Separator)
+	SelfDir   = "."
+	ParentDir = ".."
+)
+
 // FileSystem groups basic os filesystem methods.
+// It's supposed be functional subset of https://golang.org/pkg/os
 type FileSystem interface {
 	// Create a file.
-	Create(name string) (File, error)
+	Create(path string) (File, error)
 	// MkDir makes a directory.
 	Mkdir(path string) error
-	// MkDir makes a directory path, creating intervening directories.
+	// MkDirAll makes a directory path, creating intervening directories.
 	MkdirAll(path string) error
 	// RemoveAll removes path and any children it contains.
 	RemoveAll(path string) error
@@ -32,18 +37,14 @@ type FileSystem interface {
 	CleanedAbs(path string) (ConfirmedDir, string, error)
 	// Exists is true if the path exists in the file system.
 	Exists(path string) bool
-	// Glob returns the list of matching files
+	// Glob returns the list of matching files,
+	// emulating https://golang.org/pkg/path/filepath/#Glob
 	Glob(pattern string) ([]string, error)
 	// ReadFile returns the contents of the file at the given path.
 	ReadFile(path string) ([]byte, error)
-	// WriteFile writes the data to a file at the given path.
+	// WriteFile writes the data to a file at the given path,
+	// overwriting anything that's already there.
 	WriteFile(path string, data []byte) error
 	// Walk walks the file system with the given WalkFunc.
 	Walk(path string, walkFn filepath.WalkFunc) error
 }
-
-// File groups the basic os.File methods.
-type File interface {
-	io.ReadWriteCloser
-	Stat() (os.FileInfo, error)
-}

api/filesys/fsnode.go

@@ -0,0 +1,557 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var _ File = &fsNode{}
+var _ FileSystem = &fsNode{}
+
+// fsNode is either a file or a directory.
+type fsNode struct {
+	// What node owns me?
+	parent *fsNode
+
+	// Value to return as the Name() when the
+	// parent is nil.
+	nilParentName string
+
+	// A directory mapping names to nodes.
+	// If dir is nil, then self node is a file.
+	// If dir is non-nil, then self node is a directory,
+	// albeit possibly an empty directory.
+	dir map[string]*fsNode
+
+	// if this node is a file, this is the content.
+	content []byte
+
+	// if this node is a file, this tracks whether or
+	// not it is "open".
+	open bool
+}
+
+// MakeEmptyDirInMemory returns an empty directory.
+// The paths of nodes in this object will never
+// report a leading Separator, meaning they
+// aren't "absolute" in the sense defined by
+// https://golang.org/pkg/path/filepath/#IsAbs.
+func MakeEmptyDirInMemory() *fsNode {
+	return &fsNode{
+		dir: make(map[string]*fsNode),
+	}
+}
+
+// MakeFsInMemory returns an empty 'file system'.
+// The paths of nodes in this object will always
+// report a leading Separator, meaning they
+// are "absolute" in the sense defined by
+// https://golang.org/pkg/path/filepath/#IsAbs.
+// This is a relevant difference when using Walk,
+// Glob, Match, etc.
+func MakeFsInMemory() FileSystem {
+	return &fsNode{
+		nilParentName: Separator,
+		dir:           make(map[string]*fsNode),
+	}
+}
+
+// Name returns the name of the node.
+func (n *fsNode) Name() string {
+	if n.parent == nil {
+		// Unable to lookup name in parent.
+		return n.nilParentName
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir")
+	}
+	for key, value := range n.parent.dir {
+		if value == n {
+			return key
+		}
+	}
+	log.Fatal("unable to find fsNode name")
+	return ""
+}
+
+// Path returns the full path to the node.
+func (n *fsNode) Path() string {
+	if n.parent == nil {
+		return n.nilParentName
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir, structural error")
+	}
+	return filepath.Join(n.parent.Path(), n.Name())
+}
+
+// mySplit trims trailing separators from the directory
+// result of filepath.Split.
+func mySplit(s string) (string, string) {
+	dName, fName := filepath.Split(s)
+	return StripTrailingSeps(dName), fName
+}
+
+func (n *fsNode) addFile(name string, c []byte) (result *fsNode, err error) {
+	parent := n
+	dName, fileName := mySplit(name)
+	if dName != "" {
+		parent, err = parent.addDir(dName)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if !isLegalFileNameForCreation(fileName) {
+		return nil, fmt.Errorf(
+			"illegal name '%s' in file creation", fileName)
+	}
+	result, ok := parent.dir[fileName]
+	if ok {
+		// File already exists; overwrite it.
+		result.content = c
+		return result, nil
+	}
+	result = &fsNode{
+		content: c,
+		parent:  parent,
+	}
+	parent.dir[fileName] = result
+	return result, nil
+}
+
+// Create implements FileSystem.
+// Create makes an empty file.
+func (n *fsNode) Create(path string) (result File, err error) {
+	return n.AddFile(path, []byte{})
+}
+
+// WriteFile implements FileSystem.
+func (n *fsNode) WriteFile(path string, d []byte) error {
+	_, err := n.AddFile(path, d)
+	return err
+}
+
+// AddFile adds a file and any necessary containing
+// directories to the node.
+func (n *fsNode) AddFile(
+	name string, c []byte) (result *fsNode, err error) {
+	if n.dir == nil {
+		return nil, fmt.Errorf(
+			"cannot add a file to a non-directory '%s'", n.Name())
+	}
+	return n.addFile(cleanQueryPath(name), c)
+}
+
+func (n *fsNode) addDir(path string) (result *fsNode, err error) {
+	parent := n
+	dName, subDirName := mySplit(path)
+	if dName != "" {
+		parent, err = n.addDir(dName)
+		if err != nil {
+			return nil, err
+		}
+	}
+	switch subDirName {
+	case "", SelfDir:
+		return n, nil
+	case ParentDir:
+		if n.parent == nil {
+			return nil, fmt.Errorf(
+				"cannot add a directory above '%s'", n.Path())
+		}
+		return n.parent, nil
+	default:
+		if !isLegalFileNameForCreation(subDirName) {
+			return nil, fmt.Errorf(
+				"illegal name '%s' in directory creation", subDirName)
+		}
+		result, ok := parent.dir[subDirName]
+		if ok {
+			if result.isNodeADir() {
+				// it's already there.
+				return result, nil
+			}
+			return nil, fmt.Errorf(
+				"cannot make dir '%s'; a file of that name already exists in '%s'",
+				subDirName, parent.Name())
+		}
+		result = &fsNode{
+			dir:    make(map[string]*fsNode),
+			parent: parent,
+		}
+		parent.dir[subDirName] = result
+		return result, nil
+	}
+}
+
+// Mkdir implements FileSystem.
+// Mkdir creates a directory.
+func (n *fsNode) Mkdir(path string) error {
+	_, err := n.AddDir(path)
+	return err
+}
+
+// MkdirAll implements FileSystem.
+// MkdirAll creates a directory.
+func (n *fsNode) MkdirAll(path string) error {
+	_, err := n.AddDir(path)
+	return err
+}
+
+// AddDir adds a directory to the node, not complaining
+// if it is already there.
+func (n *fsNode) AddDir(path string) (result *fsNode, err error) {
+	if n.dir == nil {
+		return nil, fmt.Errorf(
+			"cannot add a directory to file node '%s'", n.Name())
+	}
+	return n.addDir(cleanQueryPath(path))
+}
+
+// CleanedAbs implements FileSystem.
+func (n *fsNode) CleanedAbs(path string) (ConfirmedDir, string, error) {
+	node, err := n.Find(path)
+	if err != nil {
+		return "", "", errors.Wrap(err, "unable to clean")
+	}
+	if node == nil {
+		return "", "", fmt.Errorf("'%s' doesn't exist", path)
+	}
+	if node.isNodeADir() {
+		return ConfirmedDir(node.Path()), "", nil
+	}
+	return ConfirmedDir(node.parent.Path()), node.Name(), nil
+}
+
+// Exists implements FileSystem.
+// Exists returns true if the path exists.
+func (n *fsNode) Exists(path string) bool {
+	if !n.isNodeADir() {
+		return n.Name() == path
+	}
+	result, err := n.Find(path)
+	if err != nil {
+		return false
+	}
+	return result != nil
+}
+
+func cleanQueryPath(path string) string {
+	// Always ignore leading separator?
+	// Remember that filepath.Clean returns "." if
+	// given an empty string argument.
+	return filepath.Clean(StripLeadingSeps(path))
+}
+
+// Find finds the given node, else nil if not found.
+// Return error on structural/argument errors.
+func (n *fsNode) Find(path string) (*fsNode, error) {
+	if !n.isNodeADir() {
+		return nil, fmt.Errorf("can only find inside a dir")
+	}
+	if path == "" {
+		// Special case; check *before* cleaning and *before*
+		// comparison to nilParentName.
+		return nil, nil
+	}
+	if (n.parent == nil && path == n.nilParentName) || path == SelfDir {
+		// Special case
+		return n, nil
+	}
+	return n.findIt(cleanQueryPath(path))
+}
+
+func (n *fsNode) findIt(path string) (result *fsNode, err error) {
+	parent := n
+	dName, item := mySplit(path)
+	if dName != "" {
+		parent, err = n.findIt(dName)
+		if err != nil {
+			return nil, err
+		}
+		if parent == nil {
+			// all done, target doesn't exist.
+			return nil, nil
+		}
+	}
+	if !parent.isNodeADir() {
+		return nil, fmt.Errorf("'%s' is not a directory", parent.Path())
+	}
+	return parent.dir[item], nil
+}
+
+// RemoveAll implements FileSystem.
+// RemoveAll removes an item and everything it contains.
+func (n *fsNode) RemoveAll(path string) error {
+	result, err := n.Find(path)
+	if err != nil {
+		return err
+	}
+	if result == nil {
+		return fmt.Errorf("cannot find '%s' to remove it", path)
+	}
+	return result.Remove()
+}
+
+// Remove drop the node, and everything it contains, from its parent.
+func (n *fsNode) Remove() error {
+	if n.parent == nil {
+		return fmt.Errorf("cannot remove a root node")
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir")
+	}
+	for key, value := range n.parent.dir {
+		if value == n {
+			delete(n.parent.dir, key)
+			return nil
+		}
+	}
+	log.Fatal("unable to find self in parent")
+	return nil
+}
+
+// isNodeADir returns true if the node is a directory.
+// Cannot collide with the poorly named "IsDir".
+func (n *fsNode) isNodeADir() bool {
+	return n.dir != nil
+}
+
+// IsDir implements FileSystem.
+// IsDir returns true if the argument resolves
+// to a directory rooted at the node.
+func (n *fsNode) IsDir(path string) bool {
+	result, err := n.Find(path)
+	if err != nil || result == nil {
+		return false
+	}
+	return result.isNodeADir()
+}
+
+// Size returns the size of the node.
+func (n *fsNode) Size() int64 {
+	if n.isNodeADir() {
+		return int64(len(n.dir))
+	}
+	return int64(len(n.content))
+}
+
+// Open implements FileSystem.
+// Open opens the node for reading (just marks it).
+func (n *fsNode) Open(path string) (File, error) {
+	result, err := n.Find(path)
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, fmt.Errorf("cannot find '%s' to open it", path)
+	}
+	result.open = true
+	return result, nil
+}
+
+// Close marks the node closed.
+func (n *fsNode) Close() error {
+	n.open = false
+	return nil
+}
+
+// ReadFile implements FileSystem.
+func (n *fsNode) ReadFile(path string) (c []byte, err error) {
+	result, err := n.Find(path)
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, fmt.Errorf("cannot find '%s' to read it", path)
+	}
+	c = make([]byte, len(result.content))
+	_, err = result.Read(c)
+	return c, err
+}
+
+// Read returns the content of the file node.
+func (n *fsNode) Read(d []byte) (c int, err error) {
+	if n.isNodeADir() {
+		return 0, fmt.Errorf(
+			"cannot read content from non-file '%s'", n.Path())
+	}
+	return copy(d, n.content), nil
+}
+
+// Write saves the contents of the argument to the file node.
+func (n *fsNode) Write(p []byte) (c int, err error) {
+	if n.isNodeADir() {
+		return 0, fmt.Errorf(
+			"cannot write content to non-file '%s'", n.Path())
+	}
+	n.content = make([]byte, len(p))
+	return copy(n.content, p), nil
+}
+
+// ContentMatches returns true if v matches fake file's content.
+func (n *fsNode) ContentMatches(v []byte) bool {
+	return bytes.Equal(v, n.content)
+}
+
+// GetContent the content of a fake file.
+func (n *fsNode) GetContent() []byte {
+	return n.content
+}
+
+// Stat returns an instance of FileInfo.
+func (n *fsNode) Stat() (os.FileInfo, error) {
+	return fileInfo{node: n}, nil
+}
+
+// Walk implements FileSystem.
+func (n *fsNode) Walk(path string, walkFn filepath.WalkFunc) error {
+	result, err := n.Find(path)
+	if err != nil {
+		return err
+	}
+	if result == nil {
+		return fmt.Errorf("cannot find '%s' to walk it", path)
+	}
+	return result.WalkMe(walkFn)
+}
+
+// Walk runs the given walkFn on each node.
+func (n *fsNode) WalkMe(walkFn filepath.WalkFunc) error {
+	fi, err := n.Stat()
+	// always visit self first
+	err = walkFn(n.Path(), fi, err)
+	if !n.isNodeADir() {
+		// it's a file, so nothing more to do
+		return err
+	}
+	// process self as a directory
+	if err == filepath.SkipDir {
+		return nil
+	}
+	// Walk is supposed to visit in lexical order.
+	for _, k := range n.sortedDirEntries() {
+		if err := n.dir[k].WalkMe(walkFn); err != nil {
+			if err == filepath.SkipDir {
+				// stop processing this directory
+				break
+			}
+			// bail out completely
+			return err
+		}
+	}
+	return nil
+}
+
+func (n *fsNode) sortedDirEntries() []string {
+	keys := make([]string, len(n.dir))
+	i := 0
+	for k := range n.dir {
+		keys[i] = k
+		i++
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// FileCount returns a count of files.
+// Directories, empty or otherwise, not counted.
+func (n *fsNode) FileCount() int {
+	count := 0
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			count++
+		}
+		return nil
+	})
+	return count
+}
+
+func (n *fsNode) DebugPrint() {
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("err '%v' at path %q\n", err, path)
+			return nil
+		}
+		if info.IsDir() {
+			if info.Size() == 0 {
+				fmt.Println("empty dir: " + path)
+			}
+		} else {
+			fmt.Println("     file: " + path)
+		}
+		return nil
+	})
+}
+
+var legalFileNamePattern = regexp.MustCompile("^[a-zA-Z0-9-_.]+$")
+
+// This rules enforced here should be simpler and tighter
+// than what's allowed on a real OS.
+// Should be fine for testing or in-memory purposes.
+func isLegalFileNameForCreation(n string) bool {
+	if n == "" || n == SelfDir || !legalFileNamePattern.MatchString(n) {
+		return false
+	}
+	return !strings.Contains(n, ParentDir)
+}
+
+// RegExpGlob returns a list of file paths matching the regexp.
+// Excludes directories.
+func (n *fsNode) RegExpGlob(pattern string) ([]string, error) {
+	var result []string
+	var expression = regexp.MustCompile(pattern)
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			if expression.MatchString(path) {
+				result = append(result, path)
+			}
+		}
+		return nil
+	})
+	sort.Strings(result)
+	return result, nil
+}
+
+// Glob implements FileSystem.
+// Glob returns the list of file paths matching
+// per filepath.Match semantics, i.e. unlike RegExpGlob,
+// Match("foo/a*") will not match sub-sub directories of foo.
+// This is how /bin/ls behaves.
+func (n *fsNode) Glob(pattern string) ([]string, error) {
+	var result []string
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			match, err := filepath.Match(pattern, path)
+			if err != nil {
+				return err
+			}
+			if match {
+				result = append(result, path)
+			}
+		}
+		return nil
+	})
+	sort.Strings(result)
+	return result, nil
+}

api/filesys/fsnode_test.go

@@ -0,0 +1,788 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+const content = `
+Lorem ipsum dolor sit amet,
+consectetur adipiscing elit,
+sed do eiusmod tempor incididunt
+ut labore et dolore magna aliqua.
+`
+const shortContent = "hi"
+
+var topCases = []pathCase{
+	{
+		what:   "dotdot",
+		arg:    ParentDir,
+		errStr: "illegal name '..' in file creation",
+	},
+	{
+		what:   "empty",
+		arg:    "",
+		name:   "",
+		errStr: "illegal name '.' in file creation",
+	},
+	{
+		what: "simple",
+		arg:  "bob",
+		name: "bob",
+		path: "bob",
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "bob"),
+	},
+	{
+		what: "longer yet",
+		arg:  filepath.Join("longer", "foo", "bar", "beans", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "foo", "bar", "beans", "bob"),
+	},
+	{
+		what: "tricky",
+		arg:  filepath.Join("bob", ParentDir, "sally"),
+		name: "sally",
+		path: "sally",
+	},
+	{
+		what: "trickier",
+		arg:  filepath.Join("bob", "sally", ParentDir, ParentDir, "jean"),
+		name: "jean",
+		path: "jean",
+	},
+}
+
+func TestMakeEmptyDirInMemory(t *testing.T) {
+	n := MakeEmptyDirInMemory()
+	if !n.isNodeADir() {
+		t.Fatalf("not a directory")
+	}
+	if n.Size() != 0 {
+		t.Fatalf("unexpected size %d", n.Size())
+	}
+	if n.Name() != "" {
+		t.Fatalf("unexpected name '%s'", n.Name())
+	}
+	if n.Path() != "" {
+		t.Fatalf("unexpected path '%s'", n.Path())
+	}
+	runBasicOperations(
+		t, "MakeEmptyDirInMemory", false, topCases, n)
+}
+
+func TestMakeFsInMemory(t *testing.T) {
+	runBasicOperations(
+		t, "MakeFsInMemory", true, topCases, MakeFsInMemory())
+}
+
+//nolint:gocyclo
+func runBasicOperations(
+	t *testing.T, tName string, isFSysRooted bool,
+	cases []pathCase, fSys FileSystem) {
+	buff := make([]byte, 500)
+	for _, c := range cases {
+		err := fSys.WriteFile(c.arg, []byte(content))
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error writing to  '%s'!", c.what, c.arg)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if !fSys.Exists(c.path) {
+			t.Fatalf("%s; expect existence of '%s'", c.what, c.path)
+		}
+		stuff, err := fSys.ReadFile(c.path)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(stuff) != content {
+			t.Fatalf("%s; unexpected content '%s'", c.what, stuff)
+		}
+		f, err := fSys.Open(c.arg)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		fi, err := f.Stat()
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if fi.Name() != c.name {
+			t.Fatalf("%s; expected name '%s', got '%s'", c.what, c.name, fi.Name())
+		}
+		count, err := f.Read(buff)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(buff[:count]) != content {
+			t.Fatalf("%s; unexpected buff '%s'", c.what, buff)
+		}
+		count, err = f.Write([]byte(shortContent))
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if count != len(shortContent) {
+			t.Fatalf("%s; unexpected count: %d", c.what, len(shortContent))
+		}
+		stuff, err = fSys.ReadFile(c.path)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(stuff) != shortContent {
+			t.Fatalf("%s; unexpected content '%s'", c.what, stuff)
+		}
+	}
+
+	var actualPaths []string
+	var err error
+	prefix := ""
+	{
+		root := SelfDir
+		if isFSysRooted {
+			root = Separator
+			prefix = Separator
+		}
+		err = fSys.Walk(root, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				fmt.Printf("err '%v' at path %q\n", err, path)
+				return nil
+			}
+			if !info.IsDir() {
+				actualPaths = append(actualPaths, path)
+			}
+			return nil
+		})
+	}
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	var expectedPaths []string
+	for _, c := range cases {
+		if c.errStr == "" {
+			expectedPaths = append(expectedPaths, prefix+c.path)
+		}
+	}
+	sort.Strings(expectedPaths)
+	assertEqualStringSlices(t, expectedPaths, actualPaths, tName)
+}
+
+type pathCase struct {
+	what   string
+	arg    string
+	name   string
+	path   string
+	errStr string
+}
+
+func TestAddDir(t *testing.T) {
+	cases := []pathCase{
+		{
+			what:   "dotdot",
+			arg:    ParentDir,
+			errStr: "cannot add a directory above ''",
+		},
+		{
+			what: "empty",
+			arg:  "",
+			name: "",
+			path: "",
+		},
+		{
+			what: "simple",
+			arg:  "bob",
+			name: "bob",
+			path: "bob",
+		},
+		{
+			what: "longer",
+			arg:  filepath.Join("longer", "bob"),
+			name: "bob",
+			path: filepath.Join("longer", "bob"),
+		},
+		{
+			what: "longer yet",
+			arg:  filepath.Join("longer", "foo", "bar", "beans", "bob"),
+			name: "bob",
+			path: filepath.Join("longer", "foo", "bar", "beans", "bob"),
+		},
+		{
+			what: "tricky",
+			arg:  filepath.Join("bob", ParentDir, "sally"),
+			name: "sally",
+			path: "sally",
+		},
+		{
+			what: "trickier",
+			arg:  filepath.Join("bob", "sally", ParentDir, ParentDir, "jean"),
+			name: "jean",
+			path: "jean",
+		},
+	}
+	for _, c := range cases {
+		n := MakeEmptyDirInMemory()
+		f, err := n.AddDir(c.arg)
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error!", c.what)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf(
+					"%s; expected error with '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		checkNode(t, c.what, f, c.name, 0, true, c.path)
+		checkOsStat(t, c.what, f, f.Name(), 0, true)
+	}
+}
+
+var bagOfCases = []pathCase{
+	{
+		what:   "empty",
+		arg:    "",
+		errStr: "illegal name '.' in file creation",
+	},
+	{
+		what: "simple",
+		arg:  "bob",
+		name: "bob",
+		path: "bob",
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "bob"),
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "sally"),
+		name: "sally",
+		path: filepath.Join("longer", "sally"),
+	},
+	{
+		what: "even longer",
+		arg:  filepath.Join("longer", "than", "the", "other", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "than", "the", "other", "bob"),
+	},
+	{
+		what: "even longer",
+		arg:  filepath.Join("even", "much", "longer", "than", "the", "other", "bob"),
+		name: "bob",
+		path: filepath.Join("even", "much", "longer", "than", "the", "other", "bob"),
+	},
+}
+
+func TestAddFile(t *testing.T) {
+	n := MakeEmptyDirInMemory()
+	if n.FileCount() != 0 {
+		t.Fatalf("expected no files, got %d", n.FileCount())
+	}
+	expectedFileCount := 0
+	for _, c := range bagOfCases {
+		f, err := n.AddFile(c.arg, []byte(content))
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error!", c.what)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error %v", c.what, err)
+		}
+		checkNode(t, c.what, f, c.name, len(content), false, c.path)
+		checkOsStat(t, c.what, f, f.Name(), len(content), false)
+
+		result, err := n.Find(c.arg)
+		if err != nil {
+			t.Fatalf("%s; unexpected find error %v", c.what, err)
+		}
+		if result != f {
+			t.Fatalf("%s; unexpected find result %v", c.what, result)
+		}
+
+		result, err = n.Find(filepath.Join("longer", "bogus"))
+		if err != nil {
+			t.Fatalf("%s; unexpected find error %v", c.what, err)
+		}
+		if result != nil {
+			t.Fatalf("%s; unexpected find result %v", c.what, result)
+		}
+
+		expectedFileCount++
+		fc := n.FileCount()
+		if fc != expectedFileCount {
+			t.Fatalf("expected file count %d, got %d",
+				expectedFileCount, fc)
+		}
+	}
+}
+
+func checkNode(
+	t *testing.T, what string, f *fsNode, name string,
+	size int, isDir bool, path string) {
+	if f.isNodeADir() != isDir {
+		t.Fatalf("%s; unexpected isNodeADir = %v", what, f.isNodeADir())
+	}
+	if f.Size() != int64(size) {
+		t.Fatalf("%s; unexpected size %d", what, f.Size())
+	}
+	if name != f.Name() {
+		t.Fatalf("%s; expected name '%s', got '%s'", what, name, f.Name())
+	}
+	if path != f.Path() {
+		t.Fatalf("%s; expected path '%s', got '%s'", what, path, f.Path())
+	}
+}
+
+func checkOsStat(
+	t *testing.T, what string, f File, name string,
+	size int, isDir bool) {
+	info, err := f.Stat()
+	if err != nil {
+		t.Fatalf("%s; unexpected stat error %v", what, err)
+	}
+	if info.IsDir() != isDir {
+		t.Fatalf("%s; unexpected info.isNodeADir = %v", what, info.IsDir())
+	}
+	if info.Size() != int64(size) {
+		t.Fatalf("%s; unexpected info.size %d", what, info.Size())
+	}
+	if info.Name() != name {
+		t.Fatalf("%s; expected name '%s', got info.Name '%s'", what, name, info.Name())
+	}
+}
+
+var bunchOfFiles = []struct {
+	path     string
+	addAsDir bool
+}{
+	{
+		path: filepath.Join("b", "e", "a", "c", "g"),
+	},
+	{
+		path: filepath.Join("z", "r", "a", "b", "g"),
+	},
+	{
+		path: filepath.Join("b", "q", "a", "c", "g"),
+	},
+	{
+		path:     filepath.Join("b", "a", "a", "m", "g"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("b", "w"),
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "m"),
+	},
+	{
+		path: filepath.Join("b", "d", "z"),
+	},
+	{
+		path: filepath.Join("b", "d", "y"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "n"),
+	},
+	{
+		path: filepath.Join("b", "d", "x"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "o"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "m"),
+	},
+	{
+		path:     filepath.Join("b", "d", "a", "c", "i"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("x"),
+	},
+	{
+		path: filepath.Join("y"),
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "i", "beans"),
+	},
+	{
+		path:     filepath.Join("b", "d", "a", "c", "r", "w"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "u"),
+	},
+}
+
+func makeLoadedFileTree(t *testing.T) *fsNode {
+	n := MakeEmptyDirInMemory()
+	var err error
+	expectedFileCount := 0
+	for _, item := range bunchOfFiles {
+		if item.addAsDir {
+			_, err = n.AddDir(item.path)
+		} else {
+			_, err = n.AddFile(item.path, []byte(content))
+			expectedFileCount++
+		}
+		if err != nil {
+			t.Fatalf("unexpected error %v", err)
+		}
+	}
+	fc := n.FileCount()
+	if fc != expectedFileCount {
+		t.Fatalf("expected file count %d, got %d",
+			expectedFileCount, fc)
+	}
+	return n
+}
+
+func TestWalkMe(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	var actualPaths []string
+	err := n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("err '%v' at path %q\n", err, path)
+			return nil
+		}
+		if info.IsDir() {
+			if info.Name() == "ignore" {
+				return filepath.SkipDir
+			}
+		} else {
+			actualPaths = append(actualPaths, path)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	var expectedPaths []string
+	for _, c := range bunchOfFiles {
+		if !c.addAsDir && !strings.Contains(c.path, "ignore") {
+			expectedPaths = append(expectedPaths, c.path)
+		}
+	}
+	sort.Strings(expectedPaths)
+	assertEqualStringSlices(t, expectedPaths, actualPaths, "testWalkMe")
+}
+
+func TestRemove(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	orgCount := n.FileCount()
+
+	// Remove the "ignore" directory and everything below it.
+	path := filepath.Join("b", "d", "ignore")
+	result, err := n.Find(path)
+	if err != nil {
+		t.Fatalf("%s; unexpected error %v", path, err)
+	}
+	if result == nil {
+		t.Fatalf("%s; expected to find '%s'", path, path)
+	}
+	if !result.isNodeADir() {
+		t.Fatalf("%s; expected to find a directory", path)
+	}
+	err = result.Remove()
+	if err != nil {
+		t.Fatalf("%s; unable to remove: %v", path, err)
+	}
+	result, err = n.Find(path)
+	if err != nil {
+		// Just because it's gone doesn't mean error.
+		t.Fatalf("%s; unexpected error %v", path, err)
+	}
+	if result != nil {
+		t.Fatalf("%s; should not have been able to find '%s'", path, path)
+	}
+
+	// There were three files below "ignore".
+	orgCount -= 3
+
+	// Now drop one more for a total of four dropped.
+	result, _ = n.Find(filepath.Join("y"))
+	err = result.Remove()
+	if err != nil {
+		t.Fatalf("%s; unable to remove: %v", path, err)
+	}
+	orgCount -= 1
+
+	fc := n.FileCount()
+	if fc != orgCount {
+		t.Fatalf("expected file count %d, got %d",
+			orgCount, fc)
+	}
+}
+
+func TestExists(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	path := filepath.Join("b", "d", "a")
+	if !n.Exists(path) {
+		t.Fatalf("expected existence at %s", path)
+	}
+	if !n.IsDir(path) {
+		t.Fatalf("expected directory at %s", path)
+	}
+}
+
+func TestRegExpGlob(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	expected := []string{
+		filepath.Join("b", "d", "a", "c", "i", "beans"),
+		filepath.Join("b", "d", "a", "c", "m"),
+		filepath.Join("b", "d", "a", "c", "u"),
+		filepath.Join("b", "d", "ignore", "c", "m"),
+		filepath.Join("b", "d", "ignore", "c", "n"),
+		filepath.Join("b", "d", "ignore", "c", "o"),
+		filepath.Join("b", "d", "x"),
+		filepath.Join("b", "d", "y"),
+		filepath.Join("b", "d", "z"),
+	}
+	paths, err := n.RegExpGlob("b/d/*")
+	if err != nil {
+		t.Fatalf("glob error: %v", err)
+	}
+	assertEqualStringSlices(t, expected, paths, "glob test")
+}
+
+func TestGlob(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	expected := []string{
+		filepath.Join("b", "d", "x"),
+		filepath.Join("b", "d", "y"),
+		filepath.Join("b", "d", "z"),
+	}
+	paths, err := n.Glob("b/d/*")
+	if err != nil {
+		t.Fatalf("glob error: %v", err)
+	}
+	assertEqualStringSlices(t, expected, paths, "glob test")
+}
+
+func assertEqualStringSlices(t *testing.T, expected, actual []string, message string) {
+	if len(expected) != len(actual) {
+		t.Fatalf(
+			"%s; unequal sizes; len(expected)=%d, len(actual)=%d\n%+v\n%+v\n",
+			message, len(expected), len(actual), expected, actual)
+	}
+	for i := range expected {
+		if expected[i] != actual[i] {
+			t.Fatalf(
+				"%s; unequal entries; expected=%s, actual=%s",
+				message, expected[i], actual[i])
+		}
+	}
+}
+
+func TestFind(t *testing.T) {
+	cases := []struct {
+		what       string
+		arg        string
+		expectDir  bool
+		expectFile bool
+		errStr     string
+	}{
+		{
+			what: "garbage",
+			arg:  "///1(*&SA",
+		},
+		{
+			what: "simple",
+			arg:  "bob",
+		},
+		{
+			what: "no directory",
+			arg:  filepath.Join("b", "rrrrrr"),
+		},
+		{
+			what:      "is a directory",
+			arg:       filepath.Join("b", "d", "ignore"),
+			expectDir: true,
+		},
+		{
+			what:       "longer, ending in file",
+			arg:        filepath.Join("b", "d", "x"),
+			expectFile: true,
+		},
+		{
+			what:       "moar longer, ending in file",
+			arg:        filepath.Join("b", "d", "a", "c", "u"),
+			expectFile: true,
+		},
+		{
+			what:      "directory",
+			arg:       filepath.Join("b"),
+			expectDir: true,
+		},
+		{
+			// Querying for the empty string could
+			// 1) be an error,
+			// 2) return no result (and no error) as with
+			//    any illegal and therefore non-existent
+			//    file name,
+			// 3) return the node itself, like running
+			//    'ls' with no argument.
+			// Going with option 2 (no result, no error),
+			// since at this low level it makes more sense
+			// if the results for the empty string query
+			// differ from the results for the "." query.
+			what: "empty name",
+			arg:  "",
+		},
+		{
+			what:      "self dir",
+			arg:       SelfDir,
+			expectDir: true,
+		},
+		{
+			what: "parent dir - doesn't exist",
+			arg:  ParentDir,
+		},
+		{
+			what: "many parents - doesn't exist",
+			arg:  filepath.Join(ParentDir, ParentDir, ParentDir),
+		},
+	}
+
+	n := makeLoadedFileTree(t)
+	for _, item := range cases {
+		result, err := n.Find(item.arg)
+		if item.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error", item.what)
+			}
+			if !strings.Contains(err.Error(), item.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					item.what, item.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", item.what, err)
+		}
+		if result == nil {
+			if item.expectDir {
+				t.Fatalf(
+					"%s; expected to find directory '%s'", item.what, item.arg)
+			}
+			if item.expectFile {
+				t.Fatalf(
+					"%s; expected to find file '%s'", item.what, item.arg)
+			}
+			continue
+		}
+		if item.expectDir {
+			if !result.isNodeADir() {
+				t.Fatalf(
+					"%s; expected '%s' to be a directory", item.what, item.arg)
+			}
+			continue
+		}
+		if item.expectFile {
+			if result.isNodeADir() {
+				t.Fatalf("%s; expected '%s' to be a file", item.what, item.arg)
+			}
+			continue
+		}
+		t.Fatalf(
+			"%s; expected nothing for '%s', but got '%s'",
+			item.what, item.arg, result.Path())
+	}
+}
+
+func TestCleanedAbs(t *testing.T) {
+	cases := []struct {
+		what   string
+		full   string
+		cDir   string
+		name   string
+		errStr string
+	}{
+		{
+			what:   "empty",
+			full:   "",
+			errStr: "doesn't exist",
+		},
+		{
+			what:   "simple",
+			full:   "bob",
+			errStr: "'bob' doesn't exist",
+		},
+		{
+			what:   "no directory",
+			full:   filepath.Join("b", "rrrrrr"),
+			errStr: "'b/rrrrrr' doesn't exist",
+		},
+		{
+			what: "longer, ending in file",
+			full: filepath.Join("b", "d", "x"),
+			cDir: filepath.Join("b", "d"),
+			name: "x",
+		},
+		{
+			what: "moar longer, ending in file",
+			full: filepath.Join("b", "d", "a", "c", "u"),
+			cDir: filepath.Join("b", "d", "a", "c"),
+			name: "u",
+		},
+		{
+			what: "directory",
+			full: filepath.Join("b", "d"),
+			cDir: filepath.Join("b", "d"),
+			name: "",
+		},
+	}
+
+	n := makeLoadedFileTree(t)
+	for _, item := range cases {
+		cDir, name, err := n.CleanedAbs(item.full)
+		if item.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error", item.what)
+			}
+			if !strings.Contains(err.Error(), item.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					item.what, item.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", item.what, err)
+		}
+		if cDir != ConfirmedDir(item.cDir) {
+			t.Fatalf("%s; expected cDir=%s, got '%s'", item.what, item.cDir, cDir)
+		}
+		if name != item.name {
+			t.Fatalf("%s; expected name=%s, got '%s'", item.what, item.name, name)
+		}
+	}
+}

api/filesys/fsondisk.go

@@ -1,7 +1,7 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package fs
+package filesys
 
 import (
 	"fmt"

api/filesys/fsondisk_test.go

@@ -1,7 +1,7 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package fs
+package filesys_test
 
 import (
 	"io/ioutil"
@@ -10,10 +10,12 @@ import (
 	"path/filepath"
 	"reflect"
 	"testing"
+
+	. "sigs.k8s.io/kustomize/api/filesys"
 )
 
 func makeTestDir(t *testing.T) (FileSystem, string) {
-	x := MakeFsOnDisk()
+	fSys := MakeFsOnDisk()
 	td, err := ioutil.TempDir("", "kustomize_testing_dir")
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
@@ -22,20 +24,20 @@ func makeTestDir(t *testing.T) (FileSystem, string) {
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
-	if !x.Exists(testDir) {
+	if !fSys.Exists(testDir) {
 		t.Fatalf("expected existence")
 	}
-	if !x.IsDir(testDir) {
+	if !fSys.IsDir(testDir) {
 		t.Fatalf("expected directory")
 	}
-	return x, testDir
+	return fSys, testDir
 }
 
 func TestCleanedAbs_1(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	d, f, err := x.CleanedAbs("")
+	d, f, err := fSys.CleanedAbs("")
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
@@ -52,10 +54,10 @@ func TestCleanedAbs_1(t *testing.T) {
 }
 
 func TestCleanedAbs_2(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	d, f, err := x.CleanedAbs("/")
+	d, f, err := fSys.CleanedAbs("/")
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
@@ -68,16 +70,16 @@ func TestCleanedAbs_2(t *testing.T) {
 }
 
 func TestCleanedAbs_3(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	err := x.WriteFile(
+	err := fSys.WriteFile(
 		filepath.Join(testDir, "foo"), []byte(`foo`))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
 
-	d, f, err := x.CleanedAbs(filepath.Join(testDir, "foo"))
+	d, f, err := fSys.CleanedAbs(filepath.Join(testDir, "foo"))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
@@ -90,21 +92,21 @@ func TestCleanedAbs_3(t *testing.T) {
 }
 
 func TestCleanedAbs_4(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	err := x.MkdirAll(filepath.Join(testDir, "d1", "d2"))
+	err := fSys.MkdirAll(filepath.Join(testDir, "d1", "d2"))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
-	err = x.WriteFile(
+	err = fSys.WriteFile(
 		filepath.Join(testDir, "d1", "d2", "bar"),
 		[]byte(`bar`))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
 
-	d, f, err := x.CleanedAbs(
+	d, f, err := fSys.CleanedAbs(
 		filepath.Join(testDir, "d1", "d2"))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
@@ -116,7 +118,7 @@ func TestCleanedAbs_4(t *testing.T) {
 		t.Fatalf("unexpected f=%s", f)
 	}
 
-	d, f, err = x.CleanedAbs(
+	d, f, err = fSys.CleanedAbs(
 		filepath.Join(testDir, "d1", "d2", "bar"))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
@@ -130,26 +132,26 @@ func TestCleanedAbs_4(t *testing.T) {
 }
 
 func TestReadFilesRealFS(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	err := x.WriteFile(path.Join(testDir, "foo"), []byte(`foo`))
+	err := fSys.WriteFile(path.Join(testDir, "foo"), []byte(`foo`))
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
-	if !x.Exists(path.Join(testDir, "foo")) {
+	if !fSys.Exists(path.Join(testDir, "foo")) {
 		t.Fatalf("expected foo")
 	}
-	if x.IsDir(path.Join(testDir, "foo")) {
+	if fSys.IsDir(path.Join(testDir, "foo")) {
 		t.Fatalf("expected foo not to be a directory")
 	}
 
-	err = x.WriteFile(path.Join(testDir, "bar"), []byte(`bar`))
+	err = fSys.WriteFile(path.Join(testDir, "bar"), []byte(`bar`))
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 
-	files, err := x.Glob(path.Join("testDir", "*"))
+	files, err := fSys.Glob(path.Join("testDir", "*"))
 	expected := []string{
 		path.Join(testDir, "bar"),
 		path.Join(testDir, "foo"),

api/filesys/util.go

@@ -0,0 +1,30 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import "path/filepath"
+
+// RootedPath returns a rooted path, e.g. "/foo/bar" as
+// opposed to "foo/bar".
+func RootedPath(elem ...string) string {
+	return Separator + filepath.Join(elem...)
+}
+
+// StripTrailingSeps trims trailing filepath separators from input.
+func StripTrailingSeps(s string) string {
+	k := len(s)
+	for k > 0 && s[k-1] == filepath.Separator {
+		k--
+	}
+	return s[:k]
+}
+
+// StripLeadingSeps trims leading filepath separators from input.
+func StripLeadingSeps(s string) string {
+	k := 0
+	for k < len(s) && s[k] == filepath.Separator {
+		k++
+	}
+	return s[k:]
+}

api/filesys/util_test.go

@@ -0,0 +1,205 @@
+package filesys_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	. "sigs.k8s.io/kustomize/api/filesys"
+)
+
+// Confirm behavior of filepath.Match
+func TestFilePathMatch(t *testing.T) {
+	cases := []struct {
+		pattern  string
+		path     string
+		expected bool
+	}{
+		{
+			pattern:  "*e*",
+			path:     "hey",
+			expected: true,
+		},
+		{
+			pattern:  "*e*",
+			path:     "hay",
+			expected: false,
+		},
+		{
+			pattern:  "*e*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: false,
+		},
+		{
+			pattern:  "*/e/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "h/e/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/e/y",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/*/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/*/*",
+			path:     filepath.Join("h", "e", "y", "there"),
+			expected: false,
+		},
+		{
+			pattern:  "*/*/*/t*e",
+			path:     filepath.Join("h", "e", "y", "there"),
+			expected: true,
+		},
+	}
+	for _, item := range cases {
+		match, err := filepath.Match(item.pattern, item.path)
+		if err != nil {
+			t.Fatalf("unexpected err: %v", err)
+		}
+		if match != item.expected {
+			t.Fatalf("'%s' '%s' %v\n", item.pattern, item.path, match)
+		}
+	}
+}
+
+// Confirm behavior of filepath.Split
+func TestFilePathSplit(t *testing.T) {
+	cases := []struct {
+		full string
+		dir  string
+		file string
+	}{
+		{
+			full: "",
+			dir:  "",
+			file: "",
+		},
+		{
+			full: SelfDir,
+			dir:  "",
+			file: SelfDir,
+		},
+		{
+			full: "rabbit.jpg",
+			dir:  "",
+			file: "rabbit.jpg",
+		},
+		{
+			full: "/beans",
+			dir:  "/",
+			file: "beans",
+		},
+		{
+			full: "/home/foo/bar",
+			dir:  "/home/foo/",
+			file: "bar",
+		},
+		{
+			full: "/usr/local/",
+			dir:  "/usr/local/",
+			file: "",
+		},
+		{
+			full: "/usr//local//go",
+			dir:  "/usr//local//",
+			file: "go",
+		},
+	}
+	for _, p := range cases {
+		dir, file := filepath.Split(p.full)
+		if dir != p.dir || file != p.file {
+			t.Fatalf(
+				"in '%s',\ngot dir='%s' (expected '%s'),\n got file='%s' (expected %s).",
+				p.full, dir, p.dir, file, p.file)
+		}
+	}
+}
+
+func TestStripTrailingSeps(t *testing.T) {
+	cases := []struct {
+		full string
+		rem  string
+	}{
+		{
+			full: "foo",
+			rem:  "foo",
+		},
+		{
+			full: "",
+			rem:  "",
+		},
+		{
+			full: "foo/",
+			rem:  "foo",
+		},
+		{
+			full: "foo///bar///",
+			rem:  "foo///bar",
+		},
+		{
+			full: "/////",
+			rem:  "",
+		},
+		{
+			full: "/",
+			rem:  "",
+		},
+	}
+	for _, p := range cases {
+		dir := StripTrailingSeps(p.full)
+		if dir != p.rem {
+			t.Fatalf(
+				"in '%s', got dir='%s' (expected '%s')",
+				p.full, dir, p.rem)
+		}
+	}
+}
+
+func TestStripLeadingSeps(t *testing.T) {
+	cases := []struct {
+		full string
+		rem  string
+	}{
+		{
+			full: "foo",
+			rem:  "foo",
+		},
+		{
+			full: "",
+			rem:  "",
+		},
+		{
+			full: "/foo",
+			rem:  "foo",
+		},
+		{
+			full: "///foo///bar///",
+			rem:  "foo///bar///",
+		},
+		{
+			full: "/////",
+			rem:  "",
+		},
+		{
+			full: "/",
+			rem:  "",
+		},
+	}
+	for _, p := range cases {
+		dir := StripLeadingSeps(p.full)
+		if dir != p.rem {
+			t.Fatalf(
+				"in '%s', got dir='%s' (expected '%s')",
+				p.full, dir, p.rem)
+		}
+	}
+}

api/go.mod

@@ -0,0 +1,18 @@
+module sigs.k8s.io/kustomize/api
+
+go 1.13
+
+require (
+	github.com/evanphx/json-patch v4.5.0+incompatible
+	github.com/go-openapi/spec v0.19.4
+	github.com/golangci/golangci-lint v1.21.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/pkg/errors v0.8.1
+	golang.org/x/tools v0.0.0-20191010075000-0337d82405ff
+	gopkg.in/yaml.v2 v2.2.4
+	k8s.io/api v0.17.0
+	k8s.io/apimachinery v0.17.0
+	k8s.io/client-go v0.17.0
+	k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a
+	sigs.k8s.io/yaml v1.1.0
+)

api/go.sum

@@ -1,4 +1,13 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -17,6 +26,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs=
+github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -33,9 +44,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w=
-github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
@@ -51,19 +64,22 @@ github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0
 github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
-github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
-github.com/go-openapi/spec v0.19.2 h1:SStNd1jRcYtfKCN7R0laGNs80WYYvn5CbBjM2sOmCrE=
-github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
+github.com/go-openapi/spec v0.19.4 h1:ixzUSnHTd6hCemgtAJgluaTSGYpLNpJY4mA2DIkdOAo=
+github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
-github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
 github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
@@ -88,16 +104,20 @@ github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2X
 github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b h1:ekuhfTjngPhisSjOJ0QWKpPQE8/rbknHaes6WVJj5Hw=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -113,14 +133,14 @@ github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgO
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.19.1 h1:g9xL8KW7UZDCkVlgHYJMA6F4Sj/sRVa0FoCeXI+Z3iM=
-github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI=
+github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk=
@@ -131,46 +151,48 @@ github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1
 github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
-github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
-github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ=
-github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
@@ -193,12 +215,12 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481 h1:IaSjLMT6WvkoZZjspGxy3rdaTEmWLoRm49WbtVUi9sA=
-github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86 h1:q6SrfsK4FojRnJ1j8+8OJzyq3g9Y1oSVyL6nYGJXXBk=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
@@ -208,32 +230,30 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.0 h1:RFDBa+tab6mW+gX4Ww2SZDc4kS6p01FwnLtgz64Il+I=
-github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E=
 github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -252,20 +272,15 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218 h1:O0yPHYL49quNL4Oj2wVq+zbGMu4dAM6iLoOQtm49TrQ=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -273,12 +288,11 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
 github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
 github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
-github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
@@ -296,19 +310,20 @@ github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/ultraware/whitespace v0.0.3 h1:S5BCRRB5sttNy0bSOhbpw+0mb+cHiCmWfrvxpEzuUk0=
-github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
+github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
@@ -316,37 +331,49 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -355,29 +382,34 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190621203818-d432491b9138 h1:t8BZD9RDjkm9/h7yYN6kE8oaeov5r9aztkB7zKA5Tkg=
-golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88=
@@ -385,17 +417,21 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -409,32 +445,33 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-k8s.io/api v0.0.0-20190313235455-40a48860b5ab h1:DG9A67baNpoeweOy2spF1OWHhnVY5KR7/Ek/+U1lVZc=
-k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
-k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1 h1:IS7K02iBkQXpCeieSiyJjGoLSdVOv2DbPaWHJ+ZtgKg=
-k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
-k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
-k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
+k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
+k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
+k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
+k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
+k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v0.3.3 h1:niceAagH1tzskmaie/icWd7ci1wbG7Bf2c6YGcQv+3c=
-k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 h1:5sW+fEHvlJI3Ngolx30CmubFulwH28DhKjGf70Xmtco=
-k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4=
+k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
+k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
+k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/pluginator v1.0.0 h1:aeLqD8CIaqr++49YrcuRUcXl5vVKYLhCSfwXUi3ifQ4=
-sigs.k8s.io/kustomize/pluginator v1.0.0/go.mod h1:i8HdU5FdH1zDjCKiFf5CNl7slsc0QffyKsY2OuPynJ0=
-sigs.k8s.io/kustomize/v3 v3.2.0/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/hasher/hasher_test.go

@@ -6,7 +6,7 @@ package hasher_test
 import (
 	"testing"
 
-	. "sigs.k8s.io/kustomize/v3/pkg/hasher"
+	. "sigs.k8s.io/kustomize/api/hasher"
 )
 
 func TestSortArrayAndComputeHash(t *testing.T) {

api/ifc/ifc.go

@@ -5,8 +5,8 @@
 package ifc
 
 import (
-	"sigs.k8s.io/kustomize/v3/pkg/gvk"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
 // Validator provides functions to validate annotations and labels
@@ -20,6 +20,12 @@ type Validator interface {
 	IsEnvVarName(k string) error
 }
 
+// KvLoader reads and validates KV pairs.
+type KvLoader interface {
+	Validator() Validator
+	Load(args types.KvPairSources) (all []types.Pair, err error)
+}
+
 // Loader interface exposes methods to read bytes.
 type Loader interface {
 	// Root returns the root location for this Loader.
@@ -30,10 +36,6 @@ type Loader interface {
 	Load(location string) ([]byte, error)
 	// Cleanup cleans the loader
 	Cleanup() error
-	// Validator validates data for use in various k8s fields.
-	Validator() Validator
-	// Loads pairs.
-	LoadKvPairs(args types.GeneratorArgs) ([]types.Pair, error)
 }
 
 // Kunstructured allows manipulation of k8s objects
@@ -53,8 +55,8 @@ type Kunstructured interface {
 	GetMap(path string) (map[string]interface{}, error)
 	MarshalJSON() ([]byte, error)
 	UnmarshalJSON([]byte) error
-	GetGvk() gvk.Gvk
-	SetGvk(gvk.Gvk)
+	GetGvk() resid.Gvk
+	SetGvk(resid.Gvk)
 	GetKind() string
 	GetName() string
 	SetName(string)
@@ -74,11 +76,11 @@ type KunstructuredFactory interface {
 	FromMap(m map[string]interface{}) Kunstructured
 	Hasher() KunstructuredHasher
 	MakeConfigMap(
-		ldr Loader,
+		kvLdr KvLoader,
 		options *types.GeneratorOptions,
 		args *types.ConfigMapArgs) (Kunstructured, error)
 	MakeSecret(
-		ldr Loader,
+		kvLdr KvLoader,
 		options *types.GeneratorOptions,
 		args *types.SecretArgs) (Kunstructured, error)
 }

api/internal/accumulator/expansion/expand.go

@@ -1,18 +1,5 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
 // Package expansion provides functions find and replace $(FOO) style variables in strings.
 package expansion

api/internal/accumulator/expansion/expand_test.go

@@ -1,18 +1,5 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
 package expansion_test
 
@@ -20,7 +7,7 @@ import (
 	"fmt"
 	"testing"
 
-	. "sigs.k8s.io/kustomize/v3/pkg/expansion"
+	. "sigs.k8s.io/kustomize/api/internal/accumulator/expansion"
 )
 
 type expected struct {

api/internal/accumulator/loadconfigfromcrds.go

@@ -1,20 +1,7 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package config
+package accumulator
 
 import (
 	"encoding/json"
@@ -23,8 +10,11 @@ import (
 	"github.com/go-openapi/spec"
 	"github.com/pkg/errors"
 	"k8s.io/kube-openapi/pkg/common"
-	"sigs.k8s.io/kustomize/v3/pkg/gvk"
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
@@ -33,8 +23,8 @@ type nameToApiMap map[string]common.OpenAPIDefinition
 
 // LoadConfigFromCRDs parse CRD schemas from paths into a TransformerConfig
 func LoadConfigFromCRDs(
-	ldr ifc.Loader, paths []string) (*TransformerConfig, error) {
-	tc := MakeEmptyConfig()
+	ldr ifc.Loader, paths []string) (*builtinconfig.TransformerConfig, error) {
+	tc := builtinconfig.MakeEmptyConfig()
 	for _, path := range paths {
 		content, err := ldr.Load(path)
 		if err != nil {
@@ -65,13 +55,13 @@ func makeNameToApiMap(content []byte) (result nameToApiMap, err error) {
 	return
 }
 
-func makeConfigFromApiMap(m nameToApiMap) (*TransformerConfig, error) {
-	result := MakeEmptyConfig()
+func makeConfigFromApiMap(m nameToApiMap) (*builtinconfig.TransformerConfig, error) {
+	result := builtinconfig.MakeEmptyConfig()
 	for name, api := range m {
 		if !looksLikeAk8sType(api.Schema.SchemaProps.Properties) {
 			continue
 		}
-		tc := MakeEmptyConfig()
+		tc := builtinconfig.MakeEmptyConfig()
 		err := loadCrdIntoConfig(
 			tc, makeGvkFromTypeName(name), m, name, []string{})
 		if err != nil {
@@ -88,10 +78,10 @@ func makeConfigFromApiMap(m nameToApiMap) (*TransformerConfig, error) {
 // TODO: Get Group and Version for CRD from the
 // openAPI definition once
 // "x-kubernetes-group-version-kind" is available in CRD
-func makeGvkFromTypeName(n string) gvk.Gvk {
-	names := strings.Split(n, ".")
+func makeGvkFromTypeName(n string) resid.Gvk {
+	names := strings.Split(n, filesys.SelfDir)
 	kind := names[len(names)-1]
-	return gvk.Gvk{Kind: kind}
+	return resid.Gvk{Kind: kind}
 }
 
 func looksLikeAk8sType(properties myProperties) bool {
@@ -104,10 +94,7 @@ func looksLikeAk8sType(properties myProperties) bool {
 		return false
 	}
 	_, ok = properties["metadata"]
-	if !ok {
-		return false
-	}
-	return true
+	return ok
 }
 
 const (
@@ -133,7 +120,7 @@ const (
 
 // loadCrdIntoConfig loads a CRD spec into a TransformerConfig
 func loadCrdIntoConfig(
-	theConfig *TransformerConfig, theGvk gvk.Gvk, theMap nameToApiMap,
+	theConfig *builtinconfig.TransformerConfig, theGvk resid.Gvk, theMap nameToApiMap,
 	typeName string, path []string) (err error) {
 	api, ok := theMap[typeName]
 	if !ok {
@@ -173,9 +160,9 @@ func loadCrdIntoConfig(
 					nameKey = "name"
 				}
 				err = theConfig.AddNamereferenceFieldSpec(
-					NameBackReferences{
-						Gvk: gvk.Gvk{Kind: kind, Version: version},
-						FieldSpecs: []FieldSpec{
+					builtinconfig.NameBackReferences{
+						Gvk: resid.Gvk{Kind: kind, Version: version},
+						FieldSpecs: []types.FieldSpec{
 							makeFs(theGvk, append(path, propName, nameKey))},
 					})
 				if err != nil {
@@ -192,8 +179,8 @@ func loadCrdIntoConfig(
 	return nil
 }
 
-func makeFs(in gvk.Gvk, path []string) FieldSpec {
-	return FieldSpec{
+func makeFs(in resid.Gvk, path []string) types.FieldSpec {
+	return types.FieldSpec{
 		CreateIfNotPresent: false,
 		Gvk:                in,
 		Path:               strings.Join(path, "/"),

api/internal/accumulator/loadconfigfromcrds_test.go

@@ -1,28 +1,19 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package config
+package accumulator_test
 
 import (
 	"reflect"
 	"testing"
 
-	"sigs.k8s.io/kustomize/v3/internal/loadertest"
-	"sigs.k8s.io/kustomize/v3/pkg/gvk"
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/loader"
+
+	. "sigs.k8s.io/kustomize/api/internal/accumulator"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
 // This defines two CRD's:  Bee and MyKind.
@@ -145,44 +136,42 @@ If it is not set we generate a secret dynamically",
 `
 )
 
-func makeLoader(t *testing.T) ifc.Loader {
-	ldr := loadertest.NewFakeLoader("/testpath")
-	err := ldr.AddFile("/testpath/crd.json", []byte(crdContent))
-	if err != nil {
-		t.Fatalf("Failed to setup fake ldr.")
-	}
-	return ldr
-}
-
 func TestLoadCRDs(t *testing.T) {
-	nbrs := []NameBackReferences{
+	nbrs := []builtinconfig.NameBackReferences{
 		{
-			Gvk: gvk.Gvk{Kind: "Secret", Version: "v1"},
-			FieldSpecs: []FieldSpec{
+			Gvk: resid.Gvk{Kind: "Secret", Version: "v1"},
+			FieldSpecs: []types.FieldSpec{
 				{
 					CreateIfNotPresent: false,
-					Gvk:                gvk.Gvk{Kind: "MyKind"},
+					Gvk:                resid.Gvk{Kind: "MyKind"},
 					Path:               "spec/secretRef/name",
 				},
 			},
 		},
 		{
-			Gvk: gvk.Gvk{Kind: "Bee", Version: "v1beta1"},
-			FieldSpecs: []FieldSpec{
+			Gvk: resid.Gvk{Kind: "Bee", Version: "v1beta1"},
+			FieldSpecs: []types.FieldSpec{
 				{
 					CreateIfNotPresent: false,
-					Gvk:                gvk.Gvk{Kind: "MyKind"},
+					Gvk:                resid.Gvk{Kind: "MyKind"},
 					Path:               "spec/beeRef/name",
 				},
 			},
 		},
 	}
 
-	expectedTc := &TransformerConfig{
+	expectedTc := &builtinconfig.TransformerConfig{
 		NameReference: nbrs,
 	}
 
-	actualTc, err := LoadConfigFromCRDs(makeLoader(t), []string{"crd.json"})
+	fSys := filesys.MakeFsInMemory()
+	fSys.WriteFile("/testpath/crd.json", []byte(crdContent))
+	ldr, err := loader.NewLoader(loader.RestrictionRootOnly, "/testpath", fSys)
+	if err != nil {
+		t.Fatalf("unexpected error:%v", err)
+	}
+
+	actualTc, err := LoadConfigFromCRDs(ldr, []string{"crd.json"})
 	if err != nil {
 		t.Fatalf("unexpected error:%v", err)
 	}

api/internal/accumulator/namereferencetransformer.go

@@ -1,28 +1,28 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package transformers
+package accumulator
 
 import (
 	"fmt"
 	"log"
 
-	"sigs.k8s.io/kustomize/v3/pkg/resource"
-
-	"sigs.k8s.io/kustomize/v3/pkg/gvk"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/transform"
 )
 
 type nameReferenceTransformer struct {
-	backRefs []config.NameBackReferences
+	backRefs []builtinconfig.NameBackReferences
 }
 
 var _ resmap.Transformer = &nameReferenceTransformer{}
 
-// NewNameReferenceTransformer constructs a nameReferenceTransformer
+// newNameReferenceTransformer constructs a nameReferenceTransformer
 // with a given slice of NameBackReferences.
-func NewNameReferenceTransformer(br []config.NameBackReferences) resmap.Transformer {
+func newNameReferenceTransformer(br []builtinconfig.NameBackReferences) resmap.Transformer {
 	if br == nil {
 		log.Fatal("backrefs not expected to be nil")
 	}
@@ -86,7 +86,7 @@ func (o *nameReferenceTransformer) Transform(m resmap.ResMap) error {
 					if candidates == nil {
 						candidates = m.SubsetThatCouldBeReferencedByResource(referrer)
 					}
-					err := MutateField(
+					err := transform.MutateField(
 						referrer.Map(),
 						fSpec.PathSlice(),
 						fSpec.CreateIfNotPresent,
@@ -115,7 +115,7 @@ func (o *nameReferenceTransformer) Transform(m resmap.ResMap) error {
 func (o *nameReferenceTransformer) selectReferral(
 	oldName string,
 	referrer *resource.Resource,
-	target gvk.Gvk,
+	target resid.Gvk,
 	referralCandidates resmap.ResMap,
 	referralCandidateSubset []*resource.Resource) (interface{}, interface{}, error) {
 
@@ -146,7 +146,7 @@ func (o *nameReferenceTransformer) selectReferral(
 func (o *nameReferenceTransformer) getSimpleNameField(
 	oldName string,
 	referrer *resource.Resource,
-	target gvk.Gvk,
+	target resid.Gvk,
 	referralCandidates resmap.ResMap,
 	referralCandidateSubset []*resource.Resource) (interface{}, error) {
 
@@ -161,7 +161,7 @@ func (o *nameReferenceTransformer) getSimpleNameField(
 func (o *nameReferenceTransformer) getNameAndNsStruct(
 	inMap map[string]interface{},
 	referrer *resource.Resource,
-	target gvk.Gvk,
+	target resid.Gvk,
 	referralCandidates resmap.ResMap) (interface{}, error) {
 
 	// Example:
@@ -209,34 +209,30 @@ func (o *nameReferenceTransformer) getNameAndNsStruct(
 
 func (o *nameReferenceTransformer) getNewNameFunc(
 	referrer *resource.Resource,
-	target gvk.Gvk,
+	target resid.Gvk,
 	referralCandidates resmap.ResMap) func(in interface{}) (interface{}, error) {
 	return func(in interface{}) (interface{}, error) {
-		switch in.(type) {
+		switch thing := in.(type) {
 		case string:
-			oldName, _ := in.(string)
-			return o.getSimpleNameField(oldName, referrer, target,
+			return o.getSimpleNameField(thing, referrer, target,
 				referralCandidates, referralCandidates.Resources())
 		case map[string]interface{}:
 			// Kind: ValidatingWebhookConfiguration
 			// FieldSpec is webhooks/clientConfig/service
-			oldMap, _ := in.(map[string]interface{})
-			return o.getNameAndNsStruct(oldMap, referrer, target,
+			return o.getNameAndNsStruct(thing, referrer, target,
 				referralCandidates)
 		case []interface{}:
-			l, _ := in.([]interface{})
-			for idx, item := range l {
-				switch item.(type) {
+			for idx, item := range thing {
+				switch value := item.(type) {
 				case string:
 					// Kind: Role/ClusterRole
 					// FieldSpec is rules.resourceNames
-					oldName, _ := item.(string)
-					newName, err := o.getSimpleNameField(oldName, referrer, target,
+					newName, err := o.getSimpleNameField(value, referrer, target,
 						referralCandidates, referralCandidates.Resources())
 					if err != nil {
 						return nil, err
 					}
-					l[idx] = newName
+					thing[idx] = newName
 				case map[string]interface{}:
 					// Kind: RoleBinding/ClusterRoleBinding
 					// FieldSpec is subjects
@@ -245,13 +241,12 @@ func (o *nameReferenceTransformer) getNewNameFunc(
 					// what get mutatefield to request the mapping of the whole
 					// map containing namespace and name instead of just a simple
 					// string field containing the name
-					oldMap, _ := item.(map[string]interface{})
-					newMap, err := o.getNameAndNsStruct(oldMap, referrer, target,
+					newMap, err := o.getNameAndNsStruct(value, referrer, target,
 						referralCandidates)
 					if err != nil {
 						return nil, err
 					}
-					l[idx] = newMap
+					thing[idx] = newMap
 				default:
 					return nil, fmt.Errorf(
 						"%#v is expected to be either a []string or a []map[string]interface{}", in)

api/internal/accumulator/namereferencetransformer_test.go

@@ -1,18 +1,18 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package transformers
+package accumulator
 
 import (
 	"strings"
 	"testing"
 
-	"sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct"
-	"sigs.k8s.io/kustomize/v3/pkg/gvk"
-	"sigs.k8s.io/kustomize/v3/pkg/resid"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/resmaptest"
-	"sigs.k8s.io/kustomize/v3/pkg/resource"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	resmaptest_test "sigs.k8s.io/kustomize/api/testutils/resmaptest"
 )
 
 func TestNameReferenceHappyRun(t *testing.T) {
@@ -463,7 +463,7 @@ func TestNameReferenceHappyRun(t *testing.T) {
 			},
 		}).ResMap()
 
-	nrt := NewNameReferenceTransformer(defaultTransformerConfig.NameReference)
+	nrt := newNameReferenceTransformer(builtinconfig.MakeDefaultConfig().NameReference)
 	err := nrt.Transform(m)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -523,7 +523,7 @@ func TestNameReferenceUnhappyRun(t *testing.T) {
 			expectedErr: "is expected to contain a name field"},
 	}
 
-	nrt := NewNameReferenceTransformer(defaultTransformerConfig.NameReference)
+	nrt := newNameReferenceTransformer(builtinconfig.MakeDefaultConfig().NameReference)
 	for _, test := range tests {
 		err := nrt.Transform(test.resMap)
 		if err == nil {
@@ -581,7 +581,7 @@ func TestNameReferencePersistentVolumeHappyRun(t *testing.T) {
 
 	m1 := resmaptest_test.NewRmBuilder(t, rf).AddR(v1).AddR(c1).ResMap()
 
-	nrt := NewNameReferenceTransformer(defaultTransformerConfig.NameReference)
+	nrt := newNameReferenceTransformer(builtinconfig.MakeDefaultConfig().NameReference)
 	if err := nrt.Transform(m1); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -720,7 +720,7 @@ func TestNameReferenceNamespace(t *testing.T) {
 		ReplaceResource(deploymentMap(ns1, prefixedname, prefixedname, prefixedname)).
 		ReplaceResource(deploymentMap(ns2, suffixedname, suffixedname, suffixedname)).ResMap()
 
-	nrt := NewNameReferenceTransformer(defaultTransformerConfig.NameReference)
+	nrt := newNameReferenceTransformer(builtinconfig.MakeDefaultConfig().NameReference)
 	err := nrt.Transform(m)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -877,13 +877,13 @@ func TestNameReferenceClusterWide(t *testing.T) {
 			}).ResMap()
 
 	clusterRoleId := resid.NewResId(
-		gvk.Gvk{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"}, modifiedname)
+		resid.Gvk{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"}, modifiedname)
 	clusterRoleBindingId := resid.NewResId(
-		gvk.Gvk{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBinding"}, modifiedname)
+		resid.Gvk{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBinding"}, modifiedname)
 	clusterRole, _ := expected.GetByCurrentId(clusterRoleId)
 	clusterRole.AppendRefBy(clusterRoleBindingId)
 
-	nrt := NewNameReferenceTransformer(defaultTransformerConfig.NameReference)
+	nrt := newNameReferenceTransformer(builtinconfig.MakeDefaultConfig().NameReference)
 	err := nrt.Transform(m)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -1005,13 +1005,13 @@ func TestNameReferenceNamespaceTransformation(t *testing.T) {
 			}).ResMap()
 
 	clusterRoleId := resid.NewResId(
-		gvk.Gvk{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"}, modifiedname)
+		resid.Gvk{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"}, modifiedname)
 	clusterRoleBindingId := resid.NewResId(
-		gvk.Gvk{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBinding"}, modifiedname)
+		resid.Gvk{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBinding"}, modifiedname)
 	clusterRole, _ := expected.GetByCurrentId(clusterRoleId)
 	clusterRole.AppendRefBy(clusterRoleBindingId)
 
-	nrt := NewNameReferenceTransformer(defaultTransformerConfig.NameReference)
+	nrt := newNameReferenceTransformer(builtinconfig.MakeDefaultConfig().NameReference)
 	err := nrt.Transform(m)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -1049,7 +1049,7 @@ func TestNameReferenceCandidateSelection(t *testing.T) {
 		ReplaceResource(deploymentMap("", "p1-deploy1", "p1-cm1-hash", "p1-secret1-hash")).
 		ResMap()
 
-	nrt := NewNameReferenceTransformer(defaultTransformerConfig.NameReference)
+	nrt := newNameReferenceTransformer(builtinconfig.MakeDefaultConfig().NameReference)
 	err := nrt.Transform(m)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)

api/internal/accumulator/refvartransformer.go

@@ -1,41 +1,31 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package transformers
+package accumulator
 
 import (
 	"fmt"
-	"sigs.k8s.io/kustomize/v3/pkg/expansion"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
+
+	expansion2 "sigs.k8s.io/kustomize/api/internal/accumulator/expansion"
+
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
-type RefVarTransformer struct {
+type refVarTransformer struct {
 	varMap            map[string]interface{}
 	replacementCounts map[string]int
-	fieldSpecs        []config.FieldSpec
+	fieldSpecs        []types.FieldSpec
 	mappingFunc       func(string) interface{}
 }
 
-// NewRefVarTransformer returns a new RefVarTransformer
+// newRefVarTransformer returns a new refVarTransformer
 // that replaces $(VAR) style variables with values.
 // The fieldSpecs are the places to look for occurrences of $(VAR).
-func NewRefVarTransformer(
-	varMap map[string]interface{}, fs []config.FieldSpec) *RefVarTransformer {
-	return &RefVarTransformer{
+func newRefVarTransformer(
+	varMap map[string]interface{}, fs []types.FieldSpec) *refVarTransformer {
+	return &refVarTransformer{
 		varMap:     varMap,
 		fieldSpecs: fs,
 	}
@@ -45,12 +35,12 @@ func NewRefVarTransformer(
 // embedded instances of $VAR style variables, e.g. a container command string.
 // The function returns the string with the variables expanded to their final
 // values.
-func (rv *RefVarTransformer) replaceVars(in interface{}) (interface{}, error) {
+func (rv *refVarTransformer) replaceVars(in interface{}) (interface{}, error) {
 	switch vt := in.(type) {
 	case []interface{}:
 		var xs []interface{}
 		for _, a := range in.([]interface{}) {
-			xs = append(xs, expansion.Expand(a.(string), rv.mappingFunc))
+			xs = append(xs, expansion2.Expand(a.(string), rv.mappingFunc))
 		}
 		return xs, nil
 	case map[string]interface{}:
@@ -67,7 +57,7 @@ func (rv *RefVarTransformer) replaceVars(in interface{}) (interface{}, error) {
 				// This field can potentially contains a $(VAR) since it is
 				// of string type. For instance .spec.replicas: $(REPLICAS)
 				// in a Deployment object
-				xs[k] = expansion.Expand(s, rv.mappingFunc)
+				xs[k] = expansion2.Expand(s, rv.mappingFunc)
 			}
 		}
 		return xs, nil
@@ -79,7 +69,10 @@ func (rv *RefVarTransformer) replaceVars(in interface{}) (interface{}, error) {
 		}
 		// This field can potentially contain a $(VAR) since it is
 		// of string type.
-		return expansion.Expand(s, rv.mappingFunc), nil
+		return expansion2.Expand(s, rv.mappingFunc), nil
+	// staticcheck erroneously claims that `case nil`
+	// is unreachable here, so suppressing it.
+	//nolint:staticcheck
 	case nil:
 		return nil, nil
 	default:
@@ -89,7 +82,7 @@ func (rv *RefVarTransformer) replaceVars(in interface{}) (interface{}, error) {
 
 // UnusedVars returns slice of Var names that were unused
 // after a Transform run.
-func (rv *RefVarTransformer) UnusedVars() []string {
+func (rv *refVarTransformer) UnusedVars() []string {
 	var unused []string
 	for k := range rv.varMap {
 		_, ok := rv.replacementCounts[k]
@@ -101,14 +94,14 @@ func (rv *RefVarTransformer) UnusedVars() []string {
 }
 
 // Transform replaces $(VAR) style variables with values.
-func (rv *RefVarTransformer) Transform(m resmap.ResMap) error {
+func (rv *refVarTransformer) Transform(m resmap.ResMap) error {
 	rv.replacementCounts = make(map[string]int)
-	rv.mappingFunc = expansion.MappingFuncFor(
+	rv.mappingFunc = expansion2.MappingFuncFor(
 		rv.replacementCounts, rv.varMap)
 	for _, res := range m.Resources() {
 		for _, fieldSpec := range rv.fieldSpecs {
 			if res.OrgId().IsSelected(&fieldSpec.Gvk) {
-				if err := MutateField(
+				if err := transform.MutateField(
 					res.Map(), fieldSpec.PathSlice(),
 					false, rv.replaceVars); err != nil {
 					return err

api/internal/accumulator/refvartransformer_test.go

@@ -1,22 +1,24 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package transformers
+package accumulator
 
 import (
 	"reflect"
 	"testing"
 
-	"sigs.k8s.io/kustomize/v3/pkg/gvk"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/resmaptest"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	resmaptest_test "sigs.k8s.io/kustomize/api/testutils/resmaptest"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
-func TestVarRef(t *testing.T) {
+func TestRefVarTransformer(t *testing.T) {
 	type given struct {
 		varMap map[string]interface{}
-		fs     []config.FieldSpec
+		fs     []types.FieldSpec
 		res    resmap.ResMap
 	}
 	type expected struct {
@@ -37,14 +39,15 @@ func TestVarRef(t *testing.T) {
 					"BAZ": int64(5),
 					"BOO": true,
 				},
-				fs: []config.FieldSpec{
-					{Gvk: gvk.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/map"},
-					{Gvk: gvk.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/slice"},
-					{Gvk: gvk.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/interface"},
-					{Gvk: gvk.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/nil"},
-					{Gvk: gvk.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/num"},
+				fs: []types.FieldSpec{
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/map"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/slice"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/interface"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/nil"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/num"},
 				},
-				res: resmaptest_test.NewRmBuilder(t, rf).
+				res: resmaptest_test.NewRmBuilder(
+					t, resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())).
 					Add(map[string]interface{}{
 						"apiVersion": "v1",
 						"kind":       "ConfigMap",
@@ -75,7 +78,8 @@ func TestVarRef(t *testing.T) {
 						}}).ResMap(),
 			},
 			expected: expected{
-				res: resmaptest_test.NewRmBuilder(t, rf).
+				res: resmaptest_test.NewRmBuilder(
+					t, resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())).
 					Add(map[string]interface{}{
 						"apiVersion": "v1",
 						"kind":       "ConfigMap",
@@ -112,7 +116,7 @@ func TestVarRef(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
 			// arrange
-			tr := NewRefVarTransformer(tc.given.varMap, tc.given.fs)
+			tr := newRefVarTransformer(tc.given.varMap, tc.given.fs)
 
 			// act
 			err := tr.Transform(tc.given.res)

api/internal/accumulator/resaccumulator.go

@@ -8,11 +8,10 @@ import (
 	"log"
 	"strings"
 
-	"sigs.k8s.io/kustomize/v3/pkg/resid"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
 // ResAccumulator accumulates resources and the rules
@@ -20,14 +19,14 @@ import (
 // plus stuff needed to modify the ResMap.
 type ResAccumulator struct {
 	resMap  resmap.ResMap
-	tConfig *config.TransformerConfig
+	tConfig *builtinconfig.TransformerConfig
 	varSet  types.VarSet
 }
 
 func MakeEmptyAccumulator() *ResAccumulator {
 	ra := &ResAccumulator{}
 	ra.resMap = resmap.New()
-	ra.tConfig = &config.TransformerConfig{}
+	ra.tConfig = &builtinconfig.TransformerConfig{}
 	ra.varSet = types.NewVarSet()
 	return ra
 }
@@ -53,12 +52,12 @@ func (ra *ResAccumulator) AbsorbAll(
 }
 
 func (ra *ResAccumulator) MergeConfig(
-	tConfig *config.TransformerConfig) (err error) {
+	tConfig *builtinconfig.TransformerConfig) (err error) {
 	ra.tConfig, err = ra.tConfig.Merge(tConfig)
 	return err
 }
 
-func (ra *ResAccumulator) GetTransformerConfig() *config.TransformerConfig {
+func (ra *ResAccumulator) GetTransformerConfig() *builtinconfig.TransformerConfig {
 	return ra.tConfig
 }
 
@@ -147,7 +146,7 @@ func (ra *ResAccumulator) ResolveVars() error {
 	if len(replacementMap) == 0 {
 		return nil
 	}
-	t := transformers.NewRefVarTransformer(
+	t := newRefVarTransformer(
 		replacementMap, ra.tConfig.VarReference)
 	err = ra.Transform(t)
 	if len(t.UnusedVars()) > 0 {
@@ -162,6 +161,6 @@ func (ra *ResAccumulator) FixBackReferences() (err error) {
 	if ra.tConfig.NameReference == nil {
 		return nil
 	}
-	return ra.Transform(transformers.NewNameReferenceTransformer(
+	return ra.Transform(newNameReferenceTransformer(
 		ra.tConfig.NameReference))
 }

api/internal/accumulator/resaccumulator_test.go

@@ -10,19 +10,19 @@ import (
 	"strings"
 	"testing"
 
-	"sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct"
-	. "sigs.k8s.io/kustomize/v3/pkg/accumulator"
-	"sigs.k8s.io/kustomize/v3/pkg/gvk"
-	"sigs.k8s.io/kustomize/v3/pkg/resmap"
-	"sigs.k8s.io/kustomize/v3/pkg/resmaptest"
-	"sigs.k8s.io/kustomize/v3/pkg/resource"
-	"sigs.k8s.io/kustomize/v3/pkg/transformers/config"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	. "sigs.k8s.io/kustomize/api/internal/accumulator"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	resmaptest_test "sigs.k8s.io/kustomize/api/testutils/resmaptest"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
 func makeResAccumulator(t *testing.T) (*ResAccumulator, *resource.Factory) {
 	ra := MakeEmptyAccumulator()
-	err := ra.MergeConfig(config.MakeDefaultConfig())
+	err := ra.MergeConfig(builtinconfig.MakeDefaultConfig())
 	if err != nil {
 		t.Fatalf("unexpected err: %v", err)
 	}
@@ -75,13 +75,13 @@ func TestResolveVarsHappy(t *testing.T) {
 		{
 			Name: "SERVICE_ONE",
 			ObjRef: types.Target{
-				Gvk:  gvk.Gvk{Version: "v1", Kind: "Service"},
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
 				Name: "backendOne"},
 		},
 		{
 			Name: "SERVICE_TWO",
 			ObjRef: types.Target{
-				Gvk:  gvk.Gvk{Version: "v1", Kind: "Service"},
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
 				Name: "backendTwo"},
 		},
 	})
@@ -104,13 +104,13 @@ func TestResolveVarsOneUnused(t *testing.T) {
 		{
 			Name: "SERVICE_ONE",
 			ObjRef: types.Target{
-				Gvk:  gvk.Gvk{Version: "v1", Kind: "Service"},
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
 				Name: "backendOne"},
 		},
 		{
 			Name: "SERVICE_UNUSED",
 			ObjRef: types.Target{
-				Gvk:  gvk.Gvk{Version: "v1", Kind: "Service"},
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
 				Name: "backendTwo"},
 		},
 	})
@@ -165,7 +165,7 @@ func TestResolveVarsVarNeedsDisambiguation(t *testing.T) {
 		{
 			Name: "SERVICE_ONE",
 			ObjRef: types.Target{
-				Gvk:  gvk.Gvk{Version: "v1", Kind: "Service"},
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
 				Name: "backendOne",
 			},
 		},
@@ -179,13 +179,94 @@ func TestResolveVarsVarNeedsDisambiguation(t *testing.T) {
 	}
 }
 
+func makeNamespacedConfigMapWithDataProviderValue(
+	namespace string,
+	value string,
+) map[string]interface{} {
+	return map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "ConfigMap",
+		"metadata": map[string]interface{}{
+			"name":      "environment",
+			"namespace": namespace,
+		},
+		"data": map[string]interface{}{
+			"provider": value,
+		},
+	}
+}
+
+func makeVarToNamepaceAndPath(
+	name string,
+	namespace string,
+	path string,
+) types.Var {
+	return types.Var{
+		Name: name,
+		ObjRef: types.Target{
+			Gvk:       resid.Gvk{Version: "v1", Kind: "ConfigMap"},
+			Name:      "environment",
+			Namespace: namespace,
+		},
+		FieldRef: types.FieldSelector{FieldPath: path},
+	}
+}
+
+func TestResolveVarConflicts(t *testing.T) {
+	rf := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())
+
+	// create configmaps in foo and bar namespaces with `data.provider` values.
+	fooAws := makeNamespacedConfigMapWithDataProviderValue("foo", "aws")
+	barAws := makeNamespacedConfigMapWithDataProviderValue("bar", "aws")
+	barGcp := makeNamespacedConfigMapWithDataProviderValue("bar", "gcp")
+
+	// create two variables with (apparently) conflicting names that point to
+	// fieldpaths that could be generalized.
+	varFoo := makeVarToNamepaceAndPath("PROVIDER", "foo", "data.provider")
+	varBar := makeVarToNamepaceAndPath("PROVIDER", "bar", "data.provider")
+
+	// create accumulators holding apparently conflicting vars that are not
+	// actually in conflict because they point to the same concrete value.
+	rm0 := resmap.New()
+	rm0.Append(rf.FromMap(fooAws))
+	ac0 := MakeEmptyAccumulator()
+	ac0.AppendAll(rm0)
+	ac0.MergeVars([]types.Var{varFoo})
+
+	rm1 := resmap.New()
+	rm1.Append(rf.FromMap(barAws))
+	ac1 := MakeEmptyAccumulator()
+	ac1.AppendAll(rm1)
+	ac1.MergeVars([]types.Var{varBar})
+
+	// validate that two vars of the same name which reference the same concrete
+	// value do not produce a conflict.
+	err := ac0.MergeAccumulator(ac1)
+	if err == nil {
+		t.Fatalf("see bug gh-1600")
+	}
+
+	// create an accumulator will have an actually conflicting value with the
+	// two above (because it contains a variable whose name is used in the other
+	// accumulators AND whose concrete values are different).
+	rm2 := resmap.New()
+	rm2.Append(rf.FromMap(barGcp))
+	ac2 := MakeEmptyAccumulator()
+	ac2.AppendAll(rm2)
+	ac2.MergeVars([]types.Var{varBar})
+	err = ac1.MergeAccumulator(ac2)
+	if err == nil {
+		t.Fatalf("dupe vars w/ different concrete values should conflict")
+	}
+}
+
 func TestResolveVarsGoodResIdBadField(t *testing.T) {
 	ra, _ := makeResAccumulator(t)
 	err := ra.MergeVars([]types.Var{
 		{
 			Name: "SERVICE_ONE",
 			ObjRef: types.Target{
-				Gvk:  gvk.Gvk{Version: "v1", Kind: "Service"},
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
 				Name: "backendOne"},
 			FieldRef: types.FieldSelector{FieldPath: "nope_nope_nope"},
 		},
@@ -210,7 +291,7 @@ func TestResolveVarsUnmappableVar(t *testing.T) {
 		{
 			Name: "SERVICE_THREE",
 			ObjRef: types.Target{
-				Gvk:  gvk.Gvk{Version: "v1", Kind: "Service"},
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
 				Name: "doesNotExist"},
 		},
 	})
@@ -234,7 +315,7 @@ func TestResolveVarsWithNoambiguation(t *testing.T) {
 		{
 			Name: "SERVICE_ONE",
 			ObjRef: types.Target{
-				Gvk:  gvk.Gvk{Version: "v1", Kind: "Service"},
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
 				Name: "backendOne",
 			},
 		},
@@ -289,7 +370,7 @@ func TestResolveVarsWithNoambiguation(t *testing.T) {
 		{
 			Name: "SUB_SERVICE_ONE",
 			ObjRef: types.Target{
-				Gvk:  gvk.Gvk{Version: "v1", Kind: "Service"},
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
 				Name: "backendOne",
 			},
 		},

api/internal/crawl/README.md

@@ -352,7 +352,7 @@ patchesJson6902:
 ```
 
 the following flattened structure would look like:
-```json
+```
 {
   "identifiers": [
     "resources",

api/internal/crawl/ROADMAP.md

@@ -70,7 +70,7 @@ Each day, when the crawler finds and indexes these structured documents,
 it should insert aggregate data to a separate index. This data could look like the
 following:
 
-```json
+```
 {
   "kind": "kustomization",
   "added_identifiers": [
@@ -82,13 +82,13 @@ following:
         "docID45",
         ...
       ],
-    }
+    },
     {
       "identifier": "another:k8s:feature",
       "documents": [
         ...
       ],
-    }
+    },
     ...
   ]
 

api/internal/crawl/backend/search_backend.go

@@ -13,7 +13,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/rs/cors"
 
-	"sigs.k8s.io/kustomize/internal/tools/index"
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
 )
 
 type kustomizeSearch struct {
@@ -44,7 +44,7 @@ type kustomizeSearch struct {
 // /register: not implemented, but meant as an endpoint for adding new
 // kustomization files to the corpus.
 func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, "kustomize")
 	if err != nil {
 		return nil, err
 	}

api/internal/crawl/cmd/backend/Dockerfile

@@ -2,11 +2,11 @@ FROM golang:1.11 AS build
 
 ARG GO111MODULE=on
 
-WORKDIR /go/src/sigs.k8s.io/kustomize/internal/tools
-COPY . /go/src/sigs.k8s.io/kustomize/internal/tools
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
 
 RUN go mod download
-RUN CGO_ENABLED=0 go install sigs.k8s.io/kustomize/internal/tools/cmd/backend/
+RUN CGO_ENABLED=0 go install sigs.k8s.io/kustomize/api/internal/crawl/cmd/backend/
 
 FROM scratch
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/

api/internal/crawl/cmd/backend/main.go

@@ -4,9 +4,8 @@ import (
 	"context"
 	"log"
 	"os"
+	server "sigs.k8s.io/kustomize/api/internal/crawl/backend"
 	"strconv"
-
-	"sigs.k8s.io/kustomize/internal/tools/backend"
 )
 
 func main() {

api/internal/crawl/cmd/crawler/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize//api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install -v ./cmd/crawler/crawler.go
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/crawler /
+ENTRYPOINT ["/crawler"]
+CMD []

api/internal/crawl/cmd/crawler/crawler.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+const (
+	githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
+	redisCacheURL        = "REDIS_CACHE_URL"
+	redisKeyURL          = "REDIS_KEY_URL"
+	retryCount           = 3
+)
+
+type CrawlMode int
+
+const (
+	CrawlUnknown CrawlMode = iota
+	// Crawl all the kustomization files in all the repositories of a Github user
+	CrawlUser
+	// Crawl all the kustomization files in a Github repo
+	CrawlRepo
+	// Crawl all the documents in the index
+	CrawlIndex
+	// Crawl all the kustomization files on Github
+	CrawlGithub
+	// Crawl all the documents in the index and crawling all the kustomization files on Github
+	CrawlIndexAndGithub
+)
+
+func NewCrawlMode(s string) CrawlMode {
+	switch s {
+	case "github-user":
+		return CrawlUser
+	case "github-repo":
+		return CrawlRepo
+	case "index+github":
+		return CrawlIndexAndGithub
+	case "index":
+		return CrawlIndex
+	case "github":
+		return CrawlGithub
+	default:
+		return CrawlUnknown
+	}
+}
+
+func main() {
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	modePtr := flag.String("mode", "index+github",
+		`The crawling mode, which can be one of [github-user, github-repo, index, github, index+github].
+  * github-user: crawl all the kustomization files in all the repositories of a Github user (--github-user must be specified for this mode).
+  * github-repo: crawl all the kustomization files in a Github repository (--github-repo must be specified for this mode).
+  * index: crawl all the documents in the index.
+  * gihub: crawl all the kustomization files on Github.
+  * index+github: crawl all the documents in the index and crawling all the kustomization files on Github.`)
+	githubUserPtr := flag.String("github-user", "",
+		"A github user name (e.g., kubernetes-sigs). This flag is required for the `github-user` mode.")
+	githubRepoPtr := flag.String("github-repo", "",
+		"A github repository name (e.g., kubernetes-sigs/kustomize). This flag is required for the `github-repo` mode.")
+	flag.Parse()
+
+	githubToken := os.Getenv(githubAccessTokenVar)
+	if githubToken == "" {
+		log.Printf("Must set the variable '%s' to make github requests.\n",
+			githubAccessTokenVar)
+		return
+	}
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	if err != nil {
+		log.Printf("Could not create an index: %v\n", err)
+		return
+	}
+
+	cacheURL := os.Getenv(redisCacheURL)
+	cache, err := redis.DialURL(cacheURL)
+	clientCache := &http.Client{}
+	if err != nil {
+		log.Printf("Error: redis could not make a connection: %v\n", err)
+	} else {
+		clientCache = httpclient.NewClient(cache)
+	}
+
+	// docConverter takes in a plain document and processes it for the index.
+	docConverter := func(d *doc.Document) (crawler.CrawledDocument, error) {
+		kdoc := doc.KustomizationDocument{
+			Document: *d,
+		}
+
+		err := kdoc.ParseYAML()
+		return &kdoc, err
+	}
+
+	// Index updates the value in the index.
+	indexFunc := func(cdoc crawler.CrawledDocument, mode index.Mode) error {
+		switch d := cdoc.(type) {
+		case *doc.KustomizationDocument:
+			switch mode {
+			case index.Delete:
+				log.Printf("Deleting: %v", d)
+				return idx.Delete(d.ID())
+			default:
+				log.Printf("Inserting: %v", d)
+				return idx.Put(d.ID(), d)
+			}
+		default:
+			return fmt.Errorf("type %T not supported", d)
+		}
+	}
+
+	// seen tracks the IDs of all the documents in the index and their corresponding file types.
+	// This helps avoid indexing a given document multiple times.
+	seen := utils.NewSeenMap()
+
+	mode := NewCrawlMode(*modePtr)
+
+	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
+		if user != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.User(user)),
+			)
+		} else if repo != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.Repo(repo)),
+			)
+		} else {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization")),
+			)
+		}
+	}
+
+	seedDocs := make(crawler.CrawlSeed, 0)
+
+	// get all the documents in the index
+	getSeedDocsFunc := func() {
+		query := []byte(`{ "query":{ "match_all":{} } }`)
+		it := idx.IterateQuery(query, 10000, 60*time.Second)
+		for it.Next() {
+			for _, hit := range it.Value().Hits.Hits {
+				seedDocs = append(seedDocs, hit.Document.Document.Copy())
+			}
+		}
+		if err := it.Err(); err != nil {
+			log.Fatalf("getSeedDocsFunc Error iterating: %v\n", err)
+		}
+	}
+
+	switch mode {
+	case CrawlIndexAndGithub:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlIndex:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+	case CrawlGithub:
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		// add all the documents in the index into seen.
+		// this greatly reduces the time overhead of CrawlGithub.
+		getSeedDocsFunc()
+		for _, d := range seedDocs {
+			seen[d.ID()] = d.FileType
+		}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUser:
+		if *githubUserPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github user with the github-user flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(*githubUserPtr, "")}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlRepo:
+		if *githubRepoPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github repository with the github-repo flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", *githubRepoPtr)}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUnknown:
+		flag.Usage()
+		log.Fatalf("The --mode flag must be one of [github-user, github-repo, index, github, index+github].")
+	}
+}

api/internal/crawl/cmd/kustomize_stats/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install ./cmd/kustomize_stats
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/kustomize_stats /
+ENTRYPOINT ["/kustomize_stats"]

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -0,0 +1,249 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"flag"
+	"fmt"
+	"log"
+	"sort"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+)
+
+// iterateArr adds each item in arr into countMap.
+func iterateArr(arr []string, countMap map[string]int) {
+	for _, item := range arr {
+		if _, ok := countMap[item]; !ok {
+			countMap[item] = 1
+		} else {
+			countMap[item]++
+		}
+	}
+
+}
+
+// SortMapKeyByValueInt takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueInt(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
+	return keys
+}
+
+// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueLen(m map[string][]string) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return len(m[keys[i]]) > len(m[keys[j]]) })
+	return keys
+}
+
+func GeneratorOrTransformerStats(docs []*doc.KustomizationDocument) {
+	n := len(docs)
+	if n == 0 {
+		return
+	}
+
+	fileType := docs[0].FileType
+	fmt.Printf("There are totally %d %s files.\n", n, fileType)
+
+	GitRepositorySummary(docs, fileType)
+
+	// key of kindToUrls: a string in the KustomizationDocument.Kinds field
+	// value of kindToUrls: a slice of string urls defining a given kind.
+	kindToUrls := make(map[string][]string)
+
+	for _, d := range docs {
+		url := fmt.Sprintf("%s/blob/%s/%s", d.RepositoryURL, d.DefaultBranch, d.FilePath)
+		for _, kind := range d.Kinds {
+			if _, ok := kindToUrls[kind]; !ok {
+				kindToUrls[kind] = []string{url}
+			} else {
+				kindToUrls[kind] = append(kindToUrls[kind], url)
+			}
+		}
+	}
+	fmt.Printf("There are totally %d kinds of %s\n", len(kindToUrls), fileType)
+	sortedKeys := SortMapKeyByValueLen(kindToUrls)
+	for _, k := range sortedKeys {
+		sort.Strings(kindToUrls[k])
+		fmt.Printf("%s kind %s appears %d times\n", fileType, k, len(kindToUrls[k]))
+		for _, url := range kindToUrls[k] {
+			fmt.Printf("%s\n", url)
+		}
+	}
+}
+
+// GitRepositorySummary counts the distribution of docs:
+// 1) how many git repositories are these docs from?
+// 2) how many docs are from each git repository?
+func GitRepositorySummary(docs []*doc.KustomizationDocument, fileType string) {
+	m := make(map[string]int)
+	for _, d := range docs {
+		if _, ok := m[d.RepositoryURL]; ok {
+			m[d.RepositoryURL]++
+		} else {
+			m[d.RepositoryURL] = 1
+		}
+	}
+	sortedKeys := SortMapKeyByValueInt(m)
+	topN := 10
+	i := 0
+	for _, k := range sortedKeys {
+		if i >= topN {
+			break
+		}
+		fmt.Printf("%d %s are from %s\n", m[k], fileType, k)
+		i++
+	}
+}
+
+func main() {
+	topKindsPtr := flag.Int(
+		"kinds", -1,
+		`the number of kubernetes object kinds to be listed according to their popularities.
+By default, all the kinds will be listed.
+If you only want to list the 10 most popular kinds, set the flag to 10.`)
+	topIdentifiersPtr := flag.Int(
+		"identifiers", -1,
+		`the number of identifiers to be listed according to their popularities.
+By default, all the identifiers will be listed.
+If you only want to list the 10 most popular identifiers, set the flag to 10.`)
+	topKustomizeFeaturesPtr := flag.Int(
+		"kustomize-features", -1,
+		`the number of kustomize features to be listed according to their popularities.
+By default, all the features will be listed.
+If you only want to list the 10 most popular features, set the flag to 10.`)
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	flag.Parse()
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	if err != nil {
+		log.Fatalf("Could not create an index: %v\n", err)
+	}
+
+	// count tracks the number of documents in the index
+	count := 0
+
+	// kustomizationFilecount tracks the number of kustomization files in the index
+	kustomizationFilecount := 0
+
+	kindsMap := make(map[string]int)
+	identifiersMap := make(map[string]int)
+	kustomizeIdentifiersMap := make(map[string]int)
+
+	// ids tracks the unique IDs of the documents in the index
+	ids := make(map[string]struct{})
+
+	// generatorFiles include all the non-kustomization files whose FileType is generator
+	generatorFiles := make([]*doc.KustomizationDocument, 0)
+
+	// transformersFiles include all the non-kustomization files whose FileType is transformer
+	transformersFiles := make([]*doc.KustomizationDocument, 0)
+
+	checksums := make(map[string]int)
+
+	// get all the documents in the index
+	query := []byte(`{ "query":{ "match_all":{} } }`)
+	it := idx.IterateQuery(query, 10000, 60*time.Second)
+	for it.Next() {
+		for _, hit := range it.Value().Hits.Hits {
+			sum := fmt.Sprintf("%x", sha256.Sum256([]byte(hit.Document.DocumentData)))
+			if _, ok := checksums[sum]; ok {
+				checksums[sum]++
+			} else {
+				checksums[sum] = 1
+			}
+
+			// check whether there is any duplicate IDs in the index
+			if _, ok := ids[hit.ID]; !ok {
+				ids[hit.ID] = struct{}{}
+			} else {
+				log.Printf("Found duplicate ID (%s)\n", hit.ID)
+			}
+
+			count++
+			iterateArr(hit.Document.Kinds, kindsMap)
+			iterateArr(hit.Document.Identifiers, identifiersMap)
+
+			if doc.IsKustomizationFile(hit.Document.FilePath) {
+				kustomizationFilecount++
+				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
+
+			} else {
+				switch hit.Document.FileType {
+				case "generator":
+					generatorFiles = append(generatorFiles, hit.Document.Copy())
+				case "transformer":
+					transformersFiles = append(transformersFiles, hit.Document.Copy())
+				}
+			}
+		}
+	}
+
+	if err := it.Err(); err != nil {
+		log.Fatalf("Error iterating: %v\n", err)
+	}
+
+	sortedKindsMapKeys := SortMapKeyByValueInt(kindsMap)
+	sortedIdentifiersMapKeys := SortMapKeyByValueInt(identifiersMap)
+	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValueInt(kustomizeIdentifiersMap)
+
+	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
+There are %d documents in the kustomize index.
+%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
+	fmt.Printf("\n")
+
+	kindCount := 0
+	for _, key := range sortedKindsMapKeys {
+		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
+			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
+			kindCount++
+		}
+	}
+
+	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
+	identifierCount := 0
+	for _, key := range sortedIdentifiersMapKeys {
+		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
+			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
+			identifierCount++
+		}
+	}
+
+	fmt.Printf(`There are %d kustomization files in the kustomize index.
+%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
+	fmt.Printf("\n")
+	kustomizeFeatureCount := 0
+	for _, key := range sortedKustomizeIdentifiersMapKeys {
+		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
+			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
+			kustomizeFeatureCount++
+		}
+	}
+
+	GeneratorOrTransformerStats(generatorFiles)
+	GeneratorOrTransformerStats(transformersFiles)
+
+	fmt.Printf("There are total %d checksums of document contents\n", len(checksums))
+	sortedChecksums := SortMapKeyByValueInt(checksums)
+	sortedChecksums = sortedChecksums[:20]
+	fmt.Printf("The top 20 checksums are:\n")
+	for _, key := range sortedChecksums {
+		fmt.Printf("checksum %s apprears %d\n", key, checksums[key])
+	}
+}

api/internal/crawl/cmd/log-parser/README.md

@@ -0,0 +1,8 @@
+This binary takes as its input a json file including GKE logs (which can be
+[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into 
+[Cloud Storage](https://cloud.google.com/storage/docs/)), 
+and extracts the `textPayload` field of each log entry.
+
+Here is an log entry example:
+
+{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"}

api/internal/crawl/cmd/log-parser/kustomize-stats-cmd

@@ -0,0 +1,7 @@
+wget <log-file-url> -O log
+go build .
+./log-parser log >out
+cat out | grep "kind \`" | cut -d\` -f2 | tail -n 50
+cat out | grep "kind \`" | awk '{print $6}' | tail -n 50
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | cut -d\` -f2
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | awk '{print $6}'

api/internal/crawl/cmd/log-parser/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatalf("The usage of the command is: \n\t%s <log-file.json>", os.Args[0])
+	}
+
+	file, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	closeFile := func(file *os.File) {
+		if err := file.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}
+	defer closeFile(file)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		var entry interface{}
+		if err := json.Unmarshal([]byte(line), &entry); err != nil {
+			log.Printf("failed to unmarshal a log entry: %s\n", line)
+		}
+
+		m := entry.(map[string]interface{})
+		if payload, ok := m["textPayload"]; ok {
+			// use fmt.Printf here instead of log.Printf to avoid the time and code location info the log package provides
+			fmt.Printf("%s", payload)
+		} else {
+			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}

api/internal/crawl/config/base/kustomization.yaml

@@ -2,5 +2,4 @@ configmapGenerator:
 - name: elasticsearch-config
   literals:
   - es-url="http://esbasic-master:9200"
-  - kustomize-index-name="kustomize"
   - plugin-index-name="plugin"

api/internal/crawl/config/crawler/base/.gitignore

@@ -0,0 +1 @@
+github_api_secret.txt

api/internal/crawl/config/crawler/base/github_api_secret.txt

@@ -0,0 +1,2 @@
+<ADD YOUR GITHUB PERSONAL ACCESS TOKEN HERE WITHOUT A TRAILING NEWLINE>
+Run: printf "<your-token>" > github_api_secret.txt

api/internal/crawl/config/crawler/base/kustomization.yaml

@@ -5,7 +5,9 @@ configmapGenerator:
 - name: crawler-http-cache
   literals:
   - redis-cache-url="redis://redis-http-cache:6379"
-
+- name: redis-keystore
+  literals:
+  - keystore-url="redis://redis-docs-keystore:6379"
 
 secretGenerator:
 - name: github-access-token

api/internal/crawl/config/crawler/cronjob/cronjob.yaml

@@ -1,9 +1,10 @@
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: crawler
+  name: crawler-cronjob
 spec:
-  schedule: "5 0 * * */1"
+  # run the cronjob at 00:00 every 7 days
+  schedule: "0 0 */7 * *"
   jobTemplate:
     spec:
       template:
@@ -11,7 +12,10 @@ spec:
           restartPolicy: OnFailure
           containers:
           - name: crawler
-            image: gcr.io/kustomize-search/crawler:latest
+            image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+            command: ["/crawler"]
+            args: ["--mode=index+github", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
+            imagePullPolicy: Always
             env:
             - name: GITHUB_ACCESS_TOKEN
               valueFrom:

api/internal/crawl/config/crawler/job/README.md

@@ -0,0 +1,53 @@
+The crawler job can run in one of the following mode:
+
+# Crawling all the documents in the index and crawling all the kustomization files on Github
+
+This is the default setting of the crawler job. The `command` and `args` field
+of the container should be:
+
+```
+        command: ["/crawler"]
+```
+
+Or
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index+github"]
+```
+
+# Crawling all the documents in the index
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index"]
+```
+
+# Crawling all the kustomization files on Github
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github"]
+```
+
+# Crawling all the kustomization files in a Github repo
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize"]
+```
+
+# Crawling all the kustomization files in all the repositories of a Github user
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--github-user", "--github-user=kubernetes-sigs"]
+```

api/internal/crawl/config/crawler/job/job.yaml

@@ -8,7 +8,10 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: crawler
-        image: gcr.io/kustomize-search/crawler:latest
+        image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+        imagePullPolicy: Always
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
         env:
         - name: GITHUB_ACCESS_TOKEN
           valueFrom:

api/internal/crawl/config/crawler/kustomize_stats/job.yaml

@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kustomize-stats
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: kustomize-stats
+        image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
+        imagePullPolicy: Always
+        command: ["/kustomize_stats"]
+        args: ["--index=kustomize", "--kinds=51", "--identifiers=50", "--kustomize-features=50"]
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url

api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml

api/internal/crawl/config/elastic/esbackup.yaml

@@ -0,0 +1,23 @@
+# ESBackup depends on ESCluster, and is depended by ESSnapshot.
+# Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the snapshots in the `kustomize-backup` snapshot repository, instead makes all the snapshots in the repository inaccessible.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the essnapshot objects depending on it, but will cause those essnapshot objects to be reconciled, which update the status of the essnapshot objects to reflect the fact that the esbackup object is missing.
+# If you delete the `kustomize-backup` snapshot repository directly without deleting `esbackup/kustomize-backbup`, the ESBackup object will not recreate the snapshot repository.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESBackup
+metadata:
+  name: kustomize-backup
+spec:
+  storage:
+    gcs:
+      # the bucket must exist for the snapshot respository to be created successfully.
+      bucket: kustomize-backup
+      # the path does not need to exist.
+      # If the path does not exist, the controller will create the folder in the GCS bucket.
+      # If the path already exists and includes snapshots, these snapshots can be used.
+      path: kustomize
+      secret:
+        name: kustomizesa
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/escluster.yaml

@@ -0,0 +1,51 @@
+# ESCluster is depended by ESBackup and ESRestore.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESCluster
+metadata:
+  name: esbasic
+spec:
+  plugin:
+   pluginList:
+    - repository-gcs
+    - ingest-user-agent
+    - ingest-geoip
+  # To set `gcpserviceaccount`,
+  # First, create and download a GCP service account into a json file, named `sakey.json` following the instruction:
+  #     https://www.elastic.co/guide/en/elasticsearch/plugins/6.5/repository-gcs-usage.html#repository-gcs-using-service-account
+  # Second, create a secret for the service account using the following command:
+  #     $ kubectl create secret generic kustomizesa --from-file=./sakey.json
+  gcpserviceaccount:
+    name: kustomizesa
+  config:
+    env:
+      example: test
+  nodegroups:
+    - name: di
+      replicas: 2
+      data: true
+      ingest: true
+      config:
+        jvm:
+          - Djava.net.preferIPv4Stack=true
+          - Xms2g
+          - Xmx2g
+        es:
+          path.repo: '["/tmp/es_backup_basic"]'
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                es/nodegroup: di
+      resources:
+        requests:
+          memory: 3Gi
+        limits:
+          memory: 3Gi
+    - name: m
+      replicas: 2
+      master: true
+      config:
+        es:
+          path.repo: '["/tmp/es_backup_basic"]'

api/internal/crawl/config/elastic/esrestore.yaml

@@ -0,0 +1,19 @@
+# ESRestore depends on both ESCluster and ESSnapshot.
+# Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`.
+# Deleting `esrestore/kustomize-restore` will not delete the restored index.
+# Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`.
+# After the restore is complete, if the `kusotmize-restore` index is deleted manually, the ESRestore object will NOT restore the `kustomize` index to it again.
+# The correct way of using ESRestore is: create a ESRestore object to restore the index; delete the ESRestore object after the restore is complete.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESRestore
+metadata:
+  name: kustomize-restore
+spec:
+  include_global_state: true
+  ignore_unavailable: true
+  rename_pattern: kustomize
+  rename_replacement: kustomize-restore
+  essnapshot:
+    name: kustomize-snapshot
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/essnapshot.yaml

@@ -0,0 +1,23 @@
+# ESSnapshot depends on ESBackup, and is depended by ESRestore.
+# Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository.
+# After being created, the `kustomize-snapshot` snapshot will not be automatically updated when the `kuostomize` index is updated.
+# If you delete `essnapshot/kustomize-snapshot` and recreate it, the new snapshot will capture the current status of the `kustomize` index.
+# Deleting `essnapshot/kustomize-snapshot` will delete the snapshot.
+# Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`.
+# If the `kustomize-snapshot` snapshot is deleted directly without deleting `essnapshot/kustomize-snapshot`, the ESSnapshot object will recreate the snapshot.
+# The correct way of using ESSnapshot is: create an ESSnapshot object to create a snapshot, keep the ESSnapshot object until the snapshot is no longer needed.
+# To update the snapshot to capture the latest version of the index, you can either:
+#   1) delete the snapshot, and wait for the ESSnapshot object to recreate the snapshot;
+#   2) delete the ESSnapshot object, and recreate the ESSnapshot object.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESSnapshot
+metadata:
+  name: kustomize-snapshot
+spec:
+  # indices are optional. If not specified all indices are selected.
+  indices:
+  - kustomize
+  include_global_state: true
+  ignore_unavailable: true
+  esbackup:
+    name: kustomize-backup

api/internal/crawl/config/elastic/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+spec:
+  selector:
+    custom-resource: v1alpha1.ESCluster
+    custom-resource-name: esbasic
+    custom-resource-namespace: default
+    es/data: "true"
+    using: escluster.Cluster
+  ports:
+  - protocol: "TCP"
+    port: 9200
+  type: LoadBalancer
+  loadBalancerIP: ""

api/internal/crawl/config/webapp/backend/deployment.yaml

@@ -17,14 +17,11 @@ spec:
       containers:
       - name: kustomize-search
         image: gcr.io/kustomize-search/backend:latest
+        imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /liveness
             port: backend-port
-        readinessProbe:
-          httpGet:
-            path: /readiness
-            port: backend-port
         ports:
         - name: backend-port
           containerPort: 8080

api/internal/crawl/config/webapp/frontend/deployment.yaml

@@ -17,6 +17,7 @@ spec:
       containers:
       - name: frontend
         image: gcr.io/kustomize-search/frontend:latest
+        imagePullPolicy: Always
         ports:
         - name: frontend-port
           containerPort: 80

api/internal/crawl/crawler/crawler.go

@@ -0,0 +1,344 @@
+// Package crawler provides helper methods and defines an interface for lauching
+// source repository crawlers that retrieve files from a source and forwards
+// to a channel for indexing and retrieval.
+package crawler
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"sync"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	_ "github.com/gomodule/redigo/redis"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+)
+
+var (
+	logger = log.New(os.Stdout, "Crawler: ", log.LstdFlags|log.LUTC|log.Llongfile)
+)
+
+// Crawler forwards documents from source repositories to index and store them
+// for searching. Each crawler is responsible for querying it's source of
+// information, and forwarding files that have not been seen before or that need
+// updating.
+type Crawler interface {
+	// Crawl returns when it is done processing. This method does not take
+	// ownership of the channel. The channel is write only, and it
+	// designates where the crawler should forward the documents.
+	Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error
+
+	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
+	FetchDocument(context.Context, *doc.Document) error
+	// Write to the document what the created time is.
+	SetCreated(context.Context, *doc.Document) error
+
+	SetDefaultBranch(*doc.Document)
+
+	Match(*doc.Document) bool
+}
+
+type CrawledDocument interface {
+	ID() string
+	GetDocument() *doc.Document
+	// Get all the Documents directly referred in a Document.
+	// For a Document representing a non-kustomization file, an empty slice will be returned.
+	// For a Document representing a kustomization file:
+	// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
+	// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
+	// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
+	GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
+	WasCached() bool
+}
+
+type CrawlSeed []*doc.Document
+
+type IndexFunc func(CrawledDocument, index.Mode) error
+type Converter func(*doc.Document) (CrawledDocument, error)
+
+func logIfErr(err error) {
+	if err == nil {
+		return
+	}
+	logger.Println("error: ", err)
+}
+
+func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
+	for _, crawl := range crawlers {
+		if crawl.Match(d) {
+			return crawl
+		}
+	}
+	return nil
+}
+
+func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed) {
+
+	seen.Set(cdoc.ID(), cdoc.GetDocument().FileType)
+
+	match.SetDefaultBranch(cdoc.GetDocument())
+
+	// Insert into index
+	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
+		logger.Printf("Failed to insert or update doc(%s): %v",
+			cdoc.GetDocument().Path(), err)
+		return
+	}
+
+	deps, err := cdoc.GetResources(true, true, true)
+	if err != nil {
+		logger.Println(err)
+		return
+	}
+
+	for _, dep := range deps {
+		if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType {
+			continue
+		}
+		*stack = append(*stack, dep)
+	}
+}
+
+func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) {
+
+	UpdatedDocCount := 0
+	seenDocCount := 0
+	cachedDocCount := 0
+	findMatchErrCount := 0
+	FetchDocumentErrCount := 0
+	SetCreatedErrCount := 0
+	convErrCount := 0
+	deleteDocCount := 0
+	crawledDocCount := 0
+
+	// During the execution of the for loop, more Documents may be added into (*docsPtr).
+	for len(*docsPtr) > 0 {
+		// get the last Document in (*docPtr), which will be crawled in this iteration.
+		tail := (*docsPtr)[len(*docsPtr)-1]
+
+		// remove the last Document in (*docPtr)
+		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
+
+		crawledDocCount++
+		logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
+
+		if seen.Seen(tail.ID()) {
+			if !updateFileType || seen.Value(tail.ID()) == tail.FileType {
+				logger.Printf("this doc has been seen before")
+				seenDocCount++
+				continue
+			}
+		}
+
+		if tail.WasCached() {
+			logger.Printf("doc(%s) is cached already", tail.Path())
+			cachedDocCount++
+			continue
+		}
+
+		match := findMatch(tail, crawlers)
+		if match == nil {
+			logIfErr(fmt.Errorf("%v could not match any crawler", tail))
+			findMatchErrCount++
+			continue
+		}
+
+		if tail.User == "" {
+			tail.User = doc.UserName(tail.RepositoryURL)
+		}
+
+		// If the Document represents a kustomization root, FetchDcoument will change
+		// the `filePath` field of the Document by adding `kustomization.yaml` or
+		// `kustomization.yml` or `kustomization` into the the field.
+		// Therefore, it is necessary to add the ID of the Document into seen before
+		// calling FetchDocument. Otherwise, the binary may enter into an infinite loop
+		// if a kustomization file points to its kustmozation root in its `resources` or
+		// `bases` field.
+		seen.Set(tail.ID(), tail.FileType)
+
+		if refreshDoc || tail.DefaultBranch == "" {
+			match.SetDefaultBranch(tail)
+		}
+
+		if refreshDoc || tail.DocumentData == "" {
+			if err := match.FetchDocument(ctx, tail); err != nil {
+				logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
+				FetchDocumentErrCount++
+				// delete the document from the index
+				cdoc := &doc.KustomizationDocument{
+					Document: *tail,
+				}
+				seen.Set(cdoc.ID(), tail.FileType)
+				if err := indx(cdoc, index.Delete); err != nil {
+					logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
+				}
+				deleteDocCount++
+				continue
+			}
+		}
+
+		if refreshDoc || tail.CreationTime == nil {
+			if err := match.SetCreated(ctx, tail); err != nil {
+				logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
+				SetCreatedErrCount++
+			}
+		}
+
+		cdoc, err := conv(tail)
+		// If conv returns an error, cdoc can still be added into the index so that
+		// cdoc.Document can be searched.
+		if err != nil {
+			logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
+			convErrCount++
+		}
+
+		UpdatedDocCount++
+		addBranches(cdoc, match, indx, seen, stack)
+	}
+	logger.Printf("Summary of doCrawl:\n")
+	logger.Printf("\t%d documents were updated\n", UpdatedDocCount)
+	logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount)
+	logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount)
+	logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount)
+	logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n",
+		FetchDocumentErrCount, deleteDocCount)
+	logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount)
+	logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount)
+}
+
+// CrawlFromSeed updates all the documents in seed, and crawls all the new
+// documents referred in the seed.
+func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
+	conv Converter, indx IndexFunc, seen utils.SeenMap) {
+
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// Exploit seed to update bulk of corpus.
+	logger.Printf("updating %d documents from seed\n", len(seed))
+	// each unique document in seed will be crawled once.
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false)
+
+	// Traverse any new documents added while updating corpus.
+	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
+	// While crawling each document in stack, the documents directly referred in the document
+	// will be added into stack.
+	// After this statement is done, stack will become empty.
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+}
+
+// CrawlGithubRunner is a blocking function and only returns once all of the
+// crawlers are finished with execution.
+//
+// This function uses the output channel to forward kustomization documents
+// from a list of crawlers. The output is to be consumed by a database/search
+// indexer for later retrieval.
+//
+// The return value is an array of errors in which each index represents the
+// index of the crawler that emitted the error. Although the errors themselves
+// can be nil, the array will always be exactly the size of the crawlers array.
+//
+// CrawlGithubRunner takes in a seed, which represents the documents stored in an
+// index somewhere. The document data is not required to be populated. If there
+// are many documents, this is preferable. The order of iteration over the seed
+// is not guaranteed, but the CrawlGithub does guarantee that every element
+// from the seed will be processed before any other documents from the
+// crawlers.
+func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
+	crawlers []Crawler, seen utils.SeenMap) []error {
+
+	errs := make([]error, len(crawlers))
+	wg := sync.WaitGroup{}
+
+	for i, crawler := range crawlers {
+		// Crawler implementations get their own channels to prevent a
+		// crawler from closing the main output channel.
+		docs := make(chan CrawledDocument)
+		wg.Add(2)
+
+		// Forward all of the documents from this crawler's channel to
+		// the main output channel.
+		go func(docs <-chan CrawledDocument) {
+			defer wg.Done()
+			for d := range docs {
+				output <- d
+			}
+		}(docs)
+
+		// Run this crawler and capture its returned error.
+		go func(idx int, crawler Crawler,
+			docs chan<- CrawledDocument) {
+
+			defer func() {
+				wg.Done()
+				if r := recover(); r != nil {
+					errs[idx] = fmt.Errorf(
+						"%+v panicked: %v, additional error %v",
+						crawler, r, errs[idx],
+					)
+				}
+			}()
+			defer close(docs)
+			errs[idx] = crawler.Crawl(ctx, docs, seen)
+		}(i, crawler, docs) // Copies the index and the crawler
+	}
+
+	wg.Wait()
+	return errs
+}
+
+// CrawlGithub crawls all the kustomization files on Github.
+func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
+	indx IndexFunc, seen utils.SeenMap) {
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// ch is channel where all the crawlers sends the crawled documents to.
+	ch := make(chan CrawledDocument, 1<<10)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		docCount := 0
+		for cdoc := range ch {
+			docCount++
+			logger.Printf("Processing doc %d found on Github", docCount)
+			// all the docs here are kustomization files found by querying Github, and
+			// their `FileType` fields all should be empty.
+			if seen.Seen(cdoc.ID()) {
+				logger.Printf("the doc has been seen before")
+				continue
+			}
+			match := findMatch(cdoc.GetDocument(), crawlers)
+			if match == nil {
+				logIfErr(fmt.Errorf(
+					"%v could not match any crawler", cdoc))
+				continue
+			}
+			addBranches(cdoc, match, indx, seen, &stack)
+		}
+	}()
+
+	logger.Println("processing the documents found from crawling github")
+	if errs := CrawlGithubRunner(ctx, ch, crawlers, seen); errs != nil {
+		for _, err := range errs {
+			logIfErr(err)
+		}
+	}
+	close(ch)
+	wg.Wait()
+
+	// Handle deps of newly discovered documents.
+	logger.Printf("crawling the %d new documents referred by other documents",
+		len(stack))
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+}

api/internal/crawl/crawler/crawler_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"reflect"
 	"sort"
 	"strings"
@@ -11,8 +12,12 @@ import (
 	"testing"
 	"time"
 
-	"sigs.k8s.io/kustomize/internal/tools/doc"
-	"sigs.k8s.io/kustomize/v3/pkg/pgmconfig"
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/konfig"
 )
 
 const (
@@ -29,29 +34,32 @@ type testCrawler struct {
 }
 
 func (c testCrawler) Match(d *doc.Document) bool {
-	return d != nil && strings.HasPrefix(d.ID(), c.matchPrefix)
+	return d != nil
 }
 
-func (c testCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
+func (c testCrawler) SetDefaultBranch(d *doc.Document) {}
+
+func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	if i, ok := c.lukp[d.ID()]; ok {
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
-	for _, suffix := range pgmconfig.KustomizationFileNames {
-		fmt.Println(d.ID(), "/", suffix)
-		i, ok := c.lukp[d.ID()+"/"+suffix]
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
+		savedFilePath := d.FilePath
+		d.FilePath += "/" + suffix
+		i, ok := c.lukp[d.ID()]
 		if !ok {
+			d.FilePath = savedFilePath
 			continue
 		}
-		d.FilePath += "/" + suffix
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
-	return fmt.Errorf("Document %v does not exist for matcher: %s",
+	return fmt.Errorf("document %v does not exist for matcher: %s",
 		d, c.matchPrefix)
 }
 
-func (c testCrawler) SetCreated(ctx context.Context, d *doc.Document) error {
+func (c testCrawler) SetCreated(_ context.Context, d *doc.Document) error {
 	d.CreationTime = &time.Time{}
 	return nil
 }
@@ -71,12 +79,12 @@ func newCrawler(matchPrefix string, err error,
 }
 
 // Crawl implements the Crawler interface for testing.
-func (c testCrawler) Crawl(ctx context.Context,
-	output chan<- CrawlerDocument) error {
+func (c testCrawler) Crawl(_ context.Context,
+	output chan<- CrawledDocument, _ utils.SeenMap) error {
 
 	for i, d := range c.docs {
 		isResource := true
-		for _, suffix := range pgmconfig.KustomizationFileNames {
+		for _, suffix := range konfig.RecognizedKustomizationFileNames() {
 			if strings.HasSuffix(d.FilePath, suffix) {
 				isResource = false
 				break
@@ -106,8 +114,8 @@ func (s sortableDocs) Len() int {
 	return len(s)
 }
 
-func TestCrawlerRunner(t *testing.T) {
-	fmt.Println("testing CrawlerRunner")
+func TestCrawlGithubRunner(t *testing.T) {
+	log.Println("testing CrawlGithubRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -169,7 +177,7 @@ func TestCrawlerRunner(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		output := make(chan CrawlerDocument)
+		output := make(chan CrawledDocument)
 		wg := sync.WaitGroup{}
 		wg.Add(1)
 
@@ -178,8 +186,9 @@ func TestCrawlerRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()
 
-			errs := CrawlerRunner(context.Background(),
-				output, test.tc)
+			seen := utils.NewSeenMap()
+			errs := CrawlGithubRunner(context.Background(),
+				output, test.tc, seen)
 
 			// Check that errors are returned as they should be.
 			if !reflect.DeepEqual(errs, test.errs) {
@@ -212,15 +221,15 @@ func TestCrawlerRunner(t *testing.T) {
 }
 
 func TestCrawlFromSeed(t *testing.T) {
-	fmt.Println("testing CrawlFromSeed")
+	log.Println("testing CrawlFromSeed")
 
 	tests := []struct {
-		seed    CrawlerSeed
+		seed    CrawlSeed
 		matcher string
 		corpus  []doc.KustomizationDocument
 	}{
 		{
-			seed: CrawlerSeed{
+			seed: CrawlSeed{
 				{
 					RepositoryURL: kustomizeRepo,
 					FilePath:      "examples/helloWorld/kustomization.yaml",
@@ -302,29 +311,6 @@ resources:
 					RepositoryURL: kustomizeRepo,
 					FilePath:      "examples/seedcrawl2/job.yaml",
 				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/base/kustomization.yaml",
-					DocumentData: `
-resources:
-- ../app
-`,
-				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/kustomization.yaml",
-					DocumentData: `
-resources:
-- resource.yaml
-`,
-				}},
-				// Visited from crawling runner imported as resource.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/resource.yaml",
-				}},
 			},
 		},
 	}
@@ -333,15 +319,16 @@ resources:
 		cr := newCrawler(tc.matcher, nil, tc.corpus)
 		visited := make(map[string]int)
 		CrawlFromSeed(context.Background(), tc.seed, []Crawler{cr},
-			func(d *doc.Document) (CrawlerDocument, error) {
+			func(d *doc.Document) (CrawledDocument, error) {
 				return &doc.KustomizationDocument{
 					Document: *d,
 				}, nil
 			},
-			func(d CrawlerDocument, cr Crawler) error {
+			func(d CrawledDocument, mode index.Mode) error {
 				visited[d.ID()]++
 				return nil
 			},
+			utils.NewSeenMap(),
 		)
 		if lv, lc := len(visited), len(tc.corpus); lv != lc {
 			t.Errorf("error: %d of %d documents visited.", lv, lc)

api/internal/crawl/crawler/github/crawler.go

@@ -16,11 +16,13 @@ import (
 	"strings"
 	"time"
 
-	"sigs.k8s.io/kustomize/internal/tools/crawler"
-	"sigs.k8s.io/kustomize/internal/tools/doc"
-	"sigs.k8s.io/kustomize/internal/tools/httpclient"
-	"sigs.k8s.io/kustomize/v3/pkg/git"
-	"sigs.k8s.io/kustomize/v3/pkg/pgmconfig"
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
+	"sigs.k8s.io/kustomize/api/internal/git"
+	"sigs.k8s.io/kustomize/api/konfig"
 )
 
 var logger = log.New(os.Stdout, "Github Crawler: ",
@@ -28,59 +30,112 @@ var logger = log.New(os.Stdout, "Github Crawler: ",
 
 // Implements crawler.Crawler.
 type githubCrawler struct {
-	client GitHubClient
+	client GhClient
 	query  Query
+	// branchMap maps github repositories to their default branches
+	branchMap map[string]string
 }
 
-type GitHubClient struct {
+type GhClient struct {
 	RequestConfig
-	retryCount uint64
-	client     *http.Client
-}
-
-func NewClient(accessToken string, retryCount uint64, client *http.Client) GitHubClient {
-	return GitHubClient{
-		retryCount: retryCount,
-		client:     client,
-		RequestConfig: RequestConfig{
-			perPage:     githubMaxPageSize,
-			accessToken: accessToken,
-		},
-	}
+	retryCount  uint64
+	client      *http.Client
+	accessToken string
 }
 
 func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 	query Query) githubCrawler {
 
 	return githubCrawler{
-		client: GitHubClient{
+		client: GhClient{
 			retryCount: retryCount,
 			client:     client,
 			RequestConfig: RequestConfig{
-				perPage:     githubMaxPageSize,
-				accessToken: accessToken,
+				perPage: githubMaxPageSize,
 			},
+			accessToken: accessToken,
 		},
-		query: query,
+		query:     query,
+		branchMap: map[string]string{},
 	}
 }
 
-// Implements crawler.Crawler.
-func (gc githubCrawler) Crawl(
-	ctx context.Context, output chan<- crawler.CrawlerDocument) error {
+func (gc githubCrawler) SetDefaultBranch(d *doc.Document) {
+	url := gc.client.ReposRequest(d.RepositoryFullName())
+	defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+	d.DefaultBranch = defaultBranch
+	gc.branchMap[d.RepositoryURL] = d.DefaultBranch
+}
 
-	noETagClient := GitHubClient{
+func (gc githubCrawler) DefaultBranch(repo string) string {
+	return gc.branchMap[repo]
+}
+
+// Implements crawler.Crawler.
+func (gc githubCrawler) Crawl(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
+
+	ranges := []RangeWithin{
+		RangeWithin{
+		start: uint64(0),
+		end:   githubMaxFileSize,
+	},
+	}
+
+	errs := make(multiError, 0)
+	for len(ranges) > 0 {
+		tailRange := ranges[len(ranges) - 1]
+		ranges = ranges[:(len(ranges) - 1)]
+		reProcessQueryRanges, err := gc.CrawlSingleRange(ctx, output, seen, tailRange.start, tailRange.end)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		ranges = append(ranges, reProcessQueryRanges...)
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	lowerBound, upperBound uint64) ([]RangeWithin, error) {
+
+	log.Printf("CrawlSingleRange [%d, %d]", lowerBound, upperBound)
+
+	noETagClient := GhClient{
 		RequestConfig: gc.client.RequestConfig,
 		client:        &http.Client{Timeout: gc.client.client.Timeout},
 		retryCount:    gc.client.retryCount,
+		accessToken:   gc.client.accessToken,
 	}
 
+	var reProcessQueryRanges []RangeWithin
+
+	var ranges []string
+	var err error
 	// Since Github returns a max of 1000 results per query, we can use
 	// multiple queries that split the search space into chunks of at most
 	// 1000 files to get all of the data.
-	ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query))
+	for i := 0; i < 5; i++ {
+		ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query),
+			lowerBound, upperBound)
+		if err == nil {
+			logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i)
+			break
+		} else {
+			time.Sleep(time.Minute)
+		}
+	}
 	if err != nil {
-		return fmt.Errorf("could not split %v into ranges, %v\n",
+		return reProcessQueryRanges, fmt.Errorf("could not split %v into ranges, %v\n",
 			gc.query, err)
 	}
 
@@ -88,21 +143,31 @@ func (gc githubCrawler) Crawl(
 
 	// Query each range for files.
 	errs := make(multiError, 0)
+	queryResult := RangeQueryResult{}
 	for _, query := range ranges {
-		err := processQuery(ctx, gc.client, query, output)
+		reProcessQuery, rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap)
 		if err != nil {
 			errs = append(errs, err)
 		}
+		queryResult.Add(rangeResult)
+		if reProcessQuery {
+			reProcessQueryRanges = append(reProcessQueryRanges, RangeSizes(query))
+		}
 	}
 
+	logger.Printf("Summary of Crawl: %s", queryResult.String())
+
 	if len(errs) > 0 {
-		return errs
+		return reProcessQueryRanges, errs
 	}
 
-	return nil
+	return reProcessQueryRanges, nil
 }
 
-func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
+// FetchDocument first tries to fetch the document with d.FilePath. If it fails,
+// it will try to add each string in konfig.RecognizedKustomizationFileNames() to
+// d.FilePath, and try to fetch the document again.
+func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
 	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
 	if err != nil {
@@ -113,9 +178,13 @@ func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) erro
 		"/" + repoSpec.Ref + "/" + repoSpec.Path
 
 	handle := func(resp *http.Response, err error, path string) error {
+		if resp == nil {
+			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
+				url, path, err)
+		}
 		if err == nil && resp.StatusCode == http.StatusOK {
 			d.IsSame = httpclient.FromCache(resp.Header)
-			defer resp.Body.Close()
+			defer CloseResponseBody(resp)
 			data, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				return err
@@ -124,26 +193,30 @@ func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) erro
 			d.FilePath = d.FilePath + path
 			return nil
 		}
+
 		return err
 	}
-	resp, err := gc.client.GetRawUserContent(url)
-	if err := handle(resp, err, ""); err == nil {
+	resp, errGetRawUserContent := gc.client.GetRawUserContent(url)
+	if err := handle(resp, errGetRawUserContent, ""); err == nil {
 		return nil
 	}
 
-	for _, file := range pgmconfig.KustomizationFileNames {
-		resp, err = gc.client.GetRawUserContent(url + "/" + file)
-		err := handle(resp, err, "/"+file)
-		if err != nil {
-			continue
+	for _, file := range konfig.RecognizedKustomizationFileNames() {
+		resp, errGetRawUserContent = gc.client.GetRawUserContent(url + "/" + file)
+		if err = handle(resp, errGetRawUserContent, "/"+file); err == nil {
+			return nil
 		}
 	}
-	return fmt.Errorf("File Not Found: %s", url)
+	return fmt.Errorf("file not found: %s, error: %v", url, err)
 }
 
-func (gc githubCrawler) SetCreated(ctx context.Context, d *doc.Document) error {
-	fs := GithubFileSpec{}
-	fs.Repository.FullName = d.RepositoryURL + "/" + d.FilePath
+func (gc githubCrawler) SetCreated(_ context.Context, d *doc.Document) error {
+	fs := GhFileSpec{
+		Path: d.FilePath,
+		Repository: GitRepository{
+			FullName: d.RepositoryFullName(),
+		},
+	}
 	creationTime, err := gc.client.GetFileCreationTime(fs)
 	if err != nil {
 		return err
@@ -163,12 +236,34 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
 	return strings.Contains(repoSpec.Host, "github.com")
 }
 
+type RangeQueryResult struct {
+	totalDocCnt uint64
+	seenDocCnt  uint64
+	newDocCnt   uint64
+	errorCnt    uint64
+}
+
+func (r *RangeQueryResult) Add(other RangeQueryResult) {
+	r.totalDocCnt += other.totalDocCnt
+	r.newDocCnt += other.newDocCnt
+	r.seenDocCnt += other.seenDocCnt
+	r.errorCnt += other.errorCnt
+}
+
+func (r *RangeQueryResult) String() string {
+	return fmt.Sprintf("got %d files from API. "+
+		"%d have been seen before. %d are new and sent to the output channel."+
+		" %d have kustomizationResultAdapter errors.",
+		r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
+}
+
 // processQuery follows all of the pages in a query, and updates/adds the
 // documents from the crawl to the datastore/index.
-func processQuery(ctx context.Context, gcl GitHubClient, query string,
-	output chan<- crawler.CrawlerDocument) error {
+func processQuery(ctx context.Context, gcl GhClient, query string,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	branchMap map[string]string) (bool, RangeQueryResult, error) {
 
-	queryPages := make(chan GithubResponseInfo)
+	queryPages := make(chan GhResponseInfo)
 
 	go func() {
 		// Forward the document metadata to the retrieval channel.
@@ -182,74 +277,109 @@ func processQuery(ctx context.Context, gcl GitHubClient, query string,
 		close(queryPages)
 	}()
 
+	reProcessQuery := false
+
 	errs := make(multiError, 0)
-	errorCnt := 0
-	totalCnt := 0
+	result := RangeQueryResult{}
+	pageID := 1
 	for page := range queryPages {
 		if page.Error != nil {
 			errs = append(errs, page.Error)
 			continue
 		}
-
+		pageResult := RangeQueryResult{}
 		for _, file := range page.Parsed.Items {
-			k, err := kustomizationResultAdapter(gcl, file)
+			k, err := kustomizationResultAdapter(gcl, file, seen, branchMap)
 			if err != nil {
+				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
-				errorCnt++
-				continue
+				pageResult.errorCnt++
 			}
-			output <- k
-			totalCnt++
+			if k != nil {
+				pageResult.newDocCnt++
+				output <- k
+			} else {
+				pageResult.seenDocCnt++
+			}
+			pageResult.totalDocCnt++
 		}
 
-		logger.Printf("got %d files out of %d from API. %d of %d had errors\n",
-			totalCnt, page.Parsed.TotalCount, errorCnt, totalCnt)
+		logger.Printf("processQuery [TotalCount %d - page %d]: %s",
+			page.Parsed.TotalCount, pageID, pageResult.String())
+		result.Add(pageResult)
+
+		pageID++
+
+		if page.Parsed.TotalCount > githubMaxResultsPerQuery {
+			reProcessQuery = true
+		}
 	}
 
-	return errs
+	logger.Printf("Summary of processQuery: %s", result.String())
+
+	return reProcessQuery, result, errs
 }
 
-func kustomizationResultAdapter(gcl GitHubClient, k GithubFileSpec) (
-	crawler.CrawlerDocument, error) {
-
-	data, err := gcl.GetFileData(k)
-	if err != nil {
-		return nil, err
-	}
-
-	if err != nil {
-		logger.Printf(
-			"(error: %v) initializing to current time.\n", err)
-	}
-
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
+	branchMap map[string]string) (crawler.CrawledDocument, error) {
 	url := gcl.ReposRequest(k.Repository.FullName)
-	defaultBranch, err := gcl.GetDefaultBranch(url)
+	defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
 	if err != nil {
 		logger.Printf(
 			"(error: %v) setting default_branch to master\n", err)
 		defaultBranch = "master"
 	}
 
-	doc := doc.KustomizationDocument{
+	// document here is a kustomization file found by querying Github, whose
+	// `FileType` field should be empty.
+	document := doc.Document{
+		FilePath:      k.Path,
+		DefaultBranch: defaultBranch,
+		RepositoryURL: k.Repository.URL,
+		User:          doc.UserName(k.Repository.URL),
+	}
+
+	if seen.Seen(document.ID()) {
+		return nil, nil
+	}
+
+	data, err := gcl.GetFileData(k)
+	if err != nil {
+		return nil, err
+	}
+
+	d := doc.KustomizationDocument{
 		Document: doc.Document{
 			DocumentData:  string(data),
 			FilePath:      k.Path,
 			DefaultBranch: defaultBranch,
 			RepositoryURL: k.Repository.URL,
+			User:          doc.UserName(k.Repository.URL),
 		},
 	}
+	creationTime, err := gcl.GetFileCreationTime(k)
+	if err != nil {
+		logger.Printf("GetFileCreationTime failed: %v", err)
+		return &d, err
+	}
+	d.CreationTime = &creationTime
 
-	return &doc, nil
+	if err := d.ParseYAML(); err != nil {
+		logger.Printf("ParseYAML failed: %v", err)
+		return &d, err
+	}
+
+	return &d, nil
 }
 
 // ForwardPaginatedQuery follows the links to the next pages and performs all of
 // the queries for a given search query, relaying the data from each request
 // back to an output channel.
-func (gcl GitHubClient) ForwardPaginatedQuery(ctx context.Context, query string,
-	output chan<- GithubResponseInfo) error {
+func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
+	output chan<- GhResponseInfo) error {
 
 	logger.Println("querying: ", query)
-	response := gcl.parseGithubResponse(query)
+	response := gcl.parseGithubResponseWithRetry(query)
 
 	if response.Error != nil {
 		return response.Error
@@ -262,7 +392,7 @@ func (gcl GitHubClient) ForwardPaginatedQuery(ctx context.Context, query string,
 		case <-ctx.Done():
 			return nil
 		default:
-			response = gcl.parseGithubResponse(response.NextURL)
+			response = gcl.parseGithubResponseWithRetry(response.NextURL)
 			if response.Error != nil {
 				return response.Error
 			}
@@ -275,7 +405,7 @@ func (gcl GitHubClient) ForwardPaginatedQuery(ctx context.Context, query string,
 }
 
 // GetFileData gets the bytes from a file.
-func (gcl GitHubClient) GetFileData(k GithubFileSpec) ([]byte, error) {
+func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 
 	url := gcl.ContentsRequest(k.Repository.FullName, k.Path)
 
@@ -290,7 +420,10 @@ func (gcl GitHubClient) GetFileData(k GithubFileSpec) ([]byte, error) {
 		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
 			k, url, err)
 	}
-	resp.Body.Close()
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}
 
 	type githubContentRawURL struct {
 		DownloadURL string `json:"download_url,omitempty"`
@@ -309,18 +442,32 @@ func (gcl GitHubClient) GetFileData(k GithubFileSpec) ([]byte, error) {
 			k, rawURL.DownloadURL, err)
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err = ioutil.ReadAll(resp.Body)
 	return data, err
 }
 
-func (gcl GitHubClient) GetDefaultBranch(url string) (string, error) {
+func CloseResponseBody(resp *http.Response) {
+	if err := resp.Body.Close(); err != nil {
+		log.Printf("failed to close response body: %v", err)
+	}
+}
+
+// GetDefaultBranch gets the default branch of a github repository.
+// m is a map which maps a github repository to its default branch.
+// If repo is already in m, the default branch for url will be obtained from m;
+// otherwise, a query will be made to github to obtain the default branch.
+func (gcl GhClient) GetDefaultBranch(url, repo string, m map[string]string) (string, error) {
+	if v, ok := m[repo]; ok {
+		return v, nil
+	}
+
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
 			"'%s' could not get default_branch: %v", url, err)
 	}
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -341,8 +488,8 @@ func (gcl GitHubClient) GetDefaultBranch(url string) (string, error) {
 }
 
 // GetFileCreationTime gets the earliest date of a file.
-func (gcl GitHubClient) GetFileCreationTime(
-	k GithubFileSpec) (time.Time, error) {
+func (gcl GhClient) GetFileCreationTime(
+	k GhFileSpec) (time.Time, error) {
 
 	url := gcl.CommitsRequest(k.Repository.FullName, k.Path)
 
@@ -372,13 +519,13 @@ func (gcl GitHubClient) GetFileCreationTime(
 		}
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return defaultTime, fmt.Errorf(
 			"%+v: failed to read metadata: %v", k, err)
 	}
-	earliestDate := []DateSpec{}
+	var earliestDate []DateSpec
 	err = json.Unmarshal(data, &earliestDate)
 	size := len(earliestDate)
 	if err != nil || size == 0 {
@@ -413,24 +560,26 @@ func throttleRepoAPI() {
 
 type multiError []error
 
-func (me multiError) Error() string {
-	size := len(me) + 2
+func (e multiError) Error() string {
+	size := len(e) + 2
 	strs := make([]string, size)
 	strs[0] = "Errors ["
-	for i, err := range me {
+	for i, err := range e {
 		strs[i+1] = "\t" + err.Error()
 	}
 	strs[size-1] = "]"
 	return strings.Join(strs, "\n")
 }
 
-type GithubFileSpec struct {
-	Path       string `json:"path,omitempty"`
-	Repository struct {
-		API      string `json:"url,omitempty"`
-		URL      string `json:"html_url,omitempty"`
-		FullName string `json:"full_name,omitempty"`
-	} `json:"repository,omitempty"`
+type GitRepository struct {
+	API      string `json:"url,omitempty"`
+	URL      string `json:"html_url,omitempty"`
+	FullName string `json:"full_name,omitempty"`
+}
+
+type GhFileSpec struct {
+	Path       string        `json:"path,omitempty"`
+	Repository GitRepository `json:"repository,omitempty"`
 }
 
 type githubResponse struct {
@@ -438,11 +587,13 @@ type githubResponse struct {
 	// This is the number of files that match the query.
 	TotalCount uint64 `json:"total_count,omitempty"`
 
+	IncompleteResults bool `json:"incomplete_results,omitempty"`
+
 	// Github representation of a file.
-	Items []GithubFileSpec `json:"items,omitempty"`
+	Items []GhFileSpec `json:"items,omitempty"`
 }
 
-type GithubResponseInfo struct {
+type GhResponseInfo struct {
 	*http.Response
 	Parsed  *githubResponse
 	Error   error
@@ -480,9 +631,20 @@ func parseGithubLinkFormat(links string) (string, string) {
 	return next, last
 }
 
-func (gcl GitHubClient) parseGithubResponse(getRequest string) GithubResponseInfo {
+func (gcl GhClient) parseGithubResponseWithRetry(getRequest string) GhResponseInfo {
+	resp := gcl.parseGithubResponse(getRequest)
+	retries := 0
+	for resp.Parsed.IncompleteResults {
+		resp = gcl.parseGithubResponse(getRequest)
+		retries++
+	}
+	log.Printf("The result of query(%s) is complete after %d retries", getRequest, retries)
+	return resp
+}
+
+func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	resp, err := gcl.SearchGithubAPI(getRequest)
-	requestInfo := GithubResponseInfo{
+	requestInfo := GhResponseInfo{
 		Response: resp,
 		Error:    err,
 		Parsed:   nil,
@@ -493,7 +655,7 @@ func (gcl GitHubClient) parseGithubResponse(getRequest string) GithubResponseInf
 	}
 
 	var data []byte
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
 	if requestInfo.Error != nil {
 		return requestInfo
@@ -527,9 +689,9 @@ func (gcl GitHubClient) parseGithubResponse(getRequest string) GithubResponseInf
 }
 
 // SearchGithubAPI performs a search query and handles rate limitting for
-// the 'code/search?' endpoint as well as timed retries in the case of abuse
+// the 'search/code?' endpoint as well as timed retries in the case of abuse
 // prevention.
-func (gcl GitHubClient) SearchGithubAPI(query string) (*http.Response, error) {
+func (gcl GhClient) SearchGithubAPI(query string) (*http.Response, error) {
 	throttleSearchAPI()
 	return gcl.getWithRetry(query)
 }
@@ -537,30 +699,45 @@ func (gcl GitHubClient) SearchGithubAPI(query string) (*http.Response, error) {
 // GetReposData performs a search query and handles rate limitting for
 // the '/repos' endpoint as well as timed retries in the case of abuse
 // prevention.
-func (gcl GitHubClient) GetReposData(query string) (*http.Response, error) {
+func (gcl GhClient) GetReposData(query string) (*http.Response, error) {
 	throttleRepoAPI()
 	return gcl.getWithRetry(query)
 }
 
 // User content (file contents) is not API rate limited, so there's no use in
 // throttling this call.
-func (gcl GitHubClient) GetRawUserContent(query string) (*http.Response, error) {
+func (gcl GhClient) GetRawUserContent(query string) (*http.Response, error) {
 	return gcl.getWithRetry(query)
 }
 
-func (gcl GitHubClient) getWithRetry(
+func (gcl GhClient) Do(query string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", query, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("token %s", gcl.accessToken))
+
+	// gcl.client.Do: a non-2xx status code doesn't cause an error.
+	// See https://golang.org/pkg/net/http/#Client.Do for more info.
+	resp, err := gcl.client.Do(req)
+	if resp != nil && resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
+			query, resp.StatusCode)
+	}
+	return resp, err
+}
+
+func (gcl GhClient) getWithRetry(
 	query string) (resp *http.Response, err error) {
 
-	resp, err = gcl.client.Get(query)
+	resp, err = gcl.Do(query)
+
 	retryCount := gcl.retryCount
 
-	for err == nil &&
-		resp.StatusCode == http.StatusForbidden &&
-		retryCount > 0 {
-
+	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
 		retryTime := resp.Header.Get("Retry-After")
-		i, err := strconv.Atoi(retryTime)
-		if err != nil {
+		i, errAtoi := strconv.Atoi(retryTime)
+		if errAtoi != nil {
 			return resp, fmt.Errorf(
 				"query '%s' forbidden without 'Retry-After'", query)
 		}
@@ -570,7 +747,7 @@ func (gcl GitHubClient) getWithRetry(
 		logger.Printf("waiting %d seconds before retrying\n", i)
 		time.Sleep(time.Second * time.Duration(i))
 		retryCount--
-		resp, err = gcl.client.Get(query)
+		resp, err = gcl.Do(query)
 	}
 
 	if err != nil {

api/internal/crawl/crawler/github/queries.go

@@ -7,12 +7,11 @@ import (
 )
 
 const (
-	perPageArg     = "per_page"
-	accessTokenArg = "access_token"
-
-	githubMaxPageSize = 100
+	perPageArg = "per_page"
 )
 
+const githubMaxPageSize = 100
+
 // Implementation detail, not important to external API.
 type queryField struct {
 	name  string
@@ -51,7 +50,7 @@ func (qf queryField) String() string {
 type Query []queryField
 
 func QueryWith(qfs ...queryField) Query {
-	return Query(qfs)
+	return qfs
 }
 
 func (q Query) String() string {
@@ -91,6 +90,17 @@ func Path(p string) queryField {
 	return queryField{name: "path", value: p}
 }
 
+// Repo takes a repository (i.e., kubernetes-sigs/kustomize) and formats
+// it according to the Github API.
+func Repo(r string) queryField {
+	return queryField{name: "repo", value: r}
+}
+
+// Path takes a github username and formats it according to the Github API.
+func User(u string) queryField {
+	return queryField{name: "user", value: u}
+}
+
 // RequestConfig stores common variables that must be present for the queries.
 // - CodeSearchRequests: ask Github to check the code indices given a query.
 // - ContentsRequests: ask Github where to download a resource given a repo and a
@@ -98,15 +108,7 @@ func Path(p string) queryField {
 // - CommitsRequests: asks Github to list commits made one a file. Useful to
 // determine the date of a file.
 type RequestConfig struct {
-	perPage     uint64
-	accessToken string
-}
-
-func NewRequestConfig(perPage uint64, accessToken string) RequestConfig {
-	return RequestConfig{
-		perPage:     perPage,
-		accessToken: accessToken,
-	}
+	perPage uint64
 }
 
 // CodeSearchRequestWith given a list of query parameters that specify the
@@ -114,9 +116,11 @@ func NewRequestConfig(perPage uint64, accessToken string) RequestConfig {
 // the URL method to get the string value of the URL. See request.CopyWith, to
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
-	req := rc.makeRequest("search/code", query)
-	req.vals.Set("sort", "indexed")
-	req.vals.Set("order", "desc")
+	vals := url.Values{
+		"sort":  []string{"indexed"},
+		"order": []string{"desc"},
+	}
+	req := rc.makeRequest("search/code", query, vals)
 	return req
 }
 
@@ -124,26 +128,25 @@ func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 // query for the Github API to find the dowload information of this filepath.
 func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 func (rc RequestConfig) ReposRequest(fullRepoName string) string {
 	uri := fmt.Sprintf("repos/%s", fullRepoName)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 // CommitsRequest given the repo name, and a filepath returns a formatted query
 // for the Github API to find the commits that affect this file.
 func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
-	return rc.makeRequest(uri, Query{Path(path)}).URL()
+	vals := url.Values{
+		"path": []string{path},
+	}
+	return rc.makeRequest(uri, Query{}, vals).URL()
 }
 
-func (rc RequestConfig) makeRequest(path string, query Query) request {
-	vals := url.Values{}
-	if rc.accessToken != "" {
-		vals.Set(accessTokenArg, rc.accessToken)
-	}
+func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
 	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
 
 	return request{
@@ -163,7 +166,7 @@ type request struct {
 	query Query
 }
 
-// CopyWith copies the requests and adds the extra query parameters. Usefull
+// CopyWith copies the requests and adds the extra query parameters. It is useful
 // for dynamically adding sizes to a filename only query without modifying it.
 func (r request) CopyWith(queryParams ...queryField) request {
 	cpy := r
@@ -185,7 +188,7 @@ func (r request) URL() string {
 	if encoded == "" && query != "" {
 		sep = "?"
 	}
-	r.url.RawQuery = encoded + sep + query
+	r.url.RawQuery = query + sep + encoded
 	return r.url.String()
 }
 

api/internal/crawl/crawler/github/queries_test.go

@@ -53,8 +53,11 @@ func TestQueryType(t *testing.T) {
 				Filename("kustomization.yaml"),
 				Keyword("keyword1"),
 				Keyword("keyword2"),
+				Repo("user1/repo1"),
+				User("user1"),
 			),
-			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2",
+			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2+" +
+				"repo:user1/repo1+user:user1",
 		},
 	}
 
@@ -68,8 +71,7 @@ func TestQueryType(t *testing.T) {
 
 func TestGithubSearchQuery(t *testing.T) {
 	const (
-		accessToken = "random_token"
-		perPage     = 100
+		perPage = 100
 	)
 
 	testCases := []struct {
@@ -83,8 +85,7 @@ func TestGithubSearchQuery(t *testing.T) {
 	}{
 		{
 			rc: RequestConfig{
-				perPage:     perPage,
-				accessToken: accessToken,
+				perPage: perPage,
 			},
 			codeQuery: Query{
 				Filename("kustomization.yaml"),
@@ -94,13 +95,33 @@ func TestGithubSearchQuery(t *testing.T) {
 			path:         "examples/helloWorld/kustomization.yaml",
 
 			expectedCodeQuery: "https://api.github.com/search/code?" +
-				"access_token=random_token&order=desc&per_page=100&sort=indexed&q=filename:kustomization.yaml+size:64..128",
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
 
 			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
-				"examples/helloWorld/kustomization.yaml?access_token=random_token&per_page=100",
+				"examples/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"access_token=random_token&per_page=100&q=path:examples/helloWorld/kustomization.yaml",
+				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+		},
+		{
+			rc: RequestConfig{
+				perPage: perPage,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples 1/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples%201/helloWorld/kustomization.yaml?per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
 		},
 	}
 

api/internal/crawl/crawler/github/split_search_ranges.go

@@ -93,13 +93,15 @@ package github
 //   apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000
 //   = apiCallsPerResult * totalResults.
 //
-// So it could very well take apiCallsPerResult * 50 times longer to acutally
+// So it could very well take apiCallsPerResult * 50 times longer to actually
 // fetch the results (assuming the quotas for the API calls are the same as the
 // search API), than it does to perform these range searches.
 
 import (
 	"fmt"
 	"math/bits"
+	"strconv"
+	"strings"
 )
 
 // Files cannot be more than 2^19 bytes, according to
@@ -112,7 +114,7 @@ const (
 // Interface instead of struct for testing purposes.
 // Not expecting to have multiple implementations.
 type cachedSearch interface {
-	CountResults(uint64) (uint64, error)
+	CountResults(uint64, uint64) (uint64, error)
 	RequestString(filesize rangeFormatter) string
 }
 
@@ -139,7 +141,7 @@ type cachedSearch interface {
 //    problematic). The current cache implementation looks at the
 //    predecessor entry to find out if the current value is monotonic.
 //    This is where the bit trick is used, since each step in the binary
-//    search is adding or ommiting to add a decreasing power of 2 to the query
+//    search is adding or omitting to add a decreasing power of 2 to the query
 //    value, we can remove the least significant set bit to find the
 //    predecessor in constant time. Ultimately since the search is rate
 //    limited, we could also easily afford to compute this in linear time
@@ -147,11 +149,11 @@ type cachedSearch interface {
 //    cache's performance.
 type githubCachedSearch struct {
 	cache       map[uint64]uint64
-	gcl         GitHubClient
+	gcl         GhClient
 	baseRequest request
 }
 
-func newCache(client GitHubClient, query Query) githubCachedSearch {
+func newCache(client GhClient, query Query) githubCachedSearch {
 	return githubCachedSearch{
 		cache: map[uint64]uint64{
 			0: 0,
@@ -161,16 +163,16 @@ func newCache(client GitHubClient, query Query) githubCachedSearch {
 	}
 }
 
-func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
+func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
 	count, cached := c.cache[upperBound]
 	if cached {
 		return count, nil
 	}
 
-	sizeRange := RangeWithin{0, upperBound}
+	sizeRange := RangeWithin{lowerBound, upperBound}
 	rangeRequest := c.RequestString(sizeRange)
 
-	result := c.gcl.parseGithubResponse(rangeRequest)
+	result := c.gcl.parseGithubResponseWithRetry(rangeRequest)
 	if result.Error != nil {
 		return count, result.Error
 	}
@@ -204,7 +206,7 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 			"Retrying query... current lower bound: %d, got: %d\n",
 			c.cache[prev], result.Parsed.TotalCount)
 
-		result = c.gcl.parseGithubResponse(rangeRequest)
+		result = c.gcl.parseGithubResponseWithRetry(rangeRequest)
 		if result.Error != nil {
 			return count, result.Error
 		}
@@ -219,8 +221,8 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 	}
 
 	count = result.Parsed.TotalCount
-	logger.Printf("Caching new query %s, with count %d\n",
-		sizeRange.RangeString(), count)
+	logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n",
+		sizeRange.RangeString(), count, result.Parsed.IncompleteResults)
 	c.cache[upperBound] = count
 	return count, nil
 }
@@ -238,16 +240,16 @@ func (c githubCachedSearch) RequestString(filesize rangeFormatter) string {
 // This would mean that the search as it is could not find all files. If queries
 // are sorted by last indexed, and retrieved on regular intervals, it should be
 // sufficient to get most if not all documents.
-func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
-	totalFiles, err := cache.CountResults(githubMaxFileSize)
+func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64) ([]string, error) {
+	totalFiles, err := cache.CountResults(lowerBound, upperBound)
 	if err != nil {
 		return nil, err
 	}
-	logger.Println("total files: ", totalFiles)
+	logger.Println("total kustomization files: ", totalFiles)
 
 	if githubMaxResultsPerQuery >= totalFiles {
 		return []string{
-			cache.RequestString(RangeWithin{0, githubMaxFileSize}),
+			cache.RequestString(RangeWithin{lowerBound, upperBound}),
 		}, nil
 	}
 
@@ -275,6 +277,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 	// range.
 	filesAccessible := uint64(0)
 	sizes := make([]uint64, 0)
+	sizes = append(sizes, lowerBound)
 	for filesAccessible < totalFiles {
 		target := filesAccessible + githubMaxResultsPerQuery
 		if target >= totalFiles {
@@ -284,22 +287,22 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 		logger.Printf("%d accessible files, next target = %d\n",
 			filesAccessible, target)
 
-		cur, err := lowerBoundFileCount(cache, target)
+		size, err := FindFileSize(cache, target, lowerBound, upperBound)
 		if err != nil {
 			return nil, err
 		}
 
 		// If there are more than 1000 files in the next bucket, we must
 		// advance anyway and lose out on some files :(.
-		if l := len(sizes); l > 0 && sizes[l-1] == cur {
-			cur++
+		if l := len(sizes); l > 0 && sizes[l-1] == size {
+			size++
 		}
 
-		nextAccessible, err := cache.CountResults(cur)
+		nextAccessible, err := cache.CountResults(lowerBound, size)
 		if err != nil {
 			return nil, fmt.Errorf(
 				"cache should be populated at %d already, got %v",
-				cur, err)
+				size, err)
 		}
 		if nextAccessible < filesAccessible {
 			return nil, fmt.Errorf(
@@ -309,31 +312,31 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 
 		filesAccessible = nextAccessible
 		if nextAccessible < totalFiles {
-			sizes = append(sizes, cur)
+			sizes = append(sizes, size)
 		}
 	}
-
+	sizes = append(sizes, upperBound)
 	return formatFilesizeRanges(cache, sizes), nil
 }
 
-// lowerBoundFileCount finds the filesize range from [0, return value] that has
+// FindFileSize finds the filesize range from [lowerBound, return value] that has
 // the largest file count that is smaller than or equal to
 // githubMaxResultsPerQuery. It is important to note that this returned value
 // could already be in a previous range if the next file size has more than 1000
 // results. It is left to the caller to handle this bit of logic and guarantee
 // forward progession in this case.
-func lowerBoundFileCount(
-	cache cachedSearch, targetFileCount uint64) (uint64, error) {
+func FindFileSize(
+	cache cachedSearch, targetFileCount, lowerBound, upperBound uint64) (uint64, error) {
 
 	// Binary search for file sizes that make up the next <=1000 element
 	// chunk.
-	cur := uint64(0)
-	increase := githubMaxFileSize / 2
+	cur := lowerBound
+	increase := (upperBound - lowerBound) / 2
 
 	for increase > 0 {
 		mid := cur + increase
 
-		count, err := cache.CountResults(mid)
+		count, err := cache.CountResults(lowerBound, mid)
 		if err != nil {
 			return count, err
 		}
@@ -353,26 +356,24 @@ func lowerBoundFileCount(
 }
 
 func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string {
-	ranges := make([]string, 0, len(sizes)+1)
-
-	if len(sizes) > 0 {
-		ranges = append(ranges, cache.RequestString(
-			RangeLessThan{sizes[0] + 1},
-		))
+	n := len(sizes)
+	if n < 2 {
+		return []string{}
 	}
 
-	for i := 0; i < len(sizes)-1; i += 1 {
-		ranges = append(ranges, cache.RequestString(
-			RangeWithin{sizes[i] + 1, sizes[i+1]},
-		))
-
-		if i != len(sizes)-2 {
-			continue
-		}
-		ranges = append(ranges, cache.RequestString(
-			RangeGreaterThan{sizes[i+1]},
-		))
+	ranges := make([]string, 0, n-1)
+	ranges = append(ranges, cache.RequestString(RangeWithin{sizes[0], sizes[1]}))
+	for i := 1; i < n-1; i++ {
+		ranges = append(ranges, cache.RequestString(RangeWithin{sizes[i] + 1, sizes[i+1]}))
 	}
-
 	return ranges
 }
+
+func RangeSizes(s string) RangeWithin {
+	start := strings.Index(s, "+size:") + len("+size:")
+	end := strings.Index(s, "&")
+	ranges := strings.Split(s[start:end], "..")
+	lowerBound, _ := strconv.ParseUint(ranges[0], 10, 64)
+	upperBound, _ := strconv.ParseUint(ranges[1], 10, 64)
+	return RangeWithin{lowerBound, upperBound}
+}

api/internal/crawl/crawler/github/split_search_ranges_test.go

@@ -2,6 +2,7 @@ package github
 
 import (
 	"fmt"
+	"log"
 	"reflect"
 	"testing"
 )
@@ -10,8 +11,8 @@ type testCachedSearch struct {
 	cache map[uint64]uint64
 }
 
-func (c testCachedSearch) CountResults(upperBound uint64) (uint64, error) {
-	fmt.Printf("CountResults(%05x)\n", upperBound)
+func (c testCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
+	log.Printf("CountResults(%05x)\n", upperBound)
 	count, ok := c.cache[upperBound]
 	if !ok {
 		return count, fmt.Errorf("cache not set at %x", upperBound)
@@ -72,19 +73,29 @@ func TestRangeSplitting(t *testing.T) {
 		},
 	}
 
-	requests, err := FindRangesForRepoSearch(cache)
+	requests, err := FindRangesForRepoSearch(cache, 0, 524288)
 	if err != nil {
 		t.Errorf("Error while finding ranges: %v", err)
 	}
 	expected := []string{
-		"<107",      // cache.RequestString(RangeLessThan{0x6b}),
-		"107..128",  // cache.RequestString(RangeWithin{0x6b, 0x80}),
-		"129..256",  // cache.RequestString(RangeWithin{0x81, 0x100}),
-		"257..4095", // cache.RequestString(RangeWithin{0x101, 0xfff}),
-		">4095",     // cache.RequestString(RangeGreaterThan{0xfff}),
+		"0..106",       // cache.RequestString(RangeWithin{0x00, 0x6a}),
+		"107..128",     // cache.RequestString(RangeWithin{0x6b, 0x80}),
+		"129..256",     // cache.RequestString(RangeWithin{0x81, 0x100}),
+		"257..4095",    // cache.RequestString(RangeWithin{0x101, 0xfff}),
+		"4096..524288", // cache.RequestString(RangeWithin{0x1000, 0x80000}),
 	}
 
 	if !reflect.DeepEqual(requests, expected) {
 		t.Errorf("Expected requests (%v) to equal (%v)", requests, expected)
 	}
 }
+
+func TestRangeSizes(t *testing.T) {
+	s := "https://api.github.com/search/code?q=filename:kustomization.yaml+filename:kustomization.yml" +
+		"+filename:kustomization+size:2365..10000&order=desc&per_page=100&sort=indexed"
+	returnedResult := RangeSizes(s)
+	expectedResult := RangeWithin{uint64(2365), uint64(10000)}
+	if !reflect.DeepEqual(returnedResult, expectedResult) {
+		t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult)
+	}
+}

api/internal/crawl/doc/doc.go

@@ -2,16 +2,18 @@ package doc
 
 import (
 	"fmt"
+	"log"
+	"path/filepath"
+	"sort"
 	"strings"
 
-	"sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct"
-	"sigs.k8s.io/kustomize/v3/pkg/ifc"
-	"sigs.k8s.io/kustomize/v3/pkg/pgmconfig"
-	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/api/konfig"
+	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 )
 
-var fileReader ifc.KunstructuredFactory = kunstruct.NewKunstructuredFactoryImpl()
+var fileReader = kunstruct.NewKunstructuredFactoryImpl()
 
 // This document is meant to be used at the elasticsearch document type.
 // Fields are serialized as-is to elasticsearch, where indices are built
@@ -44,15 +46,36 @@ type KustomizationDocument struct {
 
 type set map[string]struct{}
 
-// Implements the CrawlerDocument interface.
-func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
-	isResource := true
-	for _, suffix := range pgmconfig.KustomizationFileNames {
-		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
-			isResource = false
+func (doc *KustomizationDocument) Copy() *KustomizationDocument {
+	return &KustomizationDocument{
+		Document:    *(doc.Document.Copy()),
+		Kinds:       doc.Kinds,
+		Identifiers: doc.Identifiers,
+		Values:      doc.Values,
+	}
+}
+
+func (doc *KustomizationDocument) String() string {
+	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
+		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
+}
+
+// IsKustomizationFile determines whether a file path is a kustomization file
+func IsKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
 		}
 	}
-	if isResource {
+	return false
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *KustomizationDocument) GetResources(
+	includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
+	if !IsKustomizationFile(doc.FilePath) {
 		return []*Document{}, nil
 	}
 
@@ -70,23 +93,50 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 	}
 	k.FixKustomizationPostUnmarshalling()
 
-	res := make([]*Document, 0, len(k.Resources))
-	for _, r := range k.Resources {
-		next, err := doc.Document.FromRelativePath(r)
-		if err != nil {
-			fmt.Printf("GetResources error: %v\n", err)
-			continue
-		}
-		res = append(res, &next)
+	res := make([]*Document, 0)
+
+	if includeResources {
+		resourceDocs := doc.CollectDocuments(k.Resources, "resource")
+		res = append(res, resourceDocs...)
+	}
+
+	if includeGenerators {
+		generatorDocs := doc.CollectDocuments(k.Generators, "generator")
+		res = append(res, generatorDocs...)
+	}
+
+	if includeTransformers {
+		transformerDocs := doc.CollectDocuments(k.Transformers, "transformer")
+		res = append(res, transformerDocs...)
 	}
 
 	return res, nil
 }
 
+// CollectDocuments construct a Document for each path in paths, and return
+// a slice of Document pointers.
+func (doc *KustomizationDocument) CollectDocuments(
+	paths []string, fileType string) []*Document {
+	docs := make([]*Document, 0, len(paths))
+	for _, r := range paths {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
+		next, err := doc.Document.FromRelativePath(r)
+		if err != nil {
+			log.Printf("CollectDocuments error: %v\n", err)
+			continue
+		}
+		next.FileType = fileType
+		docs = append(docs, &next)
+	}
+	return docs
+}
+
 func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
 	data := []byte(doc.DocumentData)
 
-	for _, suffix := range pgmconfig.KustomizationFileNames {
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
 		if !strings.HasSuffix(doc.FilePath, "/"+suffix) {
 			continue
 		}
@@ -111,6 +161,8 @@ func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error)
 	return configs, nil
 }
 
+// ParseYAML parses doc.Document and sets the following fields of doc:
+// Kinds, Values, Identifiers.
 func (doc *KustomizationDocument) ParseYAML() error {
 	doc.Identifiers = make([]string, 0)
 	doc.Values = make([]string, 0)
@@ -118,6 +170,8 @@ func (doc *KustomizationDocument) ParseYAML() error {
 
 	identifierSet := make(set)
 	valueSet := make(set)
+	kindSet := make(set)
+
 	getKind := func(m map[string]interface{}) string {
 		const defaultStr = "Kustomization"
 		kind, ok := m["kind"]
@@ -136,10 +190,14 @@ func (doc *KustomizationDocument) ParseYAML() error {
 	}
 
 	for _, contents := range ks {
-		doc.Kinds = append(doc.Kinds, getKind(contents))
+		kindSet[getKind(contents)] = struct{}{}
 		createFlatStructure(identifierSet, valueSet, contents)
 	}
 
+	for val := range kindSet {
+		doc.Kinds = append(doc.Kinds, val)
+	}
+
 	for val := range valueSet {
 		doc.Values = append(doc.Values, val)
 	}
@@ -148,6 +206,13 @@ func (doc *KustomizationDocument) ParseYAML() error {
 		doc.Identifiers = append(doc.Identifiers, key)
 	}
 
+	// Without sorting these fields, every time when the string order in these fields changes,
+	// the document in the index will be updated.
+	// Sorting these fields are necessary to avoid a document being updated unnecessarily.
+	sort.Strings(doc.Kinds)
+	sort.Strings(doc.Values)
+	sort.Strings(doc.Identifiers)
+
 	return nil
 }
 

api/internal/crawl/doc/doc_test.go

@@ -121,6 +121,38 @@ metadata:
 kind: Custom
 metadata:
   name: app-crd
+`,
+		},
+		{
+			identifiers: []string{
+				"kind",
+				"metadata",
+				"metadata:name",
+			},
+			values: []string{
+				"kind=Deployment",
+				"kind=Service",
+				"metadata:name=app1",
+				"metadata:name=app2",
+			},
+			kinds: []string{
+				"Deployment",
+				"Service",
+			},
+			filepath: "resources.yaml",
+			yaml: `
+---
+kind: Deployment
+metadata:
+  name: app1
+---
+kind: Deployment
+metadata:
+  name: app2
+---
+kind: Service
+metadata:
+  name: app1
 `,
 		},
 	}
@@ -157,11 +189,13 @@ metadata:
 	}
 }
 
+type TestStructForGetResources struct {
+	doc       KustomizationDocument
+	resources []*Document
+}
+
 func TestGetResources(t *testing.T) {
-	tests := []struct {
-		doc       KustomizationDocument
-		resources []*Document
-	}{
+	tests := []TestStructForGetResources{
 		{
 			doc: KustomizationDocument{
 				Document: Document{
@@ -181,19 +215,27 @@ resources:
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/base",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/otherbase",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "https://github.com/kubernetes-sigs/kustomize",
 					FilePath:      "examples/helloWorld",
 					DefaultBranch: "v3.1.0",
+					FileType:      "resource",
+					User:          "kubernetes-sigs",
 				},
 			},
 		},
@@ -216,9 +258,12 @@ resources:
 			resources: []*Document{},
 		},
 	}
+	runTest(t, tests, true, false, false)
+}
 
+func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
 	for _, test := range tests {
-		res, err := test.doc.GetResources()
+		res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
 		if err != nil {
 			t.Errorf("Unexpected error: %v\n", err)
 			continue
@@ -252,3 +297,83 @@ resources:
 		}
 	}
 }
+
+func TestGetResourcesAndGenerators(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, false, true)
+}
+
+func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/tr.yaml",
+					FileType:      "transformer",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, true, true)
+}

api/internal/crawl/doc/docname.go

@@ -0,0 +1,124 @@
+package doc
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/git"
+)
+
+type Document struct {
+	RepositoryURL string `json:"repositoryUrl,omitempty"`
+	// User makes it easy to aggregate data in the user level instead
+	// of the repository level
+	User          string     `json:"user,omitempty"`
+	FilePath      string     `json:"filePath,omitempty"`
+	DefaultBranch string     `json:"defaultBranch,omitempty"`
+	DocumentData  string     `json:"document,omitempty"`
+	CreationTime  *time.Time `json:"creationTime,omitempty"`
+	IsSame        bool       `json:"-"`
+	// FileType can be one of the following:
+	// "generator", "transformer", "resource", "".
+	FileType string `json:"fileType,omitempty"`
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *Document) GetDocument() *Document {
+	return doc
+}
+
+func (doc *Document) Copy() *Document {
+	return &Document{
+		RepositoryURL: doc.RepositoryURL,
+		User:          doc.User,
+		FilePath:      doc.FilePath,
+		DefaultBranch: doc.DefaultBranch,
+		DocumentData:  doc.DocumentData,
+		CreationTime:  doc.CreationTime,
+		IsSame:        doc.IsSame,
+		FileType:      doc.FileType,
+	}
+}
+
+func (doc *Document) Path() string {
+	return fmt.Sprintf("repoURL: %s filePath: %s branch: %s",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch)
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *Document) WasCached() bool {
+	return doc.IsSame
+}
+
+func (doc *Document) FromRelativePath(newFile string) (Document, error) {
+	repoSpec, err := git.NewRepoSpecFromUrl(newFile)
+	if err == nil {
+		return Document{
+			RepositoryURL: repoSpec.Host + path.Clean(repoSpec.OrgRepo),
+			FilePath:      path.Clean(repoSpec.Path),
+			DefaultBranch: repoSpec.Ref,
+			User:          UserName(repoSpec.Host + path.Clean(repoSpec.OrgRepo)),
+		}, nil
+	}
+	// else document is probably relative path.
+
+	ret := Document{
+		RepositoryURL: doc.RepositoryURL,
+		DefaultBranch: doc.DefaultBranch,
+		User:          UserName(doc.RepositoryURL),
+	}
+	ogDir, _ := path.Split(doc.FilePath)
+
+	cleaned := path.Clean(newFile)
+	if !path.IsAbs(cleaned) {
+		cleaned = path.Clean(ogDir + "/" + cleaned)
+	}
+
+	ret.FilePath = cleaned
+	return ret, nil
+}
+
+func (doc *Document) ID() string {
+	sum := sha256.Sum256([]byte(strings.Join(
+		[]string{
+			doc.RepositoryURL,
+			doc.DefaultBranch,
+			doc.FilePath,
+		},
+		"---|---")))
+	return fmt.Sprintf("%x", sum)
+}
+
+func (doc *Document) RepositoryFullName() string {
+	url := TrimUrl(doc.RepositoryURL)
+	sections := strings.Split(url, "/")
+	l := len(sections)
+	if l < 2 {
+		return url
+	}
+	return path.Join(sections[l-2], sections[l-1])
+}
+
+// TrimUrl removes all the trailing slashes and the "git@github.com:" prefix (if exists).
+func TrimUrl(s string) string {
+	url := strings.TrimRight(s, "/")
+
+	gitPrefix := "git@github.com:"
+	if strings.HasPrefix(url, gitPrefix) {
+		url = url[len(gitPrefix):]
+	}
+	return url
+}
+
+func UserName(repositoryURL string) string {
+	url := TrimUrl(repositoryURL)
+	sections := strings.Split(url, "/")
+	l := len(sections)
+	if l < 2 {
+		return url
+	}
+	return sections[l-2]
+}

api/internal/crawl/doc/docname_test.go

@@ -0,0 +1,150 @@
+package doc
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestFromRelativePath(t *testing.T) {
+	type Case struct {
+		RelativePath string
+		Expected     Document
+	}
+
+	testCases := []struct {
+		BaseDoc Document
+		Cases   []Case
+	}{
+		{
+			BaseDoc: Document{
+				RepositoryURL: "example.com/repo",
+				FilePath:      "path/to/file/kustomization.yaml",
+				DefaultBranch: "master",
+			},
+			Cases: []Case{
+				{
+					RelativePath: "../other/file/resource.yaml",
+					Expected: Document{
+						RepositoryURL: "example.com/repo",
+						FilePath:      "path/to/other/file/resource.yaml",
+						DefaultBranch: "master",
+						User:          "example.com",
+					},
+				},
+				{
+					RelativePath: "../file/../../something/../to/other/file/patch.yaml",
+					Expected: Document{
+						RepositoryURL: "example.com/repo",
+						FilePath:      "path/to/other/file/patch.yaml",
+						DefaultBranch: "master",
+						User:          "example.com",
+					},
+				},
+				{
+					RelativePath: "service.yaml",
+					Expected: Document{
+						RepositoryURL: "example.com/repo",
+						FilePath:      "path/to/file/service.yaml",
+						DefaultBranch: "master",
+						User:          "example.com",
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		for _, c := range tc.Cases {
+			rd, err := tc.BaseDoc.FromRelativePath(c.RelativePath)
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+			if !reflect.DeepEqual(rd, c.Expected) {
+				t.Errorf("document mismatch expected %v, got %v", c.Expected, rd)
+			}
+		}
+	}
+}
+
+func TestDocument_RepositoryFullName(t *testing.T) {
+	testCases := []struct {
+		doc                        Document
+		expectedRepositoryFullName string
+	}{
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com/user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com//user/repo////",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "repo/",
+			},
+			expectedRepositoryFullName: "repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "",
+			},
+			expectedRepositoryFullName: "",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "git@github.com:user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedRepositoryFullName := tc.doc.RepositoryFullName()
+		if returnedRepositoryFullName != tc.expectedRepositoryFullName {
+			t.Errorf("RepositoryFullName expected %s, got %s",
+				tc.expectedRepositoryFullName,
+				returnedRepositoryFullName)
+		}
+	}
+}
+
+func TestDocument_UserName(t *testing.T) {
+	testCases := []struct {
+		repositoryURL    string
+		expectedUserName string
+	}{
+		{
+			repositoryURL:    "https://github.com/user/repo",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "https://github.com//user/repo////",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "repo/",
+			expectedUserName: "repo",
+		},
+		{
+			repositoryURL:    "",
+			expectedUserName: "",
+		},
+		{
+			repositoryURL:    "git@github.com:user/repo",
+			expectedUserName: "user",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedUserName := UserName(tc.repositoryURL)
+		if returnedUserName != tc.expectedUserName {
+			t.Errorf("UserName expected %s, got %s",
+				tc.expectedUserName, returnedUserName)
+		}
+	}
+}