Add lint-kustomize to prow-presubmit-check

To do tool installs.

.golangci-kustomize.yml

@@ -0,0 +1,50 @@
+run:
+  deadline: 5m
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - depguard
+    # - dogsled
+    - dupl
+    # - errcheck
+    # - funlen
+    # - gochecknoinits
+    - goconst
+    # - gocritic
+    - gocyclo
+    - gofmt
+    - goimports
+    - golint
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - interfacer
+    - lll
+    - misspell
+    - nakedret
+    # - scopelint
+    - staticcheck
+    - structcheck
+    # stylecheck demands that acronyms not be treated as words
+    # in camelCase, so JsonOp become JSONOp, etc.  Yuck.
+    # - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    # - whitespace
+
+linters-settings:
+  dupl:
+    threshold: 400
+  lll:
+    line-length: 170
+  gocyclo:
+    min-complexity: 15
+  golint:
+    min-confidence: 0.85

.golangci.yml

@@ -1,30 +0,0 @@
-run:
-  deadline: 5m
-
-linters:
-  disable-all: true
-  enable:
-    - dupl
-    - goconst
-    - gocyclo
-    - gofmt
-    - golint
-    - govet
-    - ineffassign
-    - interfacer
-    - lll
-    - misspell
-    - nakedret
-    - structcheck
-    - unparam
-    - varcheck
-
-linters-settings:
-  dupl:
-    threshold: 400 
-  lll:   
-    line-length: 170
-  gocyclo:
-    min-complexity: 15
-  golint:
-    min-confidence: 0.85

.travis.yml

@@ -34,9 +34,9 @@ before_install:
 install: true
 
 script:
-  - ./travis/verify-deps.sh
-  - ./travis/pre-commit.sh
+  - make verify-kustomize
   - ./travis/kyaml-pre-commit.sh
+  - ./travis/check-go-mod.sh
 
 # TBD. Suppressing for now.
 notifications:

CONTRIBUTING.md

@@ -14,7 +14,7 @@ We have full documentation on how to get started contributing here:
 
 - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests
 - [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing)
-- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet.md) - Common resources for existing developers
+- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet/README.md) - Common resources for existing developers
 
 ## Mentorship
 

Makefile

@@ -1,30 +1,47 @@
-# This Makefile is (and must be) used by
-# travis/pre-commit.sh to qualify pull requests.
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
 #
-# That script generates all the code that needs
-# to be generated, and runs all the tests.
-#
-# Functionality in that script is gradually moving here.
+# Makefile for kustomize CLI and API.
 
 MYGOBIN := $(shell go env GOPATH)/bin
-PATH := $(PATH):$(MYGOBIN)
-SHELL := env PATH=$(PATH) /bin/bash
+SHELL := /bin/bash
+export PATH := $(MYGOBIN):$(PATH)
 
 .PHONY: all
-all: pre-commit
+all: verify-kustomize
 
-# The pre-commit.sh script generates, lints and tests.
-# It uses this makefile.  For more clarity, would like
-# to stop that - any scripts invoked by targets here
-# shouldn't "call back" to the makefile.
-.PHONY: pre-commit
-pre-commit:
-	./travis/pre-commit.sh
+.PHONY: verify-kustomize
+verify-kustomize: \
+	lint-kustomize \
+	test-unit-kustomize-all \
+	test-examples-kustomize-against-HEAD \
+	test-examples-kustomize-against-latest
 
+# The following target referenced by a file in
+# https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/kustomize
+.PHONY: prow-presubmit-check
+prow-presubmit-check: \
+	lint-kustomize \
+	test-unit-kustomize-all
+
+.PHONY: verify-kustomize-e2e
+verify-kustomize-e2e: test-examples-e2e-kustomize
+
+# Other builds in this repo might want a different linter version.
+# Without one Makefile to rule them all, the different makes
+# cannot assume that golanci-lint is at the version they want
+# since everything uses the same implicit GOPATH.
+# This installs in a temp dir to avoid overwriting someone else's
+# linter, then installs in MYGOBIN with a new name.
 # Version pinned by api/go.mod
-$(MYGOBIN)/golangci-lint:
-	cd api; \
-	go install github.com/golangci/golangci-lint/cmd/golangci-lint
+$(MYGOBIN)/golangci-lint-kustomize:
+	( \
+		set -e; \
+		export GOBIN=$$(mktemp -d) \
+		cd api; \
+		GO111MODULE=on go install github.com/golangci/golangci-lint/cmd/golangci-lint; \
+		mv $$GOBIN/golangci-lint $(MYGOBIN)/golangci-lint-kustomize \
+	)
 
 # Version pinned by api/go.mod
 $(MYGOBIN)/mdrip:
@@ -41,73 +58,177 @@ $(MYGOBIN)/goimports:
 	cd api; \
 	go install golang.org/x/tools/cmd/goimports
 
-# TODO: need a new release of the API, followed by a new pluginator.
-# pluginator v1.1.0 is too old for the code currently needed in the API.
-# Can release a new one at any time, just haven't done so.
-# When one has been released,
-#  - uncomment the pluginator line in './api/internal/tools/tools.go'
-#  - pin the version tag in './api/go.mod' to match the new release
-#  - change the following to 'cd api; go install sigs.k8s.io/kustomize/pluginator'
+# Install resource from whatever is checked out.
+$(MYGOBIN)/resource:
+	cd cmd/resource; \
+	go install .
+
+# To pin pluginator, use this recipe instead:
+#   cd api;
+#   go install sigs.k8s.io/kustomize/pluginator/v2
 $(MYGOBIN)/pluginator:
 	cd pluginator; \
 	go install .
 
+# Install kustomize from whatever is checked out.
+$(MYGOBIN)/kustomize:
+	cd kustomize; \
+	go install .
+
 .PHONY: install-tools
 install-tools: \
 	$(MYGOBIN)/goimports \
-	$(MYGOBIN)/golangci-lint \
+	$(MYGOBIN)/golangci-lint-kustomize \
 	$(MYGOBIN)/mdrip \
 	$(MYGOBIN)/pluginator \
 	$(MYGOBIN)/stringer
 
-# Builtin plugins are generated code.
-# Add new items here to create new builtins.
-builtinplugins = \
-	api/builtins/annotationstransformer.go \
-	api/builtins/configmapgenerator.go \
-	api/builtins/hashtransformer.go \
-	api/builtins/imagetagtransformer.go \
-	api/builtins/inventorytransformer.go \
-	api/builtins/labeltransformer.go \
-	api/builtins/legacyordertransformer.go \
-	api/builtins/namespacetransformer.go \
-	api/builtins/patchjson6902transformer.go \
-	api/builtins/patchstrategicmergetransformer.go \
-	api/builtins/patchtransformer.go \
-	api/builtins/prefixsuffixtransformer.go \
-	api/builtins/replicacounttransformer.go \
-	api/builtins/secretgenerator.go
+### Begin kustomize plugin rules.
+#
+# The rules to deal with builtin plugins are a bit
+# complicated because
+#
+# - Every builtin plugin is a Go plugin -
+#   meaning it gets its own module directory
+#   (outside of the api module) with Go
+#   code in a 'main' package per Go plugin rules.
+# - kustomize locates plugins using the
+#   'apiVersion' and 'kind' fields from the
+#   plugin config file.
+# - k8s wants CamelCase in 'kind' fields.
+# - The module name (the last name in the path)
+#   must be the lowercased 'kind' of the
+#   plugin because Go and related tools
+#   demand lowercase in import paths, but
+#   allow CamelCase in file names.
+# - the generated code must live in the api
+#   module (it's linked into the api).
 
-.PHONY: lint
-lint: install-tools $(builtinplugins)
-	cd api; $(MYGOBIN)/golangci-lint run ./...
-	cd kustomize; $(MYGOBIN)/golangci-lint run ./...
-	cd pluginator; $(MYGOBIN)/golangci-lint run ./...
+# Where all generated builtin plugin code should go.
+pGen=api/builtins
+# Where the builtin Go plugin modules live.
+pSrc=plugin/builtin
 
-api/builtins/%.go: $(MYGOBIN)/pluginator
-	@echo "generating $*"; \
-	cd plugin/builtin/$*; \
-	go generate .; \
-	cd ../../../api/builtins; \
-	$(MYGOBIN)/goimports -w $*.go
+_builtinplugins = \
+	AnnotationsTransformer.go \
+	ConfigMapGenerator.go \
+	HashTransformer.go \
+	ImageTagTransformer.go \
+	InventoryTransformer.go \
+	LabelTransformer.go \
+	LegacyOrderTransformer.go \
+	NamespaceTransformer.go \
+	PatchJson6902Transformer.go \
+	PatchStrategicMergeTransformer.go \
+	PatchTransformer.go \
+	PrefixSuffixTransformer.go \
+	ReplicaCountTransformer.go \
+	SecretGenerator.go
 
-.PHONY: generate
-generate: $(builtinplugins)
+# Maintaining this explicit list of generated files, and
+# adding it as a dependency to a few targets, to assure
+# they get recreated if deleted.  The rules below on how
+# to make them don't, by themselves, assure they will be
+# recreated if deleted.
+builtinplugins = $(patsubst %,$(pGen)/%,$(_builtinplugins))
 
-.PHONY: unit-test-api
-unit-test-api: $(builtinplugins)
+# These rules are verbose, but assure that if a source file
+# is modified, the corresponding generated file, and only
+# that file, will be recreated.
+$(pGen)/AnnotationsTransformer.go: $(pSrc)/annotationstransformer/AnnotationsTransformer.go
+$(pGen)/ConfigMapGenerator.go: $(pSrc)/configmapgenerator/ConfigMapGenerator.go
+$(pGen)/HashTransformer.go: $(pSrc)/hashtransformer/HashTransformer.go
+$(pGen)/ImageTagTransformer.go: $(pSrc)/imagetagtransformer/ImageTagTransformer.go
+$(pGen)/InventoryTransformer.go: $(pSrc)/inventorytransformer/InventoryTransformer.go
+$(pGen)/LabelTransformer.go: $(pSrc)/labeltransformer/LabelTransformer.go
+$(pGen)/LegacyOrderTransformer.go: $(pSrc)/legacyordertransformer/LegacyOrderTransformer.go
+$(pGen)/NamespaceTransformer.go: $(pSrc)/namespacetransformer/NamespaceTransformer.go
+$(pGen)/PatchJson6902Transformer.go: $(pSrc)/patchjson6902transformer/PatchJson6902Transformer.go
+$(pGen)/PatchStrategicMergeTransformer.go: $(pSrc)/patchstrategicmergetransformer/PatchStrategicMergeTransformer.go
+$(pGen)/PatchTransformer.go: $(pSrc)/patchtransformer/PatchTransformer.go
+$(pGen)/PrefixSuffixTransformer.go: $(pSrc)/prefixsuffixtransformer/PrefixSuffixTransformer.go
+$(pGen)/ReplicaCountTransformer.go: $(pSrc)/replicacounttransformer/ReplicaCountTransformer.go
+$(pGen)/SecretGenerator.go: $(pSrc)/secretgenerator/SecretGenerator.go
+
+# The (verbose but portable) Makefile way to convert to lowercase.
+toLowerCase = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+
+$(pGen)/%.go: $(MYGOBIN)/pluginator
+	@echo "generating $*"
+	( \
+		set -e; \
+		cd $(pSrc)/$(call toLowerCase,$*); \
+		go generate .; \
+		cd ../../../$(pGen); \
+		$(MYGOBIN)/goimports -w $*.go \
+	)
+
+# Target is for debugging.
+.PHONY: generate-kustomize-builtin-plugins
+generate-kustomize-builtin-plugins: $(builtinplugins)
+
+.PHONY: kustomize-external-go-plugin-build
+kustomize-external-go-plugin-build:
+	./hack/buildExternalGoPlugins.sh ./plugin
+
+.PHONY: kustomize-external-go-plugin-clean
+kustomize-external-go-plugin-clean:
+	./hack/buildExternalGoPlugins.sh ./plugin clean
+
+### End kustomize plugin rules.
+
+.PHONY: lint-kustomize
+lint-kustomize: install-tools $(builtinplugins)
+	cd api; \
+	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
+	cd kustomize; \
+	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
+	cd pluginator; \
+	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
+
+.PHONY: test-unit-kustomize-api
+test-unit-kustomize-api: $(builtinplugins)
 	cd api; go test ./...
 
-.PHONY: unit-test-plugins
-unit-test-plugins:
-	./hack/runPluginUnitTests.sh
+.PHONY: test-unit-kustomize-plugins
+test-unit-kustomize-plugins:
+	./hack/testUnitKustomizePlugins.sh
 
-.PHONY: unit-test-kustomize
-unit-test-kustomize:
+.PHONY: test-unit-kustomize-cli
+test-unit-kustomize-cli:
 	cd kustomize; go test ./...
 
-.PHONY: unit-test-all
-unit-test-all: unit-test-api unit-test-kustomize unit-test-plugins
+.PHONY: test-unit-kustomize-all
+test-unit-kustomize-all: \
+	test-unit-kustomize-api \
+	test-unit-kustomize-cli \
+	test-unit-kustomize-plugins
+
+.PHONY:
+test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
+	./hack/testExamplesAgainstKustomize.sh HEAD
+
+.PHONY:
+test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from ."; \
+		cd kustomize; go install .; cd ..; \
+		./hack/testExamplesE2EAgainstKustomize.sh .; \
+	)
+
+.PHONY:
+test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from latest."; \
+		GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v3; \
+		./hack/testExamplesAgainstKustomize.sh latest; \
+		echo "Reinstalling kustomize from HEAD."; \
+		cd kustomize; go install .; \
+	)
 
 # linux only.
 # This is for testing an example plugin that
@@ -116,11 +237,14 @@ unit-test-all: unit-test-api unit-test-kustomize unit-test-plugins
 # to github.com/instrumenta/kubeval.
 # Instead, download the binary.
 $(MYGOBIN)/kubeval:
-	d=$(shell mktemp -d); cd $$d; \
-	wget https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz; \
-	tar xf kubeval-linux-amd64.tar.gz; \
-	mv kubeval $(MYGOBIN); \
-	rm -rf $$d
+	( \
+		set -e; \
+		d=$(shell mktemp -d); cd $$d; \
+		wget https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz; \
+		tar xf kubeval-linux-amd64.tar.gz; \
+		mv kubeval $(MYGOBIN); \
+		rm -rf $$d; \
+	)
 
 # linux only.
 # This is for testing an example plugin that
@@ -129,16 +253,32 @@ $(MYGOBIN)/kubeval:
 # to helm.
 # Instead, download the binary.
 $(MYGOBIN)/helm:
-	d=$(shell mktemp -d); cd $$d; \
-	wget https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz; \
-	tar -xvzf helm-v2.13.1-linux-amd64.tar.gz; \
-	mv linux-amd64/helm $(MYGOBIN); \
-	rm -rf $$d
+	( \
+		set -e; \
+		d=$(shell mktemp -d); cd $$d; \
+		wget https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz; \
+		tar -xvzf helm-v2.13.1-linux-amd64.tar.gz; \
+		mv linux-amd64/helm $(MYGOBIN); \
+		rm -rf $$d \
+	)
+
+$(MYGOBIN)/kind:
+	( \
+        set -e; \
+        d=$(shell mktemp -d); cd $$d; \
+        wget -O ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(shell uname)-amd64; \
+        chmod +x ./kind; \
+        mv ./kind $(MYGOBIN); \
+        rm -rf $$d; \
+	)
 
 .PHONY: clean
-clean:
+clean: kustomize-external-go-plugin-clean
+	go clean --cache
 	rm -f $(builtinplugins)
 	rm -f $(MYGOBIN)/pluginator
+	rm -f $(MYGOBIN)/kustomize
+	rm -f $(MYGOBIN)/golangci-lint-kustomize
 
 .PHONY: nuke
 nuke: clean

README.md

@@ -28,9 +28,9 @@ Since [v1.14][kubectl announcement] the kustomize build system has been included
 
 | kubectl version | kustomize version |
 |---------|--------|
-| v1.16.0 | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
-| v1.15.x | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
-| v1.14.x | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
+| v1.16.0 | [v2.0.3](/../../tree/v2.0.3) |
+| v1.15.x | [v2.0.3](/../../tree/v2.0.3) |
+| v1.14.x | [v2.0.3](/../../tree/v2.0.3) |
 
 For examples and guides for using the kubectl integration please see the [kubectl book] or the [kubernetes documentation].
 
@@ -167,7 +167,7 @@ is governed by the [Kubernetes Code of Conduct].
 [eschewed feature list]: docs/eschewedFeatures.md
 [imageBase]: docs/images/base.jpg
 [imageOverlay]: docs/images/overlay.jpg
-[kind/feature]: https://github.com/kubernetes-sigs/kustomize/labels/kind%2Ffeature
+[kind/feature]: /../../labels/kind%2Ffeature
 [kubectl announcement]: https://kubernetes.io/blog/2019/03/25/kubernetes-1-14-release-announcement
 [kubectl book]: https://kubectl.docs.kubernetes.io/pages/app_customization/introduction.html
 [kubernetes documentation]: https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/
@@ -175,12 +175,12 @@ is governed by the [Kubernetes Code of Conduct].
 [kustomization]: docs/glossary.md#kustomization
 [overlay]: docs/glossary.md#overlay
 [overlays]: docs/glossary.md#overlay
-[release page]: https://github.com/kubernetes-sigs/kustomize/releases
+[release page]: /../../releases
 [resource]: docs/glossary.md#resource
 [resources]: docs/glossary.md#resource
 [sig-cli]: https://github.com/kubernetes/community/blob/master/sig-cli/README.md
 [variant]: docs/glossary.md#variant
 [variants]: docs/glossary.md#variant
-[v2.0.3]: https://github.com/kubernetes-sigs/kustomize/releases/tag/v2.0.3
-[v2.1.0]: https://github.com/kubernetes-sigs/kustomize/releases/tag/v2.1.0
+[v2.0.3]: /../../releases/tag/v2.0.3
+[v2.1.0]: /../../releases/tag/v2.1.0
 [workflows]: docs/workflows.md

api/builtins/AnnotationsTransformer.go

@@ -0,0 +1,39 @@
+// Code generated by pluginator on AnnotationsTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+// Add the given annotations to the given field specifications.
+type AnnotationsTransformerPlugin struct {
+	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
+	FieldSpecs  []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *AnnotationsTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Annotations = nil
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *AnnotationsTransformerPlugin) Transform(m resmap.ResMap) error {
+	t, err := transform.NewMapTransformer(
+		p.FieldSpecs,
+		p.Annotations,
+	)
+	if err != nil {
+		return err
+	}
+	return t.Transform(m)
+}
+
+func NewAnnotationsTransformerPlugin() resmap.TransformerPlugin {
+	return &AnnotationsTransformerPlugin{}
+}

api/builtins/HashTransformer.go

@@ -0,0 +1,39 @@
+// Code generated by pluginator on HashTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/resmap"
+)
+
+type HashTransformerPlugin struct {
+	hasher ifc.KunstructuredHasher
+}
+
+func (p *HashTransformerPlugin) Config(
+	h *resmap.PluginHelpers, _ []byte) (err error) {
+	p.hasher = h.ResmapFactory().RF().Hasher()
+	return nil
+}
+
+// Transform appends hash to generated resources.
+func (p *HashTransformerPlugin) Transform(m resmap.ResMap) error {
+	for _, res := range m.Resources() {
+		if res.NeedHashSuffix() {
+			h, err := p.hasher.Hash(res)
+			if err != nil {
+				return err
+			}
+			res.SetName(fmt.Sprintf("%s-%s", res.GetName(), h))
+		}
+	}
+	return nil
+}
+
+func NewHashTransformerPlugin() resmap.TransformerPlugin {
+	return &HashTransformerPlugin{}
+}

api/builtins/ImageTagTransformer.go

@@ -0,0 +1,185 @@
+// Code generated by pluginator on ImageTagTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/yaml"
+)
+
+// Find matching image declarations and replace
+// the name, tag and/or digest.
+type ImageTagTransformerPlugin struct {
+	ImageTag   types.Image       `json:"imageTag,omitempty" yaml:"imageTag,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *ImageTagTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.ImageTag = types.Image{}
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *ImageTagTransformerPlugin) Transform(m resmap.ResMap) error {
+	for _, r := range m.Resources() {
+		for _, path := range p.FieldSpecs {
+			if !r.OrgId().IsSelected(&path.Gvk) {
+				continue
+			}
+			err := transform.MutateField(
+				r.Map(), path.PathSlice(), false, p.mutateImage)
+			if err != nil {
+				return err
+			}
+		}
+		// Kept for backward compatibility
+		if err := p.findAndReplaceImage(r.Map()); err != nil && r.OrgId().Kind != `CustomResourceDefinition` {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *ImageTagTransformerPlugin) mutateImage(in interface{}) (interface{}, error) {
+	original, ok := in.(string)
+	if !ok {
+		return nil, fmt.Errorf("image path is not of type string but %T", in)
+	}
+
+	if !isImageMatched(original, p.ImageTag.Name) {
+		return original, nil
+	}
+	name, tag := split(original)
+	if p.ImageTag.NewName != "" {
+		name = p.ImageTag.NewName
+	}
+	if p.ImageTag.NewTag != "" {
+		tag = ":" + p.ImageTag.NewTag
+	}
+	if p.ImageTag.Digest != "" {
+		tag = "@" + p.ImageTag.Digest
+	}
+	return name + tag, nil
+}
+
+// findAndReplaceImage replaces the image name and
+// tags inside one object.
+// It searches the object for container session
+// then loops though all images inside containers
+// session, finds matched ones and update the
+// image name and tag name
+func (p *ImageTagTransformerPlugin) findAndReplaceImage(obj map[string]interface{}) error {
+	paths := []string{"containers", "initContainers"}
+	updated := false
+	for _, path := range paths {
+		containers, found := obj[path]
+		if found && containers != nil {
+			if _, err := p.updateContainers(containers); err != nil {
+				return err
+			}
+			updated = true
+		}
+	}
+	if !updated {
+		return p.findContainers(obj)
+	}
+	return nil
+}
+
+func (p *ImageTagTransformerPlugin) updateContainers(in interface{}) (interface{}, error) {
+	containers, ok := in.([]interface{})
+	if !ok {
+		return nil, fmt.Errorf(
+			"containers path is not of type []interface{} but %T", in)
+	}
+	for i := range containers {
+		container := containers[i].(map[string]interface{})
+		containerImage, found := container["image"]
+		if !found {
+			continue
+		}
+		imageName := containerImage.(string)
+		if isImageMatched(imageName, p.ImageTag.Name) {
+			newImage, err := p.mutateImage(imageName)
+			if err != nil {
+				return nil, err
+			}
+			container["image"] = newImage
+		}
+	}
+	return containers, nil
+}
+
+func (p *ImageTagTransformerPlugin) findContainers(obj map[string]interface{}) error {
+	for key := range obj {
+		switch typedV := obj[key].(type) {
+		case map[string]interface{}:
+			err := p.findAndReplaceImage(typedV)
+			if err != nil {
+				return err
+			}
+		case []interface{}:
+			for i := range typedV {
+				item := typedV[i]
+				typedItem, ok := item.(map[string]interface{})
+				if ok {
+					err := p.findAndReplaceImage(typedItem)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func isImageMatched(s, t string) bool {
+	// Tag values are limited to [a-zA-Z0-9_.-].
+	pattern, _ := regexp.Compile("^" + t + "(@sha256)?(:[a-zA-Z0-9_.-]*)?$")
+	return pattern.MatchString(s)
+}
+
+// split separates and returns the name and tag parts
+// from the image string using either colon `:` or at `@` separators.
+// Note that the returned tag keeps its separator.
+func split(imageName string) (name string, tag string) {
+	// check if image name contains a domain
+	// if domain is present, ignore domain and check for `:`
+	ic := -1
+	if slashIndex := strings.Index(imageName, "/"); slashIndex < 0 {
+		ic = strings.LastIndex(imageName, ":")
+	} else {
+		lastIc := strings.LastIndex(imageName[slashIndex:], ":")
+		// set ic only if `:` is present
+		if lastIc > 0 {
+			ic = slashIndex + lastIc
+		}
+	}
+	ia := strings.LastIndex(imageName, "@")
+	if ic < 0 && ia < 0 {
+		return imageName, ""
+	}
+
+	i := ic
+	if ia > 0 {
+		i = ia
+	}
+
+	name = imageName[:i]
+	tag = imageName[i:]
+	return
+}
+
+func NewImageTagTransformerPlugin() resmap.TransformerPlugin {
+	return &ImageTagTransformerPlugin{}
+}

api/builtins/LabelTransformer.go

@@ -0,0 +1,39 @@
+// Code generated by pluginator on LabelTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+// Add the given labels to the given field specifications.
+type LabelTransformerPlugin struct {
+	Labels     map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *LabelTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Labels = nil
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *LabelTransformerPlugin) Transform(m resmap.ResMap) error {
+	t, err := transform.NewMapTransformer(
+		p.FieldSpecs,
+		p.Labels,
+	)
+	if err != nil {
+		return err
+	}
+	return t.Transform(m)
+}
+
+func NewLabelTransformerPlugin() resmap.TransformerPlugin {
+	return &LabelTransformerPlugin{}
+}

api/builtins/LegacyOrderTransformer.go

@@ -0,0 +1,46 @@
+// Code generated by pluginator on LegacyOrderTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"sort"
+
+	"github.com/pkg/errors"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+)
+
+// Sort the resources using an ordering defined in the Gvk class.
+// This puts cluster-wide basic resources with no
+// dependencies (like Namespace, StorageClass, etc.)
+// first, and resources with a high number of dependencies
+// (like ValidatingWebhookConfiguration) last.
+type LegacyOrderTransformerPlugin struct{}
+
+// Nothing needed for configuration.
+func (p *LegacyOrderTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, _ []byte) (err error) {
+	return nil
+}
+
+func (p *LegacyOrderTransformerPlugin) Transform(m resmap.ResMap) (err error) {
+	resources := make([]*resource.Resource, m.Size())
+	ids := m.AllIds()
+	sort.Sort(resmap.IdSlice(ids))
+	for i, id := range ids {
+		resources[i], err = m.GetByCurrentId(id)
+		if err != nil {
+			return errors.Wrap(err, "expected match for sorting")
+		}
+	}
+	m.Clear()
+	for _, r := range resources {
+		m.Append(r)
+	}
+	return nil
+}
+
+func NewLegacyOrderTransformerPlugin() resmap.TransformerPlugin {
+	return &LegacyOrderTransformerPlugin{}
+}

api/builtins/NamespaceTransformer.go

@@ -0,0 +1,131 @@
+// Code generated by pluginator on NamespaceTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/transform"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+// Change or set the namespace of non-cluster level resources.
+type NamespaceTransformerPlugin struct {
+	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+	FieldSpecs       []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *NamespaceTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Namespace = ""
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
+	if len(p.Namespace) == 0 {
+		return nil
+	}
+	for _, r := range m.Resources() {
+		if len(r.Map()) == 0 {
+			// Don't mutate empty objects?
+			continue
+		}
+
+		id := r.OrgId()
+		applicableFs := p.applicableFieldSpecs(id)
+
+		for _, fs := range applicableFs {
+			err := transform.MutateField(
+				r.Map(), fs.PathSlice(), fs.CreateIfNotPresent,
+				p.changeNamespace(r))
+			if err != nil {
+				return err
+			}
+		}
+
+		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
+		if len(matches) != 1 {
+			return fmt.Errorf("namespace tranformation produces ID conflict: %+v", matches)
+		}
+	}
+	return nil
+}
+
+const metaNamespace = "metadata/namespace"
+
+// Special casing metadata.namespace since
+// all objects have it, even "ClusterKind" objects
+// that don't exist in a namespace (the Namespace
+// object itself doesn't live in a namespace).
+func (p *NamespaceTransformerPlugin) applicableFieldSpecs(id resid.ResId) []types.FieldSpec {
+	var res []types.FieldSpec
+	for _, fs := range p.FieldSpecs {
+		if id.IsSelected(&fs.Gvk) && (fs.Path != metaNamespace || (fs.Path == metaNamespace && id.IsNamespaceableKind())) {
+			res = append(res, fs)
+		}
+	}
+	return res
+}
+
+func (p *NamespaceTransformerPlugin) changeNamespace(
+	_ *resource.Resource) func(in interface{}) (interface{}, error) {
+	return func(in interface{}) (interface{}, error) {
+		switch in.(type) {
+		case string:
+			// will happen when the metadata/namespace
+			// value is replaced
+			return p.Namespace, nil
+		case []interface{}:
+			l, _ := in.([]interface{})
+			for idx, item := range l {
+				switch item.(type) {
+				case map[string]interface{}:
+					// Will happen when mutating the subjects
+					// field of ClusterRoleBinding and RoleBinding
+					inMap, _ := item.(map[string]interface{})
+					if _, ok := inMap["name"]; !ok {
+						continue
+					}
+					name, ok := inMap["name"].(string)
+					if !ok {
+						continue
+					}
+					// The only case we need to force the namespace
+					// if for the "service account". "default" is
+					// kind of hardcoded here for right now.
+					if name != "default" {
+						continue
+					}
+					inMap["namespace"] = p.Namespace
+					l[idx] = inMap
+				default:
+					// nothing to do for right now
+				}
+			}
+			return in, nil
+		case map[string]interface{}:
+			// Will happen if the createField=true
+			// when the namespace is added to the
+			// object
+			inMap := in.(map[string]interface{})
+			if len(inMap) == 0 {
+				return p.Namespace, nil
+			} else {
+				return in, nil
+			}
+		default:
+			return in, nil
+		}
+	}
+}
+
+func NewNamespaceTransformerPlugin() resmap.TransformerPlugin {
+	return &NamespaceTransformerPlugin{}
+}

api/builtins/PrefixSuffixTransformer.go

@@ -0,0 +1,123 @@
+// Code generated by pluginator on PrefixSuffixTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"errors"
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/yaml"
+)
+
+// Add the given prefix and suffix to the field.
+type PrefixSuffixTransformerPlugin struct {
+	Prefix     string            `json:"prefix,omitempty" yaml:"prefix,omitempty"`
+	Suffix     string            `json:"suffix,omitempty" yaml:"suffix,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+// Not placed in a file yet due to lack of demand.
+var prefixSuffixFieldSpecsToSkip = []types.FieldSpec{
+	{
+		Gvk: resid.Gvk{Kind: "CustomResourceDefinition"},
+	},
+	{
+		Gvk: resid.Gvk{Group: "apiregistration.k8s.io", Kind: "APIService"},
+	},
+}
+
+func (p *PrefixSuffixTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Prefix = ""
+	p.Suffix = ""
+	p.FieldSpecs = nil
+	err = yaml.Unmarshal(c, p)
+	if err != nil {
+		return
+	}
+	if p.FieldSpecs == nil {
+		return errors.New("fieldSpecs is not expected to be nil")
+	}
+	return
+}
+
+func (p *PrefixSuffixTransformerPlugin) Transform(m resmap.ResMap) error {
+
+	// Even if both the Prefix and Suffix are empty we want
+	// to proceed with the transformation. This allows to add contextual
+	// information to the resources (AddNamePrefix and AddNameSuffix).
+
+	for _, r := range m.Resources() {
+		if p.shouldSkip(r.OrgId()) {
+			// Don't change the actual definition
+			// of a CRD.
+			continue
+		}
+		id := r.OrgId()
+		// current default configuration contains
+		// only one entry: "metadata/name" with no GVK
+		for _, path := range p.FieldSpecs {
+			if !id.IsSelected(&path.Gvk) {
+				// With the currrent default configuration,
+				// because no Gvk is specified, so a wild
+				// card
+				continue
+			}
+
+			if smellsLikeANameChange(&path) {
+				// "metadata/name" is the only field.
+				// this will add a prefix and a suffix
+				// to the resource even if those are
+				// empty
+				r.AddNamePrefix(p.Prefix)
+				r.AddNameSuffix(p.Suffix)
+			}
+
+			// the addPrefixSuffix method will not
+			// change the name if both the prefix and suffix
+			// are empty.
+			err := transform.MutateField(
+				r.Map(),
+				path.PathSlice(),
+				path.CreateIfNotPresent,
+				p.addPrefixSuffix)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func smellsLikeANameChange(fs *types.FieldSpec) bool {
+	return fs.Path == "metadata/name"
+}
+
+func (p *PrefixSuffixTransformerPlugin) shouldSkip(
+	id resid.ResId) bool {
+	for _, path := range prefixSuffixFieldSpecsToSkip {
+		if id.IsSelected(&path.Gvk) {
+			return true
+		}
+	}
+	return false
+}
+
+func (p *PrefixSuffixTransformerPlugin) addPrefixSuffix(
+	in interface{}) (interface{}, error) {
+	s, ok := in.(string)
+	if !ok {
+		return nil, fmt.Errorf("%#v is expected to be %T", in, s)
+	}
+	return fmt.Sprintf("%s%s%s", p.Prefix, s, p.Suffix), nil
+}
+
+func NewPrefixSuffixTransformerPlugin() resmap.TransformerPlugin {
+	return &PrefixSuffixTransformerPlugin{}
+}

api/builtins/ReplicaCountTransformer.go

@@ -0,0 +1,88 @@
+// Code generated by pluginator on ReplicaCountTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/transform"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+// Find matching replicas declarations and replace the count.
+// Eases the kustomization configuration of replica changes.
+type ReplicaCountTransformerPlugin struct {
+	Replica    types.Replica     `json:"replica,omitempty" yaml:"replica,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *ReplicaCountTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Replica = types.Replica{}
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *ReplicaCountTransformerPlugin) Transform(m resmap.ResMap) error {
+	found := false
+	for i, replicaSpec := range p.FieldSpecs {
+		matcher := p.createMatcher(i)
+		matchOriginal := m.GetMatchingResourcesByOriginalId(matcher)
+		matchCurrent := m.GetMatchingResourcesByCurrentId(matcher)
+
+		for _, res := range append(matchOriginal, matchCurrent...) {
+			found = true
+			err := transform.MutateField(
+				res.Map(), replicaSpec.PathSlice(),
+				replicaSpec.CreateIfNotPresent, p.addReplicas)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if !found {
+		gvks := make([]string, len(p.FieldSpecs))
+		for i, replicaSpec := range p.FieldSpecs {
+			gvks[i] = replicaSpec.Gvk.String()
+		}
+		return fmt.Errorf("resource with name %s does not match a config with the following GVK %v",
+			p.Replica.Name, gvks)
+	}
+
+	return nil
+}
+
+// Match Replica.Name and FieldSpec
+func (p *ReplicaCountTransformerPlugin) createMatcher(i int) resmap.IdMatcher {
+	return func(r resid.ResId) bool {
+		return r.Name == p.Replica.Name &&
+			r.Gvk.IsSelected(&p.FieldSpecs[i].Gvk)
+	}
+}
+
+func (p *ReplicaCountTransformerPlugin) addReplicas(in interface{}) (interface{}, error) {
+	switch m := in.(type) {
+	case int64:
+		// Was already in the field.
+	case map[string]interface{}:
+		if len(m) != 0 {
+			// A map was already in the replicas field, don't want to
+			// discard this data silently.
+			return nil, fmt.Errorf("%#v is expected to be %T", in, m)
+		}
+		// Just got added, default type is map, but we can return anything.
+	default:
+		return nil, fmt.Errorf("%#v is expected to be %T", in, m)
+	}
+	return p.Replica.Count, nil
+}
+
+func NewReplicaCountTransformerPlugin() resmap.TransformerPlugin {
+	return &ReplicaCountTransformerPlugin{}
+}

api/builtins/annotationstransformer.go

@@ -1,39 +0,0 @@
-// Code generated by pluginator on AnnotationsTransformer; DO NOT EDIT.
-// pluginator {unknown  1970-01-01T00:00:00Z  }
-
-package builtins
-
-import (
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/transform"
-	"sigs.k8s.io/kustomize/api/types"
-	"sigs.k8s.io/yaml"
-)
-
-// Add the given annotations to the given field specifications.
-type AnnotationsTransformerPlugin struct {
-	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
-	FieldSpecs  []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
-}
-
-func (p *AnnotationsTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-	p.Annotations = nil
-	p.FieldSpecs = nil
-	return yaml.Unmarshal(c, p)
-}
-
-func (p *AnnotationsTransformerPlugin) Transform(m resmap.ResMap) error {
-	t, err := transform.NewMapTransformer(
-		p.FieldSpecs,
-		p.Annotations,
-	)
-	if err != nil {
-		return err
-	}
-	return t.Transform(m)
-}
-
-func NewAnnotationsTransformerPlugin() resmap.TransformerPlugin {
-	return &AnnotationsTransformerPlugin{}
-}

api/builtins/hashtransformer.go

@@ -1,39 +0,0 @@
-// Code generated by pluginator on HashTransformer; DO NOT EDIT.
-// pluginator {unknown  1970-01-01T00:00:00Z  }
-
-package builtins
-
-import (
-	"fmt"
-
-	"sigs.k8s.io/kustomize/api/ifc"
-	"sigs.k8s.io/kustomize/api/resmap"
-)
-
-type HashTransformerPlugin struct {
-	hasher ifc.KunstructuredHasher
-}
-
-func (p *HashTransformerPlugin) Config(
-	h *resmap.PluginHelpers, config []byte) (err error) {
-	p.hasher = h.ResmapFactory().RF().Hasher()
-	return nil
-}
-
-// Transform appends hash to generated resources.
-func (p *HashTransformerPlugin) Transform(m resmap.ResMap) error {
-	for _, res := range m.Resources() {
-		if res.NeedHashSuffix() {
-			h, err := p.hasher.Hash(res)
-			if err != nil {
-				return err
-			}
-			res.SetName(fmt.Sprintf("%s-%s", res.GetName(), h))
-		}
-	}
-	return nil
-}
-
-func NewHashTransformerPlugin() resmap.TransformerPlugin {
-	return &HashTransformerPlugin{}
-}

api/builtins/imagetagtransformer.go

@@ -1,185 +0,0 @@
-// Code generated by pluginator on ImageTagTransformer; DO NOT EDIT.
-// pluginator {unknown  1970-01-01T00:00:00Z  }
-
-package builtins
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-
-	"sigs.k8s.io/kustomize/api/transform"
-	"sigs.k8s.io/kustomize/api/types"
-
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/yaml"
-)
-
-// Find matching image declarations and replace
-// the name, tag and/or digest.
-type ImageTagTransformerPlugin struct {
-	ImageTag   types.Image       `json:"imageTag,omitempty" yaml:"imageTag,omitempty"`
-	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
-}
-
-func (p *ImageTagTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-	p.ImageTag = types.Image{}
-	p.FieldSpecs = nil
-	return yaml.Unmarshal(c, p)
-}
-
-func (p *ImageTagTransformerPlugin) Transform(m resmap.ResMap) error {
-	for _, r := range m.Resources() {
-		for _, path := range p.FieldSpecs {
-			if !r.OrgId().IsSelected(&path.Gvk) {
-				continue
-			}
-			err := transform.MutateField(
-				r.Map(), path.PathSlice(), false, p.mutateImage)
-			if err != nil {
-				return err
-			}
-		}
-		// Kept for backward compatibility
-		if err := p.findAndReplaceImage(r.Map()); err != nil && r.OrgId().Kind != `CustomResourceDefinition` {
-			return err
-		}
-	}
-	return nil
-}
-
-func (p *ImageTagTransformerPlugin) mutateImage(in interface{}) (interface{}, error) {
-	original, ok := in.(string)
-	if !ok {
-		return nil, fmt.Errorf("image path is not of type string but %T", in)
-	}
-
-	if !isImageMatched(original, p.ImageTag.Name) {
-		return original, nil
-	}
-	name, tag := split(original)
-	if p.ImageTag.NewName != "" {
-		name = p.ImageTag.NewName
-	}
-	if p.ImageTag.NewTag != "" {
-		tag = ":" + p.ImageTag.NewTag
-	}
-	if p.ImageTag.Digest != "" {
-		tag = "@" + p.ImageTag.Digest
-	}
-	return name + tag, nil
-}
-
-// findAndReplaceImage replaces the image name and
-// tags inside one object.
-// It searches the object for container session
-// then loops though all images inside containers
-// session, finds matched ones and update the
-// image name and tag name
-func (p *ImageTagTransformerPlugin) findAndReplaceImage(obj map[string]interface{}) error {
-	paths := []string{"containers", "initContainers"}
-	updated := false
-	for _, path := range paths {
-		containers, found := obj[path]
-		if found {
-			if _, err := p.updateContainers(containers); err != nil {
-				return err
-			}
-			updated = true
-		}
-	}
-	if !updated {
-		return p.findContainers(obj)
-	}
-	return nil
-}
-
-func (p *ImageTagTransformerPlugin) updateContainers(in interface{}) (interface{}, error) {
-	containers, ok := in.([]interface{})
-	if !ok {
-		return nil, fmt.Errorf(
-			"containers path is not of type []interface{} but %T", in)
-	}
-	for i := range containers {
-		container := containers[i].(map[string]interface{})
-		containerImage, found := container["image"]
-		if !found {
-			continue
-		}
-		imageName := containerImage.(string)
-		if isImageMatched(imageName, p.ImageTag.Name) {
-			newImage, err := p.mutateImage(imageName)
-			if err != nil {
-				return nil, err
-			}
-			container["image"] = newImage
-		}
-	}
-	return containers, nil
-}
-
-func (p *ImageTagTransformerPlugin) findContainers(obj map[string]interface{}) error {
-	for key := range obj {
-		switch typedV := obj[key].(type) {
-		case map[string]interface{}:
-			err := p.findAndReplaceImage(typedV)
-			if err != nil {
-				return err
-			}
-		case []interface{}:
-			for i := range typedV {
-				item := typedV[i]
-				typedItem, ok := item.(map[string]interface{})
-				if ok {
-					err := p.findAndReplaceImage(typedItem)
-					if err != nil {
-						return err
-					}
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func isImageMatched(s, t string) bool {
-	// Tag values are limited to [a-zA-Z0-9_.-].
-	pattern, _ := regexp.Compile("^" + t + "(@sha256)?(:[a-zA-Z0-9_.-]*)?$")
-	return pattern.MatchString(s)
-}
-
-// split separates and returns the name and tag parts
-// from the image string using either colon `:` or at `@` separators.
-// Note that the returned tag keeps its separator.
-func split(imageName string) (name string, tag string) {
-	// check if image name contains a domain
-	// if domain is present, ignore domain and check for `:`
-	ic := -1
-	if slashIndex := strings.Index(imageName, "/"); slashIndex < 0 {
-		ic = strings.LastIndex(imageName, ":")
-	} else {
-		lastIc := strings.LastIndex(imageName[slashIndex:], ":")
-		// set ic only if `:` is present
-		if lastIc > 0 {
-			ic = slashIndex + lastIc
-		}
-	}
-	ia := strings.LastIndex(imageName, "@")
-	if ic < 0 && ia < 0 {
-		return imageName, ""
-	}
-
-	i := ic
-	if ia > 0 {
-		i = ia
-	}
-
-	name = imageName[:i]
-	tag = imageName[i:]
-	return
-}
-
-func NewImageTagTransformerPlugin() resmap.TransformerPlugin {
-	return &ImageTagTransformerPlugin{}
-}

api/builtins/labeltransformer.go

@@ -1,39 +0,0 @@
-// Code generated by pluginator on LabelTransformer; DO NOT EDIT.
-// pluginator {unknown  1970-01-01T00:00:00Z  }
-
-package builtins
-
-import (
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/transform"
-	"sigs.k8s.io/kustomize/api/types"
-	"sigs.k8s.io/yaml"
-)
-
-// Add the given labels to the given field specifications.
-type LabelTransformerPlugin struct {
-	Labels     map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
-	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
-}
-
-func (p *LabelTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-	p.Labels = nil
-	p.FieldSpecs = nil
-	return yaml.Unmarshal(c, p)
-}
-
-func (p *LabelTransformerPlugin) Transform(m resmap.ResMap) error {
-	t, err := transform.NewMapTransformer(
-		p.FieldSpecs,
-		p.Labels,
-	)
-	if err != nil {
-		return err
-	}
-	return t.Transform(m)
-}
-
-func NewLabelTransformerPlugin() resmap.TransformerPlugin {
-	return &LabelTransformerPlugin{}
-}

api/builtins/legacyordertransformer.go

@@ -1,46 +0,0 @@
-// Code generated by pluginator on LegacyOrderTransformer; DO NOT EDIT.
-// pluginator {unknown  1970-01-01T00:00:00Z  }
-
-package builtins
-
-import (
-	"sort"
-
-	"github.com/pkg/errors"
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/resource"
-)
-
-// Sort the resources using an ordering defined in the Gvk class.
-// This puts cluster-wide basic resources with no
-// dependencies (like Namespace, StorageClass, etc.)
-// first, and resources with a high number of dependencies
-// (like ValidatingWebhookConfiguration) last.
-type LegacyOrderTransformerPlugin struct{}
-
-// Nothing needed for configuration.
-func (p *LegacyOrderTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-	return nil
-}
-
-func (p *LegacyOrderTransformerPlugin) Transform(m resmap.ResMap) (err error) {
-	resources := make([]*resource.Resource, m.Size())
-	ids := m.AllIds()
-	sort.Sort(resmap.IdSlice(ids))
-	for i, id := range ids {
-		resources[i], err = m.GetByCurrentId(id)
-		if err != nil {
-			return errors.Wrap(err, "expected match for sorting")
-		}
-	}
-	m.Clear()
-	for _, r := range resources {
-		m.Append(r)
-	}
-	return nil
-}
-
-func NewLegacyOrderTransformerPlugin() resmap.TransformerPlugin {
-	return &LegacyOrderTransformerPlugin{}
-}

api/builtins/namespacetransformer.go

@@ -1,131 +0,0 @@
-// Code generated by pluginator on NamespaceTransformer; DO NOT EDIT.
-// pluginator {unknown  1970-01-01T00:00:00Z  }
-
-package builtins
-
-import (
-	"fmt"
-
-	"sigs.k8s.io/kustomize/api/transform"
-
-	"sigs.k8s.io/kustomize/api/resid"
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/resource"
-	"sigs.k8s.io/kustomize/api/types"
-	"sigs.k8s.io/yaml"
-)
-
-// Change or set the namespace of non-cluster level resources.
-type NamespaceTransformerPlugin struct {
-	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
-	FieldSpecs       []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
-}
-
-func (p *NamespaceTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-	p.Namespace = ""
-	p.FieldSpecs = nil
-	return yaml.Unmarshal(c, p)
-}
-
-func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
-	if len(p.Namespace) == 0 {
-		return nil
-	}
-	for _, r := range m.Resources() {
-		if len(r.Map()) == 0 {
-			// Don't mutate empty objects?
-			continue
-		}
-
-		id := r.OrgId()
-		applicableFs := p.applicableFieldSpecs(id)
-
-		for _, fs := range applicableFs {
-			err := transform.MutateField(
-				r.Map(), fs.PathSlice(), fs.CreateIfNotPresent,
-				p.changeNamespace(r))
-			if err != nil {
-				return err
-			}
-		}
-
-		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
-		if len(matches) != 1 {
-			return fmt.Errorf("namespace tranformation produces ID conflict: %#v", matches)
-		}
-	}
-	return nil
-}
-
-const metaNamespace = "metadata/namespace"
-
-// Special casing metadata.namespace since
-// all objects have it, even "ClusterKind" objects
-// that don't exist in a namespace (the Namespace
-// object itself doesn't live in a namespace).
-func (p *NamespaceTransformerPlugin) applicableFieldSpecs(id resid.ResId) []types.FieldSpec {
-	var res []types.FieldSpec
-	for _, fs := range p.FieldSpecs {
-		if id.IsSelected(&fs.Gvk) && (fs.Path != metaNamespace || (fs.Path == metaNamespace && id.IsNamespaceableKind())) {
-			res = append(res, fs)
-		}
-	}
-	return res
-}
-
-func (p *NamespaceTransformerPlugin) changeNamespace(
-	referrer *resource.Resource) func(in interface{}) (interface{}, error) {
-	return func(in interface{}) (interface{}, error) {
-		switch in.(type) {
-		case string:
-			// will happen when the metadata/namespace
-			// value is replaced
-			return p.Namespace, nil
-		case []interface{}:
-			l, _ := in.([]interface{})
-			for idx, item := range l {
-				switch item.(type) {
-				case map[string]interface{}:
-					// Will happen when mutating the subjects
-					// field of ClusterRoleBinding and RoleBinding
-					inMap, _ := item.(map[string]interface{})
-					if _, ok := inMap["name"]; !ok {
-						continue
-					}
-					name, ok := inMap["name"].(string)
-					if !ok {
-						continue
-					}
-					// The only case we need to force the namespace
-					// if for the "service account". "default" is
-					// kind of hardcoded here for right now.
-					if name != "default" {
-						continue
-					}
-					inMap["namespace"] = p.Namespace
-					l[idx] = inMap
-				default:
-					// nothing to do for right now
-				}
-			}
-			return in, nil
-		case map[string]interface{}:
-			// Will happen if the createField=true
-			// when the namespace is added to the
-			// object
-			inMap := in.(map[string]interface{})
-			if len(inMap) == 0 {
-				return p.Namespace, nil
-			} else {
-				return in, nil
-			}
-		default:
-			return in, nil
-		}
-	}
-}
-
-func NewNamespaceTransformerPlugin() resmap.TransformerPlugin {
-	return &NamespaceTransformerPlugin{}
-}

api/builtins/prefixsuffixtransformer.go

@@ -1,123 +0,0 @@
-// Code generated by pluginator on PrefixSuffixTransformer; DO NOT EDIT.
-// pluginator {unknown  1970-01-01T00:00:00Z  }
-
-package builtins
-
-import (
-	"errors"
-	"fmt"
-
-	"sigs.k8s.io/kustomize/api/transform"
-	"sigs.k8s.io/kustomize/api/types"
-
-	"sigs.k8s.io/kustomize/api/resid"
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/yaml"
-)
-
-// Add the given prefix and suffix to the field.
-type PrefixSuffixTransformerPlugin struct {
-	Prefix     string            `json:"prefix,omitempty" yaml:"prefix,omitempty"`
-	Suffix     string            `json:"suffix,omitempty" yaml:"suffix,omitempty"`
-	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
-}
-
-// Not placed in a file yet due to lack of demand.
-var prefixSuffixFieldSpecsToSkip = []types.FieldSpec{
-	{
-		Gvk: resid.Gvk{Kind: "CustomResourceDefinition"},
-	},
-	{
-		Gvk: resid.Gvk{Group: "apiregistration.k8s.io", Kind: "APIService"},
-	},
-}
-
-func (p *PrefixSuffixTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-	p.Prefix = ""
-	p.Suffix = ""
-	p.FieldSpecs = nil
-	err = yaml.Unmarshal(c, p)
-	if err != nil {
-		return
-	}
-	if p.FieldSpecs == nil {
-		return errors.New("fieldSpecs is not expected to be nil")
-	}
-	return
-}
-
-func (p *PrefixSuffixTransformerPlugin) Transform(m resmap.ResMap) error {
-
-	// Even if both the Prefix and Suffix are empty we want
-	// to proceed with the transformation. This allows to add contextual
-	// information to the resources (AddNamePrefix and AddNameSuffix).
-
-	for _, r := range m.Resources() {
-		if p.shouldSkip(r.OrgId()) {
-			// Don't change the actual definition
-			// of a CRD.
-			continue
-		}
-		id := r.OrgId()
-		// current default configuration contains
-		// only one entry: "metadata/name" with no GVK
-		for _, path := range p.FieldSpecs {
-			if !id.IsSelected(&path.Gvk) {
-				// With the currrent default configuration,
-				// because no Gvk is specified, so a wild
-				// card
-				continue
-			}
-
-			if smellsLikeANameChange(&path) {
-				// "metadata/name" is the only field.
-				// this will add a prefix and a suffix
-				// to the resource even if those are
-				// empty
-				r.AddNamePrefix(p.Prefix)
-				r.AddNameSuffix(p.Suffix)
-			}
-
-			// the addPrefixSuffix method will not
-			// change the name if both the prefix and suffix
-			// are empty.
-			err := transform.MutateField(
-				r.Map(),
-				path.PathSlice(),
-				path.CreateIfNotPresent,
-				p.addPrefixSuffix)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func smellsLikeANameChange(fs *types.FieldSpec) bool {
-	return fs.Path == "metadata/name"
-}
-
-func (p *PrefixSuffixTransformerPlugin) shouldSkip(
-	id resid.ResId) bool {
-	for _, path := range prefixSuffixFieldSpecsToSkip {
-		if id.IsSelected(&path.Gvk) {
-			return true
-		}
-	}
-	return false
-}
-
-func (p *PrefixSuffixTransformerPlugin) addPrefixSuffix(
-	in interface{}) (interface{}, error) {
-	s, ok := in.(string)
-	if !ok {
-		return nil, fmt.Errorf("%#v is expected to be %T", in, s)
-	}
-	return fmt.Sprintf("%s%s%s", p.Prefix, s, p.Suffix), nil
-}
-
-func NewPrefixSuffixTransformerPlugin() resmap.TransformerPlugin {
-	return &PrefixSuffixTransformerPlugin{}
-}

api/builtins/replicacounttransformer.go

@@ -1,89 +0,0 @@
-// Code generated by pluginator on ReplicaCountTransformer; DO NOT EDIT.
-// pluginator {unknown  1970-01-01T00:00:00Z  }
-
-package builtins
-
-import (
-	"fmt"
-
-	"sigs.k8s.io/kustomize/api/transform"
-
-	"sigs.k8s.io/kustomize/api/resid"
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/types"
-	"sigs.k8s.io/yaml"
-)
-
-// Find matching replicas declarations and replace the count.
-// Eases the kustomization configuration of replica changes.
-type ReplicaCountTransformerPlugin struct {
-	Replica    types.Replica     `json:"replica,omitempty" yaml:"replica,omitempty"`
-	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
-}
-
-func (p *ReplicaCountTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-
-	p.Replica = types.Replica{}
-	p.FieldSpecs = nil
-	return yaml.Unmarshal(c, p)
-}
-
-func (p *ReplicaCountTransformerPlugin) Transform(m resmap.ResMap) error {
-	found := false
-	for i, replicaSpec := range p.FieldSpecs {
-		matcher := p.createMatcher(i)
-		matchOriginal := m.GetMatchingResourcesByOriginalId(matcher)
-		matchCurrent := m.GetMatchingResourcesByCurrentId(matcher)
-
-		for _, res := range append(matchOriginal, matchCurrent...) {
-			found = true
-			err := transform.MutateField(
-				res.Map(), replicaSpec.PathSlice(),
-				replicaSpec.CreateIfNotPresent, p.addReplicas)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	if !found {
-		gvks := make([]string, len(p.FieldSpecs))
-		for i, replicaSpec := range p.FieldSpecs {
-			gvks[i] = replicaSpec.Gvk.String()
-		}
-		return fmt.Errorf("resource with name %s does not match a config with the following GVK %v",
-			p.Replica.Name, gvks)
-	}
-
-	return nil
-}
-
-// Match Replica.Name and FieldSpec
-func (p *ReplicaCountTransformerPlugin) createMatcher(i int) resmap.IdMatcher {
-	return func(r resid.ResId) bool {
-		return r.Name == p.Replica.Name &&
-			r.Gvk.IsSelected(&p.FieldSpecs[i].Gvk)
-	}
-}
-
-func (p *ReplicaCountTransformerPlugin) addReplicas(in interface{}) (interface{}, error) {
-	switch m := in.(type) {
-	case int64:
-		// Was already in the field.
-	case map[string]interface{}:
-		if len(m) != 0 {
-			// A map was already in the replicas field, don't want to
-			// discard this data silently.
-			return nil, fmt.Errorf("%#v is expected to be %T", in, m)
-		}
-		// Just got added, default type is map, but we can return anything.
-	default:
-		return nil, fmt.Errorf("%#v is expected to be %T", in, m)
-	}
-	return p.Replica.Count, nil
-}
-
-func NewReplicaCountTransformerPlugin() resmap.TransformerPlugin {
-	return &ReplicaCountTransformerPlugin{}
-}

api/filesys/file.go

@@ -6,34 +6,8 @@ package filesys
 import (
 	"io"
 	"os"
-	"time"
 )
 
-var _ os.FileInfo = &fileInfo{}
-
-// fileInfo implements os.FileInfo for a fileInMemory instance.
-type fileInfo struct {
-	*fileInMemory
-}
-
-// Name returns the name of the file
-func (fi *fileInfo) Name() string { return fi.name }
-
-// Size returns the size of the file
-func (fi *fileInfo) Size() int64 { return int64(len(fi.content)) }
-
-// Mode returns the file mode
-func (fi *fileInfo) Mode() os.FileMode { return 0777 }
-
-// ModTime returns the modification time
-func (fi *fileInfo) ModTime() time.Time { return time.Time{} }
-
-// IsDir returns if it is a directory
-func (fi *fileInfo) IsDir() bool { return fi.dir }
-
-// Sys should return underlying data source, but it now returns nil
-func (fi *fileInfo) Sys() interface{} { return nil }
-
 // File groups the basic os.File methods.
 type File interface {
 	io.ReadWriteCloser

api/filesys/fileinfo.go

@@ -0,0 +1,34 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"os"
+	"time"
+)
+
+var _ os.FileInfo = fileInfo{}
+
+// fileInfo implements os.FileInfo for a fileInMemory instance.
+type fileInfo struct {
+	node *fsNode
+}
+
+// Name returns the name of the file
+func (fi fileInfo) Name() string { return fi.node.Name() }
+
+// Size returns the size of the file
+func (fi fileInfo) Size() int64 { return fi.node.Size() }
+
+// Mode returns the file mode
+func (fi fileInfo) Mode() os.FileMode { return 0777 }
+
+// ModTime returns a bogus time
+func (fi fileInfo) ModTime() time.Time { return time.Time{} }
+
+// IsDir returns true if it is a directory
+func (fi fileInfo) IsDir() bool { return fi.node.isNodeADir() }
+
+// Sys should return underlying data source, but it now returns nil
+func (fi fileInfo) Sys() interface{} { return nil }

api/filesys/fileinmemory.go

@@ -1,56 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package filesys
-
-import (
-	"bytes"
-	"os"
-)
-
-var _ File = &fileInMemory{}
-
-// fileInMemory implements File in-memory for tests.
-type fileInMemory struct {
-	name    string
-	content []byte
-	dir     bool
-	open    bool
-}
-
-// makeDir makes a fake directory.
-func makeDir(name string) *fileInMemory {
-	return &fileInMemory{name: name, dir: true}
-}
-
-// Close marks the fake file closed.
-func (f *fileInMemory) Close() error {
-	f.open = false
-	return nil
-}
-
-// Read never fails, and doesn't mutate p.
-func (f *fileInMemory) Read(p []byte) (n int, err error) {
-	return len(p), nil
-}
-
-// Write saves the contents of the argument to memory.
-func (f *fileInMemory) Write(p []byte) (n int, err error) {
-	f.content = p
-	return len(p), nil
-}
-
-// ContentMatches returns true if v matches fake file's content.
-func (f *fileInMemory) ContentMatches(v []byte) bool {
-	return bytes.Equal(v, f.content)
-}
-
-// GetContent the content of a fake file.
-func (f *fileInMemory) GetContent() []byte {
-	return f.content
-}
-
-// Stat returns nil.
-func (f *fileInMemory) Stat() (os.FileInfo, error) {
-	return nil, nil
-}

api/filesys/filesystem.go

@@ -8,13 +8,20 @@ import (
 	"path/filepath"
 )
 
+const (
+	Separator = string(filepath.Separator)
+	SelfDir   = "."
+	ParentDir = ".."
+)
+
 // FileSystem groups basic os filesystem methods.
+// It's supposed be functional subset of https://golang.org/pkg/os
 type FileSystem interface {
 	// Create a file.
-	Create(name string) (File, error)
+	Create(path string) (File, error)
 	// MkDir makes a directory.
 	Mkdir(path string) error
-	// MkDir makes a directory path, creating intervening directories.
+	// MkDirAll makes a directory path, creating intervening directories.
 	MkdirAll(path string) error
 	// RemoveAll removes path and any children it contains.
 	RemoveAll(path string) error
@@ -30,11 +37,13 @@ type FileSystem interface {
 	CleanedAbs(path string) (ConfirmedDir, string, error)
 	// Exists is true if the path exists in the file system.
 	Exists(path string) bool
-	// Glob returns the list of matching files
+	// Glob returns the list of matching files,
+	// emulating https://golang.org/pkg/path/filepath/#Glob
 	Glob(pattern string) ([]string, error)
 	// ReadFile returns the contents of the file at the given path.
 	ReadFile(path string) ([]byte, error)
-	// WriteFile writes the data to a file at the given path.
+	// WriteFile writes the data to a file at the given path,
+	// overwriting anything that's already there.
 	WriteFile(path string, data []byte) error
 	// Walk walks the file system with the given WalkFunc.
 	Walk(path string, walkFn filepath.WalkFunc) error

api/filesys/fsinmemory.go

@@ -1,223 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package filesys
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-)
-
-var _ FileSystem = &fsInMemory{}
-
-// fsInMemory implements FileSystem using a in-memory filesystem
-// primarily for use in tests.
-type fsInMemory struct {
-	m map[string]*fileInMemory
-}
-
-// MakeFsInMemory returns an instance of fsInMemory with no files in it.
-func MakeFsInMemory() FileSystem {
-	result := &fsInMemory{m: map[string]*fileInMemory{}}
-	result.Mkdir(separator)
-	return result
-}
-
-const (
-	separator = string(filepath.Separator)
-	doubleSep = separator + separator
-)
-
-// Create assures a fake file appears in the in-memory file system.
-func (fs *fsInMemory) Create(name string) (File, error) {
-	f := &fileInMemory{}
-	f.open = true
-	fs.m[name] = f
-	return fs.m[name], nil
-}
-
-// Mkdir assures a fake directory appears in the in-memory file system.
-func (fs *fsInMemory) Mkdir(name string) error {
-	fs.m[name] = makeDir(name)
-	return nil
-}
-
-// MkdirAll delegates to Mkdir
-func (fs *fsInMemory) MkdirAll(name string) error {
-	return fs.Mkdir(name)
-}
-
-// RemoveAll presumably does rm -r on a path.
-// There's no error.
-func (fs *fsInMemory) RemoveAll(name string) error {
-	var toRemove []string
-	for k := range fs.m {
-		if strings.HasPrefix(k, name) {
-			toRemove = append(toRemove, k)
-		}
-	}
-	for _, k := range toRemove {
-		delete(fs.m, k)
-	}
-	return nil
-}
-
-// Open returns a fake file in the open state.
-func (fs *fsInMemory) Open(name string) (File, error) {
-	if _, found := fs.m[name]; !found {
-		return nil, fmt.Errorf("file %q cannot be opened", name)
-	}
-	return fs.m[name], nil
-}
-
-// CleanedAbs cannot fail.
-func (fs *fsInMemory) CleanedAbs(path string) (ConfirmedDir, string, error) {
-	if fs.IsDir(path) {
-		return ConfirmedDir(path), "", nil
-	}
-	d := filepath.Dir(path)
-	if d == path {
-		return ConfirmedDir(d), "", nil
-	}
-	return ConfirmedDir(d), filepath.Base(path), nil
-}
-
-// Exists returns true if file is known.
-func (fs *fsInMemory) Exists(name string) bool {
-	_, found := fs.m[name]
-	return found
-}
-
-// Glob returns the list of matching files
-func (fs *fsInMemory) Glob(pattern string) ([]string, error) {
-	var result []string
-	for p := range fs.m {
-		if fs.pathMatch(p, pattern) {
-			result = append(result, p)
-		}
-	}
-	sort.Strings(result)
-	return result, nil
-}
-
-// IsDir returns true if the file exists and is a directory.
-func (fs *fsInMemory) IsDir(name string) bool {
-	f, found := fs.m[name]
-	if found && f.dir {
-		return true
-	}
-	if !strings.HasSuffix(name, separator) {
-		name = name + separator
-	}
-	for k := range fs.m {
-		if strings.HasPrefix(k, name) {
-			return true
-		}
-	}
-	return false
-}
-
-// ReadFile always returns an empty bytes and error depending on content of m.
-func (fs *fsInMemory) ReadFile(name string) ([]byte, error) {
-	if ff, found := fs.m[name]; found {
-		return ff.content, nil
-	}
-	return nil, fmt.Errorf("cannot read file %q", name)
-}
-
-// WriteFile always succeeds and does nothing.
-func (fs *fsInMemory) WriteFile(name string, c []byte) error {
-	ff := &fileInMemory{}
-	ff.Write(c)
-	fs.m[name] = ff
-	return nil
-}
-
-// Walk implements filepath.Walk using the fake filesystem.
-func (fs *fsInMemory) Walk(path string, walkFn filepath.WalkFunc) error {
-	info, err := fs.lstat(path)
-	if err != nil {
-		err = walkFn(path, info, err)
-	} else {
-		err = fs.walk(path, info, walkFn)
-	}
-	if err == filepath.SkipDir {
-		return nil
-	}
-	return err
-}
-
-func (fs *fsInMemory) pathMatch(path, pattern string) bool {
-	match, _ := filepath.Match(pattern, path)
-	return match
-}
-
-func (fs *fsInMemory) lstat(path string) (*fileInfo, error) {
-	f, found := fs.m[path]
-	if !found {
-		return nil, os.ErrNotExist
-	}
-	return &fileInfo{f}, nil
-}
-
-func (fs *fsInMemory) join(elem ...string) string {
-	for i, e := range elem {
-		if e != "" {
-			return strings.Replace(
-				strings.Join(elem[i:], separator), doubleSep, separator, -1)
-		}
-	}
-	return ""
-}
-
-func (fs *fsInMemory) readDirNames(path string) []string {
-	var names []string
-	if !strings.HasSuffix(path, separator) {
-		path += separator
-	}
-	pathSegments := strings.Count(path, separator)
-	for name := range fs.m {
-		if name == path {
-			continue
-		}
-		if strings.Count(name, separator) > pathSegments {
-			continue
-		}
-		if strings.HasPrefix(name, path) {
-			names = append(names, filepath.Base(name))
-		}
-	}
-	sort.Strings(names)
-	return names
-}
-
-func (fs *fsInMemory) walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
-	if !info.IsDir() {
-		return walkFn(path, info, nil)
-	}
-
-	names := fs.readDirNames(path)
-	if err := walkFn(path, info, nil); err != nil {
-		return err
-	}
-	for _, name := range names {
-		filename := fs.join(path, name)
-		fileInfo, err := fs.lstat(filename)
-		if err != nil {
-			if err := walkFn(filename, fileInfo, os.ErrNotExist); err != nil && err != filepath.SkipDir {
-				return err
-			}
-		} else {
-			err = fs.walk(filename, fileInfo, walkFn)
-			if err != nil {
-				if !fileInfo.IsDir() || err != filepath.SkipDir {
-					return err
-				}
-			}
-		}
-	}
-	return nil
-}

api/filesys/fsinmemory_test.go

@@ -1,149 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package filesys_test
-
-import (
-	"bytes"
-	"reflect"
-	"testing"
-
-	. "sigs.k8s.io/kustomize/api/filesys"
-)
-
-func TestExists(t *testing.T) {
-	fSys := MakeFsInMemory()
-	if fSys.Exists("foo") {
-		t.Fatalf("expected no foo")
-	}
-	fSys.Mkdir("/")
-	if !fSys.IsDir("/") {
-		t.Fatalf("expected dir at /")
-	}
-}
-
-func TestIsDir(t *testing.T) {
-	fSys := MakeFsInMemory()
-	expectedName := "my-dir"
-	err := fSys.Mkdir(expectedName)
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	shouldExist(t, fSys, expectedName)
-	if !fSys.IsDir(expectedName) {
-		t.Fatalf(expectedName + " should be a dir")
-	}
-}
-
-func shouldExist(t *testing.T, fSys FileSystem, name string) {
-	if !fSys.Exists(name) {
-		t.Fatalf(name + " should exist")
-	}
-}
-
-func shouldNotExist(t *testing.T, fSys FileSystem, name string) {
-	if fSys.Exists(name) {
-		t.Fatalf(name + " should not exist")
-	}
-}
-
-func TestRemoveAll(t *testing.T) {
-	fSys := MakeFsInMemory()
-	fSys.WriteFile("/foo/project/file.yaml", []byte("Unused"))
-	fSys.WriteFile("/foo/project/subdir/file.yaml", []byte("Unused"))
-	fSys.WriteFile("/foo/apple/subdir/file.yaml", []byte("Unused"))
-	shouldExist(t, fSys, "/foo/project/file.yaml")
-	shouldExist(t, fSys, "/foo/project/subdir/file.yaml")
-	shouldExist(t, fSys, "/foo/apple/subdir/file.yaml")
-	err := fSys.RemoveAll("/foo/project")
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	shouldNotExist(t, fSys, "/foo/project/file.yaml")
-	shouldNotExist(t, fSys, "/foo/project/subdir/file.yaml")
-	shouldExist(t, fSys, "/foo/apple/subdir/file.yaml")
-}
-
-func TestIsDirDeeper(t *testing.T) {
-	fSys := MakeFsInMemory()
-	fSys.WriteFile("/foo/project/file.yaml", []byte("Unused"))
-	fSys.WriteFile("/foo/project/subdir/file.yaml", []byte("Unused"))
-	if !fSys.IsDir("/") {
-		t.Fatalf("/ should be a dir")
-	}
-	if !fSys.IsDir("/foo") {
-		t.Fatalf("/foo should be a dir")
-	}
-	if !fSys.IsDir("/foo/project") {
-		t.Fatalf("/foo/project should be a dir")
-	}
-	if fSys.IsDir("/fo") {
-		t.Fatalf("/fo should not be a dir")
-	}
-	if fSys.IsDir("/x") {
-		t.Fatalf("/x should not be a dir")
-	}
-}
-
-func TestCreate(t *testing.T) {
-	fSys := MakeFsInMemory()
-	f, err := fSys.Create("foo")
-	if f == nil {
-		t.Fatalf("expected file")
-	}
-	if err != nil {
-		t.Fatalf("unexpected error")
-	}
-	shouldExist(t, fSys, "foo")
-}
-
-func TestReadFile(t *testing.T) {
-	fSys := MakeFsInMemory()
-	f, err := fSys.Create("foo")
-	if f == nil {
-		t.Fatalf("expected file")
-	}
-	if err != nil {
-		t.Fatalf("unexpected error")
-	}
-	content, err := fSys.ReadFile("foo")
-	if len(content) != 0 {
-		t.Fatalf("expected no content")
-	}
-	if err != nil {
-		t.Fatalf("expected no error")
-	}
-}
-
-func TestWriteFile(t *testing.T) {
-	fSys := MakeFsInMemory()
-	c := []byte("heybuddy")
-	err := fSys.WriteFile("foo", c)
-	if err != nil {
-		t.Fatalf("expected no error")
-	}
-	content, err := fSys.ReadFile("foo")
-	if err != nil {
-		t.Fatalf("expected read to work: %v", err)
-	}
-	if bytes.Compare(c, content) != 0 {
-		t.Fatalf("incorrect content: %v", content)
-	}
-}
-
-func TestGlob(t *testing.T) {
-	fSys := MakeFsInMemory()
-	fSys.Create("dir/foo")
-	fSys.Create("dir/bar")
-	files, err := fSys.Glob("dir/*")
-	if err != nil {
-		t.Fatalf("expected no error")
-	}
-	expected := []string{
-		"dir/bar",
-		"dir/foo",
-	}
-	if !reflect.DeepEqual(files, expected) {
-		t.Fatalf("incorrect files found by glob: %v", files)
-	}
-}

api/filesys/fsnode.go

@@ -0,0 +1,557 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var _ File = &fsNode{}
+var _ FileSystem = &fsNode{}
+
+// fsNode is either a file or a directory.
+type fsNode struct {
+	// What node owns me?
+	parent *fsNode
+
+	// Value to return as the Name() when the
+	// parent is nil.
+	nilParentName string
+
+	// A directory mapping names to nodes.
+	// If dir is nil, then self node is a file.
+	// If dir is non-nil, then self node is a directory,
+	// albeit possibly an empty directory.
+	dir map[string]*fsNode
+
+	// if this node is a file, this is the content.
+	content []byte
+
+	// if this node is a file, this tracks whether or
+	// not it is "open".
+	open bool
+}
+
+// MakeEmptyDirInMemory returns an empty directory.
+// The paths of nodes in this object will never
+// report a leading Separator, meaning they
+// aren't "absolute" in the sense defined by
+// https://golang.org/pkg/path/filepath/#IsAbs.
+func MakeEmptyDirInMemory() *fsNode {
+	return &fsNode{
+		dir: make(map[string]*fsNode),
+	}
+}
+
+// MakeFsInMemory returns an empty 'file system'.
+// The paths of nodes in this object will always
+// report a leading Separator, meaning they
+// are "absolute" in the sense defined by
+// https://golang.org/pkg/path/filepath/#IsAbs.
+// This is a relevant difference when using Walk,
+// Glob, Match, etc.
+func MakeFsInMemory() FileSystem {
+	return &fsNode{
+		nilParentName: Separator,
+		dir:           make(map[string]*fsNode),
+	}
+}
+
+// Name returns the name of the node.
+func (n *fsNode) Name() string {
+	if n.parent == nil {
+		// Unable to lookup name in parent.
+		return n.nilParentName
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir")
+	}
+	for key, value := range n.parent.dir {
+		if value == n {
+			return key
+		}
+	}
+	log.Fatal("unable to find fsNode name")
+	return ""
+}
+
+// Path returns the full path to the node.
+func (n *fsNode) Path() string {
+	if n.parent == nil {
+		return n.nilParentName
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir, structural error")
+	}
+	return filepath.Join(n.parent.Path(), n.Name())
+}
+
+// mySplit trims trailing separators from the directory
+// result of filepath.Split.
+func mySplit(s string) (string, string) {
+	dName, fName := filepath.Split(s)
+	return StripTrailingSeps(dName), fName
+}
+
+func (n *fsNode) addFile(name string, c []byte) (result *fsNode, err error) {
+	parent := n
+	dName, fileName := mySplit(name)
+	if dName != "" {
+		parent, err = parent.addDir(dName)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if !isLegalFileNameForCreation(fileName) {
+		return nil, fmt.Errorf(
+			"illegal name '%s' in file creation", fileName)
+	}
+	result, ok := parent.dir[fileName]
+	if ok {
+		// File already exists; overwrite it.
+		result.content = c
+		return result, nil
+	}
+	result = &fsNode{
+		content: c,
+		parent:  parent,
+	}
+	parent.dir[fileName] = result
+	return result, nil
+}
+
+// Create implements FileSystem.
+// Create makes an empty file.
+func (n *fsNode) Create(path string) (result File, err error) {
+	return n.AddFile(path, []byte{})
+}
+
+// WriteFile implements FileSystem.
+func (n *fsNode) WriteFile(path string, d []byte) error {
+	_, err := n.AddFile(path, d)
+	return err
+}
+
+// AddFile adds a file and any necessary containing
+// directories to the node.
+func (n *fsNode) AddFile(
+	name string, c []byte) (result *fsNode, err error) {
+	if n.dir == nil {
+		return nil, fmt.Errorf(
+			"cannot add a file to a non-directory '%s'", n.Name())
+	}
+	return n.addFile(cleanQueryPath(name), c)
+}
+
+func (n *fsNode) addDir(path string) (result *fsNode, err error) {
+	parent := n
+	dName, subDirName := mySplit(path)
+	if dName != "" {
+		parent, err = n.addDir(dName)
+		if err != nil {
+			return nil, err
+		}
+	}
+	switch subDirName {
+	case "", SelfDir:
+		return n, nil
+	case ParentDir:
+		if n.parent == nil {
+			return nil, fmt.Errorf(
+				"cannot add a directory above '%s'", n.Path())
+		}
+		return n.parent, nil
+	default:
+		if !isLegalFileNameForCreation(subDirName) {
+			return nil, fmt.Errorf(
+				"illegal name '%s' in directory creation", subDirName)
+		}
+		result, ok := parent.dir[subDirName]
+		if ok {
+			if result.isNodeADir() {
+				// it's already there.
+				return result, nil
+			}
+			return nil, fmt.Errorf(
+				"cannot make dir '%s'; a file of that name already exists in '%s'",
+				subDirName, parent.Name())
+		}
+		result = &fsNode{
+			dir:    make(map[string]*fsNode),
+			parent: parent,
+		}
+		parent.dir[subDirName] = result
+		return result, nil
+	}
+}
+
+// Mkdir implements FileSystem.
+// Mkdir creates a directory.
+func (n *fsNode) Mkdir(path string) error {
+	_, err := n.AddDir(path)
+	return err
+}
+
+// MkdirAll implements FileSystem.
+// MkdirAll creates a directory.
+func (n *fsNode) MkdirAll(path string) error {
+	_, err := n.AddDir(path)
+	return err
+}
+
+// AddDir adds a directory to the node, not complaining
+// if it is already there.
+func (n *fsNode) AddDir(path string) (result *fsNode, err error) {
+	if n.dir == nil {
+		return nil, fmt.Errorf(
+			"cannot add a directory to file node '%s'", n.Name())
+	}
+	return n.addDir(cleanQueryPath(path))
+}
+
+// CleanedAbs implements FileSystem.
+func (n *fsNode) CleanedAbs(path string) (ConfirmedDir, string, error) {
+	node, err := n.Find(path)
+	if err != nil {
+		return "", "", errors.Wrap(err, "unable to clean")
+	}
+	if node == nil {
+		return "", "", fmt.Errorf("'%s' doesn't exist", path)
+	}
+	if node.isNodeADir() {
+		return ConfirmedDir(node.Path()), "", nil
+	}
+	return ConfirmedDir(node.parent.Path()), node.Name(), nil
+}
+
+// Exists implements FileSystem.
+// Exists returns true if the path exists.
+func (n *fsNode) Exists(path string) bool {
+	if !n.isNodeADir() {
+		return n.Name() == path
+	}
+	result, err := n.Find(path)
+	if err != nil {
+		return false
+	}
+	return result != nil
+}
+
+func cleanQueryPath(path string) string {
+	// Always ignore leading separator?
+	// Remember that filepath.Clean returns "." if
+	// given an empty string argument.
+	return filepath.Clean(StripLeadingSeps(path))
+}
+
+// Find finds the given node, else nil if not found.
+// Return error on structural/argument errors.
+func (n *fsNode) Find(path string) (*fsNode, error) {
+	if !n.isNodeADir() {
+		return nil, fmt.Errorf("can only find inside a dir")
+	}
+	if path == "" {
+		// Special case; check *before* cleaning and *before*
+		// comparison to nilParentName.
+		return nil, nil
+	}
+	if (n.parent == nil && path == n.nilParentName) || path == SelfDir {
+		// Special case
+		return n, nil
+	}
+	return n.findIt(cleanQueryPath(path))
+}
+
+func (n *fsNode) findIt(path string) (result *fsNode, err error) {
+	parent := n
+	dName, item := mySplit(path)
+	if dName != "" {
+		parent, err = n.findIt(dName)
+		if err != nil {
+			return nil, err
+		}
+		if parent == nil {
+			// all done, target doesn't exist.
+			return nil, nil
+		}
+	}
+	if !parent.isNodeADir() {
+		return nil, fmt.Errorf("'%s' is not a directory", parent.Path())
+	}
+	return parent.dir[item], nil
+}
+
+// RemoveAll implements FileSystem.
+// RemoveAll removes an item and everything it contains.
+func (n *fsNode) RemoveAll(path string) error {
+	result, err := n.Find(path)
+	if err != nil {
+		return err
+	}
+	if result == nil {
+		return fmt.Errorf("cannot find '%s' to remove it", path)
+	}
+	return result.Remove()
+}
+
+// Remove drop the node, and everything it contains, from its parent.
+func (n *fsNode) Remove() error {
+	if n.parent == nil {
+		return fmt.Errorf("cannot remove a root node")
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir")
+	}
+	for key, value := range n.parent.dir {
+		if value == n {
+			delete(n.parent.dir, key)
+			return nil
+		}
+	}
+	log.Fatal("unable to find self in parent")
+	return nil
+}
+
+// isNodeADir returns true if the node is a directory.
+// Cannot collide with the poorly named "IsDir".
+func (n *fsNode) isNodeADir() bool {
+	return n.dir != nil
+}
+
+// IsDir implements FileSystem.
+// IsDir returns true if the argument resolves
+// to a directory rooted at the node.
+func (n *fsNode) IsDir(path string) bool {
+	result, err := n.Find(path)
+	if err != nil || result == nil {
+		return false
+	}
+	return result.isNodeADir()
+}
+
+// Size returns the size of the node.
+func (n *fsNode) Size() int64 {
+	if n.isNodeADir() {
+		return int64(len(n.dir))
+	}
+	return int64(len(n.content))
+}
+
+// Open implements FileSystem.
+// Open opens the node for reading (just marks it).
+func (n *fsNode) Open(path string) (File, error) {
+	result, err := n.Find(path)
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, fmt.Errorf("cannot find '%s' to open it", path)
+	}
+	result.open = true
+	return result, nil
+}
+
+// Close marks the node closed.
+func (n *fsNode) Close() error {
+	n.open = false
+	return nil
+}
+
+// ReadFile implements FileSystem.
+func (n *fsNode) ReadFile(path string) (c []byte, err error) {
+	result, err := n.Find(path)
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, fmt.Errorf("cannot find '%s' to read it", path)
+	}
+	c = make([]byte, len(result.content))
+	_, err = result.Read(c)
+	return c, err
+}
+
+// Read returns the content of the file node.
+func (n *fsNode) Read(d []byte) (c int, err error) {
+	if n.isNodeADir() {
+		return 0, fmt.Errorf(
+			"cannot read content from non-file '%s'", n.Path())
+	}
+	return copy(d, n.content), nil
+}
+
+// Write saves the contents of the argument to the file node.
+func (n *fsNode) Write(p []byte) (c int, err error) {
+	if n.isNodeADir() {
+		return 0, fmt.Errorf(
+			"cannot write content to non-file '%s'", n.Path())
+	}
+	n.content = make([]byte, len(p))
+	return copy(n.content, p), nil
+}
+
+// ContentMatches returns true if v matches fake file's content.
+func (n *fsNode) ContentMatches(v []byte) bool {
+	return bytes.Equal(v, n.content)
+}
+
+// GetContent the content of a fake file.
+func (n *fsNode) GetContent() []byte {
+	return n.content
+}
+
+// Stat returns an instance of FileInfo.
+func (n *fsNode) Stat() (os.FileInfo, error) {
+	return fileInfo{node: n}, nil
+}
+
+// Walk implements FileSystem.
+func (n *fsNode) Walk(path string, walkFn filepath.WalkFunc) error {
+	result, err := n.Find(path)
+	if err != nil {
+		return err
+	}
+	if result == nil {
+		return fmt.Errorf("cannot find '%s' to walk it", path)
+	}
+	return result.WalkMe(walkFn)
+}
+
+// Walk runs the given walkFn on each node.
+func (n *fsNode) WalkMe(walkFn filepath.WalkFunc) error {
+	fi, err := n.Stat()
+	// always visit self first
+	err = walkFn(n.Path(), fi, err)
+	if !n.isNodeADir() {
+		// it's a file, so nothing more to do
+		return err
+	}
+	// process self as a directory
+	if err == filepath.SkipDir {
+		return nil
+	}
+	// Walk is supposed to visit in lexical order.
+	for _, k := range n.sortedDirEntries() {
+		if err := n.dir[k].WalkMe(walkFn); err != nil {
+			if err == filepath.SkipDir {
+				// stop processing this directory
+				break
+			}
+			// bail out completely
+			return err
+		}
+	}
+	return nil
+}
+
+func (n *fsNode) sortedDirEntries() []string {
+	keys := make([]string, len(n.dir))
+	i := 0
+	for k := range n.dir {
+		keys[i] = k
+		i++
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// FileCount returns a count of files.
+// Directories, empty or otherwise, not counted.
+func (n *fsNode) FileCount() int {
+	count := 0
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			count++
+		}
+		return nil
+	})
+	return count
+}
+
+func (n *fsNode) DebugPrint() {
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("err '%v' at path %q\n", err, path)
+			return nil
+		}
+		if info.IsDir() {
+			if info.Size() == 0 {
+				fmt.Println("empty dir: " + path)
+			}
+		} else {
+			fmt.Println("     file: " + path)
+		}
+		return nil
+	})
+}
+
+var legalFileNamePattern = regexp.MustCompile("^[a-zA-Z0-9-_.]+$")
+
+// This rules enforced here should be simpler and tighter
+// than what's allowed on a real OS.
+// Should be fine for testing or in-memory purposes.
+func isLegalFileNameForCreation(n string) bool {
+	if n == "" || n == SelfDir || !legalFileNamePattern.MatchString(n) {
+		return false
+	}
+	return !strings.Contains(n, ParentDir)
+}
+
+// RegExpGlob returns a list of file paths matching the regexp.
+// Excludes directories.
+func (n *fsNode) RegExpGlob(pattern string) ([]string, error) {
+	var result []string
+	var expression = regexp.MustCompile(pattern)
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			if expression.MatchString(path) {
+				result = append(result, path)
+			}
+		}
+		return nil
+	})
+	sort.Strings(result)
+	return result, nil
+}
+
+// Glob implements FileSystem.
+// Glob returns the list of file paths matching
+// per filepath.Match semantics, i.e. unlike RegExpGlob,
+// Match("foo/a*") will not match sub-sub directories of foo.
+// This is how /bin/ls behaves.
+func (n *fsNode) Glob(pattern string) ([]string, error) {
+	var result []string
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			match, err := filepath.Match(pattern, path)
+			if err != nil {
+				return err
+			}
+			if match {
+				result = append(result, path)
+			}
+		}
+		return nil
+	})
+	sort.Strings(result)
+	return result, nil
+}

api/filesys/fsnode_test.go

@@ -0,0 +1,788 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+const content = `
+Lorem ipsum dolor sit amet,
+consectetur adipiscing elit,
+sed do eiusmod tempor incididunt
+ut labore et dolore magna aliqua.
+`
+const shortContent = "hi"
+
+var topCases = []pathCase{
+	{
+		what:   "dotdot",
+		arg:    ParentDir,
+		errStr: "illegal name '..' in file creation",
+	},
+	{
+		what:   "empty",
+		arg:    "",
+		name:   "",
+		errStr: "illegal name '.' in file creation",
+	},
+	{
+		what: "simple",
+		arg:  "bob",
+		name: "bob",
+		path: "bob",
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "bob"),
+	},
+	{
+		what: "longer yet",
+		arg:  filepath.Join("longer", "foo", "bar", "beans", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "foo", "bar", "beans", "bob"),
+	},
+	{
+		what: "tricky",
+		arg:  filepath.Join("bob", ParentDir, "sally"),
+		name: "sally",
+		path: "sally",
+	},
+	{
+		what: "trickier",
+		arg:  filepath.Join("bob", "sally", ParentDir, ParentDir, "jean"),
+		name: "jean",
+		path: "jean",
+	},
+}
+
+func TestMakeEmptyDirInMemory(t *testing.T) {
+	n := MakeEmptyDirInMemory()
+	if !n.isNodeADir() {
+		t.Fatalf("not a directory")
+	}
+	if n.Size() != 0 {
+		t.Fatalf("unexpected size %d", n.Size())
+	}
+	if n.Name() != "" {
+		t.Fatalf("unexpected name '%s'", n.Name())
+	}
+	if n.Path() != "" {
+		t.Fatalf("unexpected path '%s'", n.Path())
+	}
+	runBasicOperations(
+		t, "MakeEmptyDirInMemory", false, topCases, n)
+}
+
+func TestMakeFsInMemory(t *testing.T) {
+	runBasicOperations(
+		t, "MakeFsInMemory", true, topCases, MakeFsInMemory())
+}
+
+//nolint:gocyclo
+func runBasicOperations(
+	t *testing.T, tName string, isFSysRooted bool,
+	cases []pathCase, fSys FileSystem) {
+	buff := make([]byte, 500)
+	for _, c := range cases {
+		err := fSys.WriteFile(c.arg, []byte(content))
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error writing to  '%s'!", c.what, c.arg)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if !fSys.Exists(c.path) {
+			t.Fatalf("%s; expect existence of '%s'", c.what, c.path)
+		}
+		stuff, err := fSys.ReadFile(c.path)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(stuff) != content {
+			t.Fatalf("%s; unexpected content '%s'", c.what, stuff)
+		}
+		f, err := fSys.Open(c.arg)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		fi, err := f.Stat()
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if fi.Name() != c.name {
+			t.Fatalf("%s; expected name '%s', got '%s'", c.what, c.name, fi.Name())
+		}
+		count, err := f.Read(buff)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(buff[:count]) != content {
+			t.Fatalf("%s; unexpected buff '%s'", c.what, buff)
+		}
+		count, err = f.Write([]byte(shortContent))
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if count != len(shortContent) {
+			t.Fatalf("%s; unexpected count: %d", c.what, len(shortContent))
+		}
+		stuff, err = fSys.ReadFile(c.path)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(stuff) != shortContent {
+			t.Fatalf("%s; unexpected content '%s'", c.what, stuff)
+		}
+	}
+
+	var actualPaths []string
+	var err error
+	prefix := ""
+	{
+		root := SelfDir
+		if isFSysRooted {
+			root = Separator
+			prefix = Separator
+		}
+		err = fSys.Walk(root, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				fmt.Printf("err '%v' at path %q\n", err, path)
+				return nil
+			}
+			if !info.IsDir() {
+				actualPaths = append(actualPaths, path)
+			}
+			return nil
+		})
+	}
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	var expectedPaths []string
+	for _, c := range cases {
+		if c.errStr == "" {
+			expectedPaths = append(expectedPaths, prefix+c.path)
+		}
+	}
+	sort.Strings(expectedPaths)
+	assertEqualStringSlices(t, expectedPaths, actualPaths, tName)
+}
+
+type pathCase struct {
+	what   string
+	arg    string
+	name   string
+	path   string
+	errStr string
+}
+
+func TestAddDir(t *testing.T) {
+	cases := []pathCase{
+		{
+			what:   "dotdot",
+			arg:    ParentDir,
+			errStr: "cannot add a directory above ''",
+		},
+		{
+			what: "empty",
+			arg:  "",
+			name: "",
+			path: "",
+		},
+		{
+			what: "simple",
+			arg:  "bob",
+			name: "bob",
+			path: "bob",
+		},
+		{
+			what: "longer",
+			arg:  filepath.Join("longer", "bob"),
+			name: "bob",
+			path: filepath.Join("longer", "bob"),
+		},
+		{
+			what: "longer yet",
+			arg:  filepath.Join("longer", "foo", "bar", "beans", "bob"),
+			name: "bob",
+			path: filepath.Join("longer", "foo", "bar", "beans", "bob"),
+		},
+		{
+			what: "tricky",
+			arg:  filepath.Join("bob", ParentDir, "sally"),
+			name: "sally",
+			path: "sally",
+		},
+		{
+			what: "trickier",
+			arg:  filepath.Join("bob", "sally", ParentDir, ParentDir, "jean"),
+			name: "jean",
+			path: "jean",
+		},
+	}
+	for _, c := range cases {
+		n := MakeEmptyDirInMemory()
+		f, err := n.AddDir(c.arg)
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error!", c.what)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf(
+					"%s; expected error with '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		checkNode(t, c.what, f, c.name, 0, true, c.path)
+		checkOsStat(t, c.what, f, f.Name(), 0, true)
+	}
+}
+
+var bagOfCases = []pathCase{
+	{
+		what:   "empty",
+		arg:    "",
+		errStr: "illegal name '.' in file creation",
+	},
+	{
+		what: "simple",
+		arg:  "bob",
+		name: "bob",
+		path: "bob",
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "bob"),
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "sally"),
+		name: "sally",
+		path: filepath.Join("longer", "sally"),
+	},
+	{
+		what: "even longer",
+		arg:  filepath.Join("longer", "than", "the", "other", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "than", "the", "other", "bob"),
+	},
+	{
+		what: "even longer",
+		arg:  filepath.Join("even", "much", "longer", "than", "the", "other", "bob"),
+		name: "bob",
+		path: filepath.Join("even", "much", "longer", "than", "the", "other", "bob"),
+	},
+}
+
+func TestAddFile(t *testing.T) {
+	n := MakeEmptyDirInMemory()
+	if n.FileCount() != 0 {
+		t.Fatalf("expected no files, got %d", n.FileCount())
+	}
+	expectedFileCount := 0
+	for _, c := range bagOfCases {
+		f, err := n.AddFile(c.arg, []byte(content))
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error!", c.what)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error %v", c.what, err)
+		}
+		checkNode(t, c.what, f, c.name, len(content), false, c.path)
+		checkOsStat(t, c.what, f, f.Name(), len(content), false)
+
+		result, err := n.Find(c.arg)
+		if err != nil {
+			t.Fatalf("%s; unexpected find error %v", c.what, err)
+		}
+		if result != f {
+			t.Fatalf("%s; unexpected find result %v", c.what, result)
+		}
+
+		result, err = n.Find(filepath.Join("longer", "bogus"))
+		if err != nil {
+			t.Fatalf("%s; unexpected find error %v", c.what, err)
+		}
+		if result != nil {
+			t.Fatalf("%s; unexpected find result %v", c.what, result)
+		}
+
+		expectedFileCount++
+		fc := n.FileCount()
+		if fc != expectedFileCount {
+			t.Fatalf("expected file count %d, got %d",
+				expectedFileCount, fc)
+		}
+	}
+}
+
+func checkNode(
+	t *testing.T, what string, f *fsNode, name string,
+	size int, isDir bool, path string) {
+	if f.isNodeADir() != isDir {
+		t.Fatalf("%s; unexpected isNodeADir = %v", what, f.isNodeADir())
+	}
+	if f.Size() != int64(size) {
+		t.Fatalf("%s; unexpected size %d", what, f.Size())
+	}
+	if name != f.Name() {
+		t.Fatalf("%s; expected name '%s', got '%s'", what, name, f.Name())
+	}
+	if path != f.Path() {
+		t.Fatalf("%s; expected path '%s', got '%s'", what, path, f.Path())
+	}
+}
+
+func checkOsStat(
+	t *testing.T, what string, f File, name string,
+	size int, isDir bool) {
+	info, err := f.Stat()
+	if err != nil {
+		t.Fatalf("%s; unexpected stat error %v", what, err)
+	}
+	if info.IsDir() != isDir {
+		t.Fatalf("%s; unexpected info.isNodeADir = %v", what, info.IsDir())
+	}
+	if info.Size() != int64(size) {
+		t.Fatalf("%s; unexpected info.size %d", what, info.Size())
+	}
+	if info.Name() != name {
+		t.Fatalf("%s; expected name '%s', got info.Name '%s'", what, name, info.Name())
+	}
+}
+
+var bunchOfFiles = []struct {
+	path     string
+	addAsDir bool
+}{
+	{
+		path: filepath.Join("b", "e", "a", "c", "g"),
+	},
+	{
+		path: filepath.Join("z", "r", "a", "b", "g"),
+	},
+	{
+		path: filepath.Join("b", "q", "a", "c", "g"),
+	},
+	{
+		path:     filepath.Join("b", "a", "a", "m", "g"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("b", "w"),
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "m"),
+	},
+	{
+		path: filepath.Join("b", "d", "z"),
+	},
+	{
+		path: filepath.Join("b", "d", "y"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "n"),
+	},
+	{
+		path: filepath.Join("b", "d", "x"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "o"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "m"),
+	},
+	{
+		path:     filepath.Join("b", "d", "a", "c", "i"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("x"),
+	},
+	{
+		path: filepath.Join("y"),
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "i", "beans"),
+	},
+	{
+		path:     filepath.Join("b", "d", "a", "c", "r", "w"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "u"),
+	},
+}
+
+func makeLoadedFileTree(t *testing.T) *fsNode {
+	n := MakeEmptyDirInMemory()
+	var err error
+	expectedFileCount := 0
+	for _, item := range bunchOfFiles {
+		if item.addAsDir {
+			_, err = n.AddDir(item.path)
+		} else {
+			_, err = n.AddFile(item.path, []byte(content))
+			expectedFileCount++
+		}
+		if err != nil {
+			t.Fatalf("unexpected error %v", err)
+		}
+	}
+	fc := n.FileCount()
+	if fc != expectedFileCount {
+		t.Fatalf("expected file count %d, got %d",
+			expectedFileCount, fc)
+	}
+	return n
+}
+
+func TestWalkMe(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	var actualPaths []string
+	err := n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("err '%v' at path %q\n", err, path)
+			return nil
+		}
+		if info.IsDir() {
+			if info.Name() == "ignore" {
+				return filepath.SkipDir
+			}
+		} else {
+			actualPaths = append(actualPaths, path)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	var expectedPaths []string
+	for _, c := range bunchOfFiles {
+		if !c.addAsDir && !strings.Contains(c.path, "ignore") {
+			expectedPaths = append(expectedPaths, c.path)
+		}
+	}
+	sort.Strings(expectedPaths)
+	assertEqualStringSlices(t, expectedPaths, actualPaths, "testWalkMe")
+}
+
+func TestRemove(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	orgCount := n.FileCount()
+
+	// Remove the "ignore" directory and everything below it.
+	path := filepath.Join("b", "d", "ignore")
+	result, err := n.Find(path)
+	if err != nil {
+		t.Fatalf("%s; unexpected error %v", path, err)
+	}
+	if result == nil {
+		t.Fatalf("%s; expected to find '%s'", path, path)
+	}
+	if !result.isNodeADir() {
+		t.Fatalf("%s; expected to find a directory", path)
+	}
+	err = result.Remove()
+	if err != nil {
+		t.Fatalf("%s; unable to remove: %v", path, err)
+	}
+	result, err = n.Find(path)
+	if err != nil {
+		// Just because it's gone doesn't mean error.
+		t.Fatalf("%s; unexpected error %v", path, err)
+	}
+	if result != nil {
+		t.Fatalf("%s; should not have been able to find '%s'", path, path)
+	}
+
+	// There were three files below "ignore".
+	orgCount -= 3
+
+	// Now drop one more for a total of four dropped.
+	result, _ = n.Find(filepath.Join("y"))
+	err = result.Remove()
+	if err != nil {
+		t.Fatalf("%s; unable to remove: %v", path, err)
+	}
+	orgCount -= 1
+
+	fc := n.FileCount()
+	if fc != orgCount {
+		t.Fatalf("expected file count %d, got %d",
+			orgCount, fc)
+	}
+}
+
+func TestExists(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	path := filepath.Join("b", "d", "a")
+	if !n.Exists(path) {
+		t.Fatalf("expected existence at %s", path)
+	}
+	if !n.IsDir(path) {
+		t.Fatalf("expected directory at %s", path)
+	}
+}
+
+func TestRegExpGlob(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	expected := []string{
+		filepath.Join("b", "d", "a", "c", "i", "beans"),
+		filepath.Join("b", "d", "a", "c", "m"),
+		filepath.Join("b", "d", "a", "c", "u"),
+		filepath.Join("b", "d", "ignore", "c", "m"),
+		filepath.Join("b", "d", "ignore", "c", "n"),
+		filepath.Join("b", "d", "ignore", "c", "o"),
+		filepath.Join("b", "d", "x"),
+		filepath.Join("b", "d", "y"),
+		filepath.Join("b", "d", "z"),
+	}
+	paths, err := n.RegExpGlob("b/d/*")
+	if err != nil {
+		t.Fatalf("glob error: %v", err)
+	}
+	assertEqualStringSlices(t, expected, paths, "glob test")
+}
+
+func TestGlob(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	expected := []string{
+		filepath.Join("b", "d", "x"),
+		filepath.Join("b", "d", "y"),
+		filepath.Join("b", "d", "z"),
+	}
+	paths, err := n.Glob("b/d/*")
+	if err != nil {
+		t.Fatalf("glob error: %v", err)
+	}
+	assertEqualStringSlices(t, expected, paths, "glob test")
+}
+
+func assertEqualStringSlices(t *testing.T, expected, actual []string, message string) {
+	if len(expected) != len(actual) {
+		t.Fatalf(
+			"%s; unequal sizes; len(expected)=%d, len(actual)=%d\n%+v\n%+v\n",
+			message, len(expected), len(actual), expected, actual)
+	}
+	for i := range expected {
+		if expected[i] != actual[i] {
+			t.Fatalf(
+				"%s; unequal entries; expected=%s, actual=%s",
+				message, expected[i], actual[i])
+		}
+	}
+}
+
+func TestFind(t *testing.T) {
+	cases := []struct {
+		what       string
+		arg        string
+		expectDir  bool
+		expectFile bool
+		errStr     string
+	}{
+		{
+			what: "garbage",
+			arg:  "///1(*&SA",
+		},
+		{
+			what: "simple",
+			arg:  "bob",
+		},
+		{
+			what: "no directory",
+			arg:  filepath.Join("b", "rrrrrr"),
+		},
+		{
+			what:      "is a directory",
+			arg:       filepath.Join("b", "d", "ignore"),
+			expectDir: true,
+		},
+		{
+			what:       "longer, ending in file",
+			arg:        filepath.Join("b", "d", "x"),
+			expectFile: true,
+		},
+		{
+			what:       "moar longer, ending in file",
+			arg:        filepath.Join("b", "d", "a", "c", "u"),
+			expectFile: true,
+		},
+		{
+			what:      "directory",
+			arg:       filepath.Join("b"),
+			expectDir: true,
+		},
+		{
+			// Querying for the empty string could
+			// 1) be an error,
+			// 2) return no result (and no error) as with
+			//    any illegal and therefore non-existent
+			//    file name,
+			// 3) return the node itself, like running
+			//    'ls' with no argument.
+			// Going with option 2 (no result, no error),
+			// since at this low level it makes more sense
+			// if the results for the empty string query
+			// differ from the results for the "." query.
+			what: "empty name",
+			arg:  "",
+		},
+		{
+			what:      "self dir",
+			arg:       SelfDir,
+			expectDir: true,
+		},
+		{
+			what: "parent dir - doesn't exist",
+			arg:  ParentDir,
+		},
+		{
+			what: "many parents - doesn't exist",
+			arg:  filepath.Join(ParentDir, ParentDir, ParentDir),
+		},
+	}
+
+	n := makeLoadedFileTree(t)
+	for _, item := range cases {
+		result, err := n.Find(item.arg)
+		if item.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error", item.what)
+			}
+			if !strings.Contains(err.Error(), item.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					item.what, item.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", item.what, err)
+		}
+		if result == nil {
+			if item.expectDir {
+				t.Fatalf(
+					"%s; expected to find directory '%s'", item.what, item.arg)
+			}
+			if item.expectFile {
+				t.Fatalf(
+					"%s; expected to find file '%s'", item.what, item.arg)
+			}
+			continue
+		}
+		if item.expectDir {
+			if !result.isNodeADir() {
+				t.Fatalf(
+					"%s; expected '%s' to be a directory", item.what, item.arg)
+			}
+			continue
+		}
+		if item.expectFile {
+			if result.isNodeADir() {
+				t.Fatalf("%s; expected '%s' to be a file", item.what, item.arg)
+			}
+			continue
+		}
+		t.Fatalf(
+			"%s; expected nothing for '%s', but got '%s'",
+			item.what, item.arg, result.Path())
+	}
+}
+
+func TestCleanedAbs(t *testing.T) {
+	cases := []struct {
+		what   string
+		full   string
+		cDir   string
+		name   string
+		errStr string
+	}{
+		{
+			what:   "empty",
+			full:   "",
+			errStr: "doesn't exist",
+		},
+		{
+			what:   "simple",
+			full:   "bob",
+			errStr: "'bob' doesn't exist",
+		},
+		{
+			what:   "no directory",
+			full:   filepath.Join("b", "rrrrrr"),
+			errStr: "'b/rrrrrr' doesn't exist",
+		},
+		{
+			what: "longer, ending in file",
+			full: filepath.Join("b", "d", "x"),
+			cDir: filepath.Join("b", "d"),
+			name: "x",
+		},
+		{
+			what: "moar longer, ending in file",
+			full: filepath.Join("b", "d", "a", "c", "u"),
+			cDir: filepath.Join("b", "d", "a", "c"),
+			name: "u",
+		},
+		{
+			what: "directory",
+			full: filepath.Join("b", "d"),
+			cDir: filepath.Join("b", "d"),
+			name: "",
+		},
+	}
+
+	n := makeLoadedFileTree(t)
+	for _, item := range cases {
+		cDir, name, err := n.CleanedAbs(item.full)
+		if item.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error", item.what)
+			}
+			if !strings.Contains(err.Error(), item.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					item.what, item.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", item.what, err)
+		}
+		if cDir != ConfirmedDir(item.cDir) {
+			t.Fatalf("%s; expected cDir=%s, got '%s'", item.what, item.cDir, cDir)
+		}
+		if name != item.name {
+			t.Fatalf("%s; expected name=%s, got '%s'", item.what, item.name, name)
+		}
+	}
+}

api/filesys/rpath.go

@@ -1,12 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package filesys
-
-import "path/filepath"
-
-// RootedPath returns a rooted path, e.g. "/foo/bar" as
-// opposed to "foo/bar".
-func RootedPath(elem ...string) string {
-	return separator + filepath.Join(elem...)
-}

api/filesys/util.go

@@ -0,0 +1,30 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import "path/filepath"
+
+// RootedPath returns a rooted path, e.g. "/foo/bar" as
+// opposed to "foo/bar".
+func RootedPath(elem ...string) string {
+	return Separator + filepath.Join(elem...)
+}
+
+// StripTrailingSeps trims trailing filepath separators from input.
+func StripTrailingSeps(s string) string {
+	k := len(s)
+	for k > 0 && s[k-1] == filepath.Separator {
+		k--
+	}
+	return s[:k]
+}
+
+// StripLeadingSeps trims leading filepath separators from input.
+func StripLeadingSeps(s string) string {
+	k := 0
+	for k < len(s) && s[k] == filepath.Separator {
+		k++
+	}
+	return s[k:]
+}

api/filesys/util_test.go

@@ -0,0 +1,205 @@
+package filesys_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	. "sigs.k8s.io/kustomize/api/filesys"
+)
+
+// Confirm behavior of filepath.Match
+func TestFilePathMatch(t *testing.T) {
+	cases := []struct {
+		pattern  string
+		path     string
+		expected bool
+	}{
+		{
+			pattern:  "*e*",
+			path:     "hey",
+			expected: true,
+		},
+		{
+			pattern:  "*e*",
+			path:     "hay",
+			expected: false,
+		},
+		{
+			pattern:  "*e*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: false,
+		},
+		{
+			pattern:  "*/e/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "h/e/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/e/y",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/*/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/*/*",
+			path:     filepath.Join("h", "e", "y", "there"),
+			expected: false,
+		},
+		{
+			pattern:  "*/*/*/t*e",
+			path:     filepath.Join("h", "e", "y", "there"),
+			expected: true,
+		},
+	}
+	for _, item := range cases {
+		match, err := filepath.Match(item.pattern, item.path)
+		if err != nil {
+			t.Fatalf("unexpected err: %v", err)
+		}
+		if match != item.expected {
+			t.Fatalf("'%s' '%s' %v\n", item.pattern, item.path, match)
+		}
+	}
+}
+
+// Confirm behavior of filepath.Split
+func TestFilePathSplit(t *testing.T) {
+	cases := []struct {
+		full string
+		dir  string
+		file string
+	}{
+		{
+			full: "",
+			dir:  "",
+			file: "",
+		},
+		{
+			full: SelfDir,
+			dir:  "",
+			file: SelfDir,
+		},
+		{
+			full: "rabbit.jpg",
+			dir:  "",
+			file: "rabbit.jpg",
+		},
+		{
+			full: "/beans",
+			dir:  "/",
+			file: "beans",
+		},
+		{
+			full: "/home/foo/bar",
+			dir:  "/home/foo/",
+			file: "bar",
+		},
+		{
+			full: "/usr/local/",
+			dir:  "/usr/local/",
+			file: "",
+		},
+		{
+			full: "/usr//local//go",
+			dir:  "/usr//local//",
+			file: "go",
+		},
+	}
+	for _, p := range cases {
+		dir, file := filepath.Split(p.full)
+		if dir != p.dir || file != p.file {
+			t.Fatalf(
+				"in '%s',\ngot dir='%s' (expected '%s'),\n got file='%s' (expected %s).",
+				p.full, dir, p.dir, file, p.file)
+		}
+	}
+}
+
+func TestStripTrailingSeps(t *testing.T) {
+	cases := []struct {
+		full string
+		rem  string
+	}{
+		{
+			full: "foo",
+			rem:  "foo",
+		},
+		{
+			full: "",
+			rem:  "",
+		},
+		{
+			full: "foo/",
+			rem:  "foo",
+		},
+		{
+			full: "foo///bar///",
+			rem:  "foo///bar",
+		},
+		{
+			full: "/////",
+			rem:  "",
+		},
+		{
+			full: "/",
+			rem:  "",
+		},
+	}
+	for _, p := range cases {
+		dir := StripTrailingSeps(p.full)
+		if dir != p.rem {
+			t.Fatalf(
+				"in '%s', got dir='%s' (expected '%s')",
+				p.full, dir, p.rem)
+		}
+	}
+}
+
+func TestStripLeadingSeps(t *testing.T) {
+	cases := []struct {
+		full string
+		rem  string
+	}{
+		{
+			full: "foo",
+			rem:  "foo",
+		},
+		{
+			full: "",
+			rem:  "",
+		},
+		{
+			full: "/foo",
+			rem:  "foo",
+		},
+		{
+			full: "///foo///bar///",
+			rem:  "foo///bar///",
+		},
+		{
+			full: "/////",
+			rem:  "",
+		},
+		{
+			full: "/",
+			rem:  "",
+		},
+	}
+	for _, p := range cases {
+		dir := StripLeadingSeps(p.full)
+		if dir != p.rem {
+			t.Fatalf(
+				"in '%s', got dir='%s' (expected '%s')",
+				p.full, dir, p.rem)
+		}
+	}
+}

api/go.mod

@@ -3,16 +3,16 @@ module sigs.k8s.io/kustomize/api
 go 1.13
 
 require (
-	github.com/emicklei/go-restful v2.9.6+incompatible // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/go-openapi/spec v0.19.4
-	github.com/golangci/golangci-lint v1.19.1
-	github.com/googleapis/gnostic v0.3.0 // indirect
-	github.com/monopole/mdrip v1.0.0
+	github.com/golangci/golangci-lint v1.21.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/pkg/errors v0.8.1
-	golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678
+	golang.org/x/tools v0.0.0-20191010075000-0337d82405ff
 	gopkg.in/yaml.v2 v2.2.4
+	k8s.io/api v0.17.0
+	k8s.io/apimachinery v0.17.0
+	k8s.io/client-go v0.17.0
 	k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a
-	sigs.k8s.io/kustomize/pseudo/k8s v0.1.0
 	sigs.k8s.io/yaml v1.1.0
 )

api/go.sum

@@ -1,14 +1,11 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -29,6 +26,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs=
+github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -45,12 +44,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w=
-github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
@@ -106,15 +104,18 @@ github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2X
 github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b h1:ekuhfTjngPhisSjOJ0QWKpPQE8/rbknHaes6WVJj5Hw=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -132,14 +133,14 @@ github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgO
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.19.1 h1:g9xL8KW7UZDCkVlgHYJMA6F4Sj/sRVa0FoCeXI+Z3iM=
-github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI=
+github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk=
@@ -150,37 +151,33 @@ github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1
 github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
-github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0 h1:UykbtMB/w5No2LmE16gINgLj+r/vbziTgaoERQv6U+0=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a h1:YH0IojQwndMQdeRWdw1aPT8bkbiWaYR3WD+Zf5e09DU=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741 h1:OuuPl66BpF1q3OEkaPpp+VfzxrBBY62ATGdWqql/XX8=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
-github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -192,6 +189,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -219,10 +217,10 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86 h1:q6SrfsK4FojRnJ1j8+8OJzyq3g9Y1oSVyL6nYGJXXBk=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
@@ -239,8 +237,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.0 h1:RFDBa+tab6mW+gX4Ww2SZDc4kS6p01FwnLtgz64Il+I=
-github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -275,21 +271,16 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218 h1:O0yPHYL49quNL4Oj2wVq+zbGMu4dAM6iLoOQtm49TrQ=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -322,15 +313,17 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/ultraware/whitespace v0.0.3 h1:S5BCRRB5sttNy0bSOhbpw+0mb+cHiCmWfrvxpEzuUk0=
-github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
+github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
@@ -338,16 +331,22 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -358,19 +357,23 @@ golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -379,19 +382,22 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -400,21 +406,28 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -436,8 +449,15 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
+k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
+k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
+k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
+k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
@@ -445,17 +465,13 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/pseudo/k8s v0.0.0-20191108212413-1f86a0ca5d6c h1:t7fk+ljA3Ru4pro+/0RuOAZcODDhByL1fvIdyHLhjTY=
-sigs.k8s.io/kustomize/pseudo/k8s v0.0.0-20191108212413-1f86a0ca5d6c/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0 h1:otg4dLFc03c3gzl+2CV8GPGcd1kk8wjXwD+UhhcCn5I=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/internal/accumulator/loadconfigfromcrds.go

@@ -10,6 +10,7 @@ import (
 	"github.com/go-openapi/spec"
 	"github.com/pkg/errors"
 	"k8s.io/kube-openapi/pkg/common"
+	"sigs.k8s.io/kustomize/api/filesys"
 	"sigs.k8s.io/kustomize/api/ifc"
 	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
 	"sigs.k8s.io/kustomize/api/resid"
@@ -78,7 +79,7 @@ func makeConfigFromApiMap(m nameToApiMap) (*builtinconfig.TransformerConfig, err
 // openAPI definition once
 // "x-kubernetes-group-version-kind" is available in CRD
 func makeGvkFromTypeName(n string) resid.Gvk {
-	names := strings.Split(n, ".")
+	names := strings.Split(n, filesys.SelfDir)
 	kind := names[len(names)-1]
 	return resid.Gvk{Kind: kind}
 }
@@ -93,10 +94,7 @@ func looksLikeAk8sType(properties myProperties) bool {
 		return false
 	}
 	_, ok = properties["metadata"]
-	if !ok {
-		return false
-	}
-	return true
+	return ok
 }
 
 const (

api/internal/accumulator/loadconfigfromcrds_test.go

@@ -7,9 +7,10 @@ import (
 	"reflect"
 	"testing"
 
-	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/loader"
+
 	. "sigs.k8s.io/kustomize/api/internal/accumulator"
-	"sigs.k8s.io/kustomize/api/internal/loadertest"
 	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/types"
@@ -41,7 +42,7 @@ More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#ty
 					"type": "string"
 				},
 				"metadata": {
-					"$ref": "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+					"$ref": "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 				},
 				"spec": {
 					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec"
@@ -54,7 +55,7 @@ More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#ty
 		"Dependencies": [
 			"github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec",
 			"github.com/example/pkg/apis/jingfang/v1beta1.BeeStatus",
-			"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+			"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 		]
 	},
 	"github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec": {
@@ -86,7 +87,7 @@ In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-con
 					"type": "string"
 				},
 				"metadata": {
-					"$ref": "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+					"$ref": "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 				},
 				"spec": {
 					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec"
@@ -99,7 +100,7 @@ In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-con
 		"Dependencies": [
 			"github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec",
 			"github.com/example/pkg/apis/jingfang/v1beta1.MyKindStatus",
-			"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+			"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 		]
 	},
 	"github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec": {
@@ -116,13 +117,13 @@ In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-con
 If it is not set we generate a secret dynamically",
 					"x-kubernetes-object-ref-api-version": "v1",
 					"x-kubernetes-object-ref-kind": "Secret",
-					"$ref": "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1.LocalObjectReference"
+					"$ref": "k8s.io/api/core/v1.LocalObjectReference"
 				}
 			}
 		},
 		"Dependencies": [
 			"github.com/example/pkg/apis/jingfang/v1beta1.Bee",
-			"sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1.LocalObjectReference"
+			"k8s.io/api/core/v1.LocalObjectReference"
 		]
 	},
 	"github.com/example/pkg/apis/jingfang/v1beta1.MyKindStatus": {
@@ -135,15 +136,6 @@ If it is not set we generate a secret dynamically",
 `
 )
 
-func makeLoader(t *testing.T) ifc.Loader {
-	ldr := loadertest.NewFakeLoader("/testpath")
-	err := ldr.AddFile("/testpath/crd.json", []byte(crdContent))
-	if err != nil {
-		t.Fatalf("Failed to setup fake ldr.")
-	}
-	return ldr
-}
-
 func TestLoadCRDs(t *testing.T) {
 	nbrs := []builtinconfig.NameBackReferences{
 		{
@@ -172,7 +164,14 @@ func TestLoadCRDs(t *testing.T) {
 		NameReference: nbrs,
 	}
 
-	actualTc, err := LoadConfigFromCRDs(makeLoader(t), []string{"crd.json"})
+	fSys := filesys.MakeFsInMemory()
+	fSys.WriteFile("/testpath/crd.json", []byte(crdContent))
+	ldr, err := loader.NewLoader(loader.RestrictionRootOnly, "/testpath", fSys)
+	if err != nil {
+		t.Fatalf("unexpected error:%v", err)
+	}
+
+	actualTc, err := LoadConfigFromCRDs(ldr, []string{"crd.json"})
 	if err != nil {
 		t.Fatalf("unexpected error:%v", err)
 	}

api/internal/accumulator/namereferencetransformer.go

@@ -212,31 +212,27 @@ func (o *nameReferenceTransformer) getNewNameFunc(
 	target resid.Gvk,
 	referralCandidates resmap.ResMap) func(in interface{}) (interface{}, error) {
 	return func(in interface{}) (interface{}, error) {
-		switch in.(type) {
+		switch thing := in.(type) {
 		case string:
-			oldName, _ := in.(string)
-			return o.getSimpleNameField(oldName, referrer, target,
+			return o.getSimpleNameField(thing, referrer, target,
 				referralCandidates, referralCandidates.Resources())
 		case map[string]interface{}:
 			// Kind: ValidatingWebhookConfiguration
 			// FieldSpec is webhooks/clientConfig/service
-			oldMap, _ := in.(map[string]interface{})
-			return o.getNameAndNsStruct(oldMap, referrer, target,
+			return o.getNameAndNsStruct(thing, referrer, target,
 				referralCandidates)
 		case []interface{}:
-			l, _ := in.([]interface{})
-			for idx, item := range l {
-				switch item.(type) {
+			for idx, item := range thing {
+				switch value := item.(type) {
 				case string:
 					// Kind: Role/ClusterRole
 					// FieldSpec is rules.resourceNames
-					oldName, _ := item.(string)
-					newName, err := o.getSimpleNameField(oldName, referrer, target,
+					newName, err := o.getSimpleNameField(value, referrer, target,
 						referralCandidates, referralCandidates.Resources())
 					if err != nil {
 						return nil, err
 					}
-					l[idx] = newName
+					thing[idx] = newName
 				case map[string]interface{}:
 					// Kind: RoleBinding/ClusterRoleBinding
 					// FieldSpec is subjects
@@ -245,13 +241,12 @@ func (o *nameReferenceTransformer) getNewNameFunc(
 					// what get mutatefield to request the mapping of the whole
 					// map containing namespace and name instead of just a simple
 					// string field containing the name
-					oldMap, _ := item.(map[string]interface{})
-					newMap, err := o.getNameAndNsStruct(oldMap, referrer, target,
+					newMap, err := o.getNameAndNsStruct(value, referrer, target,
 						referralCandidates)
 					if err != nil {
 						return nil, err
 					}
-					l[idx] = newMap
+					thing[idx] = newMap
 				default:
 					return nil, fmt.Errorf(
 						"%#v is expected to be either a []string or a []map[string]interface{}", in)

api/internal/accumulator/namereferencetransformer_test.go

@@ -12,7 +12,7 @@ import (
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
-	"sigs.k8s.io/kustomize/api/testutils/resmaptest"
+	resmaptest_test "sigs.k8s.io/kustomize/api/testutils/resmaptest"
 )
 
 func TestNameReferenceHappyRun(t *testing.T) {

api/internal/accumulator/refvartransformer.go

@@ -70,6 +70,9 @@ func (rv *refVarTransformer) replaceVars(in interface{}) (interface{}, error) {
 		// This field can potentially contain a $(VAR) since it is
 		// of string type.
 		return expansion2.Expand(s, rv.mappingFunc), nil
+	// staticcheck erroneously claims that `case nil`
+	// is unreachable here, so suppressing it.
+	//nolint:staticcheck
 	case nil:
 		return nil, nil
 	default:

api/internal/accumulator/refvartransformer_test.go

@@ -11,7 +11,7 @@ import (
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
-	"sigs.k8s.io/kustomize/api/testutils/resmaptest"
+	resmaptest_test "sigs.k8s.io/kustomize/api/testutils/resmaptest"
 	"sigs.k8s.io/kustomize/api/types"
 )
 

api/internal/accumulator/resaccumulator_test.go

@@ -16,7 +16,7 @@ import (
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
-	"sigs.k8s.io/kustomize/api/testutils/resmaptest"
+	resmaptest_test "sigs.k8s.io/kustomize/api/testutils/resmaptest"
 	"sigs.k8s.io/kustomize/api/types"
 )
 

api/internal/crawl/backend/search_backend.go

@@ -13,7 +13,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/rs/cors"
 
-	"sigs.k8s.io/kustomize/hack/crawl/index"
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
 )
 
 type kustomizeSearch struct {
@@ -44,7 +44,7 @@ type kustomizeSearch struct {
 // /register: not implemented, but meant as an endpoint for adding new
 // kustomization files to the corpus.
 func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, "kustomize")
 	if err != nil {
 		return nil, err
 	}

api/internal/crawl/cmd/backend/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install sigs.k8s.io/kustomize/api/internal/crawl/cmd/backend/
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/backend /
+ENTRYPOINT ["/backend"]

api/internal/crawl/cmd/backend/main.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	server "sigs.k8s.io/kustomize/api/internal/crawl/backend"
+	"strconv"
+)
+
+func main() {
+	portStr := os.Getenv("PORT")
+	port, err := strconv.Atoi(portStr)
+	if portStr == "" || err != nil {
+		log.Fatalf("$PORT(%s) must be set to an integer\n", portStr)
+	}
+
+	ctx := context.Background()
+
+	ks, err := server.NewKustomizeSearch(ctx)
+	if err != nil {
+		log.Fatalf("Error creating kustomize server: %v", ks)
+	}
+
+	err = ks.Serve(port)
+	if err != nil {
+		log.Fatalf("Error while running server: %v", err)
+	}
+}

api/internal/crawl/cmd/crawler/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize//api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install -v ./cmd/crawler/crawler.go
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/crawler /
+ENTRYPOINT ["/crawler"]
+CMD []

api/internal/crawl/cmd/crawler/crawler.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+const (
+	githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
+	redisCacheURL        = "REDIS_CACHE_URL"
+	redisKeyURL          = "REDIS_KEY_URL"
+	retryCount           = 3
+)
+
+type CrawlMode int
+
+const (
+	CrawlUnknown CrawlMode = iota
+	// Crawl all the kustomization files in all the repositories of a Github user
+	CrawlUser
+	// Crawl all the kustomization files in a Github repo
+	CrawlRepo
+	// Crawl all the documents in the index
+	CrawlIndex
+	// Crawl all the kustomization files on Github
+	CrawlGithub
+	// Crawl all the documents in the index and crawling all the kustomization files on Github
+	CrawlIndexAndGithub
+)
+
+func NewCrawlMode(s string) CrawlMode {
+	switch s {
+	case "github-user":
+		return CrawlUser
+	case "github-repo":
+		return CrawlRepo
+	case "index+github":
+		return CrawlIndexAndGithub
+	case "index":
+		return CrawlIndex
+	case "github":
+		return CrawlGithub
+	default:
+		return CrawlUnknown
+	}
+}
+
+func main() {
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	modePtr := flag.String("mode", "index+github",
+		`The crawling mode, which can be one of [github-user, github-repo, index, github, index+github].
+  * github-user: crawl all the kustomization files in all the repositories of a Github user (--github-user must be specified for this mode).
+  * github-repo: crawl all the kustomization files in a Github repository (--github-repo must be specified for this mode).
+  * index: crawl all the documents in the index.
+  * gihub: crawl all the kustomization files on Github.
+  * index+github: crawl all the documents in the index and crawling all the kustomization files on Github.`)
+	githubUserPtr := flag.String("github-user", "",
+		"A github user name (e.g., kubernetes-sigs). This flag is required for the `github-user` mode.")
+	githubRepoPtr := flag.String("github-repo", "",
+		"A github repository name (e.g., kubernetes-sigs/kustomize). This flag is required for the `github-repo` mode.")
+	flag.Parse()
+
+	githubToken := os.Getenv(githubAccessTokenVar)
+	if githubToken == "" {
+		log.Printf("Must set the variable '%s' to make github requests.\n",
+			githubAccessTokenVar)
+		return
+	}
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	if err != nil {
+		log.Printf("Could not create an index: %v\n", err)
+		return
+	}
+
+	cacheURL := os.Getenv(redisCacheURL)
+	cache, err := redis.DialURL(cacheURL)
+	clientCache := &http.Client{}
+	if err != nil {
+		log.Printf("Error: redis could not make a connection: %v\n", err)
+	} else {
+		clientCache = httpclient.NewClient(cache)
+	}
+
+	// docConverter takes in a plain document and processes it for the index.
+	docConverter := func(d *doc.Document) (crawler.CrawledDocument, error) {
+		kdoc := doc.KustomizationDocument{
+			Document: *d,
+		}
+
+		err := kdoc.ParseYAML()
+		return &kdoc, err
+	}
+
+	// Index updates the value in the index.
+	indexFunc := func(cdoc crawler.CrawledDocument, mode index.Mode) error {
+		switch d := cdoc.(type) {
+		case *doc.KustomizationDocument:
+			switch mode {
+			case index.Delete:
+				log.Printf("Deleting: %v", d)
+				return idx.Delete(d.ID())
+			default:
+				log.Printf("Inserting: %v", d)
+				return idx.Put(d.ID(), d)
+			}
+		default:
+			return fmt.Errorf("type %T not supported", d)
+		}
+	}
+
+	// seen tracks the IDs of all the documents in the index and their corresponding file types.
+	// This helps avoid indexing a given document multiple times.
+	seen := utils.NewSeenMap()
+
+	mode := NewCrawlMode(*modePtr)
+
+	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
+		if user != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.User(user)),
+			)
+		} else if repo != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.Repo(repo)),
+			)
+		} else {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization")),
+			)
+		}
+	}
+
+	seedDocs := make(crawler.CrawlSeed, 0)
+
+	// get all the documents in the index
+	getSeedDocsFunc := func() {
+		query := []byte(`{ "query":{ "match_all":{} } }`)
+		it := idx.IterateQuery(query, 10000, 60*time.Second)
+		for it.Next() {
+			for _, hit := range it.Value().Hits.Hits {
+				seedDocs = append(seedDocs, hit.Document.Document.Copy())
+			}
+		}
+		if err := it.Err(); err != nil {
+			log.Fatalf("getSeedDocsFunc Error iterating: %v\n", err)
+		}
+	}
+
+	switch mode {
+	case CrawlIndexAndGithub:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlIndex:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+	case CrawlGithub:
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		// add all the documents in the index into seen.
+		// this greatly reduces the time overhead of CrawlGithub.
+		getSeedDocsFunc()
+		for _, d := range seedDocs {
+			seen[d.ID()] = d.FileType
+		}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUser:
+		if *githubUserPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github user with the github-user flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(*githubUserPtr, "")}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlRepo:
+		if *githubRepoPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github repository with the github-repo flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", *githubRepoPtr)}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUnknown:
+		flag.Usage()
+		log.Fatalf("The --mode flag must be one of [github-user, github-repo, index, github, index+github].")
+	}
+}

api/internal/crawl/cmd/kustomize_stats/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install ./cmd/kustomize_stats
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/kustomize_stats /
+ENTRYPOINT ["/kustomize_stats"]

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -0,0 +1,249 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"flag"
+	"fmt"
+	"log"
+	"sort"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+)
+
+// iterateArr adds each item in arr into countMap.
+func iterateArr(arr []string, countMap map[string]int) {
+	for _, item := range arr {
+		if _, ok := countMap[item]; !ok {
+			countMap[item] = 1
+		} else {
+			countMap[item]++
+		}
+	}
+
+}
+
+// SortMapKeyByValueInt takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueInt(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
+	return keys
+}
+
+// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueLen(m map[string][]string) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return len(m[keys[i]]) > len(m[keys[j]]) })
+	return keys
+}
+
+func GeneratorOrTransformerStats(docs []*doc.KustomizationDocument) {
+	n := len(docs)
+	if n == 0 {
+		return
+	}
+
+	fileType := docs[0].FileType
+	fmt.Printf("There are totally %d %s files.\n", n, fileType)
+
+	GitRepositorySummary(docs, fileType)
+
+	// key of kindToUrls: a string in the KustomizationDocument.Kinds field
+	// value of kindToUrls: a slice of string urls defining a given kind.
+	kindToUrls := make(map[string][]string)
+
+	for _, d := range docs {
+		url := fmt.Sprintf("%s/blob/%s/%s", d.RepositoryURL, d.DefaultBranch, d.FilePath)
+		for _, kind := range d.Kinds {
+			if _, ok := kindToUrls[kind]; !ok {
+				kindToUrls[kind] = []string{url}
+			} else {
+				kindToUrls[kind] = append(kindToUrls[kind], url)
+			}
+		}
+	}
+	fmt.Printf("There are totally %d kinds of %s\n", len(kindToUrls), fileType)
+	sortedKeys := SortMapKeyByValueLen(kindToUrls)
+	for _, k := range sortedKeys {
+		sort.Strings(kindToUrls[k])
+		fmt.Printf("%s kind %s appears %d times\n", fileType, k, len(kindToUrls[k]))
+		for _, url := range kindToUrls[k] {
+			fmt.Printf("%s\n", url)
+		}
+	}
+}
+
+// GitRepositorySummary counts the distribution of docs:
+// 1) how many git repositories are these docs from?
+// 2) how many docs are from each git repository?
+func GitRepositorySummary(docs []*doc.KustomizationDocument, fileType string) {
+	m := make(map[string]int)
+	for _, d := range docs {
+		if _, ok := m[d.RepositoryURL]; ok {
+			m[d.RepositoryURL]++
+		} else {
+			m[d.RepositoryURL] = 1
+		}
+	}
+	sortedKeys := SortMapKeyByValueInt(m)
+	topN := 10
+	i := 0
+	for _, k := range sortedKeys {
+		if i >= topN {
+			break
+		}
+		fmt.Printf("%d %s are from %s\n", m[k], fileType, k)
+		i++
+	}
+}
+
+func main() {
+	topKindsPtr := flag.Int(
+		"kinds", -1,
+		`the number of kubernetes object kinds to be listed according to their popularities.
+By default, all the kinds will be listed.
+If you only want to list the 10 most popular kinds, set the flag to 10.`)
+	topIdentifiersPtr := flag.Int(
+		"identifiers", -1,
+		`the number of identifiers to be listed according to their popularities.
+By default, all the identifiers will be listed.
+If you only want to list the 10 most popular identifiers, set the flag to 10.`)
+	topKustomizeFeaturesPtr := flag.Int(
+		"kustomize-features", -1,
+		`the number of kustomize features to be listed according to their popularities.
+By default, all the features will be listed.
+If you only want to list the 10 most popular features, set the flag to 10.`)
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	flag.Parse()
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	if err != nil {
+		log.Fatalf("Could not create an index: %v\n", err)
+	}
+
+	// count tracks the number of documents in the index
+	count := 0
+
+	// kustomizationFilecount tracks the number of kustomization files in the index
+	kustomizationFilecount := 0
+
+	kindsMap := make(map[string]int)
+	identifiersMap := make(map[string]int)
+	kustomizeIdentifiersMap := make(map[string]int)
+
+	// ids tracks the unique IDs of the documents in the index
+	ids := make(map[string]struct{})
+
+	// generatorFiles include all the non-kustomization files whose FileType is generator
+	generatorFiles := make([]*doc.KustomizationDocument, 0)
+
+	// transformersFiles include all the non-kustomization files whose FileType is transformer
+	transformersFiles := make([]*doc.KustomizationDocument, 0)
+
+	checksums := make(map[string]int)
+
+	// get all the documents in the index
+	query := []byte(`{ "query":{ "match_all":{} } }`)
+	it := idx.IterateQuery(query, 10000, 60*time.Second)
+	for it.Next() {
+		for _, hit := range it.Value().Hits.Hits {
+			sum := fmt.Sprintf("%x", sha256.Sum256([]byte(hit.Document.DocumentData)))
+			if _, ok := checksums[sum]; ok {
+				checksums[sum]++
+			} else {
+				checksums[sum] = 1
+			}
+
+			// check whether there is any duplicate IDs in the index
+			if _, ok := ids[hit.ID]; !ok {
+				ids[hit.ID] = struct{}{}
+			} else {
+				log.Printf("Found duplicate ID (%s)\n", hit.ID)
+			}
+
+			count++
+			iterateArr(hit.Document.Kinds, kindsMap)
+			iterateArr(hit.Document.Identifiers, identifiersMap)
+
+			if doc.IsKustomizationFile(hit.Document.FilePath) {
+				kustomizationFilecount++
+				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
+
+			} else {
+				switch hit.Document.FileType {
+				case "generator":
+					generatorFiles = append(generatorFiles, hit.Document.Copy())
+				case "transformer":
+					transformersFiles = append(transformersFiles, hit.Document.Copy())
+				}
+			}
+		}
+	}
+
+	if err := it.Err(); err != nil {
+		log.Fatalf("Error iterating: %v\n", err)
+	}
+
+	sortedKindsMapKeys := SortMapKeyByValueInt(kindsMap)
+	sortedIdentifiersMapKeys := SortMapKeyByValueInt(identifiersMap)
+	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValueInt(kustomizeIdentifiersMap)
+
+	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
+There are %d documents in the kustomize index.
+%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
+	fmt.Printf("\n")
+
+	kindCount := 0
+	for _, key := range sortedKindsMapKeys {
+		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
+			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
+			kindCount++
+		}
+	}
+
+	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
+	identifierCount := 0
+	for _, key := range sortedIdentifiersMapKeys {
+		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
+			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
+			identifierCount++
+		}
+	}
+
+	fmt.Printf(`There are %d kustomization files in the kustomize index.
+%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
+	fmt.Printf("\n")
+	kustomizeFeatureCount := 0
+	for _, key := range sortedKustomizeIdentifiersMapKeys {
+		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
+			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
+			kustomizeFeatureCount++
+		}
+	}
+
+	GeneratorOrTransformerStats(generatorFiles)
+	GeneratorOrTransformerStats(transformersFiles)
+
+	fmt.Printf("There are total %d checksums of document contents\n", len(checksums))
+	sortedChecksums := SortMapKeyByValueInt(checksums)
+	sortedChecksums = sortedChecksums[:20]
+	fmt.Printf("The top 20 checksums are:\n")
+	for _, key := range sortedChecksums {
+		fmt.Printf("checksum %s apprears %d\n", key, checksums[key])
+	}
+}

api/internal/crawl/cmd/log-parser/README.md

@@ -0,0 +1,8 @@
+This binary takes as its input a json file including GKE logs (which can be
+[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into 
+[Cloud Storage](https://cloud.google.com/storage/docs/)), 
+and extracts the `textPayload` field of each log entry.
+
+Here is an log entry example:
+
+{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"}

api/internal/crawl/cmd/log-parser/kustomize-stats-cmd

@@ -0,0 +1,7 @@
+wget <log-file-url> -O log
+go build .
+./log-parser log >out
+cat out | grep "kind \`" | cut -d\` -f2 | tail -n 50
+cat out | grep "kind \`" | awk '{print $6}' | tail -n 50
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | cut -d\` -f2
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | awk '{print $6}'

api/internal/crawl/cmd/log-parser/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatalf("The usage of the command is: \n\t%s <log-file.json>", os.Args[0])
+	}
+
+	file, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	closeFile := func(file *os.File) {
+		if err := file.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}
+	defer closeFile(file)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		var entry interface{}
+		if err := json.Unmarshal([]byte(line), &entry); err != nil {
+			log.Printf("failed to unmarshal a log entry: %s\n", line)
+		}
+
+		m := entry.(map[string]interface{})
+		if payload, ok := m["textPayload"]; ok {
+			// use fmt.Printf here instead of log.Printf to avoid the time and code location info the log package provides
+			fmt.Printf("%s", payload)
+		} else {
+			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}

api/internal/crawl/config/base/kustomization.yaml

@@ -0,0 +1,5 @@
+configmapGenerator:
+- name: elasticsearch-config
+  literals:
+  - es-url="http://esbasic-master:9200"
+  - plugin-index-name="plugin"

api/internal/crawl/config/crawler/base/.gitignore

@@ -0,0 +1 @@
+github_api_secret.txt

api/internal/crawl/config/crawler/base/github_api_secret.txt

@@ -0,0 +1,2 @@
+<ADD YOUR GITHUB PERSONAL ACCESS TOKEN HERE WITHOUT A TRAILING NEWLINE>
+Run: printf "<your-token>" > github_api_secret.txt

api/internal/crawl/config/crawler/base/kustomization.yaml

@@ -0,0 +1,15 @@
+resources:
+- ../../base
+
+configmapGenerator:
+- name: crawler-http-cache
+  literals:
+  - redis-cache-url="redis://redis-http-cache:6379"
+- name: redis-keystore
+  literals:
+  - keystore-url="redis://redis-docs-keystore:6379"
+
+secretGenerator:
+- name: github-access-token
+  files:
+  - token=github_api_secret.txt

api/internal/crawl/config/crawler/cronjob/cronjob.yaml

@@ -0,0 +1,34 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: crawler-cronjob
+spec:
+  # run the cronjob at 00:00 every 7 days
+  schedule: "0 0 */7 * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler
+            image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+            command: ["/crawler"]
+            args: ["--mode=index+github", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
+            imagePullPolicy: Always
+            env:
+            - name: GITHUB_ACCESS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: github-access-token
+                  key: token
+            - name: ELASTICSEARCH_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: elasticsearch-config
+                  key: es-url
+            - name: REDIS_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: crawler-http-cache
+                  key: redis-cache-url

api/internal/crawl/config/crawler/job/README.md

@@ -0,0 +1,53 @@
+The crawler job can run in one of the following mode:
+
+# Crawling all the documents in the index and crawling all the kustomization files on Github
+
+This is the default setting of the crawler job. The `command` and `args` field
+of the container should be:
+
+```
+        command: ["/crawler"]
+```
+
+Or
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index+github"]
+```
+
+# Crawling all the documents in the index
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index"]
+```
+
+# Crawling all the kustomization files on Github
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github"]
+```
+
+# Crawling all the kustomization files in a Github repo
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize"]
+```
+
+# Crawling all the kustomization files in all the repositories of a Github user
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--github-user", "--github-user=kubernetes-sigs"]
+```

api/internal/crawl/config/crawler/job/job.yaml

@@ -0,0 +1,35 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: crawler
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: crawler
+        image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+        imagePullPolicy: Always
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
+        env:
+        - name: GITHUB_ACCESS_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: github-access-token
+              key: token
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url
+        - name: REDIS_CACHE_URL
+          valueFrom:
+            configMapKeyRef:
+              name: crawler-http-cache
+              key: redis-cache-url
+        - name: REDIS_KEY_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-keystore
+              key: keystore-url

api/internal/crawl/config/crawler/kustomize_stats/job.yaml

@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kustomize-stats
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: kustomize-stats
+        image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
+        imagePullPolicy: Always
+        command: ["/kustomize_stats"]
+        args: ["--index=kustomize", "--kinds=51", "--identifiers=50", "--kustomize-features=50"]
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url

api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml

api/internal/crawl/config/elastic/esbackup.yaml

@@ -0,0 +1,23 @@
+# ESBackup depends on ESCluster, and is depended by ESSnapshot.
+# Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the snapshots in the `kustomize-backup` snapshot repository, instead makes all the snapshots in the repository inaccessible.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the essnapshot objects depending on it, but will cause those essnapshot objects to be reconciled, which update the status of the essnapshot objects to reflect the fact that the esbackup object is missing.
+# If you delete the `kustomize-backup` snapshot repository directly without deleting `esbackup/kustomize-backbup`, the ESBackup object will not recreate the snapshot repository.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESBackup
+metadata:
+  name: kustomize-backup
+spec:
+  storage:
+    gcs:
+      # the bucket must exist for the snapshot respository to be created successfully.
+      bucket: kustomize-backup
+      # the path does not need to exist.
+      # If the path does not exist, the controller will create the folder in the GCS bucket.
+      # If the path already exists and includes snapshots, these snapshots can be used.
+      path: kustomize
+      secret:
+        name: kustomizesa
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/escluster.yaml

@@ -0,0 +1,51 @@
+# ESCluster is depended by ESBackup and ESRestore.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESCluster
+metadata:
+  name: esbasic
+spec:
+  plugin:
+   pluginList:
+    - repository-gcs
+    - ingest-user-agent
+    - ingest-geoip
+  # To set `gcpserviceaccount`,
+  # First, create and download a GCP service account into a json file, named `sakey.json` following the instruction:
+  #     https://www.elastic.co/guide/en/elasticsearch/plugins/6.5/repository-gcs-usage.html#repository-gcs-using-service-account
+  # Second, create a secret for the service account using the following command:
+  #     $ kubectl create secret generic kustomizesa --from-file=./sakey.json
+  gcpserviceaccount:
+    name: kustomizesa
+  config:
+    env:
+      example: test
+  nodegroups:
+    - name: di
+      replicas: 2
+      data: true
+      ingest: true
+      config:
+        jvm:
+          - Djava.net.preferIPv4Stack=true
+          - Xms2g
+          - Xmx2g
+        es:
+          path.repo: '["/tmp/es_backup_basic"]'
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                es/nodegroup: di
+      resources:
+        requests:
+          memory: 3Gi
+        limits:
+          memory: 3Gi
+    - name: m
+      replicas: 2
+      master: true
+      config:
+        es:
+          path.repo: '["/tmp/es_backup_basic"]'

api/internal/crawl/config/elastic/esrestore.yaml

@@ -0,0 +1,19 @@
+# ESRestore depends on both ESCluster and ESSnapshot.
+# Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`.
+# Deleting `esrestore/kustomize-restore` will not delete the restored index.
+# Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`.
+# After the restore is complete, if the `kusotmize-restore` index is deleted manually, the ESRestore object will NOT restore the `kustomize` index to it again.
+# The correct way of using ESRestore is: create a ESRestore object to restore the index; delete the ESRestore object after the restore is complete.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESRestore
+metadata:
+  name: kustomize-restore
+spec:
+  include_global_state: true
+  ignore_unavailable: true
+  rename_pattern: kustomize
+  rename_replacement: kustomize-restore
+  essnapshot:
+    name: kustomize-snapshot
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/essnapshot.yaml

@@ -0,0 +1,23 @@
+# ESSnapshot depends on ESBackup, and is depended by ESRestore.
+# Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository.
+# After being created, the `kustomize-snapshot` snapshot will not be automatically updated when the `kuostomize` index is updated.
+# If you delete `essnapshot/kustomize-snapshot` and recreate it, the new snapshot will capture the current status of the `kustomize` index.
+# Deleting `essnapshot/kustomize-snapshot` will delete the snapshot.
+# Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`.
+# If the `kustomize-snapshot` snapshot is deleted directly without deleting `essnapshot/kustomize-snapshot`, the ESSnapshot object will recreate the snapshot.
+# The correct way of using ESSnapshot is: create an ESSnapshot object to create a snapshot, keep the ESSnapshot object until the snapshot is no longer needed.
+# To update the snapshot to capture the latest version of the index, you can either:
+#   1) delete the snapshot, and wait for the ESSnapshot object to recreate the snapshot;
+#   2) delete the ESSnapshot object, and recreate the ESSnapshot object.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESSnapshot
+metadata:
+  name: kustomize-snapshot
+spec:
+  # indices are optional. If not specified all indices are selected.
+  indices:
+  - kustomize
+  include_global_state: true
+  ignore_unavailable: true
+  esbackup:
+    name: kustomize-backup

api/internal/crawl/config/elastic/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+spec:
+  selector:
+    custom-resource: v1alpha1.ESCluster
+    custom-resource-name: esbasic
+    custom-resource-namespace: default
+    es/data: "true"
+    using: escluster.Cluster
+  ports:
+  - protocol: "TCP"
+    port: 9200
+  type: LoadBalancer
+  loadBalancerIP: ""

api/internal/crawl/config/webapp/backend/deployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kustomize-search
+spec:
+  selector:
+    matchLabels:
+      app: kustomize-search
+      tier: backend
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: kustomize-search
+        tier: backend
+    spec:
+      containers:
+      - name: kustomize-search
+        image: gcr.io/kustomize-search/backend:latest
+        imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /liveness
+            port: backend-port
+        ports:
+        - name: backend-port
+          containerPort: 8080
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url
+        - name: PORT
+          value: "8080"

api/internal/crawl/config/webapp/frontend/deployment.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kustomize-search-ui
+spec:
+  selector:
+    matchLabels:
+      app: kustomize-search
+      tier: frontend
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: kustomize-search
+        tier: frontend
+    spec:
+      containers:
+      - name: frontend
+        image: gcr.io/kustomize-search/frontend:latest
+        imagePullPolicy: Always
+        ports:
+        - name: frontend-port
+          containerPort: 80

api/internal/crawl/crawler/crawler.go

@@ -0,0 +1,344 @@
+// Package crawler provides helper methods and defines an interface for lauching
+// source repository crawlers that retrieve files from a source and forwards
+// to a channel for indexing and retrieval.
+package crawler
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"sync"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	_ "github.com/gomodule/redigo/redis"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+)
+
+var (
+	logger = log.New(os.Stdout, "Crawler: ", log.LstdFlags|log.LUTC|log.Llongfile)
+)
+
+// Crawler forwards documents from source repositories to index and store them
+// for searching. Each crawler is responsible for querying it's source of
+// information, and forwarding files that have not been seen before or that need
+// updating.
+type Crawler interface {
+	// Crawl returns when it is done processing. This method does not take
+	// ownership of the channel. The channel is write only, and it
+	// designates where the crawler should forward the documents.
+	Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error
+
+	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
+	FetchDocument(context.Context, *doc.Document) error
+	// Write to the document what the created time is.
+	SetCreated(context.Context, *doc.Document) error
+
+	SetDefaultBranch(*doc.Document)
+
+	Match(*doc.Document) bool
+}
+
+type CrawledDocument interface {
+	ID() string
+	GetDocument() *doc.Document
+	// Get all the Documents directly referred in a Document.
+	// For a Document representing a non-kustomization file, an empty slice will be returned.
+	// For a Document representing a kustomization file:
+	// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
+	// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
+	// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
+	GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
+	WasCached() bool
+}
+
+type CrawlSeed []*doc.Document
+
+type IndexFunc func(CrawledDocument, index.Mode) error
+type Converter func(*doc.Document) (CrawledDocument, error)
+
+func logIfErr(err error) {
+	if err == nil {
+		return
+	}
+	logger.Println("error: ", err)
+}
+
+func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
+	for _, crawl := range crawlers {
+		if crawl.Match(d) {
+			return crawl
+		}
+	}
+	return nil
+}
+
+func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed) {
+
+	seen.Set(cdoc.ID(), cdoc.GetDocument().FileType)
+
+	match.SetDefaultBranch(cdoc.GetDocument())
+
+	// Insert into index
+	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
+		logger.Printf("Failed to insert or update doc(%s): %v",
+			cdoc.GetDocument().Path(), err)
+		return
+	}
+
+	deps, err := cdoc.GetResources(true, true, true)
+	if err != nil {
+		logger.Println(err)
+		return
+	}
+
+	for _, dep := range deps {
+		if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType {
+			continue
+		}
+		*stack = append(*stack, dep)
+	}
+}
+
+func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) {
+
+	UpdatedDocCount := 0
+	seenDocCount := 0
+	cachedDocCount := 0
+	findMatchErrCount := 0
+	FetchDocumentErrCount := 0
+	SetCreatedErrCount := 0
+	convErrCount := 0
+	deleteDocCount := 0
+	crawledDocCount := 0
+
+	// During the execution of the for loop, more Documents may be added into (*docsPtr).
+	for len(*docsPtr) > 0 {
+		// get the last Document in (*docPtr), which will be crawled in this iteration.
+		tail := (*docsPtr)[len(*docsPtr)-1]
+
+		// remove the last Document in (*docPtr)
+		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
+
+		crawledDocCount++
+		logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
+
+		if seen.Seen(tail.ID()) {
+			if !updateFileType || seen.Value(tail.ID()) == tail.FileType {
+				logger.Printf("this doc has been seen before")
+				seenDocCount++
+				continue
+			}
+		}
+
+		if tail.WasCached() {
+			logger.Printf("doc(%s) is cached already", tail.Path())
+			cachedDocCount++
+			continue
+		}
+
+		match := findMatch(tail, crawlers)
+		if match == nil {
+			logIfErr(fmt.Errorf("%v could not match any crawler", tail))
+			findMatchErrCount++
+			continue
+		}
+
+		if tail.User == "" {
+			tail.User = doc.UserName(tail.RepositoryURL)
+		}
+
+		// If the Document represents a kustomization root, FetchDcoument will change
+		// the `filePath` field of the Document by adding `kustomization.yaml` or
+		// `kustomization.yml` or `kustomization` into the the field.
+		// Therefore, it is necessary to add the ID of the Document into seen before
+		// calling FetchDocument. Otherwise, the binary may enter into an infinite loop
+		// if a kustomization file points to its kustmozation root in its `resources` or
+		// `bases` field.
+		seen.Set(tail.ID(), tail.FileType)
+
+		if refreshDoc || tail.DefaultBranch == "" {
+			match.SetDefaultBranch(tail)
+		}
+
+		if refreshDoc || tail.DocumentData == "" {
+			if err := match.FetchDocument(ctx, tail); err != nil {
+				logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
+				FetchDocumentErrCount++
+				// delete the document from the index
+				cdoc := &doc.KustomizationDocument{
+					Document: *tail,
+				}
+				seen.Set(cdoc.ID(), tail.FileType)
+				if err := indx(cdoc, index.Delete); err != nil {
+					logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
+				}
+				deleteDocCount++
+				continue
+			}
+		}
+
+		if refreshDoc || tail.CreationTime == nil {
+			if err := match.SetCreated(ctx, tail); err != nil {
+				logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
+				SetCreatedErrCount++
+			}
+		}
+
+		cdoc, err := conv(tail)
+		// If conv returns an error, cdoc can still be added into the index so that
+		// cdoc.Document can be searched.
+		if err != nil {
+			logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
+			convErrCount++
+		}
+
+		UpdatedDocCount++
+		addBranches(cdoc, match, indx, seen, stack)
+	}
+	logger.Printf("Summary of doCrawl:\n")
+	logger.Printf("\t%d documents were updated\n", UpdatedDocCount)
+	logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount)
+	logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount)
+	logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount)
+	logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n",
+		FetchDocumentErrCount, deleteDocCount)
+	logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount)
+	logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount)
+}
+
+// CrawlFromSeed updates all the documents in seed, and crawls all the new
+// documents referred in the seed.
+func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
+	conv Converter, indx IndexFunc, seen utils.SeenMap) {
+
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// Exploit seed to update bulk of corpus.
+	logger.Printf("updating %d documents from seed\n", len(seed))
+	// each unique document in seed will be crawled once.
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false)
+
+	// Traverse any new documents added while updating corpus.
+	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
+	// While crawling each document in stack, the documents directly referred in the document
+	// will be added into stack.
+	// After this statement is done, stack will become empty.
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+}
+
+// CrawlGithubRunner is a blocking function and only returns once all of the
+// crawlers are finished with execution.
+//
+// This function uses the output channel to forward kustomization documents
+// from a list of crawlers. The output is to be consumed by a database/search
+// indexer for later retrieval.
+//
+// The return value is an array of errors in which each index represents the
+// index of the crawler that emitted the error. Although the errors themselves
+// can be nil, the array will always be exactly the size of the crawlers array.
+//
+// CrawlGithubRunner takes in a seed, which represents the documents stored in an
+// index somewhere. The document data is not required to be populated. If there
+// are many documents, this is preferable. The order of iteration over the seed
+// is not guaranteed, but the CrawlGithub does guarantee that every element
+// from the seed will be processed before any other documents from the
+// crawlers.
+func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
+	crawlers []Crawler, seen utils.SeenMap) []error {
+
+	errs := make([]error, len(crawlers))
+	wg := sync.WaitGroup{}
+
+	for i, crawler := range crawlers {
+		// Crawler implementations get their own channels to prevent a
+		// crawler from closing the main output channel.
+		docs := make(chan CrawledDocument)
+		wg.Add(2)
+
+		// Forward all of the documents from this crawler's channel to
+		// the main output channel.
+		go func(docs <-chan CrawledDocument) {
+			defer wg.Done()
+			for d := range docs {
+				output <- d
+			}
+		}(docs)
+
+		// Run this crawler and capture its returned error.
+		go func(idx int, crawler Crawler,
+			docs chan<- CrawledDocument) {
+
+			defer func() {
+				wg.Done()
+				if r := recover(); r != nil {
+					errs[idx] = fmt.Errorf(
+						"%+v panicked: %v, additional error %v",
+						crawler, r, errs[idx],
+					)
+				}
+			}()
+			defer close(docs)
+			errs[idx] = crawler.Crawl(ctx, docs, seen)
+		}(i, crawler, docs) // Copies the index and the crawler
+	}
+
+	wg.Wait()
+	return errs
+}
+
+// CrawlGithub crawls all the kustomization files on Github.
+func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
+	indx IndexFunc, seen utils.SeenMap) {
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// ch is channel where all the crawlers sends the crawled documents to.
+	ch := make(chan CrawledDocument, 1<<10)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		docCount := 0
+		for cdoc := range ch {
+			docCount++
+			logger.Printf("Processing doc %d found on Github", docCount)
+			// all the docs here are kustomization files found by querying Github, and
+			// their `FileType` fields all should be empty.
+			if seen.Seen(cdoc.ID()) {
+				logger.Printf("the doc has been seen before")
+				continue
+			}
+			match := findMatch(cdoc.GetDocument(), crawlers)
+			if match == nil {
+				logIfErr(fmt.Errorf(
+					"%v could not match any crawler", cdoc))
+				continue
+			}
+			addBranches(cdoc, match, indx, seen, &stack)
+		}
+	}()
+
+	logger.Println("processing the documents found from crawling github")
+	if errs := CrawlGithubRunner(ctx, ch, crawlers, seen); errs != nil {
+		for _, err := range errs {
+			logIfErr(err)
+		}
+	}
+	close(ch)
+	wg.Wait()
+
+	// Handle deps of newly discovered documents.
+	logger.Printf("crawling the %d new documents referred by other documents",
+		len(stack))
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+}

api/internal/crawl/crawler/crawler_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"reflect"
 	"sort"
 	"strings"
@@ -11,8 +12,12 @@ import (
 	"testing"
 	"time"
 
-	"sigs.k8s.io/kustomize/api/pgmconfig"
-	"sigs.k8s.io/kustomize/hack/crawl/doc"
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/konfig"
 )
 
 const (
@@ -29,21 +34,24 @@ type testCrawler struct {
 }
 
 func (c testCrawler) Match(d *doc.Document) bool {
-	return d != nil && strings.HasPrefix(d.ID(), c.matchPrefix)
+	return d != nil
 }
 
-func (c testCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
+func (c testCrawler) SetDefaultBranch(d *doc.Document) {}
+
+func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	if i, ok := c.lukp[d.ID()]; ok {
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
-	for _, suffix := range pgmconfig.RecognizedKustomizationFileNames() {
-		fmt.Println(d.ID(), "/", suffix)
-		i, ok := c.lukp[d.ID()+"/"+suffix]
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
+		savedFilePath := d.FilePath
+		d.FilePath += "/" + suffix
+		i, ok := c.lukp[d.ID()]
 		if !ok {
+			d.FilePath = savedFilePath
 			continue
 		}
-		d.FilePath += "/" + suffix
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
@@ -51,7 +59,7 @@ func (c testCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
 		d, c.matchPrefix)
 }
 
-func (c testCrawler) SetCreated(ctx context.Context, d *doc.Document) error {
+func (c testCrawler) SetCreated(_ context.Context, d *doc.Document) error {
 	d.CreationTime = &time.Time{}
 	return nil
 }
@@ -71,12 +79,12 @@ func newCrawler(matchPrefix string, err error,
 }
 
 // Crawl implements the Crawler interface for testing.
-func (c testCrawler) Crawl(ctx context.Context,
-	output chan<- CrawledDocument) error {
+func (c testCrawler) Crawl(_ context.Context,
+	output chan<- CrawledDocument, _ utils.SeenMap) error {
 
 	for i, d := range c.docs {
 		isResource := true
-		for _, suffix := range pgmconfig.RecognizedKustomizationFileNames() {
+		for _, suffix := range konfig.RecognizedKustomizationFileNames() {
 			if strings.HasSuffix(d.FilePath, suffix) {
 				isResource = false
 				break
@@ -106,8 +114,8 @@ func (s sortableDocs) Len() int {
 	return len(s)
 }
 
-func TestCrawlerRunner(t *testing.T) {
-	fmt.Println("testing CRunner")
+func TestCrawlGithubRunner(t *testing.T) {
+	log.Println("testing CrawlGithubRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -178,8 +186,9 @@ func TestCrawlerRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()
 
-			errs := CRunner(context.Background(),
-				output, test.tc)
+			seen := utils.NewSeenMap()
+			errs := CrawlGithubRunner(context.Background(),
+				output, test.tc, seen)
 
 			// Check that errors are returned as they should be.
 			if !reflect.DeepEqual(errs, test.errs) {
@@ -212,7 +221,7 @@ func TestCrawlerRunner(t *testing.T) {
 }
 
 func TestCrawlFromSeed(t *testing.T) {
-	fmt.Println("testing CrawlFromSeed")
+	log.Println("testing CrawlFromSeed")
 
 	tests := []struct {
 		seed    CrawlSeed
@@ -302,29 +311,6 @@ resources:
 					RepositoryURL: kustomizeRepo,
 					FilePath:      "examples/seedcrawl2/job.yaml",
 				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/base/kustomization.yaml",
-					DocumentData: `
-resources:
-- ../app
-`,
-				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/kustomization.yaml",
-					DocumentData: `
-resources:
-- resource.yaml
-`,
-				}},
-				// Visited from crawling runner imported as resource.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/resource.yaml",
-				}},
 			},
 		},
 	}
@@ -338,10 +324,11 @@ resources:
 					Document: *d,
 				}, nil
 			},
-			func(d CrawledDocument, cr Crawler) error {
+			func(d CrawledDocument, mode index.Mode) error {
 				visited[d.ID()]++
 				return nil
 			},
+			utils.NewSeenMap(),
 		)
 		if lv, lc := len(visited), len(tc.corpus); lv != lc {
 			t.Errorf("error: %d of %d documents visited.", lv, lc)

api/internal/crawl/crawler/github/crawler.go

@@ -0,0 +1,759 @@
+// Package github implements the crawler.Crawler interface, getting data
+// from the Github search API.
+package github
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math"
+	"net/http"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
+	"sigs.k8s.io/kustomize/api/internal/git"
+	"sigs.k8s.io/kustomize/api/konfig"
+)
+
+var logger = log.New(os.Stdout, "Github Crawler: ",
+	log.LstdFlags|log.LUTC|log.Llongfile)
+
+// Implements crawler.Crawler.
+type githubCrawler struct {
+	client GhClient
+	query  Query
+	// branchMap maps github repositories to their default branches
+	branchMap map[string]string
+}
+
+type GhClient struct {
+	RequestConfig
+	retryCount  uint64
+	client      *http.Client
+	accessToken string
+}
+
+func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
+	query Query) githubCrawler {
+
+	return githubCrawler{
+		client: GhClient{
+			retryCount: retryCount,
+			client:     client,
+			RequestConfig: RequestConfig{
+				perPage: githubMaxPageSize,
+			},
+			accessToken: accessToken,
+		},
+		query:     query,
+		branchMap: map[string]string{},
+	}
+}
+
+func (gc githubCrawler) SetDefaultBranch(d *doc.Document) {
+	url := gc.client.ReposRequest(d.RepositoryFullName())
+	defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+	d.DefaultBranch = defaultBranch
+	gc.branchMap[d.RepositoryURL] = d.DefaultBranch
+}
+
+func (gc githubCrawler) DefaultBranch(repo string) string {
+	return gc.branchMap[repo]
+}
+
+// Implements crawler.Crawler.
+func (gc githubCrawler) Crawl(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
+
+	ranges := []RangeWithin{
+		RangeWithin{
+		start: uint64(0),
+		end:   githubMaxFileSize,
+	},
+	}
+
+	errs := make(multiError, 0)
+	for len(ranges) > 0 {
+		tailRange := ranges[len(ranges) - 1]
+		ranges = ranges[:(len(ranges) - 1)]
+		reProcessQueryRanges, err := gc.CrawlSingleRange(ctx, output, seen, tailRange.start, tailRange.end)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		ranges = append(ranges, reProcessQueryRanges...)
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	lowerBound, upperBound uint64) ([]RangeWithin, error) {
+
+	log.Printf("CrawlSingleRange [%d, %d]", lowerBound, upperBound)
+
+	noETagClient := GhClient{
+		RequestConfig: gc.client.RequestConfig,
+		client:        &http.Client{Timeout: gc.client.client.Timeout},
+		retryCount:    gc.client.retryCount,
+		accessToken:   gc.client.accessToken,
+	}
+
+	var reProcessQueryRanges []RangeWithin
+
+	var ranges []string
+	var err error
+	// Since Github returns a max of 1000 results per query, we can use
+	// multiple queries that split the search space into chunks of at most
+	// 1000 files to get all of the data.
+	for i := 0; i < 5; i++ {
+		ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query),
+			lowerBound, upperBound)
+		if err == nil {
+			logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i)
+			break
+		} else {
+			time.Sleep(time.Minute)
+		}
+	}
+	if err != nil {
+		return reProcessQueryRanges, fmt.Errorf("could not split %v into ranges, %v\n",
+			gc.query, err)
+	}
+
+	logger.Println("ranges: ", ranges)
+
+	// Query each range for files.
+	errs := make(multiError, 0)
+	queryResult := RangeQueryResult{}
+	for _, query := range ranges {
+		reProcessQuery, rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		queryResult.Add(rangeResult)
+		if reProcessQuery {
+			reProcessQueryRanges = append(reProcessQueryRanges, RangeSizes(query))
+		}
+	}
+
+	logger.Printf("Summary of Crawl: %s", queryResult.String())
+
+	if len(errs) > 0 {
+		return reProcessQueryRanges, errs
+	}
+
+	return reProcessQueryRanges, nil
+}
+
+// FetchDocument first tries to fetch the document with d.FilePath. If it fails,
+// it will try to add each string in konfig.RecognizedKustomizationFileNames() to
+// d.FilePath, and try to fetch the document again.
+func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
+	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
+	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
+	if err != nil {
+		return fmt.Errorf("invalid repospec: %v", err)
+	}
+
+	url := "https://raw.githubusercontent.com/" + repoSpec.OrgRepo +
+		"/" + repoSpec.Ref + "/" + repoSpec.Path
+
+	handle := func(resp *http.Response, err error, path string) error {
+		if resp == nil {
+			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
+				url, path, err)
+		}
+		if err == nil && resp.StatusCode == http.StatusOK {
+			d.IsSame = httpclient.FromCache(resp.Header)
+			defer CloseResponseBody(resp)
+			data, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+			d.DocumentData = string(data)
+			d.FilePath = d.FilePath + path
+			return nil
+		}
+
+		return err
+	}
+	resp, errGetRawUserContent := gc.client.GetRawUserContent(url)
+	if err := handle(resp, errGetRawUserContent, ""); err == nil {
+		return nil
+	}
+
+	for _, file := range konfig.RecognizedKustomizationFileNames() {
+		resp, errGetRawUserContent = gc.client.GetRawUserContent(url + "/" + file)
+		if err = handle(resp, errGetRawUserContent, "/"+file); err == nil {
+			return nil
+		}
+	}
+	return fmt.Errorf("file not found: %s, error: %v", url, err)
+}
+
+func (gc githubCrawler) SetCreated(_ context.Context, d *doc.Document) error {
+	fs := GhFileSpec{
+		Path: d.FilePath,
+		Repository: GitRepository{
+			FullName: d.RepositoryFullName(),
+		},
+	}
+	creationTime, err := gc.client.GetFileCreationTime(fs)
+	if err != nil {
+		return err
+	}
+	d.CreationTime = &creationTime
+	return nil
+}
+
+func (gc githubCrawler) Match(d *doc.Document) bool {
+	url := d.RepositoryURL + "/" + d.FilePath + "?ref=" + "/" +
+		d.DefaultBranch
+	repoSpec, err := git.NewRepoSpecFromUrl(url)
+	if err != nil {
+		return false
+	}
+
+	return strings.Contains(repoSpec.Host, "github.com")
+}
+
+type RangeQueryResult struct {
+	totalDocCnt uint64
+	seenDocCnt  uint64
+	newDocCnt   uint64
+	errorCnt    uint64
+}
+
+func (r *RangeQueryResult) Add(other RangeQueryResult) {
+	r.totalDocCnt += other.totalDocCnt
+	r.newDocCnt += other.newDocCnt
+	r.seenDocCnt += other.seenDocCnt
+	r.errorCnt += other.errorCnt
+}
+
+func (r *RangeQueryResult) String() string {
+	return fmt.Sprintf("got %d files from API. "+
+		"%d have been seen before. %d are new and sent to the output channel."+
+		" %d have kustomizationResultAdapter errors.",
+		r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
+}
+
+// processQuery follows all of the pages in a query, and updates/adds the
+// documents from the crawl to the datastore/index.
+func processQuery(ctx context.Context, gcl GhClient, query string,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	branchMap map[string]string) (bool, RangeQueryResult, error) {
+
+	queryPages := make(chan GhResponseInfo)
+
+	go func() {
+		// Forward the document metadata to the retrieval channel.
+		// This separation allows for concurrent requests for the code
+		// search, and the retrieval portions of the API.
+		err := gcl.ForwardPaginatedQuery(ctx, query, queryPages)
+		if err != nil {
+			// TODO(damienr74) handle this error with redis?
+			logger.Println(err)
+		}
+		close(queryPages)
+	}()
+
+	reProcessQuery := false
+
+	errs := make(multiError, 0)
+	result := RangeQueryResult{}
+	pageID := 1
+	for page := range queryPages {
+		if page.Error != nil {
+			errs = append(errs, page.Error)
+			continue
+		}
+		pageResult := RangeQueryResult{}
+		for _, file := range page.Parsed.Items {
+			k, err := kustomizationResultAdapter(gcl, file, seen, branchMap)
+			if err != nil {
+				logger.Printf("kustomizationResultAdapter failed: %v", err)
+				errs = append(errs, err)
+				pageResult.errorCnt++
+			}
+			if k != nil {
+				pageResult.newDocCnt++
+				output <- k
+			} else {
+				pageResult.seenDocCnt++
+			}
+			pageResult.totalDocCnt++
+		}
+
+		logger.Printf("processQuery [TotalCount %d - page %d]: %s",
+			page.Parsed.TotalCount, pageID, pageResult.String())
+		result.Add(pageResult)
+
+		pageID++
+
+		if page.Parsed.TotalCount > githubMaxResultsPerQuery {
+			reProcessQuery = true
+		}
+	}
+
+	logger.Printf("Summary of processQuery: %s", result.String())
+
+	return reProcessQuery, result, errs
+}
+
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
+	branchMap map[string]string) (crawler.CrawledDocument, error) {
+	url := gcl.ReposRequest(k.Repository.FullName)
+	defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+
+	// document here is a kustomization file found by querying Github, whose
+	// `FileType` field should be empty.
+	document := doc.Document{
+		FilePath:      k.Path,
+		DefaultBranch: defaultBranch,
+		RepositoryURL: k.Repository.URL,
+		User:          doc.UserName(k.Repository.URL),
+	}
+
+	if seen.Seen(document.ID()) {
+		return nil, nil
+	}
+
+	data, err := gcl.GetFileData(k)
+	if err != nil {
+		return nil, err
+	}
+
+	d := doc.KustomizationDocument{
+		Document: doc.Document{
+			DocumentData:  string(data),
+			FilePath:      k.Path,
+			DefaultBranch: defaultBranch,
+			RepositoryURL: k.Repository.URL,
+			User:          doc.UserName(k.Repository.URL),
+		},
+	}
+	creationTime, err := gcl.GetFileCreationTime(k)
+	if err != nil {
+		logger.Printf("GetFileCreationTime failed: %v", err)
+		return &d, err
+	}
+	d.CreationTime = &creationTime
+
+	if err := d.ParseYAML(); err != nil {
+		logger.Printf("ParseYAML failed: %v", err)
+		return &d, err
+	}
+
+	return &d, nil
+}
+
+// ForwardPaginatedQuery follows the links to the next pages and performs all of
+// the queries for a given search query, relaying the data from each request
+// back to an output channel.
+func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
+	output chan<- GhResponseInfo) error {
+
+	logger.Println("querying: ", query)
+	response := gcl.parseGithubResponseWithRetry(query)
+
+	if response.Error != nil {
+		return response.Error
+	}
+
+	output <- response
+
+	for response.LastURL != "" && response.NextURL != "" {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			response = gcl.parseGithubResponseWithRetry(response.NextURL)
+			if response.Error != nil {
+				return response.Error
+			}
+
+			output <- response
+		}
+	}
+
+	return nil
+}
+
+// GetFileData gets the bytes from a file.
+func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
+
+	url := gcl.ContentsRequest(k.Repository.FullName, k.Path)
+
+	resp, err := gcl.GetReposData(url)
+	if err != nil {
+		return nil, fmt.Errorf("%+v: could not get '%s' metadata: %v",
+			k, url, err)
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
+			k, url, err)
+	}
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}
+
+	type githubContentRawURL struct {
+		DownloadURL string `json:"download_url,omitempty"`
+	}
+	var rawURL githubContentRawURL
+	err = json.Unmarshal(data, &rawURL)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"%+v: could not get 'download_url' from '%s' response: %v",
+			k, data, err)
+	}
+
+	resp, err = gcl.GetRawUserContent(rawURL.DownloadURL)
+	if err != nil {
+		return nil, fmt.Errorf("%+v: could not fetch file raw data '%s': %v",
+			k, rawURL.DownloadURL, err)
+	}
+
+	defer CloseResponseBody(resp)
+	data, err = ioutil.ReadAll(resp.Body)
+	return data, err
+}
+
+func CloseResponseBody(resp *http.Response) {
+	if err := resp.Body.Close(); err != nil {
+		log.Printf("failed to close response body: %v", err)
+	}
+}
+
+// GetDefaultBranch gets the default branch of a github repository.
+// m is a map which maps a github repository to its default branch.
+// If repo is already in m, the default branch for url will be obtained from m;
+// otherwise, a query will be made to github to obtain the default branch.
+func (gcl GhClient) GetDefaultBranch(url, repo string, m map[string]string) (string, error) {
+	if v, ok := m[repo]; ok {
+		return v, nil
+	}
+
+	resp, err := gcl.GetReposData(url)
+	if err != nil {
+		return "", fmt.Errorf(
+			"'%s' could not get default_branch: %v", url, err)
+	}
+	defer CloseResponseBody(resp)
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf(
+			"could not read default_branch: %v", err)
+	}
+
+	type defaultBranch struct {
+		DefaultBranch string `json:"default_branch,omitempty"`
+	}
+	var branch defaultBranch
+	err = json.Unmarshal(data, &branch)
+	if err != nil {
+		return "", fmt.Errorf(
+			"default_branch json malformed: %v", err)
+	}
+
+	return branch.DefaultBranch, nil
+}
+
+// GetFileCreationTime gets the earliest date of a file.
+func (gcl GhClient) GetFileCreationTime(
+	k GhFileSpec) (time.Time, error) {
+
+	url := gcl.CommitsRequest(k.Repository.FullName, k.Path)
+
+	defaultTime := time.Now()
+
+	resp, err := gcl.GetReposData(url)
+	if err != nil {
+		return defaultTime, fmt.Errorf(
+			"%+v: '%s' could not get metadata: %v", k, url, err)
+	}
+
+	type DateSpec struct {
+		Commit struct {
+			Author struct {
+				Date string `json:"date,omitempty"`
+			} `json:"author,omitempty"`
+		} `json:"commit,omitempty"`
+	}
+
+	_, lastURL := parseGithubLinkFormat(resp.Header.Get("link"))
+	if lastURL != "" {
+		resp, err = gcl.GetReposData(lastURL)
+		if err != nil {
+			return defaultTime, fmt.Errorf(
+				"%+v: '%s' could not get metadata: %v",
+				k, lastURL, err)
+		}
+	}
+
+	defer CloseResponseBody(resp)
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return defaultTime, fmt.Errorf(
+			"%+v: failed to read metadata: %v", k, err)
+	}
+	var earliestDate []DateSpec
+	err = json.Unmarshal(data, &earliestDate)
+	size := len(earliestDate)
+	if err != nil || size == 0 {
+		return defaultTime, fmt.Errorf(
+			"%+v: server response '%s' not in expected format: %v",
+			k, data, err)
+	}
+
+	return time.Parse(time.RFC3339, earliestDate[size-1].Commit.Author.Date)
+}
+
+// TODO(damienr74) change the tickers to actually check api rate limits, reset
+// times, and throttle requests dynamically based off of current utilization,
+// instead of hardcoding the documented values, these calls are not quota'd.
+// This is now especially important, since caching the API requests will reduce
+// API quota use (so we can actually make more requests in the allotted time
+// period).
+//
+// See https://developer.github.com/v3/rate_limit/ for details.
+var (
+	searchRateTicker  = time.NewTicker(time.Second * 2)
+	contentRateTicker = time.NewTicker(time.Second * 1)
+)
+
+func throttleSearchAPI() {
+	<-searchRateTicker.C
+}
+
+func throttleRepoAPI() {
+	<-contentRateTicker.C
+}
+
+type multiError []error
+
+func (e multiError) Error() string {
+	size := len(e) + 2
+	strs := make([]string, size)
+	strs[0] = "Errors ["
+	for i, err := range e {
+		strs[i+1] = "\t" + err.Error()
+	}
+	strs[size-1] = "]"
+	return strings.Join(strs, "\n")
+}
+
+type GitRepository struct {
+	API      string `json:"url,omitempty"`
+	URL      string `json:"html_url,omitempty"`
+	FullName string `json:"full_name,omitempty"`
+}
+
+type GhFileSpec struct {
+	Path       string        `json:"path,omitempty"`
+	Repository GitRepository `json:"repository,omitempty"`
+}
+
+type githubResponse struct {
+	// MaxUint is reserved as a sentinel value.
+	// This is the number of files that match the query.
+	TotalCount uint64 `json:"total_count,omitempty"`
+
+	IncompleteResults bool `json:"incomplete_results,omitempty"`
+
+	// Github representation of a file.
+	Items []GhFileSpec `json:"items,omitempty"`
+}
+
+type GhResponseInfo struct {
+	*http.Response
+	Parsed  *githubResponse
+	Error   error
+	NextURL string
+	LastURL string
+}
+
+func parseGithubLinkFormat(links string) (string, string) {
+	const (
+		linkNext    = "next"
+		linkLast    = "last"
+		linkInfoURL = 1
+		linkInfoRel = 2
+	)
+
+	next, last := "", ""
+	linkInfo := regexp.MustCompile(`<(.*)>.*; rel="(last|next)"`)
+
+	for _, link := range strings.Split(links, ",") {
+		linkParse := linkInfo.FindStringSubmatch(link)
+		if len(linkParse) != 3 {
+			continue
+		}
+
+		url := linkParse[linkInfoURL]
+		switch linkParse[linkInfoRel] {
+		case linkNext:
+			next = url
+		case linkLast:
+			last = url
+		default:
+		}
+	}
+
+	return next, last
+}
+
+func (gcl GhClient) parseGithubResponseWithRetry(getRequest string) GhResponseInfo {
+	resp := gcl.parseGithubResponse(getRequest)
+	retries := 0
+	for resp.Parsed.IncompleteResults {
+		resp = gcl.parseGithubResponse(getRequest)
+		retries++
+	}
+	log.Printf("The result of query(%s) is complete after %d retries", getRequest, retries)
+	return resp
+}
+
+func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
+	resp, err := gcl.SearchGithubAPI(getRequest)
+	requestInfo := GhResponseInfo{
+		Response: resp,
+		Error:    err,
+		Parsed:   nil,
+	}
+
+	if err != nil || resp == nil {
+		return requestInfo
+	}
+
+	var data []byte
+	defer CloseResponseBody(resp)
+	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
+	if requestInfo.Error != nil {
+		return requestInfo
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		logger.Println("query: ", getRequest)
+		logger.Println("status not OK at the source")
+		logger.Println("header dump", resp.Header)
+		logger.Println("body dump", string(data))
+		requestInfo.Error = fmt.Errorf("request rejected, status '%s'",
+			resp.Status)
+		return requestInfo
+	}
+
+	requestInfo.NextURL, requestInfo.LastURL =
+		parseGithubLinkFormat(resp.Header.Get("link"))
+
+	resultCount := githubResponse{
+		TotalCount: math.MaxUint64,
+	}
+	requestInfo.Error = json.Unmarshal(data, &resultCount)
+	if requestInfo.Error != nil {
+		return requestInfo
+	}
+
+	requestInfo.Parsed = &resultCount
+
+	return requestInfo
+
+}
+
+// SearchGithubAPI performs a search query and handles rate limitting for
+// the 'search/code?' endpoint as well as timed retries in the case of abuse
+// prevention.
+func (gcl GhClient) SearchGithubAPI(query string) (*http.Response, error) {
+	throttleSearchAPI()
+	return gcl.getWithRetry(query)
+}
+
+// GetReposData performs a search query and handles rate limitting for
+// the '/repos' endpoint as well as timed retries in the case of abuse
+// prevention.
+func (gcl GhClient) GetReposData(query string) (*http.Response, error) {
+	throttleRepoAPI()
+	return gcl.getWithRetry(query)
+}
+
+// User content (file contents) is not API rate limited, so there's no use in
+// throttling this call.
+func (gcl GhClient) GetRawUserContent(query string) (*http.Response, error) {
+	return gcl.getWithRetry(query)
+}
+
+func (gcl GhClient) Do(query string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", query, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("token %s", gcl.accessToken))
+
+	// gcl.client.Do: a non-2xx status code doesn't cause an error.
+	// See https://golang.org/pkg/net/http/#Client.Do for more info.
+	resp, err := gcl.client.Do(req)
+	if resp != nil && resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
+			query, resp.StatusCode)
+	}
+	return resp, err
+}
+
+func (gcl GhClient) getWithRetry(
+	query string) (resp *http.Response, err error) {
+
+	resp, err = gcl.Do(query)
+
+	retryCount := gcl.retryCount
+
+	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
+		retryTime := resp.Header.Get("Retry-After")
+		i, errAtoi := strconv.Atoi(retryTime)
+		if errAtoi != nil {
+			return resp, fmt.Errorf(
+				"query '%s' forbidden without 'Retry-After'", query)
+		}
+		logger.Printf(
+			"status forbidden, retring %d more times\n", retryCount)
+
+		logger.Printf("waiting %d seconds before retrying\n", i)
+		time.Sleep(time.Second * time.Duration(i))
+		retryCount--
+		resp, err = gcl.Do(query)
+	}
+
+	if err != nil {
+		return resp, fmt.Errorf("query '%s' could not be processed, %v",
+			query, err)
+	}
+
+	return resp, err
+}

api/internal/crawl/crawler/github/queries.go

@@ -7,10 +7,11 @@ import (
 )
 
 const (
-	perPageArg     = "per_page"
-	accessTokenArg = "access_token"
+	perPageArg = "per_page"
 )
 
+const githubMaxPageSize = 100
+
 // Implementation detail, not important to external API.
 type queryField struct {
 	name  string
@@ -89,6 +90,17 @@ func Path(p string) queryField {
 	return queryField{name: "path", value: p}
 }
 
+// Repo takes a repository (i.e., kubernetes-sigs/kustomize) and formats
+// it according to the Github API.
+func Repo(r string) queryField {
+	return queryField{name: "repo", value: r}
+}
+
+// Path takes a github username and formats it according to the Github API.
+func User(u string) queryField {
+	return queryField{name: "user", value: u}
+}
+
 // RequestConfig stores common variables that must be present for the queries.
 // - CodeSearchRequests: ask Github to check the code indices given a query.
 // - ContentsRequests: ask Github where to download a resource given a repo and a
@@ -96,15 +108,7 @@ func Path(p string) queryField {
 // - CommitsRequests: asks Github to list commits made one a file. Useful to
 // determine the date of a file.
 type RequestConfig struct {
-	perPage     uint64
-	accessToken string
-}
-
-func NewRequestConfig(perPage uint64, accessToken string) RequestConfig {
-	return RequestConfig{
-		perPage:     perPage,
-		accessToken: accessToken,
-	}
+	perPage uint64
 }
 
 // CodeSearchRequestWith given a list of query parameters that specify the
@@ -112,9 +116,11 @@ func NewRequestConfig(perPage uint64, accessToken string) RequestConfig {
 // the URL method to get the string value of the URL. See request.CopyWith, to
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
-	req := rc.makeRequest("search/code", query)
-	req.vals.Set("sort", "indexed")
-	req.vals.Set("order", "desc")
+	vals := url.Values{
+		"sort":  []string{"indexed"},
+		"order": []string{"desc"},
+	}
+	req := rc.makeRequest("search/code", query, vals)
 	return req
 }
 
@@ -122,26 +128,25 @@ func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 // query for the Github API to find the dowload information of this filepath.
 func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 func (rc RequestConfig) ReposRequest(fullRepoName string) string {
 	uri := fmt.Sprintf("repos/%s", fullRepoName)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 // CommitsRequest given the repo name, and a filepath returns a formatted query
 // for the Github API to find the commits that affect this file.
 func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
-	return rc.makeRequest(uri, Query{Path(path)}).URL()
+	vals := url.Values{
+		"path": []string{path},
+	}
+	return rc.makeRequest(uri, Query{}, vals).URL()
 }
 
-func (rc RequestConfig) makeRequest(path string, query Query) request {
-	vals := url.Values{}
-	if rc.accessToken != "" {
-		vals.Set(accessTokenArg, rc.accessToken)
-	}
+func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
 	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
 
 	return request{
@@ -161,7 +166,7 @@ type request struct {
 	query Query
 }
 
-// CopyWith copies the requests and adds the extra query parameters. Usefull
+// CopyWith copies the requests and adds the extra query parameters. It is useful
 // for dynamically adding sizes to a filename only query without modifying it.
 func (r request) CopyWith(queryParams ...queryField) request {
 	cpy := r
@@ -183,7 +188,7 @@ func (r request) URL() string {
 	if encoded == "" && query != "" {
 		sep = "?"
 	}
-	r.url.RawQuery = encoded + sep + query
+	r.url.RawQuery = query + sep + encoded
 	return r.url.String()
 }
 

api/internal/crawl/crawler/github/queries_test.go

@@ -0,0 +1,140 @@
+package github
+
+import (
+	"testing"
+)
+
+func TestQueryFields(t *testing.T) {
+	testCases := []struct {
+		formatter queryField
+		expected  string
+	}{
+		{
+			formatter: Keyword("keyword"),
+			expected:  "keyword",
+		},
+		{
+			formatter: Filesize(RangeLessThan{23}),
+			expected:  "size:<23",
+		},
+		{
+			formatter: Filesize(RangeWithin{24, 64}),
+			expected:  "size:24..64",
+		},
+		{
+			formatter: Filesize(RangeGreaterThan{64}),
+			expected:  "size:>64",
+		},
+		{
+			formatter: Path("some/path/to/file"),
+			expected:  "path:some/path/to/file",
+		},
+		{
+			formatter: Filename("kustomization.yaml"),
+			expected:  "filename:kustomization.yaml",
+		},
+	}
+
+	for _, test := range testCases {
+		if result := test.formatter.String(); result != test.expected {
+			t.Errorf("got (%#v = %s), expected %s", test.formatter, result, test.expected)
+		}
+	}
+}
+
+func TestQueryType(t *testing.T) {
+	testCases := []struct {
+		query    Query
+		expected string
+	}{
+		{
+			query: QueryWith(
+				Filesize(RangeWithin{24, 64}),
+				Filename("kustomization.yaml"),
+				Keyword("keyword1"),
+				Keyword("keyword2"),
+				Repo("user1/repo1"),
+				User("user1"),
+			),
+			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2+" +
+				"repo:user1/repo1+user:user1",
+		},
+	}
+
+	for _, test := range testCases {
+		if queryStr := test.query.String(); queryStr != test.expected {
+			t.Errorf("got (%#v = %s), expected %s", test.query, queryStr, test.expected)
+		}
+
+	}
+}
+
+func TestGithubSearchQuery(t *testing.T) {
+	const (
+		perPage = 100
+	)
+
+	testCases := []struct {
+		rc                    RequestConfig
+		codeQuery             Query
+		fullRepoName          string
+		path                  string
+		expectedCodeQuery     string
+		expectedContentsQuery string
+		expectedCommitsQuery  string
+	}{
+		{
+			rc: RequestConfig{
+				perPage: perPage,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples/helloWorld/kustomization.yaml?per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+		},
+		{
+			rc: RequestConfig{
+				perPage: perPage,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples 1/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples%201/helloWorld/kustomization.yaml?per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+		},
+	}
+
+	for _, test := range testCases {
+		if result := test.rc.CodeSearchRequestWith(test.codeQuery).URL(); result != test.expectedCodeQuery {
+			t.Errorf("Got code query: %s, expected %s", result, test.expectedCodeQuery)
+		}
+
+		if result := test.rc.ContentsRequest(test.fullRepoName, test.path); result != test.expectedContentsQuery {
+			t.Errorf("Got contents query: %s, expected %s", result, test.expectedContentsQuery)
+		}
+		if result := test.rc.CommitsRequest(test.fullRepoName, test.path); result != test.expectedCommitsQuery {
+			t.Errorf("Got commits query: %s, expected %s", result, test.expectedCommitsQuery)
+		}
+	}
+}

api/internal/crawl/crawler/github/split_search_ranges.go

@@ -93,13 +93,15 @@ package github
 //   apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000
 //   = apiCallsPerResult * totalResults.
 //
-// So it could very well take apiCallsPerResult * 50 times longer to acutally
+// So it could very well take apiCallsPerResult * 50 times longer to actually
 // fetch the results (assuming the quotas for the API calls are the same as the
 // search API), than it does to perform these range searches.
 
 import (
 	"fmt"
 	"math/bits"
+	"strconv"
+	"strings"
 )
 
 // Files cannot be more than 2^19 bytes, according to
@@ -112,7 +114,7 @@ const (
 // Interface instead of struct for testing purposes.
 // Not expecting to have multiple implementations.
 type cachedSearch interface {
-	CountResults(uint64) (uint64, error)
+	CountResults(uint64, uint64) (uint64, error)
 	RequestString(filesize rangeFormatter) string
 }
 
@@ -139,7 +141,7 @@ type cachedSearch interface {
 //    problematic). The current cache implementation looks at the
 //    predecessor entry to find out if the current value is monotonic.
 //    This is where the bit trick is used, since each step in the binary
-//    search is adding or ommiting to add a decreasing power of 2 to the query
+//    search is adding or omitting to add a decreasing power of 2 to the query
 //    value, we can remove the least significant set bit to find the
 //    predecessor in constant time. Ultimately since the search is rate
 //    limited, we could also easily afford to compute this in linear time
@@ -161,16 +163,16 @@ func newCache(client GhClient, query Query) githubCachedSearch {
 	}
 }
 
-func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
+func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
 	count, cached := c.cache[upperBound]
 	if cached {
 		return count, nil
 	}
 
-	sizeRange := RangeWithin{0, upperBound}
+	sizeRange := RangeWithin{lowerBound, upperBound}
 	rangeRequest := c.RequestString(sizeRange)
 
-	result := c.gcl.parseGithubResponse(rangeRequest)
+	result := c.gcl.parseGithubResponseWithRetry(rangeRequest)
 	if result.Error != nil {
 		return count, result.Error
 	}
@@ -204,7 +206,7 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 			"Retrying query... current lower bound: %d, got: %d\n",
 			c.cache[prev], result.Parsed.TotalCount)
 
-		result = c.gcl.parseGithubResponse(rangeRequest)
+		result = c.gcl.parseGithubResponseWithRetry(rangeRequest)
 		if result.Error != nil {
 			return count, result.Error
 		}
@@ -219,8 +221,8 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 	}
 
 	count = result.Parsed.TotalCount
-	logger.Printf("Caching new query %s, with count %d\n",
-		sizeRange.RangeString(), count)
+	logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n",
+		sizeRange.RangeString(), count, result.Parsed.IncompleteResults)
 	c.cache[upperBound] = count
 	return count, nil
 }
@@ -238,16 +240,16 @@ func (c githubCachedSearch) RequestString(filesize rangeFormatter) string {
 // This would mean that the search as it is could not find all files. If queries
 // are sorted by last indexed, and retrieved on regular intervals, it should be
 // sufficient to get most if not all documents.
-func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
-	totalFiles, err := cache.CountResults(githubMaxFileSize)
+func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64) ([]string, error) {
+	totalFiles, err := cache.CountResults(lowerBound, upperBound)
 	if err != nil {
 		return nil, err
 	}
-	logger.Println("total files: ", totalFiles)
+	logger.Println("total kustomization files: ", totalFiles)
 
 	if githubMaxResultsPerQuery >= totalFiles {
 		return []string{
-			cache.RequestString(RangeWithin{0, githubMaxFileSize}),
+			cache.RequestString(RangeWithin{lowerBound, upperBound}),
 		}, nil
 	}
 
@@ -275,6 +277,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 	// range.
 	filesAccessible := uint64(0)
 	sizes := make([]uint64, 0)
+	sizes = append(sizes, lowerBound)
 	for filesAccessible < totalFiles {
 		target := filesAccessible + githubMaxResultsPerQuery
 		if target >= totalFiles {
@@ -284,22 +287,22 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 		logger.Printf("%d accessible files, next target = %d\n",
 			filesAccessible, target)
 
-		cur, err := lowerBoundFileCount(cache, target)
+		size, err := FindFileSize(cache, target, lowerBound, upperBound)
 		if err != nil {
 			return nil, err
 		}
 
 		// If there are more than 1000 files in the next bucket, we must
 		// advance anyway and lose out on some files :(.
-		if l := len(sizes); l > 0 && sizes[l-1] == cur {
-			cur++
+		if l := len(sizes); l > 0 && sizes[l-1] == size {
+			size++
 		}
 
-		nextAccessible, err := cache.CountResults(cur)
+		nextAccessible, err := cache.CountResults(lowerBound, size)
 		if err != nil {
 			return nil, fmt.Errorf(
 				"cache should be populated at %d already, got %v",
-				cur, err)
+				size, err)
 		}
 		if nextAccessible < filesAccessible {
 			return nil, fmt.Errorf(
@@ -309,31 +312,31 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 
 		filesAccessible = nextAccessible
 		if nextAccessible < totalFiles {
-			sizes = append(sizes, cur)
+			sizes = append(sizes, size)
 		}
 	}
-
+	sizes = append(sizes, upperBound)
 	return formatFilesizeRanges(cache, sizes), nil
 }
 
-// lowerBoundFileCount finds the filesize range from [0, return value] that has
+// FindFileSize finds the filesize range from [lowerBound, return value] that has
 // the largest file count that is smaller than or equal to
 // githubMaxResultsPerQuery. It is important to note that this returned value
 // could already be in a previous range if the next file size has more than 1000
 // results. It is left to the caller to handle this bit of logic and guarantee
 // forward progession in this case.
-func lowerBoundFileCount(
-	cache cachedSearch, targetFileCount uint64) (uint64, error) {
+func FindFileSize(
+	cache cachedSearch, targetFileCount, lowerBound, upperBound uint64) (uint64, error) {
 
 	// Binary search for file sizes that make up the next <=1000 element
 	// chunk.
-	cur := uint64(0)
-	increase := githubMaxFileSize / 2
+	cur := lowerBound
+	increase := (upperBound - lowerBound) / 2
 
 	for increase > 0 {
 		mid := cur + increase
 
-		count, err := cache.CountResults(mid)
+		count, err := cache.CountResults(lowerBound, mid)
 		if err != nil {
 			return count, err
 		}
@@ -353,26 +356,24 @@ func lowerBoundFileCount(
 }
 
 func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string {
-	ranges := make([]string, 0, len(sizes)+1)
-
-	if len(sizes) > 0 {
-		ranges = append(ranges, cache.RequestString(
-			RangeLessThan{sizes[0] + 1},
-		))
+	n := len(sizes)
+	if n < 2 {
+		return []string{}
 	}
 
-	for i := 0; i < len(sizes)-1; i += 1 {
-		ranges = append(ranges, cache.RequestString(
-			RangeWithin{sizes[i] + 1, sizes[i+1]},
-		))
-
-		if i != len(sizes)-2 {
-			continue
-		}
-		ranges = append(ranges, cache.RequestString(
-			RangeGreaterThan{sizes[i+1]},
-		))
+	ranges := make([]string, 0, n-1)
+	ranges = append(ranges, cache.RequestString(RangeWithin{sizes[0], sizes[1]}))
+	for i := 1; i < n-1; i++ {
+		ranges = append(ranges, cache.RequestString(RangeWithin{sizes[i] + 1, sizes[i+1]}))
 	}
-
 	return ranges
 }
+
+func RangeSizes(s string) RangeWithin {
+	start := strings.Index(s, "+size:") + len("+size:")
+	end := strings.Index(s, "&")
+	ranges := strings.Split(s[start:end], "..")
+	lowerBound, _ := strconv.ParseUint(ranges[0], 10, 64)
+	upperBound, _ := strconv.ParseUint(ranges[1], 10, 64)
+	return RangeWithin{lowerBound, upperBound}
+}

api/internal/crawl/crawler/github/split_search_ranges_test.go

@@ -0,0 +1,101 @@
+package github
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+	"testing"
+)
+
+type testCachedSearch struct {
+	cache map[uint64]uint64
+}
+
+func (c testCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
+	log.Printf("CountResults(%05x)\n", upperBound)
+	count, ok := c.cache[upperBound]
+	if !ok {
+		return count, fmt.Errorf("cache not set at %x", upperBound)
+	}
+	return count, nil
+}
+
+func (c testCachedSearch) RequestString(filesize rangeFormatter) string {
+	return filesize.RangeString()
+}
+
+// TODO(damienr74) make tests easier to write.. I'm thinking I can make the test
+// cache take in a list of (filesize, count) pairs and it can populate the cache
+// without relying on how the implementation will create queries. This was only
+// a quick and dirty test to make sure that modifications are not going to break
+// the functionality.
+func TestRangeSplitting(t *testing.T) {
+	// Keys follow the binary search depending on whether or not the range
+	// is too small/large to find close to optimal filesize ranges. This
+	// test is heavily tied to the fact that the search is using powers of two
+	// to make progress in the search (hence the use of hexadecimal values).
+	cache := testCachedSearch{
+		map[uint64]uint64{
+			0x80000: 5000,
+			0x40000: 5000,
+			0x20000: 5000,
+			0x10000: 5000,
+			0x08000: 5000,
+			0x04000: 5000,
+			0x02000: 5000,
+			0x01000: 5000,
+			0x00fff: 3950,
+			0x00ffe: 3950,
+			0x00ffc: 3950,
+			0x00ff8: 3950,
+			0x00ff0: 3950,
+			0x00fe0: 3950,
+			0x00fc0: 3950,
+			0x00f80: 3950,
+			0x00f00: 3950,
+			0x00e00: 3950,
+			0x00c00: 3950,
+			0x00800: 3950,
+			0x00400: 3950,
+			0x00200: 3688,
+			0x00180: 3028,
+			0x00100: 2999,
+			0x000c0: 2448,
+			0x00080: 1999,
+			0x00070: 1600,
+			0x0006c: 1003,
+			0x0006b: 1001,
+			0x0006a: 999,
+			0x00068: 999,
+			0x00060: 999,
+			0x00040: 999,
+			0x00000: 0,
+		},
+	}
+
+	requests, err := FindRangesForRepoSearch(cache, 0, 524288)
+	if err != nil {
+		t.Errorf("Error while finding ranges: %v", err)
+	}
+	expected := []string{
+		"0..106",       // cache.RequestString(RangeWithin{0x00, 0x6a}),
+		"107..128",     // cache.RequestString(RangeWithin{0x6b, 0x80}),
+		"129..256",     // cache.RequestString(RangeWithin{0x81, 0x100}),
+		"257..4095",    // cache.RequestString(RangeWithin{0x101, 0xfff}),
+		"4096..524288", // cache.RequestString(RangeWithin{0x1000, 0x80000}),
+	}
+
+	if !reflect.DeepEqual(requests, expected) {
+		t.Errorf("Expected requests (%v) to equal (%v)", requests, expected)
+	}
+}
+
+func TestRangeSizes(t *testing.T) {
+	s := "https://api.github.com/search/code?q=filename:kustomization.yaml+filename:kustomization.yml" +
+		"+filename:kustomization+size:2365..10000&order=desc&per_page=100&sort=indexed"
+	returnedResult := RangeSizes(s)
+	expectedResult := RangeWithin{uint64(2365), uint64(10000)}
+	if !reflect.DeepEqual(returnedResult, expectedResult) {
+		t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult)
+	}
+}

api/internal/crawl/doc/doc.go

@@ -0,0 +1,267 @@
+package doc
+
+import (
+	"fmt"
+	"log"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/api/konfig"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+var fileReader = kunstruct.NewKunstructuredFactoryImpl()
+
+// This document is meant to be used at the elasticsearch document type.
+// Fields are serialized as-is to elasticsearch, where indices are built
+// to facilitate text search queries. Identifiers, Values, FilePath,
+// RepositoryURL and DocumentData are meant to be searched for text queries
+// directly, while the other fields can either be used as a filter, or as
+// additional metadata displayed in the UI.
+//
+// The fields of the document and their purpose are listed below:
+// - DocumentData contains the contents of the kustomization file.
+// - Kinds Represents the kubernetes Kinds that are in this file.
+// - Identifiers are a list of (partial and full) identifier paths that can be
+//   found by users. Each part of a path is delimited by ":" e.g. spec:replicas.
+// - Values are a list of identifier paths and their values that can be found by
+//   search queries. The path is delimited by ":" and the value follows the "="
+//   symbol e.g. spec:replicas=4.
+// - FilePath is the path of the file.
+// - RepositoryURL is the URL of the source repository.
+// - CreationTime is the time at which the file was created.
+//
+// Representing each Identifier and Value as a flat string representation
+// facilitates the use of complex text search features from elasticsearch such
+// as fuzzy searching, regex, wildcards, etc.
+type KustomizationDocument struct {
+	Document
+	Kinds       []string `json:"kinds,omitempty"`
+	Identifiers []string `json:"identifiers,omitempty"`
+	Values      []string `json:"values,omitempty"`
+}
+
+type set map[string]struct{}
+
+func (doc *KustomizationDocument) Copy() *KustomizationDocument {
+	return &KustomizationDocument{
+		Document:    *(doc.Document.Copy()),
+		Kinds:       doc.Kinds,
+		Identifiers: doc.Identifiers,
+		Values:      doc.Values,
+	}
+}
+
+func (doc *KustomizationDocument) String() string {
+	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
+		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
+}
+
+// IsKustomizationFile determines whether a file path is a kustomization file
+func IsKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
+		}
+	}
+	return false
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *KustomizationDocument) GetResources(
+	includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
+	if !IsKustomizationFile(doc.FilePath) {
+		return []*Document{}, nil
+	}
+
+	content := []byte(doc.DocumentData)
+	content, err := FixKustomizationPreUnmarshallingNonFatal(content)
+	if err != nil {
+		return nil, fmt.Errorf("could not fix kustomize file: %v", err)
+	}
+
+	var k types.Kustomization
+	err = yaml.Unmarshal(content, &k)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"could not parse kustomization: %v", err)
+	}
+	k.FixKustomizationPostUnmarshalling()
+
+	res := make([]*Document, 0)
+
+	if includeResources {
+		resourceDocs := doc.CollectDocuments(k.Resources, "resource")
+		res = append(res, resourceDocs...)
+	}
+
+	if includeGenerators {
+		generatorDocs := doc.CollectDocuments(k.Generators, "generator")
+		res = append(res, generatorDocs...)
+	}
+
+	if includeTransformers {
+		transformerDocs := doc.CollectDocuments(k.Transformers, "transformer")
+		res = append(res, transformerDocs...)
+	}
+
+	return res, nil
+}
+
+// CollectDocuments construct a Document for each path in paths, and return
+// a slice of Document pointers.
+func (doc *KustomizationDocument) CollectDocuments(
+	paths []string, fileType string) []*Document {
+	docs := make([]*Document, 0, len(paths))
+	for _, r := range paths {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
+		next, err := doc.Document.FromRelativePath(r)
+		if err != nil {
+			log.Printf("CollectDocuments error: %v\n", err)
+			continue
+		}
+		next.FileType = fileType
+		docs = append(docs, &next)
+	}
+	return docs
+}
+
+func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
+	data := []byte(doc.DocumentData)
+
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
+		if !strings.HasSuffix(doc.FilePath, "/"+suffix) {
+			continue
+		}
+		var config map[string]interface{}
+		err := yaml.Unmarshal(data, &config)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"unable to parse kustomization: %v", err)
+		}
+		return []map[string]interface{}{config}, nil
+	}
+
+	configs := make([]map[string]interface{}, 0)
+	ks, err := fileReader.SliceFromBytes(data)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse resource: %v", err)
+	}
+	for _, k := range ks {
+		configs = append(configs, k.Map())
+	}
+
+	return configs, nil
+}
+
+// ParseYAML parses doc.Document and sets the following fields of doc:
+// Kinds, Values, Identifiers.
+func (doc *KustomizationDocument) ParseYAML() error {
+	doc.Identifiers = make([]string, 0)
+	doc.Values = make([]string, 0)
+	doc.Kinds = make([]string, 0, 1)
+
+	identifierSet := make(set)
+	valueSet := make(set)
+	kindSet := make(set)
+
+	getKind := func(m map[string]interface{}) string {
+		const defaultStr = "Kustomization"
+		kind, ok := m["kind"]
+		if !ok {
+			return defaultStr
+		}
+		if str, ok := kind.(string); ok && str != "" {
+			return str
+		}
+		return defaultStr
+	}
+
+	ks, err := doc.readBytes()
+	if err != nil {
+		return err
+	}
+
+	for _, contents := range ks {
+		kindSet[getKind(contents)] = struct{}{}
+		createFlatStructure(identifierSet, valueSet, contents)
+	}
+
+	for val := range kindSet {
+		doc.Kinds = append(doc.Kinds, val)
+	}
+
+	for val := range valueSet {
+		doc.Values = append(doc.Values, val)
+	}
+
+	for key := range identifierSet {
+		doc.Identifiers = append(doc.Identifiers, key)
+	}
+
+	// Without sorting these fields, every time when the string order in these fields changes,
+	// the document in the index will be updated.
+	// Sorting these fields are necessary to avoid a document being updated unnecessarily.
+	sort.Strings(doc.Kinds)
+	sort.Strings(doc.Values)
+	sort.Strings(doc.Identifiers)
+
+	return nil
+}
+
+func createFlatStructure(identifierSet set, valueSet set, contents map[string]interface{}) {
+	type Map struct {
+		data   map[string]interface{}
+		prefix string
+	}
+
+	toVisit := []Map{
+		{
+			data:   contents,
+			prefix: "",
+		},
+	}
+
+	for i := 0; i < len(toVisit); i++ {
+		visiting := toVisit[i]
+		for k, v := range visiting.data {
+			identifier := fmt.Sprintf("%s:%s", visiting.prefix, k)
+			// noop after the first iteration.
+			identifier = strings.TrimLeft(identifier, ":")
+
+			// Recursive function traverses structure to find
+			// identifiers and values. These later get formatted
+			// into doc.Identifiers and doc.Values respectively.
+			var traverseStructure func(interface{})
+			traverseStructure = func(arg interface{}) {
+				switch value := arg.(type) {
+				case map[string]interface{}:
+					toVisit = append(toVisit, Map{
+						data:   value,
+						prefix: identifier,
+					})
+				case []interface{}:
+					for _, val := range value {
+						traverseStructure(val)
+					}
+				case interface{}:
+					esc := fmt.Sprintf("%v", value)
+
+					valuePath := fmt.Sprintf("%s=%v",
+						identifier, esc)
+					valueSet[valuePath] = struct{}{}
+				}
+			}
+			traverseStructure(v)
+
+			identifierSet[identifier] = struct{}{}
+		}
+	}
+}

api/internal/crawl/doc/doc_test.go

@@ -0,0 +1,379 @@
+package doc
+
+import (
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestParseYAML(t *testing.T) {
+	testCases := []struct {
+		identifiers []string
+		values      []string
+		kinds       []string
+		filepath    string
+		yaml        string
+	}{
+		{
+			identifiers: []string{
+				"namePrefix",
+				"metadata",
+				"metadata:name",
+				"kind",
+			},
+			values: []string{
+				"kind=",
+				"namePrefix=dev-",
+				"metadata:name=app",
+			},
+			kinds: []string{
+				"Kustomization",
+			},
+			filepath: "some/path/to/kustomization.yaml",
+			yaml: `
+namePrefix: dev-
+metadata:
+  name: app
+kind: ""
+`,
+		},
+		{
+			identifiers: []string{
+				"namePrefix",
+				"metadata",
+				"metadata:name",
+				"metadata:spec",
+				"metadata:spec:replicas",
+				"kind",
+				"replicas",
+				"replicas:name",
+				"replicas:count",
+				"resource",
+			},
+			values: []string{
+				"namePrefix=dev-",
+				"metadata:name=n1",
+				"metadata:spec:replicas=3",
+				"kind=Kustomization",
+				"replicas:name=n1",
+				"replicas:name=n2",
+				"replicas:count=3",
+				"resource=file1.yaml",
+				"resource=file2.yaml",
+			},
+			kinds: []string{
+				"Kustomization",
+			},
+			filepath: "./kustomization.yaml",
+			yaml: `
+namePrefix: dev-
+# map of map
+metadata:
+  name: n1
+  spec:
+    replicas: 3
+kind: Kustomization
+
+#list of map
+replicas:
+- name: n1
+  count: 3
+- name: n2
+  count: 3
+
+# list
+resource:
+- file1.yaml
+- file2.yaml
+`,
+		},
+		{
+			identifiers: []string{
+				"kind",
+				"metadata",
+				"metadata:name",
+			},
+			values: []string{
+				"kind=Deployment",
+				"kind=Service",
+				"kind=Custom",
+				"metadata:name=app",
+				"metadata:name=app-service",
+				"metadata:name=app-crd",
+			},
+			kinds: []string{
+				"Deployment",
+				"Service",
+				"Custom",
+			},
+			filepath: "resources.yaml",
+			yaml: `
+---
+kind: Deployment
+metadata:
+  name: app
+---
+kind: Service
+metadata:
+  name: app-service
+---
+kind: Custom
+metadata:
+  name: app-crd
+`,
+		},
+		{
+			identifiers: []string{
+				"kind",
+				"metadata",
+				"metadata:name",
+			},
+			values: []string{
+				"kind=Deployment",
+				"kind=Service",
+				"metadata:name=app1",
+				"metadata:name=app2",
+			},
+			kinds: []string{
+				"Deployment",
+				"Service",
+			},
+			filepath: "resources.yaml",
+			yaml: `
+---
+kind: Deployment
+metadata:
+  name: app1
+---
+kind: Deployment
+metadata:
+  name: app2
+---
+kind: Service
+metadata:
+  name: app1
+`,
+		},
+	}
+
+	for _, test := range testCases {
+		doc := KustomizationDocument{
+			Document: Document{
+				DocumentData: test.yaml,
+				FilePath:     test.filepath,
+			},
+		}
+
+		err := doc.ParseYAML()
+		if err != nil {
+			t.Errorf("Document error error: %s", err)
+		}
+
+		cmpStrings := func(got, expected []string, label string) {
+			sort.Strings(got)
+			sort.Strings(expected)
+
+			if !reflect.DeepEqual(got, expected) {
+				t.Errorf("Expected %s (%v) to be equal to (%v)\n",
+					label,
+					strings.Join(got, ","),
+					strings.Join(expected, ","))
+			}
+
+		}
+
+		cmpStrings(doc.Identifiers, test.identifiers, "identifiers")
+		cmpStrings(doc.Values, test.values, "values")
+		cmpStrings(doc.Kinds, test.kinds, "kinds")
+	}
+}
+
+type TestStructForGetResources struct {
+	doc       KustomizationDocument
+	resources []*Document
+}
+
+func TestGetResources(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+bases:
+- ../base
+- ../otherbase
+
+resources:
+- file.yaml
+- https://github.com/kubernetes-sigs/kustomize/examples/helloWorld?ref=v3.1.0
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/base",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/otherbase",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "https://github.com/kubernetes-sigs/kustomize",
+					FilePath:      "examples/helloWorld",
+					DefaultBranch: "v3.1.0",
+					FileType:      "resource",
+					User:          "kubernetes-sigs",
+				},
+			},
+		},
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "https://github.com/some/repo",
+					FilePath:      "some/resource.yaml",
+					DocumentData: `
+bases:
+- ../base
+- ../overlay
+
+resources:
+- https://github.com/kubernetes-sigs/kustomize/examples/helloWorld?ref=v3.1.0
+- some/file.yaml
+`,
+				},
+			},
+			resources: []*Document{},
+		},
+	}
+	runTest(t, tests, true, false, false)
+}
+
+func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
+	for _, test := range tests {
+		res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
+		if err != nil {
+			t.Errorf("Unexpected error: %v\n", err)
+			continue
+		}
+		if len(test.resources) != len(res) {
+			t.Errorf("Number of resources does not match.")
+			continue
+		}
+		cmp := func(docs []*Document) func(i, j int) bool {
+			return func(i, j int) bool {
+				if docs[i].RepositoryURL != docs[j].RepositoryURL {
+					return docs[i].RepositoryURL <
+						docs[j].RepositoryURL
+				}
+
+				if docs[i].FilePath != docs[j].FilePath {
+					return docs[i].FilePath <
+						docs[j].FilePath
+				}
+
+				return docs[i].DefaultBranch < docs[j].DefaultBranch
+			}
+		}
+		sort.Slice(test.resources, cmp(test.resources))
+		sort.Slice(res, cmp(res))
+		for i, r := range test.resources {
+			if !reflect.DeepEqual(res[i], r) {
+				t.Errorf("Expected '%+v' to equal '%+v'\n",
+					res[i], r)
+			}
+		}
+	}
+}
+
+func TestGetResourcesAndGenerators(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, false, true)
+}
+
+func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/tr.yaml",
+					FileType:      "transformer",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, true, true)
+}

api/internal/crawl/doc/docname.go

@@ -0,0 +1,124 @@
+package doc
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/git"
+)
+
+type Document struct {
+	RepositoryURL string `json:"repositoryUrl,omitempty"`
+	// User makes it easy to aggregate data in the user level instead
+	// of the repository level
+	User          string     `json:"user,omitempty"`
+	FilePath      string     `json:"filePath,omitempty"`
+	DefaultBranch string     `json:"defaultBranch,omitempty"`
+	DocumentData  string     `json:"document,omitempty"`
+	CreationTime  *time.Time `json:"creationTime,omitempty"`
+	IsSame        bool       `json:"-"`
+	// FileType can be one of the following:
+	// "generator", "transformer", "resource", "".
+	FileType string `json:"fileType,omitempty"`
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *Document) GetDocument() *Document {
+	return doc
+}
+
+func (doc *Document) Copy() *Document {
+	return &Document{
+		RepositoryURL: doc.RepositoryURL,
+		User:          doc.User,
+		FilePath:      doc.FilePath,
+		DefaultBranch: doc.DefaultBranch,
+		DocumentData:  doc.DocumentData,
+		CreationTime:  doc.CreationTime,
+		IsSame:        doc.IsSame,
+		FileType:      doc.FileType,
+	}
+}
+
+func (doc *Document) Path() string {
+	return fmt.Sprintf("repoURL: %s filePath: %s branch: %s",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch)
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *Document) WasCached() bool {
+	return doc.IsSame
+}
+
+func (doc *Document) FromRelativePath(newFile string) (Document, error) {
+	repoSpec, err := git.NewRepoSpecFromUrl(newFile)
+	if err == nil {
+		return Document{
+			RepositoryURL: repoSpec.Host + path.Clean(repoSpec.OrgRepo),
+			FilePath:      path.Clean(repoSpec.Path),
+			DefaultBranch: repoSpec.Ref,
+			User:          UserName(repoSpec.Host + path.Clean(repoSpec.OrgRepo)),
+		}, nil
+	}
+	// else document is probably relative path.
+
+	ret := Document{
+		RepositoryURL: doc.RepositoryURL,
+		DefaultBranch: doc.DefaultBranch,
+		User:          UserName(doc.RepositoryURL),
+	}
+	ogDir, _ := path.Split(doc.FilePath)
+
+	cleaned := path.Clean(newFile)
+	if !path.IsAbs(cleaned) {
+		cleaned = path.Clean(ogDir + "/" + cleaned)
+	}
+
+	ret.FilePath = cleaned
+	return ret, nil
+}
+
+func (doc *Document) ID() string {
+	sum := sha256.Sum256([]byte(strings.Join(
+		[]string{
+			doc.RepositoryURL,
+			doc.DefaultBranch,
+			doc.FilePath,
+		},
+		"---|---")))
+	return fmt.Sprintf("%x", sum)
+}
+
+func (doc *Document) RepositoryFullName() string {
+	url := TrimUrl(doc.RepositoryURL)
+	sections := strings.Split(url, "/")
+	l := len(sections)
+	if l < 2 {
+		return url
+	}
+	return path.Join(sections[l-2], sections[l-1])
+}
+
+// TrimUrl removes all the trailing slashes and the "git@github.com:" prefix (if exists).
+func TrimUrl(s string) string {
+	url := strings.TrimRight(s, "/")
+
+	gitPrefix := "git@github.com:"
+	if strings.HasPrefix(url, gitPrefix) {
+		url = url[len(gitPrefix):]
+	}
+	return url
+}
+
+func UserName(repositoryURL string) string {
+	url := TrimUrl(repositoryURL)
+	sections := strings.Split(url, "/")
+	l := len(sections)
+	if l < 2 {
+		return url
+	}
+	return sections[l-2]
+}

api/internal/crawl/doc/docname_test.go

@@ -0,0 +1,150 @@
+package doc
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestFromRelativePath(t *testing.T) {
+	type Case struct {
+		RelativePath string
+		Expected     Document
+	}
+
+	testCases := []struct {
+		BaseDoc Document
+		Cases   []Case
+	}{
+		{
+			BaseDoc: Document{
+				RepositoryURL: "example.com/repo",
+				FilePath:      "path/to/file/kustomization.yaml",
+				DefaultBranch: "master",
+			},
+			Cases: []Case{
+				{
+					RelativePath: "../other/file/resource.yaml",
+					Expected: Document{
+						RepositoryURL: "example.com/repo",
+						FilePath:      "path/to/other/file/resource.yaml",
+						DefaultBranch: "master",
+						User:          "example.com",
+					},
+				},
+				{
+					RelativePath: "../file/../../something/../to/other/file/patch.yaml",
+					Expected: Document{
+						RepositoryURL: "example.com/repo",
+						FilePath:      "path/to/other/file/patch.yaml",
+						DefaultBranch: "master",
+						User:          "example.com",
+					},
+				},
+				{
+					RelativePath: "service.yaml",
+					Expected: Document{
+						RepositoryURL: "example.com/repo",
+						FilePath:      "path/to/file/service.yaml",
+						DefaultBranch: "master",
+						User:          "example.com",
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		for _, c := range tc.Cases {
+			rd, err := tc.BaseDoc.FromRelativePath(c.RelativePath)
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+			if !reflect.DeepEqual(rd, c.Expected) {
+				t.Errorf("document mismatch expected %v, got %v", c.Expected, rd)
+			}
+		}
+	}
+}
+
+func TestDocument_RepositoryFullName(t *testing.T) {
+	testCases := []struct {
+		doc                        Document
+		expectedRepositoryFullName string
+	}{
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com/user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com//user/repo////",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "repo/",
+			},
+			expectedRepositoryFullName: "repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "",
+			},
+			expectedRepositoryFullName: "",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "git@github.com:user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedRepositoryFullName := tc.doc.RepositoryFullName()
+		if returnedRepositoryFullName != tc.expectedRepositoryFullName {
+			t.Errorf("RepositoryFullName expected %s, got %s",
+				tc.expectedRepositoryFullName,
+				returnedRepositoryFullName)
+		}
+	}
+}
+
+func TestDocument_UserName(t *testing.T) {
+	testCases := []struct {
+		repositoryURL    string
+		expectedUserName string
+	}{
+		{
+			repositoryURL:    "https://github.com/user/repo",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "https://github.com//user/repo////",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "repo/",
+			expectedUserName: "repo",
+		},
+		{
+			repositoryURL:    "",
+			expectedUserName: "",
+		},
+		{
+			repositoryURL:    "git@github.com:user/repo",
+			expectedUserName: "user",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedUserName := UserName(tc.repositoryURL)
+		if returnedUserName != tc.expectedUserName {
+			t.Errorf("UserName expected %s, got %s",
+				tc.expectedUserName, returnedUserName)
+		}
+	}
+}