Add lint-kustomize to prow-presubmit-check

To do tool installs.
Merge pull request #2218 from pwittrock/master
2026-06-14 18:40:55 +00:00 · 2020-02-24 10:17:52 -08:00 · 2020-02-21 14:35:25 -08:00 · 2020-02-21 10:18:52 -08:00 · 2020-02-21 10:16:21 -08:00 · 2020-02-21 08:09:48 -08:00
341 changed files with 45607 additions and 9748 deletions
--- a/.golangci-kustomize.yml
+++ b/.golangci-kustomize.yml
@@ -7,8 +7,13 @@ linters:
    - bodyclose
    - deadcode
    - depguard
+    # - dogsled
    - dupl
+    # - errcheck
+    # - funlen
+    # - gochecknoinits
    - goconst
+    # - gocritic
    - gocyclo
    - gofmt
    - goimports
@@ -21,6 +26,7 @@ linters:
    - lll
    - misspell
    - nakedret
+    # - scopelint
    - staticcheck
    - structcheck
    # stylecheck demands that acronyms not be treated as words
@@ -28,9 +34,10 @@ linters:
    # - stylecheck
    - typecheck
    - unconvert
-    - unused
    - unparam
+    - unused
    - varcheck
+    # - whitespace

 linters-settings:
  dupl:
--- a/.travis.yml
+++ b/.travis.yml
@@ -36,6 +36,7 @@ install: true
 script:
  - make verify-kustomize
  - ./travis/kyaml-pre-commit.sh
+  - ./travis/check-go-mod.sh

 # TBD. Suppressing for now.
 notifications:
--- a/35
+++ b/35
@@ -17,6 +17,16 @@ verify-kustomize: \
 	test-examples-kustomize-against-HEAD \
 	test-examples-kustomize-against-latest

+# The following target referenced by a file in
+# https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/kustomize
+.PHONY: prow-presubmit-check
+prow-presubmit-check: \
+	lint-kustomize \
+	test-unit-kustomize-all
+
+.PHONY: verify-kustomize-e2e
+verify-kustomize-e2e: test-examples-e2e-kustomize
+
 # Other builds in this repo might want a different linter version.
 # Without one Makefile to rule them all, the different makes
 # cannot assume that golanci-lint is at the version they want
@@ -48,6 +58,11 @@ $(MYGOBIN)/goimports:
 	cd api; \
 	go install golang.org/x/tools/cmd/goimports

+# Install resource from whatever is checked out.
+$(MYGOBIN)/resource:
+	cd cmd/resource; \
+	go install .
+
 # To pin pluginator, use this recipe instead:
 #   cd api;
 #   go install sigs.k8s.io/kustomize/pluginator/v2
@@ -193,6 +208,16 @@ test-unit-kustomize-all: \
 test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
 	./hack/testExamplesAgainstKustomize.sh HEAD

+.PHONY:
+test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from ."; \
+		cd kustomize; go install .; cd ..; \
+		./hack/testExamplesE2EAgainstKustomize.sh .; \
+	)
+
 .PHONY:
 test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
 	( \
@@ -237,6 +262,16 @@ $(MYGOBIN)/helm:
 		rm -rf $$d \
 	)

+$(MYGOBIN)/kind:
+	( \
+        set -e; \
+        d=$(shell mktemp -d); cd $$d; \
+        wget -O ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(shell uname)-amd64; \
+        chmod +x ./kind; \
+        mv ./kind $(MYGOBIN); \
+        rm -rf $$d; \
+	)
+
 .PHONY: clean
 clean: kustomize-external-go-plugin-clean
 	go clean --cache
--- a/api/builtins/NamespaceTransformer.go
+++ b/api/builtins/NamespaceTransformer.go
@@ -52,7 +52,7 @@ func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {

 		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
 		if len(matches) != 1 {
-			return fmt.Errorf("namespace tranformation produces ID conflict: %#v", matches)
+			return fmt.Errorf("namespace tranformation produces ID conflict: %+v", matches)
 		}
 	}
 	return nil
--- a/api/go.mod
+++ b/api/go.mod
@@ -7,7 +7,6 @@ require (
 	github.com/go-openapi/spec v0.19.4
 	github.com/golangci/golangci-lint v1.21.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
-	github.com/monopole/mdrip v1.0.1
 	github.com/pkg/errors v0.8.1
 	golang.org/x/tools v0.0.0-20191010075000-0337d82405ff
 	gopkg.in/yaml.v2 v2.2.4
--- a/api/go.sum
+++ b/api/go.sum
@@ -2,14 +2,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -49,14 +45,9 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w=
-github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -119,13 +110,10 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -145,15 +133,12 @@ github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgO
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps=
 github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI=
 github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
@@ -169,9 +154,8 @@ github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -184,31 +168,16 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
-github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
-github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0 h1:UykbtMB/w5No2LmE16gINgLj+r/vbziTgaoERQv6U+0=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a h1:YH0IojQwndMQdeRWdw1aPT8bkbiWaYR3WD+Zf5e09DU=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741 h1:OuuPl66BpF1q3OEkaPpp+VfzxrBBY62ATGdWqql/XX8=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -248,10 +217,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
 github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
 github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
@@ -270,10 +237,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.0 h1:RFDBa+tab6mW+gX4Ww2SZDc4kS6p01FwnLtgz64Il+I=
-github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM=
-github.com/monopole/mdrip v1.0.1 h1:LzwNWb8ge3+4iBZmxIE6VfUR5KIxhF7DKdf85t8Yvos=
-github.com/monopole/mdrip v1.0.1/go.mod h1:/7E04hlzRG9Jrp6WILZfYYm/REoJWL2l+MlsCO1eH74=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -308,12 +271,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
@@ -322,8 +281,6 @@ github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -356,7 +313,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -364,7 +320,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
 github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs=
@@ -387,7 +342,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -410,8 +364,6 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -437,7 +389,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -447,7 +398,6 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -462,12 +412,11 @@ golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs=
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -492,8 +441,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
-gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
@@ -518,7 +465,6 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
@@ -526,10 +472,6 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphD
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4=
-sigs.k8s.io/kustomize/pluginator/v2 v2.0.0 h1:sES7e09G19Q0VjRp4ATSYKpTXoWaX8WMSHfw6u3G2Ok=
-sigs.k8s.io/kustomize/pluginator/v2 v2.0.0/go.mod h1:zrXhTv8BAKt0egmZX/8AtMOSFUSWM9YuoHvvqz8/eHE=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
--- a/api/internal/crawl/backend/search_backend.go
+++ b/api/internal/crawl/backend/search_backend.go
@@ -44,7 +44,7 @@ type kustomizeSearch struct {
 // /register: not implemented, but meant as an endpoint for adding new
 // kustomization files to the corpus.
 func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, "kustomize")
 	if err != nil {
 		return nil, err
 	}
--- a/api/internal/crawl/cmd/crawler/crawler.go
+++ b/api/internal/crawl/cmd/crawler/crawler.go
@@ -2,11 +2,15 @@ package main

 import (
 	"context"
+	"flag"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"time"

+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -23,71 +27,189 @@ const (
 	retryCount           = 3
 )

+type CrawlMode int
+
+const (
+	CrawlUnknown CrawlMode = iota
+	// Crawl all the kustomization files in all the repositories of a Github user
+	CrawlUser
+	// Crawl all the kustomization files in a Github repo
+	CrawlRepo
+	// Crawl all the documents in the index
+	CrawlIndex
+	// Crawl all the kustomization files on Github
+	CrawlGithub
+	// Crawl all the documents in the index and crawling all the kustomization files on Github
+	CrawlIndexAndGithub
+)
+
+func NewCrawlMode(s string) CrawlMode {
+	switch s {
+	case "github-user":
+		return CrawlUser
+	case "github-repo":
+		return CrawlRepo
+	case "index+github":
+		return CrawlIndexAndGithub
+	case "index":
+		return CrawlIndex
+	case "github":
+		return CrawlGithub
+	default:
+		return CrawlUnknown
+	}
+}
+
 func main() {
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	modePtr := flag.String("mode", "index+github",
+		`The crawling mode, which can be one of [github-user, github-repo, index, github, index+github].
+  * github-user: crawl all the kustomization files in all the repositories of a Github user (--github-user must be specified for this mode).
+  * github-repo: crawl all the kustomization files in a Github repository (--github-repo must be specified for this mode).
+  * index: crawl all the documents in the index.
+  * gihub: crawl all the kustomization files on Github.
+  * index+github: crawl all the documents in the index and crawling all the kustomization files on Github.`)
+	githubUserPtr := flag.String("github-user", "",
+		"A github user name (e.g., kubernetes-sigs). This flag is required for the `github-user` mode.")
+	githubRepoPtr := flag.String("github-repo", "",
+		"A github repository name (e.g., kubernetes-sigs/kustomize). This flag is required for the `github-repo` mode.")
+	flag.Parse()
+
 	githubToken := os.Getenv(githubAccessTokenVar)
 	if githubToken == "" {
-		fmt.Printf("Must set the variable '%s' to make github requests.\n",
+		log.Printf("Must set the variable '%s' to make github requests.\n",
 			githubAccessTokenVar)
 		return
 	}

 	ctx := context.Background()
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
 	if err != nil {
-		fmt.Printf("Could not create an index: %v\n", err)
+		log.Printf("Could not create an index: %v\n", err)
 		return
 	}

 	cacheURL := os.Getenv(redisCacheURL)
-
-	query := []byte(`{ "query":{ "match_all":{} } }`)
-	it := idx.IterateQuery(query, 10000, 60*time.Second)
-	docs := make(crawler.CrawlSeed, 0)
-	for it.Next() {
-		for _, hit := range it.Value().Hits.Hits {
-			docs = append(docs, hit.Document.Copy())
-		}
-	}
-
-	if err := it.Err(); err != nil {
-		fmt.Printf("Error iterating: %v\n", err)
-	}
-
 	cache, err := redis.DialURL(cacheURL)
 	clientCache := &http.Client{}
 	if err != nil {
-		fmt.Printf("Error: redis could not make a connection: %v\n", err)
+		log.Printf("Error: redis could not make a connection: %v\n", err)
 	} else {
 		clientCache = httpclient.NewClient(cache)
 	}

-	ghCrawler := github.NewCrawler(githubToken, retryCount, clientCache,
-		github.QueryWith(
-			github.Filename("kustomization.yaml"),
-			github.Filename("kustomization.yml")),
-	)
+	// docConverter takes in a plain document and processes it for the index.
+	docConverter := func(d *doc.Document) (crawler.CrawledDocument, error) {
+		kdoc := doc.KustomizationDocument{
+			Document: *d,
+		}

-	crawler.CrawlFromSeed(ctx, docs, []crawler.Crawler{ghCrawler},
-		// Converter takes in a plain document and processes it for the
-		// index.
-		func(d *doc.Document) (crawler.CrawledDocument, error) {
-			kdoc := doc.KustomizationDocument{
-				Document: *d,
-			}
+		err := kdoc.ParseYAML()
+		return &kdoc, err
+	}

-			err := kdoc.ParseYAML()
-			return &kdoc, err
-		},
-		// IndexFunc updates the value in the index.
-		func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler) error {
-			switch d := cdoc.(type) {
-			case *doc.KustomizationDocument:
-				fmt.Println("Inserting: ", d.ID(), d)
-				_, err := idx.Put(d.ID(), d)
-				return err
+	// Index updates the value in the index.
+	indexFunc := func(cdoc crawler.CrawledDocument, mode index.Mode) error {
+		switch d := cdoc.(type) {
+		case *doc.KustomizationDocument:
+			switch mode {
+			case index.Delete:
+				log.Printf("Deleting: %v", d)
+				return idx.Delete(d.ID())
 			default:
-				return fmt.Errorf("type %T not supported", d)
+				log.Printf("Inserting: %v", d)
+				return idx.Put(d.ID(), d)
 			}
-		},
-	)
+		default:
+			return fmt.Errorf("type %T not supported", d)
+		}
+	}
+
+	// seen tracks the IDs of all the documents in the index and their corresponding file types.
+	// This helps avoid indexing a given document multiple times.
+	seen := utils.NewSeenMap()
+
+	mode := NewCrawlMode(*modePtr)
+
+	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
+		if user != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.User(user)),
+			)
+		} else if repo != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.Repo(repo)),
+			)
+		} else {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization")),
+			)
+		}
+	}
+
+	seedDocs := make(crawler.CrawlSeed, 0)
+
+	// get all the documents in the index
+	getSeedDocsFunc := func() {
+		query := []byte(`{ "query":{ "match_all":{} } }`)
+		it := idx.IterateQuery(query, 10000, 60*time.Second)
+		for it.Next() {
+			for _, hit := range it.Value().Hits.Hits {
+				seedDocs = append(seedDocs, hit.Document.Document.Copy())
+			}
+		}
+		if err := it.Err(); err != nil {
+			log.Fatalf("getSeedDocsFunc Error iterating: %v\n", err)
+		}
+	}
+
+	switch mode {
+	case CrawlIndexAndGithub:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlIndex:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+	case CrawlGithub:
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		// add all the documents in the index into seen.
+		// this greatly reduces the time overhead of CrawlGithub.
+		getSeedDocsFunc()
+		for _, d := range seedDocs {
+			seen[d.ID()] = d.FileType
+		}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUser:
+		if *githubUserPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github user with the github-user flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(*githubUserPtr, "")}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlRepo:
+		if *githubRepoPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github repository with the github-repo flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", *githubRepoPtr)}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUnknown:
+		flag.Usage()
+		log.Fatalf("The --mode flag must be one of [github-user, github-repo, index, github, index+github].")
+	}
 }
--- a/api/internal/crawl/cmd/kustomize_stats/Dockerfile
+++ b/api/internal/crawl/cmd/kustomize_stats/Dockerfile
@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install ./cmd/kustomize_stats
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/kustomize_stats /
+ENTRYPOINT ["/kustomize_stats"]
--- a/api/internal/crawl/cmd/kustomize_stats/main.go
+++ b/api/internal/crawl/cmd/kustomize_stats/main.go
@@ -0,0 +1,249 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"flag"
+	"fmt"
+	"log"
+	"sort"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+)
+
+// iterateArr adds each item in arr into countMap.
+func iterateArr(arr []string, countMap map[string]int) {
+	for _, item := range arr {
+		if _, ok := countMap[item]; !ok {
+			countMap[item] = 1
+		} else {
+			countMap[item]++
+		}
+	}
+
+}
+
+// SortMapKeyByValueInt takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueInt(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
+	return keys
+}
+
+// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueLen(m map[string][]string) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return len(m[keys[i]]) > len(m[keys[j]]) })
+	return keys
+}
+
+func GeneratorOrTransformerStats(docs []*doc.KustomizationDocument) {
+	n := len(docs)
+	if n == 0 {
+		return
+	}
+
+	fileType := docs[0].FileType
+	fmt.Printf("There are totally %d %s files.\n", n, fileType)
+
+	GitRepositorySummary(docs, fileType)
+
+	// key of kindToUrls: a string in the KustomizationDocument.Kinds field
+	// value of kindToUrls: a slice of string urls defining a given kind.
+	kindToUrls := make(map[string][]string)
+
+	for _, d := range docs {
+		url := fmt.Sprintf("%s/blob/%s/%s", d.RepositoryURL, d.DefaultBranch, d.FilePath)
+		for _, kind := range d.Kinds {
+			if _, ok := kindToUrls[kind]; !ok {
+				kindToUrls[kind] = []string{url}
+			} else {
+				kindToUrls[kind] = append(kindToUrls[kind], url)
+			}
+		}
+	}
+	fmt.Printf("There are totally %d kinds of %s\n", len(kindToUrls), fileType)
+	sortedKeys := SortMapKeyByValueLen(kindToUrls)
+	for _, k := range sortedKeys {
+		sort.Strings(kindToUrls[k])
+		fmt.Printf("%s kind %s appears %d times\n", fileType, k, len(kindToUrls[k]))
+		for _, url := range kindToUrls[k] {
+			fmt.Printf("%s\n", url)
+		}
+	}
+}
+
+// GitRepositorySummary counts the distribution of docs:
+// 1) how many git repositories are these docs from?
+// 2) how many docs are from each git repository?
+func GitRepositorySummary(docs []*doc.KustomizationDocument, fileType string) {
+	m := make(map[string]int)
+	for _, d := range docs {
+		if _, ok := m[d.RepositoryURL]; ok {
+			m[d.RepositoryURL]++
+		} else {
+			m[d.RepositoryURL] = 1
+		}
+	}
+	sortedKeys := SortMapKeyByValueInt(m)
+	topN := 10
+	i := 0
+	for _, k := range sortedKeys {
+		if i >= topN {
+			break
+		}
+		fmt.Printf("%d %s are from %s\n", m[k], fileType, k)
+		i++
+	}
+}
+
+func main() {
+	topKindsPtr := flag.Int(
+		"kinds", -1,
+		`the number of kubernetes object kinds to be listed according to their popularities.
+By default, all the kinds will be listed.
+If you only want to list the 10 most popular kinds, set the flag to 10.`)
+	topIdentifiersPtr := flag.Int(
+		"identifiers", -1,
+		`the number of identifiers to be listed according to their popularities.
+By default, all the identifiers will be listed.
+If you only want to list the 10 most popular identifiers, set the flag to 10.`)
+	topKustomizeFeaturesPtr := flag.Int(
+		"kustomize-features", -1,
+		`the number of kustomize features to be listed according to their popularities.
+By default, all the features will be listed.
+If you only want to list the 10 most popular features, set the flag to 10.`)
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	flag.Parse()
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	if err != nil {
+		log.Fatalf("Could not create an index: %v\n", err)
+	}
+
+	// count tracks the number of documents in the index
+	count := 0
+
+	// kustomizationFilecount tracks the number of kustomization files in the index
+	kustomizationFilecount := 0
+
+	kindsMap := make(map[string]int)
+	identifiersMap := make(map[string]int)
+	kustomizeIdentifiersMap := make(map[string]int)
+
+	// ids tracks the unique IDs of the documents in the index
+	ids := make(map[string]struct{})
+
+	// generatorFiles include all the non-kustomization files whose FileType is generator
+	generatorFiles := make([]*doc.KustomizationDocument, 0)
+
+	// transformersFiles include all the non-kustomization files whose FileType is transformer
+	transformersFiles := make([]*doc.KustomizationDocument, 0)
+
+	checksums := make(map[string]int)
+
+	// get all the documents in the index
+	query := []byte(`{ "query":{ "match_all":{} } }`)
+	it := idx.IterateQuery(query, 10000, 60*time.Second)
+	for it.Next() {
+		for _, hit := range it.Value().Hits.Hits {
+			sum := fmt.Sprintf("%x", sha256.Sum256([]byte(hit.Document.DocumentData)))
+			if _, ok := checksums[sum]; ok {
+				checksums[sum]++
+			} else {
+				checksums[sum] = 1
+			}
+
+			// check whether there is any duplicate IDs in the index
+			if _, ok := ids[hit.ID]; !ok {
+				ids[hit.ID] = struct{}{}
+			} else {
+				log.Printf("Found duplicate ID (%s)\n", hit.ID)
+			}
+
+			count++
+			iterateArr(hit.Document.Kinds, kindsMap)
+			iterateArr(hit.Document.Identifiers, identifiersMap)
+
+			if doc.IsKustomizationFile(hit.Document.FilePath) {
+				kustomizationFilecount++
+				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
+
+			} else {
+				switch hit.Document.FileType {
+				case "generator":
+					generatorFiles = append(generatorFiles, hit.Document.Copy())
+				case "transformer":
+					transformersFiles = append(transformersFiles, hit.Document.Copy())
+				}
+			}
+		}
+	}
+
+	if err := it.Err(); err != nil {
+		log.Fatalf("Error iterating: %v\n", err)
+	}
+
+	sortedKindsMapKeys := SortMapKeyByValueInt(kindsMap)
+	sortedIdentifiersMapKeys := SortMapKeyByValueInt(identifiersMap)
+	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValueInt(kustomizeIdentifiersMap)
+
+	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
+There are %d documents in the kustomize index.
+%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
+	fmt.Printf("\n")
+
+	kindCount := 0
+	for _, key := range sortedKindsMapKeys {
+		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
+			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
+			kindCount++
+		}
+	}
+
+	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
+	identifierCount := 0
+	for _, key := range sortedIdentifiersMapKeys {
+		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
+			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
+			identifierCount++
+		}
+	}
+
+	fmt.Printf(`There are %d kustomization files in the kustomize index.
+%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
+	fmt.Printf("\n")
+	kustomizeFeatureCount := 0
+	for _, key := range sortedKustomizeIdentifiersMapKeys {
+		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
+			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
+			kustomizeFeatureCount++
+		}
+	}
+
+	GeneratorOrTransformerStats(generatorFiles)
+	GeneratorOrTransformerStats(transformersFiles)
+
+	fmt.Printf("There are total %d checksums of document contents\n", len(checksums))
+	sortedChecksums := SortMapKeyByValueInt(checksums)
+	sortedChecksums = sortedChecksums[:20]
+	fmt.Printf("The top 20 checksums are:\n")
+	for _, key := range sortedChecksums {
+		fmt.Printf("checksum %s apprears %d\n", key, checksums[key])
+	}
+}
--- a/api/internal/crawl/cmd/log-parser/README.md
+++ b/api/internal/crawl/cmd/log-parser/README.md
@@ -0,0 +1,8 @@
+This binary takes as its input a json file including GKE logs (which can be
+[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into 
+[Cloud Storage](https://cloud.google.com/storage/docs/)), 
+and extracts the `textPayload` field of each log entry.
+
+Here is an log entry example:
+
+{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"}
--- a/api/internal/crawl/cmd/log-parser/kustomize-stats-cmd
+++ b/api/internal/crawl/cmd/log-parser/kustomize-stats-cmd
@@ -0,0 +1,7 @@
+wget <log-file-url> -O log
+go build .
+./log-parser log >out
+cat out | grep "kind \`" | cut -d\` -f2 | tail -n 50
+cat out | grep "kind \`" | awk '{print $6}' | tail -n 50
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | cut -d\` -f2
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | awk '{print $6}'
--- a/api/internal/crawl/cmd/log-parser/main.go
+++ b/api/internal/crawl/cmd/log-parser/main.go
@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatalf("The usage of the command is: \n\t%s <log-file.json>", os.Args[0])
+	}
+
+	file, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	closeFile := func(file *os.File) {
+		if err := file.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}
+	defer closeFile(file)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		var entry interface{}
+		if err := json.Unmarshal([]byte(line), &entry); err != nil {
+			log.Printf("failed to unmarshal a log entry: %s\n", line)
+		}
+
+		m := entry.(map[string]interface{})
+		if payload, ok := m["textPayload"]; ok {
+			// use fmt.Printf here instead of log.Printf to avoid the time and code location info the log package provides
+			fmt.Printf("%s", payload)
+		} else {
+			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}
--- a/api/internal/crawl/config/base/kustomization.yaml
+++ b/api/internal/crawl/config/base/kustomization.yaml
@@ -2,5 +2,4 @@ configmapGenerator:
 - name: elasticsearch-config
  literals:
  - es-url="http://esbasic-master:9200"
-  - kustomize-index-name="kustomize"
  - plugin-index-name="plugin"
--- a/api/internal/crawl/config/crawler/base/.gitignore
+++ b/api/internal/crawl/config/crawler/base/.gitignore
@@ -0,0 +1 @@
+github_api_secret.txt
--- a/api/internal/crawl/config/crawler/cronjob/cronjob.yaml
+++ b/api/internal/crawl/config/crawler/cronjob/cronjob.yaml
@@ -1,9 +1,10 @@
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: crawler
+  name: crawler-cronjob
 spec:
-  schedule: "5 0 * * */1"
+  # run the cronjob at 00:00 every 7 days
+  schedule: "0 0 */7 * *"
  jobTemplate:
    spec:
      template:
@@ -11,7 +12,9 @@ spec:
          restartPolicy: OnFailure
          containers:
          - name: crawler
-            image: gcr.io/kustomize-search/crawler:latest
+            image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+            command: ["/crawler"]
+            args: ["--mode=index+github", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
            imagePullPolicy: Always
            env:
            - name: GITHUB_ACCESS_TOKEN
--- a/api/internal/crawl/config/crawler/job/README.md
+++ b/api/internal/crawl/config/crawler/job/README.md
@@ -0,0 +1,53 @@
+The crawler job can run in one of the following mode:
+
+# Crawling all the documents in the index and crawling all the kustomization files on Github
+
+This is the default setting of the crawler job. The `command` and `args` field
+of the container should be:
+
+```
+        command: ["/crawler"]
+```
+
+Or
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index+github"]
+```
+
+# Crawling all the documents in the index
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index"]
+```
+
+# Crawling all the kustomization files on Github
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github"]
+```
+
+# Crawling all the kustomization files in a Github repo
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize"]
+```
+
+# Crawling all the kustomization files in all the repositories of a Github user
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--github-user", "--github-user=kubernetes-sigs"]
+```
--- a/api/internal/crawl/config/crawler/job/job.yaml
+++ b/api/internal/crawl/config/crawler/job/job.yaml
@@ -8,8 +8,10 @@ spec:
      restartPolicy: OnFailure
      containers:
      - name: crawler
-        image: gcr.io/kustomize-search/crawler:latest
+        image: gcr.io/haiyanmeng-gke-dev/crawler:v1
        imagePullPolicy: Always
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
        env:
        - name: GITHUB_ACCESS_TOKEN
          valueFrom:
--- a/api/internal/crawl/config/crawler/kustomize_stats/job.yaml
+++ b/api/internal/crawl/config/crawler/kustomize_stats/job.yaml
@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kustomize-stats
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: kustomize-stats
+        image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
+        imagePullPolicy: Always
+        command: ["/kustomize_stats"]
+        args: ["--index=kustomize", "--kinds=51", "--identifiers=50", "--kustomize-features=50"]
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url
--- a/api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml
+++ b/api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml
--- a/api/internal/crawl/config/elastic/esbackup.yaml
+++ b/api/internal/crawl/config/elastic/esbackup.yaml
@@ -0,0 +1,23 @@
+# ESBackup depends on ESCluster, and is depended by ESSnapshot.
+# Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the snapshots in the `kustomize-backup` snapshot repository, instead makes all the snapshots in the repository inaccessible.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the essnapshot objects depending on it, but will cause those essnapshot objects to be reconciled, which update the status of the essnapshot objects to reflect the fact that the esbackup object is missing.
+# If you delete the `kustomize-backup` snapshot repository directly without deleting `esbackup/kustomize-backbup`, the ESBackup object will not recreate the snapshot repository.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESBackup
+metadata:
+  name: kustomize-backup
+spec:
+  storage:
+    gcs:
+      # the bucket must exist for the snapshot respository to be created successfully.
+      bucket: kustomize-backup
+      # the path does not need to exist.
+      # If the path does not exist, the controller will create the folder in the GCS bucket.
+      # If the path already exists and includes snapshots, these snapshots can be used.
+      path: kustomize
+      secret:
+        name: kustomizesa
+  escluster:
+    name: esbasic
--- a/api/internal/crawl/config/elastic/escluster.yaml
+++ b/api/internal/crawl/config/elastic/escluster.yaml
@@ -1,3 +1,4 @@
+# ESCluster is depended by ESBackup and ESRestore.
 apiVersion: elasticsearch.cloud.google.com/v1alpha1
 kind: ESCluster
 metadata:
@@ -8,6 +9,13 @@ spec:
    - repository-gcs
    - ingest-user-agent
    - ingest-geoip
+  # To set `gcpserviceaccount`,
+  # First, create and download a GCP service account into a json file, named `sakey.json` following the instruction:
+  #     https://www.elastic.co/guide/en/elasticsearch/plugins/6.5/repository-gcs-usage.html#repository-gcs-using-service-account
+  # Second, create a secret for the service account using the following command:
+  #     $ kubectl create secret generic kustomizesa --from-file=./sakey.json
+  gcpserviceaccount:
+    name: kustomizesa
  config:
    env:
      example: test
--- a/api/internal/crawl/config/elastic/esrestore.yaml
+++ b/api/internal/crawl/config/elastic/esrestore.yaml
@@ -0,0 +1,19 @@
+# ESRestore depends on both ESCluster and ESSnapshot.
+# Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`.
+# Deleting `esrestore/kustomize-restore` will not delete the restored index.
+# Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`.
+# After the restore is complete, if the `kusotmize-restore` index is deleted manually, the ESRestore object will NOT restore the `kustomize` index to it again.
+# The correct way of using ESRestore is: create a ESRestore object to restore the index; delete the ESRestore object after the restore is complete.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESRestore
+metadata:
+  name: kustomize-restore
+spec:
+  include_global_state: true
+  ignore_unavailable: true
+  rename_pattern: kustomize
+  rename_replacement: kustomize-restore
+  essnapshot:
+    name: kustomize-snapshot
+  escluster:
+    name: esbasic
--- a/api/internal/crawl/config/elastic/essnapshot.yaml
+++ b/api/internal/crawl/config/elastic/essnapshot.yaml
@@ -0,0 +1,23 @@
+# ESSnapshot depends on ESBackup, and is depended by ESRestore.
+# Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository.
+# After being created, the `kustomize-snapshot` snapshot will not be automatically updated when the `kuostomize` index is updated.
+# If you delete `essnapshot/kustomize-snapshot` and recreate it, the new snapshot will capture the current status of the `kustomize` index.
+# Deleting `essnapshot/kustomize-snapshot` will delete the snapshot.
+# Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`.
+# If the `kustomize-snapshot` snapshot is deleted directly without deleting `essnapshot/kustomize-snapshot`, the ESSnapshot object will recreate the snapshot.
+# The correct way of using ESSnapshot is: create an ESSnapshot object to create a snapshot, keep the ESSnapshot object until the snapshot is no longer needed.
+# To update the snapshot to capture the latest version of the index, you can either:
+#   1) delete the snapshot, and wait for the ESSnapshot object to recreate the snapshot;
+#   2) delete the ESSnapshot object, and recreate the ESSnapshot object.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESSnapshot
+metadata:
+  name: kustomize-snapshot
+spec:
+  # indices are optional. If not specified all indices are selected.
+  indices:
+  - kustomize
+  include_global_state: true
+  ignore_unavailable: true
+  esbackup:
+    name: kustomize-backup
--- a/api/internal/crawl/crawler/crawler.go
+++ b/api/internal/crawl/crawler/crawler.go
@@ -10,6 +10,10 @@ import (
 	"os"
 	"sync"

+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
 	_ "github.com/gomodule/redigo/redis"

 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -27,157 +31,210 @@ type Crawler interface {
 	// Crawl returns when it is done processing. This method does not take
 	// ownership of the channel. The channel is write only, and it
 	// designates where the crawler should forward the documents.
-	Crawl(ctx context.Context, output chan<- CrawledDocument) error
+	Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error

 	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
 	FetchDocument(context.Context, *doc.Document) error
 	// Write to the document what the created time is.
 	SetCreated(context.Context, *doc.Document) error

+	SetDefaultBranch(*doc.Document)
+
 	Match(*doc.Document) bool
 }

 type CrawledDocument interface {
 	ID() string
 	GetDocument() *doc.Document
-	GetResources() ([]*doc.Document, error)
+	// Get all the Documents directly referred in a Document.
+	// For a Document representing a non-kustomization file, an empty slice will be returned.
+	// For a Document representing a kustomization file:
+	// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
+	// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
+	// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
+	GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
 	WasCached() bool
 }

 type CrawlSeed []*doc.Document

-type IndexFunc func(CrawledDocument, Crawler) error
+type IndexFunc func(CrawledDocument, index.Mode) error
 type Converter func(*doc.Document) (CrawledDocument, error)

-// Cleaner, more efficient, and more extensible crawler implementation.
-// The seed must include the ids of each document in the index.
-func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
-	crawlers []Crawler, conv Converter, indx IndexFunc) {
+func logIfErr(err error) {
+	if err == nil {
+		return
+	}
+	logger.Println("error: ", err)
+}

-	seen := make(map[string]struct{})
-
-	logIfErr := func(err error) {
-		if err == nil {
-			return
+func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
+	for _, crawl := range crawlers {
+		if crawl.Match(d) {
+			return crawl
 		}
-		logger.Println("error: ", err)
+	}
+	return nil
+}
+
+func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed) {
+
+	seen.Set(cdoc.ID(), cdoc.GetDocument().FileType)
+
+	match.SetDefaultBranch(cdoc.GetDocument())
+
+	// Insert into index
+	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
+		logger.Printf("Failed to insert or update doc(%s): %v",
+			cdoc.GetDocument().Path(), err)
+		return
 	}

-	stack := make(CrawlSeed, 0)
-
-	findMatch := func(d *doc.Document) Crawler {
-		for _, crawl := range crawlers {
-			if crawl.Match(d) {
-				return crawl
-			}
-		}
-
-		return nil
+	deps, err := cdoc.GetResources(true, true, true)
+	if err != nil {
+		logger.Println(err)
+		return
 	}

-	addBranches := func(cdoc CrawledDocument, match Crawler) {
-		if _, ok := seen[cdoc.ID()]; ok {
-			return
+	for _, dep := range deps {
+		if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType {
+			continue
 		}
+		*stack = append(*stack, dep)
+	}
+}

-		seen[cdoc.ID()] = struct{}{}
-		// Insert into index
-		err := indx(cdoc, match)
-		logIfErr(err)
-		if err != nil {
-			return
-		}
+func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) {

-		deps, err := cdoc.GetResources()
-		logIfErr(err)
-		if err != nil {
-			return
-		}
-		for _, dep := range deps {
-			if _, ok := seen[dep.ID()]; ok {
+	UpdatedDocCount := 0
+	seenDocCount := 0
+	cachedDocCount := 0
+	findMatchErrCount := 0
+	FetchDocumentErrCount := 0
+	SetCreatedErrCount := 0
+	convErrCount := 0
+	deleteDocCount := 0
+	crawledDocCount := 0
+
+	// During the execution of the for loop, more Documents may be added into (*docsPtr).
+	for len(*docsPtr) > 0 {
+		// get the last Document in (*docPtr), which will be crawled in this iteration.
+		tail := (*docsPtr)[len(*docsPtr)-1]
+
+		// remove the last Document in (*docPtr)
+		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
+
+		crawledDocCount++
+		logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
+
+		if seen.Seen(tail.ID()) {
+			if !updateFileType || seen.Value(tail.ID()) == tail.FileType {
+				logger.Printf("this doc has been seen before")
+				seenDocCount++
 				continue
 			}
-			stack = append(stack, dep)
 		}
-	}

-	doCrawl := func(docsPtr *CrawlSeed) {
-		n := len(*docsPtr)
-		for i := 0; i < n; i++ {
-			next := (*docsPtr)[i]
-			match := findMatch(next)
-			if match == nil {
-				logIfErr(fmt.Errorf(
-					"%v could not match any crawler", next))
-				continue
-			}
+		if tail.WasCached() {
+			logger.Printf("doc(%s) is cached already", tail.Path())
+			cachedDocCount++
+			continue
+		}

-			logger.Println("Crawling ", next.RepositoryURL, next.FilePath)
-			err := match.FetchDocument(ctx, next)
-			logIfErr(err)
-			// If there was no change or there is an error, we don't have
-			// to branch out, since the dependencies are already in the
-			// index, or we cannot find the document.
-			if err != nil || next.WasCached() {
-				if next.WasCached() {
-					logger.Println(next.RepositoryURL, next.FilePath, "is cached already")
+		match := findMatch(tail, crawlers)
+		if match == nil {
+			logIfErr(fmt.Errorf("%v could not match any crawler", tail))
+			findMatchErrCount++
+			continue
+		}
+
+		if tail.User == "" {
+			tail.User = doc.UserName(tail.RepositoryURL)
+		}
+
+		// If the Document represents a kustomization root, FetchDcoument will change
+		// the `filePath` field of the Document by adding `kustomization.yaml` or
+		// `kustomization.yml` or `kustomization` into the the field.
+		// Therefore, it is necessary to add the ID of the Document into seen before
+		// calling FetchDocument. Otherwise, the binary may enter into an infinite loop
+		// if a kustomization file points to its kustmozation root in its `resources` or
+		// `bases` field.
+		seen.Set(tail.ID(), tail.FileType)
+
+		if refreshDoc || tail.DefaultBranch == "" {
+			match.SetDefaultBranch(tail)
+		}
+
+		if refreshDoc || tail.DocumentData == "" {
+			if err := match.FetchDocument(ctx, tail); err != nil {
+				logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
+				FetchDocumentErrCount++
+				// delete the document from the index
+				cdoc := &doc.KustomizationDocument{
+					Document: *tail,
 				}
+				seen.Set(cdoc.ID(), tail.FileType)
+				if err := indx(cdoc, index.Delete); err != nil {
+					logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
+				}
+				deleteDocCount++
 				continue
 			}
-
-			logIfErr(match.SetCreated(ctx, next))
-
-			cdoc, err := conv(next)
-			logIfErr(err)
-
-			addBranches(cdoc, match)
 		}
+
+		if refreshDoc || tail.CreationTime == nil {
+			if err := match.SetCreated(ctx, tail); err != nil {
+				logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
+				SetCreatedErrCount++
+			}
+		}
+
+		cdoc, err := conv(tail)
+		// If conv returns an error, cdoc can still be added into the index so that
+		// cdoc.Document can be searched.
+		if err != nil {
+			logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
+			convErrCount++
+		}
+
+		UpdatedDocCount++
+		addBranches(cdoc, match, indx, seen, stack)
 	}
+	logger.Printf("Summary of doCrawl:\n")
+	logger.Printf("\t%d documents were updated\n", UpdatedDocCount)
+	logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount)
+	logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount)
+	logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount)
+	logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n",
+		FetchDocumentErrCount, deleteDocCount)
+	logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount)
+	logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount)
+}
+
+// CrawlFromSeed updates all the documents in seed, and crawls all the new
+// documents referred in the seed.
+func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
+	conv Converter, indx IndexFunc, seen utils.SeenMap) {
+
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)

 	// Exploit seed to update bulk of corpus.
 	logger.Printf("updating %d documents from seed\n", len(seed))
-	doCrawl(&seed)
-	// Traverse any new links added while updating corpus.
+	// each unique document in seed will be crawled once.
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false)
+
+	// Traverse any new documents added while updating corpus.
 	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
-	doCrawl(&stack)
-
-	ch := make(chan CrawledDocument, 1<<10)
-	wg := sync.WaitGroup{}
-
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for cdoc := range ch {
-			if _, ok := seen[cdoc.ID()]; ok {
-				continue
-			}
-			match := findMatch(cdoc.GetDocument())
-			if match == nil {
-				logIfErr(fmt.Errorf(
-					"%v could not match any crawler", cdoc))
-				continue
-			}
-			addBranches(cdoc, match)
-		}
-	}()
-
-	// Exploration through APIs.
-	errs := CRunner(ctx, ch, crawlers)
-	if errs != nil {
-		for _, err := range errs {
-			logIfErr(err)
-		}
-	}
-	close(ch)
-	logger.Println("Processing the new documents from the crawlers' exploration.")
-	wg.Wait()
-	// Handle deps of newly discovered documents.
-	logger.Printf("crawling the %d new documents from the crawlers' exploration.",
-		len(stack))
-	doCrawl(&stack)
+	// While crawling each document in stack, the documents directly referred in the document
+	// will be added into stack.
+	// After this statement is done, stack will become empty.
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
 }

-// CRunner is a blocking function and only returns once all of the
+// CrawlGithubRunner is a blocking function and only returns once all of the
 // crawlers are finished with execution.
 //
 // This function uses the output channel to forward kustomization documents
@@ -188,14 +245,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
 // index of the crawler that emitted the error. Although the errors themselves
 // can be nil, the array will always be exactly the size of the crawlers array.
 //
-// CRunner takes in a seed, which represents the documents stored in an
+// CrawlGithubRunner takes in a seed, which represents the documents stored in an
 // index somewhere. The document data is not required to be populated. If there
 // are many documents, this is preferable. The order of iteration over the seed
-// is not guaranteed, but the CRunner does guarantee that every element
+// is not guaranteed, but the CrawlGithub does guarantee that every element
 // from the seed will be processed before any other documents from the
 // crawlers.
-func CRunner(ctx context.Context,
-	output chan<- CrawledDocument, crawlers []Crawler) []error {
+func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
+	crawlers []Crawler, seen utils.SeenMap) []error {

 	errs := make([]error, len(crawlers))
 	wg := sync.WaitGroup{}
@@ -229,10 +286,59 @@ func CRunner(ctx context.Context,
 				}
 			}()
 			defer close(docs)
-			errs[idx] = crawler.Crawl(ctx, docs)
+			errs[idx] = crawler.Crawl(ctx, docs, seen)
 		}(i, crawler, docs) // Copies the index and the crawler
 	}

 	wg.Wait()
 	return errs
 }
+
+// CrawlGithub crawls all the kustomization files on Github.
+func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
+	indx IndexFunc, seen utils.SeenMap) {
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// ch is channel where all the crawlers sends the crawled documents to.
+	ch := make(chan CrawledDocument, 1<<10)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		docCount := 0
+		for cdoc := range ch {
+			docCount++
+			logger.Printf("Processing doc %d found on Github", docCount)
+			// all the docs here are kustomization files found by querying Github, and
+			// their `FileType` fields all should be empty.
+			if seen.Seen(cdoc.ID()) {
+				logger.Printf("the doc has been seen before")
+				continue
+			}
+			match := findMatch(cdoc.GetDocument(), crawlers)
+			if match == nil {
+				logIfErr(fmt.Errorf(
+					"%v could not match any crawler", cdoc))
+				continue
+			}
+			addBranches(cdoc, match, indx, seen, &stack)
+		}
+	}()
+
+	logger.Println("processing the documents found from crawling github")
+	if errs := CrawlGithubRunner(ctx, ch, crawlers, seen); errs != nil {
+		for _, err := range errs {
+			logIfErr(err)
+		}
+	}
+	close(ch)
+	wg.Wait()
+
+	// Handle deps of newly discovered documents.
+	logger.Printf("crawling the %d new documents referred by other documents",
+		len(stack))
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+}
--- a/api/internal/crawl/crawler/crawler_test.go
+++ b/api/internal/crawl/crawler/crawler_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"reflect"
 	"sort"
 	"strings"
@@ -11,6 +12,10 @@ import (
 	"testing"
 	"time"

+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
 	"sigs.k8s.io/kustomize/api/konfig"
 )
@@ -32,18 +37,21 @@ func (c testCrawler) Match(d *doc.Document) bool {
 	return d != nil
 }

+func (c testCrawler) SetDefaultBranch(d *doc.Document) {}
+
 func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	if i, ok := c.lukp[d.ID()]; ok {
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
 	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
-		fmt.Println(d.ID(), "/", suffix)
-		i, ok := c.lukp[d.ID()+"/"+suffix]
+		savedFilePath := d.FilePath
+		d.FilePath += "/" + suffix
+		i, ok := c.lukp[d.ID()]
 		if !ok {
+			d.FilePath = savedFilePath
 			continue
 		}
-		d.FilePath += "/" + suffix
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
@@ -72,7 +80,7 @@ func newCrawler(matchPrefix string, err error,

 // Crawl implements the Crawler interface for testing.
 func (c testCrawler) Crawl(_ context.Context,
-	output chan<- CrawledDocument) error {
+	output chan<- CrawledDocument, _ utils.SeenMap) error {

 	for i, d := range c.docs {
 		isResource := true
@@ -106,8 +114,8 @@ func (s sortableDocs) Len() int {
 	return len(s)
 }

-func TestCrawlerRunner(t *testing.T) {
-	fmt.Println("testing CRunner")
+func TestCrawlGithubRunner(t *testing.T) {
+	log.Println("testing CrawlGithubRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -178,8 +186,9 @@ func TestCrawlerRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()

-			errs := CRunner(context.Background(),
-				output, test.tc)
+			seen := utils.NewSeenMap()
+			errs := CrawlGithubRunner(context.Background(),
+				output, test.tc, seen)

 			// Check that errors are returned as they should be.
 			if !reflect.DeepEqual(errs, test.errs) {
@@ -212,7 +221,7 @@ func TestCrawlerRunner(t *testing.T) {
 }

 func TestCrawlFromSeed(t *testing.T) {
-	fmt.Println("testing CrawlFromSeed")
+	log.Println("testing CrawlFromSeed")

 	tests := []struct {
 		seed    CrawlSeed
@@ -302,29 +311,6 @@ resources:
 					RepositoryURL: kustomizeRepo,
 					FilePath:      "examples/seedcrawl2/job.yaml",
 				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/base/kustomization.yaml",
-					DocumentData: `
-resources:
- ../app
-`,
-				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/kustomization.yaml",
-					DocumentData: `
-resources:
- resource.yaml
-`,
-				}},
-				// Visited from crawling runner imported as resource.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/resource.yaml",
-				}},
 			},
 		},
 	}
@@ -338,10 +324,11 @@ resources:
 					Document: *d,
 				}, nil
 			},
-			func(d CrawledDocument, cr Crawler) error {
+			func(d CrawledDocument, mode index.Mode) error {
 				visited[d.ID()]++
 				return nil
 			},
+			utils.NewSeenMap(),
 		)
 		if lv, lc := len(visited), len(tc.corpus); lv != lc {
 			t.Errorf("error: %d of %d documents visited.", lv, lc)
--- a/api/internal/crawl/crawler/github/crawler.go
+++ b/api/internal/crawl/crawler/github/crawler.go
@@ -16,6 +16,8 @@ import (
 	"strings"
 	"time"

+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
 	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
@@ -30,6 +32,8 @@ var logger = log.New(os.Stdout, "Github Crawler: ",
 type githubCrawler struct {
 	client GhClient
 	query  Query
+	// branchMap maps github repositories to their default branches
+	branchMap map[string]string
 }

 type GhClient struct {
@@ -51,13 +55,60 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 			},
 			accessToken: accessToken,
 		},
-		query: query,
+		query:     query,
+		branchMap: map[string]string{},
 	}
 }

+func (gc githubCrawler) SetDefaultBranch(d *doc.Document) {
+	url := gc.client.ReposRequest(d.RepositoryFullName())
+	defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+	d.DefaultBranch = defaultBranch
+	gc.branchMap[d.RepositoryURL] = d.DefaultBranch
+}
+
+func (gc githubCrawler) DefaultBranch(repo string) string {
+	return gc.branchMap[repo]
+}
+
 // Implements crawler.Crawler.
-func (gc githubCrawler) Crawl(
-	ctx context.Context, output chan<- crawler.CrawledDocument) error {
+func (gc githubCrawler) Crawl(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
+
+	ranges := []RangeWithin{
+		RangeWithin{
+		start: uint64(0),
+		end:   githubMaxFileSize,
+	},
+	}
+
+	errs := make(multiError, 0)
+	for len(ranges) > 0 {
+		tailRange := ranges[len(ranges) - 1]
+		ranges = ranges[:(len(ranges) - 1)]
+		reProcessQueryRanges, err := gc.CrawlSingleRange(ctx, output, seen, tailRange.start, tailRange.end)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		ranges = append(ranges, reProcessQueryRanges...)
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	lowerBound, upperBound uint64) ([]RangeWithin, error) {
+
+	log.Printf("CrawlSingleRange [%d, %d]", lowerBound, upperBound)

 	noETagClient := GhClient{
 		RequestConfig: gc.client.RequestConfig,
@@ -66,12 +117,25 @@ func (gc githubCrawler) Crawl(
 		accessToken:   gc.client.accessToken,
 	}

+	var reProcessQueryRanges []RangeWithin
+
+	var ranges []string
+	var err error
 	// Since Github returns a max of 1000 results per query, we can use
 	// multiple queries that split the search space into chunks of at most
 	// 1000 files to get all of the data.
-	ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query))
+	for i := 0; i < 5; i++ {
+		ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query),
+			lowerBound, upperBound)
+		if err == nil {
+			logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i)
+			break
+		} else {
+			time.Sleep(time.Minute)
+		}
+	}
 	if err != nil {
-		return fmt.Errorf("could not split %v into ranges, %v\n",
+		return reProcessQueryRanges, fmt.Errorf("could not split %v into ranges, %v\n",
 			gc.query, err)
 	}

@@ -79,20 +143,30 @@ func (gc githubCrawler) Crawl(

 	// Query each range for files.
 	errs := make(multiError, 0)
+	queryResult := RangeQueryResult{}
 	for _, query := range ranges {
-		err := processQuery(ctx, gc.client, query, output)
+		reProcessQuery, rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap)
 		if err != nil {
 			errs = append(errs, err)
 		}
+		queryResult.Add(rangeResult)
+		if reProcessQuery {
+			reProcessQueryRanges = append(reProcessQueryRanges, RangeSizes(query))
+		}
 	}

+	logger.Printf("Summary of Crawl: %s", queryResult.String())
+
 	if len(errs) > 0 {
-		return errs
+		return reProcessQueryRanges, errs
 	}

-	return nil
+	return reProcessQueryRanges, nil
 }

+// FetchDocument first tries to fetch the document with d.FilePath. If it fails,
+// it will try to add each string in konfig.RecognizedKustomizationFileNames() to
+// d.FilePath, and try to fetch the document again.
 func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
 	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
@@ -104,9 +178,13 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 		"/" + repoSpec.Ref + "/" + repoSpec.Path

 	handle := func(resp *http.Response, err error, path string) error {
+		if resp == nil {
+			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
+				url, path, err)
+		}
 		if err == nil && resp.StatusCode == http.StatusOK {
 			d.IsSame = httpclient.FromCache(resp.Header)
-			defer resp.Body.Close()
+			defer CloseResponseBody(resp)
 			data, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				return err
@@ -115,18 +193,18 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 			d.FilePath = d.FilePath + path
 			return nil
 		}
+
 		return err
 	}
-	resp, err := gc.client.GetRawUserContent(url)
-	if err := handle(resp, err, ""); err == nil {
+	resp, errGetRawUserContent := gc.client.GetRawUserContent(url)
+	if err := handle(resp, errGetRawUserContent, ""); err == nil {
 		return nil
 	}

 	for _, file := range konfig.RecognizedKustomizationFileNames() {
-		resp, err = gc.client.GetRawUserContent(url + "/" + file)
-		err := handle(resp, err, "/"+file)
-		if err != nil {
-			continue
+		resp, errGetRawUserContent = gc.client.GetRawUserContent(url + "/" + file)
+		if err = handle(resp, errGetRawUserContent, "/"+file); err == nil {
+			return nil
 		}
 	}
 	return fmt.Errorf("file not found: %s, error: %v", url, err)
@@ -158,10 +236,32 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
 	return strings.Contains(repoSpec.Host, "github.com")
 }

+type RangeQueryResult struct {
+	totalDocCnt uint64
+	seenDocCnt  uint64
+	newDocCnt   uint64
+	errorCnt    uint64
+}
+
+func (r *RangeQueryResult) Add(other RangeQueryResult) {
+	r.totalDocCnt += other.totalDocCnt
+	r.newDocCnt += other.newDocCnt
+	r.seenDocCnt += other.seenDocCnt
+	r.errorCnt += other.errorCnt
+}
+
+func (r *RangeQueryResult) String() string {
+	return fmt.Sprintf("got %d files from API. "+
+		"%d have been seen before. %d are new and sent to the output channel."+
+		" %d have kustomizationResultAdapter errors.",
+		r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
+}
+
 // processQuery follows all of the pages in a query, and updates/adds the
 // documents from the crawl to the datastore/index.
 func processQuery(ctx context.Context, gcl GhClient, query string,
-	output chan<- crawler.CrawledDocument) error {
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	branchMap map[string]string) (bool, RangeQueryResult, error) {

 	queryPages := make(chan GhResponseInfo)

@@ -177,60 +277,86 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
 		close(queryPages)
 	}()

+	reProcessQuery := false
+
 	errs := make(multiError, 0)
-	errorCnt := 0
-	totalCnt := 0
+	result := RangeQueryResult{}
+	pageID := 1
 	for page := range queryPages {
 		if page.Error != nil {
 			errs = append(errs, page.Error)
 			continue
 		}
-
+		pageResult := RangeQueryResult{}
 		for _, file := range page.Parsed.Items {
-			k, err := kustomizationResultAdapter(gcl, file)
+			k, err := kustomizationResultAdapter(gcl, file, seen, branchMap)
 			if err != nil {
 				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
-				errorCnt++
+				pageResult.errorCnt++
 			}
 			if k != nil {
+				pageResult.newDocCnt++
 				output <- k
+			} else {
+				pageResult.seenDocCnt++
 			}
-			totalCnt++
+			pageResult.totalDocCnt++
 		}

-		logger.Printf("got %d files out of %d from API. %d of %d had errors\n",
-			totalCnt, page.Parsed.TotalCount, errorCnt, totalCnt)
+		logger.Printf("processQuery [TotalCount %d - page %d]: %s",
+			page.Parsed.TotalCount, pageID, pageResult.String())
+		result.Add(pageResult)
+
+		pageID++
+
+		if page.Parsed.TotalCount > githubMaxResultsPerQuery {
+			reProcessQuery = true
+		}
 	}

-	return errs
+	logger.Printf("Summary of processQuery: %s", result.String())
+
+	return reProcessQuery, result, errs
 }

-func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
-	crawler.CrawledDocument, error) {
-
-	data, err := gcl.GetFileData(k)
-	if err != nil {
-		return nil, err
-	}
-
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
+	branchMap map[string]string) (crawler.CrawledDocument, error) {
 	url := gcl.ReposRequest(k.Repository.FullName)
-	defaultBranch, err := gcl.GetDefaultBranch(url)
+	defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
 	if err != nil {
 		logger.Printf(
 			"(error: %v) setting default_branch to master\n", err)
 		defaultBranch = "master"
 	}

+	// document here is a kustomization file found by querying Github, whose
+	// `FileType` field should be empty.
+	document := doc.Document{
+		FilePath:      k.Path,
+		DefaultBranch: defaultBranch,
+		RepositoryURL: k.Repository.URL,
+		User:          doc.UserName(k.Repository.URL),
+	}
+
+	if seen.Seen(document.ID()) {
+		return nil, nil
+	}
+
+	data, err := gcl.GetFileData(k)
+	if err != nil {
+		return nil, err
+	}
+
 	d := doc.KustomizationDocument{
 		Document: doc.Document{
 			DocumentData:  string(data),
 			FilePath:      k.Path,
 			DefaultBranch: defaultBranch,
 			RepositoryURL: k.Repository.URL,
+			User:          doc.UserName(k.Repository.URL),
 		},
 	}
-	logger.Printf("Set the creationTime field")
 	creationTime, err := gcl.GetFileCreationTime(k)
 	if err != nil {
 		logger.Printf("GetFileCreationTime failed: %v", err)
@@ -253,7 +379,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 	output chan<- GhResponseInfo) error {

 	logger.Println("querying: ", query)
-	response := gcl.parseGithubResponse(query)
+	response := gcl.parseGithubResponseWithRetry(query)

 	if response.Error != nil {
 		return response.Error
@@ -266,7 +392,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 		case <-ctx.Done():
 			return nil
 		default:
-			response = gcl.parseGithubResponse(response.NextURL)
+			response = gcl.parseGithubResponseWithRetry(response.NextURL)
 			if response.Error != nil {
 				return response.Error
 			}
@@ -294,7 +420,10 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
 			k, url, err)
 	}
-	resp.Body.Close()
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}

 	type githubContentRawURL struct {
 		DownloadURL string `json:"download_url,omitempty"`
@@ -313,18 +442,32 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 			k, rawURL.DownloadURL, err)
 	}

-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err = ioutil.ReadAll(resp.Body)
 	return data, err
 }

-func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
+func CloseResponseBody(resp *http.Response) {
+	if err := resp.Body.Close(); err != nil {
+		log.Printf("failed to close response body: %v", err)
+	}
+}
+
+// GetDefaultBranch gets the default branch of a github repository.
+// m is a map which maps a github repository to its default branch.
+// If repo is already in m, the default branch for url will be obtained from m;
+// otherwise, a query will be made to github to obtain the default branch.
+func (gcl GhClient) GetDefaultBranch(url, repo string, m map[string]string) (string, error) {
+	if v, ok := m[repo]; ok {
+		return v, nil
+	}
+
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
 			"'%s' could not get default_branch: %v", url, err)
 	}
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -376,7 +519,7 @@ func (gcl GhClient) GetFileCreationTime(
 		}
 	}

-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return defaultTime, fmt.Errorf(
@@ -444,6 +587,8 @@ type githubResponse struct {
 	// This is the number of files that match the query.
 	TotalCount uint64 `json:"total_count,omitempty"`

+	IncompleteResults bool `json:"incomplete_results,omitempty"`
+
 	// Github representation of a file.
 	Items []GhFileSpec `json:"items,omitempty"`
 }
@@ -486,6 +631,17 @@ func parseGithubLinkFormat(links string) (string, string) {
 	return next, last
 }

+func (gcl GhClient) parseGithubResponseWithRetry(getRequest string) GhResponseInfo {
+	resp := gcl.parseGithubResponse(getRequest)
+	retries := 0
+	for resp.Parsed.IncompleteResults {
+		resp = gcl.parseGithubResponse(getRequest)
+		retries++
+	}
+	log.Printf("The result of query(%s) is complete after %d retries", getRequest, retries)
+	return resp
+}
+
 func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	resp, err := gcl.SearchGithubAPI(getRequest)
 	requestInfo := GhResponseInfo{
@@ -499,7 +655,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	}

 	var data []byte
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
 	if requestInfo.Error != nil {
 		return requestInfo
@@ -533,7 +689,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 }

 // SearchGithubAPI performs a search query and handles rate limitting for
-// the 'code/search?' endpoint as well as timed retries in the case of abuse
+// the 'search/code?' endpoint as well as timed retries in the case of abuse
 // prevention.
 func (gcl GhClient) SearchGithubAPI(query string) (*http.Response, error) {
 	throttleSearchAPI()
@@ -560,7 +716,15 @@ func (gcl GhClient) Do(query string) (*http.Response, error) {
 		return nil, err
 	}
 	req.Header.Add("Authorization", fmt.Sprintf("token %s", gcl.accessToken))
-	return gcl.client.Do(req)
+
+	// gcl.client.Do: a non-2xx status code doesn't cause an error.
+	// See https://golang.org/pkg/net/http/#Client.Do for more info.
+	resp, err := gcl.client.Do(req)
+	if resp != nil && resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
+			query, resp.StatusCode)
+	}
+	return resp, err
 }

 func (gcl GhClient) getWithRetry(
@@ -570,13 +734,10 @@ func (gcl GhClient) getWithRetry(

 	retryCount := gcl.retryCount

-	for err == nil &&
-		resp.StatusCode == http.StatusForbidden &&
-		retryCount > 0 {
-
+	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
 		retryTime := resp.Header.Get("Retry-After")
-		i, err := strconv.Atoi(retryTime)
-		if err != nil {
+		i, errAtoi := strconv.Atoi(retryTime)
+		if errAtoi != nil {
 			return resp, fmt.Errorf(
 				"query '%s' forbidden without 'Retry-After'", query)
 		}
--- a/api/internal/crawl/crawler/github/queries.go
+++ b/api/internal/crawl/crawler/github/queries.go
@@ -90,6 +90,17 @@ func Path(p string) queryField {
 	return queryField{name: "path", value: p}
 }

+// Repo takes a repository (i.e., kubernetes-sigs/kustomize) and formats
+// it according to the Github API.
+func Repo(r string) queryField {
+	return queryField{name: "repo", value: r}
+}
+
+// Path takes a github username and formats it according to the Github API.
+func User(u string) queryField {
+	return queryField{name: "user", value: u}
+}
+
 // RequestConfig stores common variables that must be present for the queries.
 // - CodeSearchRequests: ask Github to check the code indices given a query.
 // - ContentsRequests: ask Github where to download a resource given a repo and a
@@ -105,9 +116,11 @@ type RequestConfig struct {
 // the URL method to get the string value of the URL. See request.CopyWith, to
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
-	req := rc.makeRequest("search/code", query)
-	req.vals.Set("sort", "indexed")
-	req.vals.Set("order", "desc")
+	vals := url.Values{
+		"sort":  []string{"indexed"},
+		"order": []string{"desc"},
+	}
+	req := rc.makeRequest("search/code", query, vals)
 	return req
 }

@@ -115,23 +128,25 @@ func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 // query for the Github API to find the dowload information of this filepath.
 func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }

 func (rc RequestConfig) ReposRequest(fullRepoName string) string {
 	uri := fmt.Sprintf("repos/%s", fullRepoName)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }

 // CommitsRequest given the repo name, and a filepath returns a formatted query
 // for the Github API to find the commits that affect this file.
 func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
-	return rc.makeRequest(uri, Query{Path(path)}).URL()
+	vals := url.Values{
+		"path": []string{path},
+	}
+	return rc.makeRequest(uri, Query{}, vals).URL()
 }

-func (rc RequestConfig) makeRequest(path string, query Query) request {
-	vals := url.Values{}
+func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
 	vals.Set(perPageArg, fmt.Sprint(rc.perPage))

 	return request{
@@ -151,7 +166,7 @@ type request struct {
 	query Query
 }

-// CopyWith copies the requests and adds the extra query parameters. Usefull
+// CopyWith copies the requests and adds the extra query parameters. It is useful
 // for dynamically adding sizes to a filename only query without modifying it.
 func (r request) CopyWith(queryParams ...queryField) request {
 	cpy := r
--- a/api/internal/crawl/crawler/github/queries_test.go
+++ b/api/internal/crawl/crawler/github/queries_test.go
@@ -53,8 +53,11 @@ func TestQueryType(t *testing.T) {
 				Filename("kustomization.yaml"),
 				Keyword("keyword1"),
 				Keyword("keyword2"),
+				Repo("user1/repo1"),
+				User("user1"),
 			),
-			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2",
+			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2+" +
+				"repo:user1/repo1+user:user1",
 		},
 	}

@@ -98,7 +101,27 @@ func TestGithubSearchQuery(t *testing.T) {
 				"examples/helloWorld/kustomization.yaml?per_page=100",

 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"q=path:examples/helloWorld/kustomization.yaml&per_page=100",
+				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+		},
+		{
+			rc: RequestConfig{
+				perPage: perPage,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples 1/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples%201/helloWorld/kustomization.yaml?per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
 		},
 	}

--- a/api/internal/crawl/crawler/github/split_search_ranges.go
+++ b/api/internal/crawl/crawler/github/split_search_ranges.go
@@ -93,13 +93,15 @@ package github
 //   apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000
 //   = apiCallsPerResult * totalResults.
 //
-// So it could very well take apiCallsPerResult * 50 times longer to acutally
+// So it could very well take apiCallsPerResult * 50 times longer to actually
 // fetch the results (assuming the quotas for the API calls are the same as the
 // search API), than it does to perform these range searches.

 import (
 	"fmt"
 	"math/bits"
+	"strconv"
+	"strings"
 )

 // Files cannot be more than 2^19 bytes, according to
@@ -112,7 +114,7 @@ const (
 // Interface instead of struct for testing purposes.
 // Not expecting to have multiple implementations.
 type cachedSearch interface {
-	CountResults(uint64) (uint64, error)
+	CountResults(uint64, uint64) (uint64, error)
 	RequestString(filesize rangeFormatter) string
 }

@@ -139,7 +141,7 @@ type cachedSearch interface {
 //    problematic). The current cache implementation looks at the
 //    predecessor entry to find out if the current value is monotonic.
 //    This is where the bit trick is used, since each step in the binary
-//    search is adding or ommiting to add a decreasing power of 2 to the query
+//    search is adding or omitting to add a decreasing power of 2 to the query
 //    value, we can remove the least significant set bit to find the
 //    predecessor in constant time. Ultimately since the search is rate
 //    limited, we could also easily afford to compute this in linear time
@@ -161,16 +163,16 @@ func newCache(client GhClient, query Query) githubCachedSearch {
 	}
 }

-func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
+func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
 	count, cached := c.cache[upperBound]
 	if cached {
 		return count, nil
 	}

-	sizeRange := RangeWithin{0, upperBound}
+	sizeRange := RangeWithin{lowerBound, upperBound}
 	rangeRequest := c.RequestString(sizeRange)

-	result := c.gcl.parseGithubResponse(rangeRequest)
+	result := c.gcl.parseGithubResponseWithRetry(rangeRequest)
 	if result.Error != nil {
 		return count, result.Error
 	}
@@ -204,7 +206,7 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 			"Retrying query... current lower bound: %d, got: %d\n",
 			c.cache[prev], result.Parsed.TotalCount)

-		result = c.gcl.parseGithubResponse(rangeRequest)
+		result = c.gcl.parseGithubResponseWithRetry(rangeRequest)
 		if result.Error != nil {
 			return count, result.Error
 		}
@@ -219,8 +221,8 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 	}

 	count = result.Parsed.TotalCount
-	logger.Printf("Caching new query %s, with count %d\n",
-		sizeRange.RangeString(), count)
+	logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n",
+		sizeRange.RangeString(), count, result.Parsed.IncompleteResults)
 	c.cache[upperBound] = count
 	return count, nil
 }
@@ -238,16 +240,16 @@ func (c githubCachedSearch) RequestString(filesize rangeFormatter) string {
 // This would mean that the search as it is could not find all files. If queries
 // are sorted by last indexed, and retrieved on regular intervals, it should be
 // sufficient to get most if not all documents.
-func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
-	totalFiles, err := cache.CountResults(githubMaxFileSize)
+func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64) ([]string, error) {
+	totalFiles, err := cache.CountResults(lowerBound, upperBound)
 	if err != nil {
 		return nil, err
 	}
-	logger.Println("total files: ", totalFiles)
+	logger.Println("total kustomization files: ", totalFiles)

 	if githubMaxResultsPerQuery >= totalFiles {
 		return []string{
-			cache.RequestString(RangeWithin{0, githubMaxFileSize}),
+			cache.RequestString(RangeWithin{lowerBound, upperBound}),
 		}, nil
 	}

@@ -275,6 +277,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 	// range.
 	filesAccessible := uint64(0)
 	sizes := make([]uint64, 0)
+	sizes = append(sizes, lowerBound)
 	for filesAccessible < totalFiles {
 		target := filesAccessible + githubMaxResultsPerQuery
 		if target >= totalFiles {
@@ -284,22 +287,22 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 		logger.Printf("%d accessible files, next target = %d\n",
 			filesAccessible, target)

-		cur, err := lowerBoundFileCount(cache, target)
+		size, err := FindFileSize(cache, target, lowerBound, upperBound)
 		if err != nil {
 			return nil, err
 		}

 		// If there are more than 1000 files in the next bucket, we must
 		// advance anyway and lose out on some files :(.
-		if l := len(sizes); l > 0 && sizes[l-1] == cur {
-			cur++
+		if l := len(sizes); l > 0 && sizes[l-1] == size {
+			size++
 		}

-		nextAccessible, err := cache.CountResults(cur)
+		nextAccessible, err := cache.CountResults(lowerBound, size)
 		if err != nil {
 			return nil, fmt.Errorf(
 				"cache should be populated at %d already, got %v",
-				cur, err)
+				size, err)
 		}
 		if nextAccessible < filesAccessible {
 			return nil, fmt.Errorf(
@@ -309,31 +312,31 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {

 		filesAccessible = nextAccessible
 		if nextAccessible < totalFiles {
-			sizes = append(sizes, cur)
+			sizes = append(sizes, size)
 		}
 	}
-
+	sizes = append(sizes, upperBound)
 	return formatFilesizeRanges(cache, sizes), nil
 }

-// lowerBoundFileCount finds the filesize range from [0, return value] that has
+// FindFileSize finds the filesize range from [lowerBound, return value] that has
 // the largest file count that is smaller than or equal to
 // githubMaxResultsPerQuery. It is important to note that this returned value
 // could already be in a previous range if the next file size has more than 1000
 // results. It is left to the caller to handle this bit of logic and guarantee
 // forward progession in this case.
-func lowerBoundFileCount(
-	cache cachedSearch, targetFileCount uint64) (uint64, error) {
+func FindFileSize(
+	cache cachedSearch, targetFileCount, lowerBound, upperBound uint64) (uint64, error) {

 	// Binary search for file sizes that make up the next <=1000 element
 	// chunk.
-	cur := uint64(0)
-	increase := githubMaxFileSize / 2
+	cur := lowerBound
+	increase := (upperBound - lowerBound) / 2

 	for increase > 0 {
 		mid := cur + increase

-		count, err := cache.CountResults(mid)
+		count, err := cache.CountResults(lowerBound, mid)
 		if err != nil {
 			return count, err
 		}
@@ -353,26 +356,24 @@ func lowerBoundFileCount(
 }

 func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string {
-	ranges := make([]string, 0, len(sizes)+1)
-
-	if len(sizes) > 0 {
-		ranges = append(ranges, cache.RequestString(
-			RangeLessThan{sizes[0] + 1},
-		))
+	n := len(sizes)
+	if n < 2 {
+		return []string{}
 	}

-	for i := 0; i < len(sizes)-1; i += 1 {
-		ranges = append(ranges, cache.RequestString(
-			RangeWithin{sizes[i] + 1, sizes[i+1]},
-		))
-
-		if i != len(sizes)-2 {
-			continue
-		}
-		ranges = append(ranges, cache.RequestString(
-			RangeGreaterThan{sizes[i+1]},
-		))
+	ranges := make([]string, 0, n-1)
+	ranges = append(ranges, cache.RequestString(RangeWithin{sizes[0], sizes[1]}))
+	for i := 1; i < n-1; i++ {
+		ranges = append(ranges, cache.RequestString(RangeWithin{sizes[i] + 1, sizes[i+1]}))
 	}
-
 	return ranges
 }
+
+func RangeSizes(s string) RangeWithin {
+	start := strings.Index(s, "+size:") + len("+size:")
+	end := strings.Index(s, "&")
+	ranges := strings.Split(s[start:end], "..")
+	lowerBound, _ := strconv.ParseUint(ranges[0], 10, 64)
+	upperBound, _ := strconv.ParseUint(ranges[1], 10, 64)
+	return RangeWithin{lowerBound, upperBound}
+}
--- a/api/internal/crawl/crawler/github/split_search_ranges_test.go
+++ b/api/internal/crawl/crawler/github/split_search_ranges_test.go
@@ -2,6 +2,7 @@ package github

 import (
 	"fmt"
+	"log"
 	"reflect"
 	"testing"
 )
@@ -10,8 +11,8 @@ type testCachedSearch struct {
 	cache map[uint64]uint64
 }

-func (c testCachedSearch) CountResults(upperBound uint64) (uint64, error) {
-	fmt.Printf("CountResults(%05x)\n", upperBound)
+func (c testCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
+	log.Printf("CountResults(%05x)\n", upperBound)
 	count, ok := c.cache[upperBound]
 	if !ok {
 		return count, fmt.Errorf("cache not set at %x", upperBound)
@@ -72,19 +73,29 @@ func TestRangeSplitting(t *testing.T) {
 		},
 	}

-	requests, err := FindRangesForRepoSearch(cache)
+	requests, err := FindRangesForRepoSearch(cache, 0, 524288)
 	if err != nil {
 		t.Errorf("Error while finding ranges: %v", err)
 	}
 	expected := []string{
-		"<107",      // cache.RequestString(RangeLessThan{0x6b}),
-		"107..128",  // cache.RequestString(RangeWithin{0x6b, 0x80}),
-		"129..256",  // cache.RequestString(RangeWithin{0x81, 0x100}),
-		"257..4095", // cache.RequestString(RangeWithin{0x101, 0xfff}),
-		">4095",     // cache.RequestString(RangeGreaterThan{0xfff}),
+		"0..106",       // cache.RequestString(RangeWithin{0x00, 0x6a}),
+		"107..128",     // cache.RequestString(RangeWithin{0x6b, 0x80}),
+		"129..256",     // cache.RequestString(RangeWithin{0x81, 0x100}),
+		"257..4095",    // cache.RequestString(RangeWithin{0x101, 0xfff}),
+		"4096..524288", // cache.RequestString(RangeWithin{0x1000, 0x80000}),
 	}

 	if !reflect.DeepEqual(requests, expected) {
 		t.Errorf("Expected requests (%v) to equal (%v)", requests, expected)
 	}
 }
+
+func TestRangeSizes(t *testing.T) {
+	s := "https://api.github.com/search/code?q=filename:kustomization.yaml+filename:kustomization.yml" +
+		"+filename:kustomization+size:2365..10000&order=desc&per_page=100&sort=indexed"
+	returnedResult := RangeSizes(s)
+	expectedResult := RangeWithin{uint64(2365), uint64(10000)}
+	if !reflect.DeepEqual(returnedResult, expectedResult) {
+		t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult)
+	}
+}
--- a/api/internal/crawl/doc/doc.go
+++ b/api/internal/crawl/doc/doc.go
@@ -2,6 +2,9 @@ package doc

 import (
 	"fmt"
+	"log"
+	"path/filepath"
+	"sort"
 	"strings"

 	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
@@ -43,21 +46,36 @@ type KustomizationDocument struct {

 type set map[string]struct{}

+func (doc *KustomizationDocument) Copy() *KustomizationDocument {
+	return &KustomizationDocument{
+		Document:    *(doc.Document.Copy()),
+		Kinds:       doc.Kinds,
+		Identifiers: doc.Identifiers,
+		Values:      doc.Values,
+	}
+}
+
 func (doc *KustomizationDocument) String() string {
 	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
 		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
-		doc.IsSame,	doc.Kinds, len(doc.Identifiers), len(doc.Values))
+		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
+}
+
+// IsKustomizationFile determines whether a file path is a kustomization file
+func IsKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
+		}
+	}
+	return false
 }

 // Implements the CrawlerDocument interface.
-func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
-	isResource := true
-	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
-		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
-			isResource = false
-		}
-	}
-	if isResource {
+func (doc *KustomizationDocument) GetResources(
+	includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
+	if !IsKustomizationFile(doc.FilePath) {
 		return []*Document{}, nil
 	}

@@ -75,19 +93,46 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 	}
 	k.FixKustomizationPostUnmarshalling()

-	res := make([]*Document, 0, len(k.Resources))
-	for _, r := range k.Resources {
-		next, err := doc.Document.FromRelativePath(r)
-		if err != nil {
-			fmt.Printf("GetResources error: %v\n", err)
-			continue
-		}
-		res = append(res, &next)
+	res := make([]*Document, 0)
+
+	if includeResources {
+		resourceDocs := doc.CollectDocuments(k.Resources, "resource")
+		res = append(res, resourceDocs...)
+	}
+
+	if includeGenerators {
+		generatorDocs := doc.CollectDocuments(k.Generators, "generator")
+		res = append(res, generatorDocs...)
+	}
+
+	if includeTransformers {
+		transformerDocs := doc.CollectDocuments(k.Transformers, "transformer")
+		res = append(res, transformerDocs...)
 	}

 	return res, nil
 }

+// CollectDocuments construct a Document for each path in paths, and return
+// a slice of Document pointers.
+func (doc *KustomizationDocument) CollectDocuments(
+	paths []string, fileType string) []*Document {
+	docs := make([]*Document, 0, len(paths))
+	for _, r := range paths {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
+		next, err := doc.Document.FromRelativePath(r)
+		if err != nil {
+			log.Printf("CollectDocuments error: %v\n", err)
+			continue
+		}
+		next.FileType = fileType
+		docs = append(docs, &next)
+	}
+	return docs
+}
+
 func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
 	data := []byte(doc.DocumentData)

@@ -116,6 +161,8 @@ func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error)
 	return configs, nil
 }

+// ParseYAML parses doc.Document and sets the following fields of doc:
+// Kinds, Values, Identifiers.
 func (doc *KustomizationDocument) ParseYAML() error {
 	doc.Identifiers = make([]string, 0)
 	doc.Values = make([]string, 0)
@@ -159,6 +206,13 @@ func (doc *KustomizationDocument) ParseYAML() error {
 		doc.Identifiers = append(doc.Identifiers, key)
 	}

+	// Without sorting these fields, every time when the string order in these fields changes,
+	// the document in the index will be updated.
+	// Sorting these fields are necessary to avoid a document being updated unnecessarily.
+	sort.Strings(doc.Kinds)
+	sort.Strings(doc.Values)
+	sort.Strings(doc.Identifiers)
+
 	return nil
 }

--- a/api/internal/crawl/doc/doc_test.go
+++ b/api/internal/crawl/doc/doc_test.go
@@ -189,11 +189,13 @@ metadata:
 	}
 }

+type TestStructForGetResources struct {
+	doc       KustomizationDocument
+	resources []*Document
+}
+
 func TestGetResources(t *testing.T) {
-	tests := []struct {
-		doc       KustomizationDocument
-		resources []*Document
-	}{
+	tests := []TestStructForGetResources{
 		{
 			doc: KustomizationDocument{
 				Document: Document{
@@ -213,19 +215,27 @@ resources:
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/base",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/otherbase",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "https://github.com/kubernetes-sigs/kustomize",
 					FilePath:      "examples/helloWorld",
 					DefaultBranch: "v3.1.0",
+					FileType:      "resource",
+					User:          "kubernetes-sigs",
 				},
 			},
 		},
@@ -248,9 +258,12 @@ resources:
 			resources: []*Document{},
 		},
 	}
+	runTest(t, tests, true, false, false)
+}

+func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
 	for _, test := range tests {
-		res, err := test.doc.GetResources()
+		res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
 		if err != nil {
 			t.Errorf("Unexpected error: %v\n", err)
 			continue
@@ -284,3 +297,83 @@ resources:
 		}
 	}
 }
+
+func TestGetResourcesAndGenerators(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, false, true)
+}
+
+func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/tr.yaml",
+					FileType:      "transformer",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, true, true)
+}
--- a/api/internal/crawl/doc/docname.go
+++ b/api/internal/crawl/doc/docname.go
@@ -11,12 +11,18 @@ import (
 )

 type Document struct {
-	RepositoryURL string     `json:"repositoryUrl,omitempty"`
+	RepositoryURL string `json:"repositoryUrl,omitempty"`
+	// User makes it easy to aggregate data in the user level instead
+	// of the repository level
+	User          string     `json:"user,omitempty"`
 	FilePath      string     `json:"filePath,omitempty"`
 	DefaultBranch string     `json:"defaultBranch,omitempty"`
 	DocumentData  string     `json:"document,omitempty"`
 	CreationTime  *time.Time `json:"creationTime,omitempty"`
 	IsSame        bool       `json:"-"`
+	// FileType can be one of the following:
+	// "generator", "transformer", "resource", "".
+	FileType string `json:"fileType,omitempty"`
 }

 // Implements the CrawlerDocument interface.
@@ -27,14 +33,21 @@ func (doc *Document) GetDocument() *Document {
 func (doc *Document) Copy() *Document {
 	return &Document{
 		RepositoryURL: doc.RepositoryURL,
+		User:          doc.User,
 		FilePath:      doc.FilePath,
 		DefaultBranch: doc.DefaultBranch,
 		DocumentData:  doc.DocumentData,
 		CreationTime:  doc.CreationTime,
 		IsSame:        doc.IsSame,
+		FileType:      doc.FileType,
 	}
 }

+func (doc *Document) Path() string {
+	return fmt.Sprintf("repoURL: %s filePath: %s branch: %s",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch)
+}
+
 // Implements the CrawlerDocument interface.
 func (doc *Document) WasCached() bool {
 	return doc.IsSame
@@ -47,6 +60,7 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 			RepositoryURL: repoSpec.Host + path.Clean(repoSpec.OrgRepo),
 			FilePath:      path.Clean(repoSpec.Path),
 			DefaultBranch: repoSpec.Ref,
+			User:          UserName(repoSpec.Host + path.Clean(repoSpec.OrgRepo)),
 		}, nil
 	}
 	// else document is probably relative path.
@@ -54,6 +68,7 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 	ret := Document{
 		RepositoryURL: doc.RepositoryURL,
 		DefaultBranch: doc.DefaultBranch,
+		User:          UserName(doc.RepositoryURL),
 	}
 	ogDir, _ := path.Split(doc.FilePath)

@@ -78,11 +93,32 @@ func (doc *Document) ID() string {
 }

 func (doc *Document) RepositoryFullName() string {
-	doc.RepositoryURL = strings.TrimRight(doc.RepositoryURL, "/")
-	sections := strings.Split(doc.RepositoryURL, "/")
+	url := TrimUrl(doc.RepositoryURL)
+	sections := strings.Split(url, "/")
 	l := len(sections)
 	if l < 2 {
-		return doc.RepositoryURL
+		return url
 	}
 	return path.Join(sections[l-2], sections[l-1])
 }
+
+// TrimUrl removes all the trailing slashes and the "git@github.com:" prefix (if exists).
+func TrimUrl(s string) string {
+	url := strings.TrimRight(s, "/")
+
+	gitPrefix := "git@github.com:"
+	if strings.HasPrefix(url, gitPrefix) {
+		url = url[len(gitPrefix):]
+	}
+	return url
+}
+
+func UserName(repositoryURL string) string {
+	url := TrimUrl(repositoryURL)
+	sections := strings.Split(url, "/")
+	l := len(sections)
+	if l < 2 {
+		return url
+	}
+	return sections[l-2]
+}
--- a/api/internal/crawl/doc/docname_test.go
+++ b/api/internal/crawl/doc/docname_test.go
@@ -28,6 +28,7 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/other/file/resource.yaml",
 						DefaultBranch: "master",
+						User:          "example.com",
 					},
 				},
 				{
@@ -36,6 +37,7 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/other/file/patch.yaml",
 						DefaultBranch: "master",
+						User:          "example.com",
 					},
 				},
 				{
@@ -44,6 +46,7 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/file/service.yaml",
 						DefaultBranch: "master",
+						User:          "example.com",
 					},
 				},
 			},
@@ -65,7 +68,7 @@ func TestFromRelativePath(t *testing.T) {

 func TestDocument_RepositoryFullName(t *testing.T) {
 	testCases := []struct {
-		doc Document
+		doc                        Document
 		expectedRepositoryFullName string
 	}{
 		{
@@ -92,6 +95,12 @@ func TestDocument_RepositoryFullName(t *testing.T) {
 			},
 			expectedRepositoryFullName: "",
 		},
+		{
+			doc: Document{
+				RepositoryURL: "git@github.com:user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
 	}

 	for _, tc := range testCases {
@@ -102,4 +111,40 @@ func TestDocument_RepositoryFullName(t *testing.T) {
 				returnedRepositoryFullName)
 		}
 	}
-}
+}
+
+func TestDocument_UserName(t *testing.T) {
+	testCases := []struct {
+		repositoryURL    string
+		expectedUserName string
+	}{
+		{
+			repositoryURL:    "https://github.com/user/repo",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "https://github.com//user/repo////",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "repo/",
+			expectedUserName: "repo",
+		},
+		{
+			repositoryURL:    "",
+			expectedUserName: "",
+		},
+		{
+			repositoryURL:    "git@github.com:user/repo",
+			expectedUserName: "user",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedUserName := UserName(tc.repositoryURL)
+		if returnedUserName != tc.expectedUserName {
+			t.Errorf("UserName expected %s, got %s",
+				tc.expectedUserName, returnedUserName)
+		}
+	}
+}
--- a/api/internal/crawl/doc/unique_doc.go
+++ b/api/internal/crawl/doc/unique_doc.go
@@ -0,0 +1,36 @@
+package doc
+
+import (
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+)
+
+// UniqueDocuments make sure a Document with a given ID appears only once
+type UniqueDocuments struct {
+	docs   []*Document
+	docIDs utils.SeenMap
+}
+
+func NewUniqueDocuments() UniqueDocuments {
+	return UniqueDocuments{
+		docs:   []*Document{},
+		docIDs: utils.NewSeenMap(),
+	}
+}
+
+func (uds *UniqueDocuments) Add(d *Document) {
+	if uds.docIDs.Seen(d.ID()) {
+		return
+	}
+	uds.docs = append(uds.docs, d)
+	uds.docIDs.Set(d.ID(), "")
+}
+
+func (uds *UniqueDocuments) AddDocuments(docs []*Document) {
+	for _, d := range docs {
+		uds.Add(d)
+	}
+}
+
+func (uds *UniqueDocuments) Documents() []*Document {
+	return uds.docs
+}
--- a/api/internal/crawl/go.mod
+++ b/api/internal/crawl/go.mod
@@ -1,13 +1,13 @@
 module sigs.k8s.io/kustomize/api/internal/crawl

-go 1.13
+go 1.12

 require (
-	github.com/elastic/go-elasticsearch/v6 v6.8.2
+	github.com/elastic/go-elasticsearch/v6 v6.8.5
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/gorilla/mux v1.7.3
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/rs/cors v1.7.0
-	sigs.k8s.io/kustomize/api v0.2.0
+	sigs.k8s.io/kustomize/api v0.3.1
 	sigs.k8s.io/yaml v1.1.0
 )
--- a/api/internal/crawl/go.sum
+++ b/api/internal/crawl/go.sum
@@ -1,14 +1,11 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -25,6 +22,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -41,13 +39,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/elastic/go-elasticsearch/v6 v6.8.2 h1:rp5DGrd63V5c6nHLjF6QEXUpZSvs0+QM3ld7m9VhV2g=
-github.com/elastic/go-elasticsearch/v6 v6.8.2/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/elastic/go-elasticsearch/v6 v6.8.5 h1:U2HtkBseC1FNBmDr0TR2tKltL6FxoY+niDAlj5M8TK8=
+github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -87,14 +84,16 @@ github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Il
 github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
 github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -106,10 +105,10 @@ github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvL
 github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
 github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
@@ -117,34 +116,35 @@ github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunE
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
 github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
-github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -154,6 +154,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -177,8 +178,8 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -192,7 +193,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -225,18 +225,15 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -261,12 +258,13 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
@@ -274,16 +272,22 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -294,19 +298,23 @@ golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -315,19 +323,22 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -336,20 +347,26 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -371,7 +388,14 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
+k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
+k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
+k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
+k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
@@ -379,14 +403,12 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.2.0 h1:e++6JpysnnlUbHmFrv6jvfF5rFlgQ103bS1DO7r5bWA=
-sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0 h1:otg4dLFc03c3gzl+2CV8GPGcd1kk8wjXwD+UhhcCn5I=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
+sigs.k8s.io/kustomize/api v0.3.1 h1:oqMIXvS6tFEUVuKIRUKDa05eC4Hh+cb9JYg8Zhp2d24=
+sigs.k8s.io/kustomize/api v0.3.1/go.mod h1:A+ATnlHqzictQfQC1q3KB/T6MSr0UWQsrrLxMWkge2E=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
--- a/api/internal/crawl/index/elasticsearch.go
+++ b/api/internal/crawl/index/elasticsearch.go
@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"time"

 	es "github.com/elastic/go-elasticsearch/v6"
@@ -21,12 +20,18 @@ const IndexConfig = `
 				"repositoryUrl": {
 					"type": "keyword"
 				},
+				"user": {
+					"type": "keyword"
+				},
 				"filePath": {
 					"type": "keyword"
 				},
 				"defaultBranch": {
 					"type": "keyword"
 				},
+				"fileType": {
+					"type": "keyword"
+				},
 				"document": {
 					"type": "text"
 				},
@@ -88,7 +93,7 @@ func (idx *index) responseErrorOrNil(info string, res *esapi.Response,

 	defer res.Body.Close()
 	if res.IsError() {
-		return fmt.Errorf("%s: %s", messageStart, res.String())
+		return fmt.Errorf("%s: %s [%d]", messageStart, res.String(), res.StatusCode)
 	}

 	if reader != nil {
@@ -179,44 +184,47 @@ func (idx *index) DeleteIndex() error {
 }

 // Insert or update the document by ID.
-func (idx *index) Put(uniqueID string, doc interface{}) (string, error) {
-	body, err := json.Marshal(doc)
+func (idx *index) Put(uniqueID string, doc interface{}) error {
+	exists, err := idx.Exists(uniqueID)
 	if err != nil {
-		return "", err
+		return err
 	}

-	req := esapi.IndexRequest{
-		Index:      idx.name,
-		Body:       bytes.NewReader(body),
-		DocumentID: uniqueID,
-	}
-	res, err := req.Do(idx.ctx, idx.client)
-
-	var id string
-	readId := func(reader io.Reader) error {
-		type InsertResult struct {
-			ID string `json:"_id,omitempty"`
+	if exists {
+		docBytes, err := json.Marshal(doc)
+		if err != nil {
+			return err
 		}
-		var ir InsertResult
-		data, err := ioutil.ReadAll(reader)
+		body := byteJoin(`{"doc":`, docBytes, `}`)
+
+		// For a document with a given id, every call of IndexRequest.Do will increase the version of a document.
+		// To avoid increasing the document version unnecessarily, use UpdateRequest here.
+		req := esapi.UpdateRequest{
+			Index:      idx.name,
+			Body:       bytes.NewReader(body),
+			DocumentID: uniqueID,
+		}
+		res, err := req.Do(idx.ctx, idx.client)
+
+		err = idx.responseErrorOrNil("could not update document",
+			res, err, ignoreResponseBody)
+	} else {
+		body, err := json.Marshal(doc)
 		if err != nil {
 			return err
 		}

-		err = json.Unmarshal(data, &ir)
-		if err != nil {
-			return err
+		req := esapi.IndexRequest{
+			Index:      idx.name,
+			Body:       bytes.NewReader(body),
+			DocumentID: uniqueID,
 		}
-		id = ir.ID
+		res, err := req.Do(idx.ctx, idx.client)

-		return nil
+		err = idx.responseErrorOrNil("could not insert document",
+			res, err, ignoreResponseBody)
 	}
-
-	// populates the id field.
-	err = idx.responseErrorOrNil("could not insert document",
-		res, err, readId)
-
-	return id, err
+	return err
 }

 type scrollUpdater func(string, readerFunc) error
@@ -296,3 +304,24 @@ func (idx *index) Delete(id string) error {
 		fmt.Sprintf("could not delete id(%s) from index(%s)", id, idx.name),
 		res, err, ignoreResponseBody)
 }
+
+// Check whether a given document id is in the index
+func (idx *index) Exists(id string) (bool, error) {
+	op := idx.client.Exists
+	res, err := op(
+		idx.name,
+		id,
+		op.WithContext(idx.ctx),
+		op.WithPretty(),
+	)
+
+	if res != nil && !res.IsError() {
+		return true, nil
+	} else if res != nil && res.StatusCode == 404 {
+		return false, nil
+	} else {
+		return false, idx.responseErrorOrNil(
+			fmt.Sprintf("could not check the existence of id(%s) from index(%s)", id, idx.name),
+			res, err, ignoreResponseBody)
+	}
+}
--- a/api/internal/crawl/index/kustomize.go
+++ b/api/internal/crawl/index/kustomize.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"strings"
 	"time"

@@ -16,6 +17,13 @@ const (
 	AggregationKeyword = "aggs"
 )

+type Mode int
+
+const (
+	InsertOrUpdate = iota
+	Delete
+)
+
 // Redefinition of Hits structure. Must match the json string of
 // KustomizeResult.Hits.Hits. Declared as a convenience for iteration.
 type KustomizeHits []struct {
@@ -91,14 +99,14 @@ type KustomizeIndex struct {
 }

 // Create index reference to the index containing the kustomize documents.
-func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
-	idx, err := newIndex(ctx, "kustomize")
+func NewKustomizeIndex(ctx context.Context, indexName string) (*KustomizeIndex, error) {
+	idx, err := newIndex(ctx, indexName)
 	if err != nil {
 		return nil, err
 	}

 	indicesExistsOp := idx.client.Indices.Exists
-	resp, err := indicesExistsOp([]string{"kustomize"},
+	resp, err := indicesExistsOp([]string{indexName},
 		indicesExistsOp.WithContext(idx.ctx),
 		indicesExistsOp.WithPretty())
 	if err != nil {
@@ -106,9 +114,9 @@ func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
 	}

 	if resp.StatusCode == 200 {
-		fmt.Printf("The kustomize index already exists\n")
+		log.Printf("The %s index already exists", indexName)
 	} else {
-		fmt.Printf("Creating the kustomize index\n")
+		log.Printf("Creating the %s index\n", indexName)
 		if err := idx.CreateIndex([]byte(IndexConfig)); err != nil {
 			return nil, err
 		}
@@ -246,7 +254,7 @@ func (it *KustomizeIterator) Next() bool {
 	}

 	if it.err == nil {
-		fmt.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
+		log.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
 		it.err = it.update(*it.scrollImpl.ScrollID, reader)
 	}

@@ -262,7 +270,7 @@ func (it *KustomizeIterator) Value() KustomizeResult {
 	return it.scrollImpl
 }

-// Check if any errors have occured.
+// Check if any errors have occurred.
 func (it *KustomizeIterator) Err() error {
 	return it.err
 }
@@ -293,12 +301,13 @@ func (ki *KustomizeIndex) IterateQuery(query []byte, batchSize int,
 }

 // type specific Put for inserting structured kustomization documents.
-func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) (string, error) {
-	id, err := ki.index.Put(id, doc)
-	if err != nil {
-		return id, fmt.Errorf("could not insert in elastic: %v", err)
-	}
-	return id, nil
+func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) error {
+	return ki.index.Put(id, doc)
+}
+
+// Delete a document with a given id from the kustomize index.
+func (ki *KustomizeIndex) Delete(id string) error {
+	return ki.index.Delete(id)
 }

 // Kustomize search options: What metrics should be returned? Kind Aggregation,
@@ -334,7 +343,7 @@ func (ki *KustomizeIndex) Search(query string,
 	if err != nil {
 		return nil, fmt.Errorf("failed to format query %s", query)
 	}
-	fmt.Printf("formated query: %s\n", data)
+	log.Printf("formated query: %s\n", data)

 	var kr ElasticKustomizeResult
 	err = ki.index.Search(data, opts.SearchOptions, func(results io.Reader) error {
--- a/api/internal/crawl/search_cmds/creationTime.md
+++ b/api/internal/crawl/search_cmds/creationTime.md
@@ -0,0 +1,411 @@
+Find out the largest value of the `creationTime` field:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "max_creationTime" : { "max" : { "field" : "creationTime" } }
+    }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "min_creationTime" : { "min" : { "field" : "creationTime" } }
+    }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all the kustomization files:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+        { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize resource files:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }, 
+      "filter": [
+        { "regexp": { "fileType": "resource"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize generator files:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }, 
+      "filter": [
+        { "regexp": { "fileType": "generator"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize transformer files:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }, 
+      "filter": [
+        { "regexp": { "fileType": "transformer"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Query all the documents whose `creationTime` <= `2016-07-29T17:38:26.000Z`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "range": {
+      "creationTime": {
+       "lte": "2016-07-29T17:38:26.000Z"
+      }
+    }
+  }
+}
+'
+```
+
+Query all the documents whose `creationTime` falls within the specific range:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "range": {
+      "creationTime": {
+       "gte": "2016-07-29T17:38:26.000Z",
+       "lte": "2016-08-29T17:38:26.000Z"
+      }
+    }
+  }
+}
+'
+```
+
+Query all the kustomization files whose `creationTime` falls within the specific range:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 20,
+  "query": {
+    "bool": {
+      "filter": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }
+      },
+      "must": {
+        "range": {
+          "creationTime": {
+            "gte": "2017-09-24T15:49:57.000Z",
+            "lte": "2017-09-24T15:49:57.000Z"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```
+
+Aggregate how many new kustomization files were added into Github each month:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize resource files were added into Github each month:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize generator files were added into Github each month:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize transformer files were added into Github each month:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomization files were added into Github each year:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize resource files were added into Github each year:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]            
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize generator files were added into Github each year:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]            
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize transformer files were added into Github each year:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]            
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Find the generator files created within the given time range:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "range": {
+          "creationTime": {
+           "gte": "2019-04-26T16:40:02.000Z",
+           "lte": "2019-04-26T16:40:02.000Z"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```
+
+Find the transformer files created within the given time range:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "range": {
+          "creationTime": {
+           "gte": "2019-04-26T16:40:02.000Z",
+           "lte": "2019-04-26T16:40:02.000Z"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/defaultBranch.md
+++ b/api/internal/crawl/search_cmds/defaultBranch.md
@@ -0,0 +1,32 @@
+Count distinct values of the `defaultBranch` field:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "defaultBranch_count" : {
+            "cardinality" : {
+                "field" : "defaultBranch",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the github branches where kustomization files and kustomize resource files live, 
+and how many kustomization files and kustomize resource files live in each branch:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "defaultBranch" : {
+            "terms" : {
+              "field" : "defaultBranch",
+              "size": 41
+            }
+        }
+    }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/fieldExistence.md
+++ b/api/internal/crawl/search_cmds/fieldExistence.md
@@ -0,0 +1,55 @@
+Count the documents whose `document` field is empty (The reason why the `document` field
+of a document is empty is because of empty documents):
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "document"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Find all the documents having the `creationTime` field set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "creationTime"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `creationTime` field is not set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "creationTime"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+The following fields of a document in the kustomize index are always non-empty:
+`repositoryUrl`, `filePath`, `defaultBranch`.
+
+The following fields of a document in the kustomize index may be empty:
+`kinds`, `identifiers`, `values`.
--- a/api/internal/crawl/search_cmds/fileType.md
+++ b/api/internal/crawl/search_cmds/fileType.md
@@ -0,0 +1,301 @@
+Find all the documents having the `fileType` field set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "fileType"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `fileType` field is not set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "fileType"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Search for all the documents whose `fileType` field is `resource`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "resource" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files whose `fileType` field is `resource`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
+       { "regexp": { "fileType": "resource" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "resource" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search all the kustomization files including a `generators` field:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": {
+        "match" : {
+          "identifiers" : {
+            "query" : "generators"
+          }
+        }
+      },
+      "filter": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents whose `fileType` field is `generator`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files whose `fileType` field is `generator`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
+       { "regexp": { "fileType": "generator" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize generator files:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search all the kustomization files including a `transformers` field:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": {
+        "match" : {
+          "identifiers" : {
+            "query" : "transformers"
+          }
+        }
+      },
+      "filter": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents whose `fileType` field is `transformer`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files whose `fileType` field is `transformer`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
+       { "regexp": { "fileType": "transformer" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize transformer files:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Count distinct values of the `fileType` field:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "fileType_count" : {
+            "cardinality" : {
+                "field" : "fileType",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the values of the `fileType` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "fileType" : {
+            "terms" : {
+              "field" : "fileType"
+            }
+        }
+    }
+}
+'
+```
+
+
+For all the kustomization files in the index, list all the values of the
+`fileType` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+             ]
+        }
+    },
+    "aggs" : {
+        "fileType" : {
+            "terms" : {
+              "field" : "fileType"
+            }
+        }
+    }
+}
+'
+```
+
+For all the non-kustomization files in the index, list all the values of the
+`fileType` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            }
+        }
+    },
+    "aggs" : {
+        "fileType" : {
+            "terms" : {
+              "field" : "fileType"
+            }
+        }
+    }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/generator.md
+++ b/api/internal/crawl/search_cmds/generator.md
@@ -0,0 +1,29 @@
+Find all the generator files whose `kinds` field includes `ChartRenderer`, and
+only output certain fields of each document:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 200,
+  "_source": {
+    "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"]
+  },
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "match" : {
+          "kinds" : {
+            "query" : "ChartRenderer"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/id.md
+++ b/api/internal/crawl/search_cmds/id.md
@@ -0,0 +1,12 @@
+Find the document with the given `_id`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "terms": {
+      "_id": [ "b3a03f3327841617db696e2d6abc30e1a1bd653f1a2bbce05637f7dcae1a43f7" ] 
+    }
+  }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/keyword_search.md
+++ b/api/internal/crawl/search_cmds/keyword_search.md
@@ -0,0 +1,82 @@
+Count the documents in the index whose `repositoryUrl` field starts with
+`https://github.com/`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Count the documents in the index whose `repositoryUrl` field does not start with
+`https://github.com/`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": [
+       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents matching the given `repositoryUrl` and `filePath`, and return 
+a version for each search hit:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "version": true,
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "repositoryUrl": "git@github.com:talos-systems/talos-controller-manager" }},
+       { "regexp": { "filePath": "hack/config.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents whose filePath ends with one of these following three filenames:
+`kustomization.yaml`, `kustomization.yml`, `kustomization`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents whose filePath does not end with any of these following
+three filenames: `kustomization.yaml`, `kustomization.yml`, `kustomization`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+      ]
+    }
+  }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/misc.md
+++ b/api/internal/crawl/search_cmds/misc.md
@@ -0,0 +1,32 @@
+Check the health status of an ElasticSearch cluster:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_cat/health?v&pretty"
+```
+
+Check the indices in an ElasticSearch cluster:
+```
+curl -s "${ElasticSearchURL}:9200/_cat/indices?v"
+```
+
+Get the mapping of the index:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty"
+```
+
+Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**):
+```
+curl -s -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty"
+```
+
+Add a new field into an existing index.
+```
+curl -s -X PUT "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping/_doc?pretty" -H 'Content-Type: application/json' -d'
+{
+  "properties": {
+    "fileType": {
+      "type": "keyword"
+    }
+  }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/repositoryUrl.md
+++ b/api/internal/crawl/search_cmds/repositoryUrl.md
@@ -0,0 +1,255 @@
+Count distinct values of the `repositoryUrl` field:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count how many Github repositories include kustomization files:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+             ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize resource files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize generator files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize transformer files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize resource dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "resource" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize generator dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "generator" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize transformer dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "transformer" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the github repositories including kustomization files and kustomize resource files,
+and how many kustomization files and kustomize resource files each github repository includes
+(the github repository including the most kustomization files is listed first):
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 2082
+            }
+        }
+    }
+}
+'
+```
+
+List the top 20 Github repositories including the most amount of kustomization files:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+             ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+List the top 20 Github repositories including the most amount of kustomize resource files:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 20
+            }
+        }
+    }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/snapshot.md
+++ b/api/internal/crawl/search_cmds/snapshot.md
@@ -0,0 +1,29 @@
+Retrieve information about all registered snapshot repositories:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_snapshot?pretty"
+```
+
+Retrieve information about a given snapshot repository, `kustomize-backup`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup?pretty"
+```
+
+Verify a snapshot repository, `kustomize-backup`, manually:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/_verify?pretty"
+```
+
+List all the snapshots in a given snapshot repository:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_cat/snapshots/kustomize-backup?v&s=id&pretty"
+```
+
+Retrieve a summary information about a given snapshot:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot?pretty"
+```
+
+Retrieve a detailed information about a given snapshot:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot/_status?pretty"
+```
--- a/api/internal/crawl/search_cmds/stats.md
+++ b/api/internal/crawl/search_cmds/stats.md
--- a/api/internal/crawl/search_cmds/text_search.md
+++ b/api/internal/crawl/search_cmds/text_search.md
@@ -0,0 +1,148 @@
+Search for all the kustomize resource files including a Deployment object:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files including a Deployment object, but only
+including the `kinds` field in the result:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "_source": {
+    "includes": ["kinds"]
+  },
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files including both a Deployment object and 
+a Service object:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment Service",
+        "operator" : "and"
+      }
+    }
+  }
+}
+'
+```
+
+Count the number of documents including Deployment and the number of documents 
+including Service:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 0,
+  "aggs" : {
+    "messages" : {
+      "filters" : {
+        "filters" : {
+          "Deployment" : { "match" : { "kinds" : "Deployment"   }},
+          "Service" : { "match" : { "kinds" : "Service" }}
+        }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files involving CRDs:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "crds"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files defining configMapGenerator:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "configMapGenerator"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents having a `kind` field:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "match" : { "identifiers" : { "query" : "kind" }}}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kuostmization files having a `kind` field:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)" }},
+       { "match" : { "identifiers" : { "query" : "kind" }}}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files defining the `generatorOptions:disableNameSuffixHash` feature:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "generatorOptions:disableNameSuffixHash"
+      }
+    }
+  }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/transformer.md
+++ b/api/internal/crawl/search_cmds/transformer.md
@@ -0,0 +1,29 @@
+Find all the trasnformer files whose `kinds` field includes `HelmValues`, and
+only output certain fields of each document:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 200,
+  "_source": {
+    "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"]
+  },
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "match" : {
+          "kinds" : {
+            "query" : "HelmValues"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```
--- a/api/internal/crawl/search_cmds/user.md
+++ b/api/internal/crawl/search_cmds/user.md
@@ -0,0 +1,380 @@
+Find all the documents having the `user` field set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "user"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `user` field is not set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "user"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Search for all the documents whose `user` field is `kubernetes-sigs`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "user": "kubernetes-sigs" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Count distinct values of the `user` field:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the values of the `user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size" : 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomization files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+             ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomization files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+             ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize resource files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize resource files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize generator files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize generator files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+              "field" : "user",
+              "size": 20  
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize transformer files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize transformer files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize generator dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "generator" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize generator dirs in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "generator" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+              "field" : "user",
+              "size": 20  
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize transformer dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "transformer" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize transformer dirs in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "transformer" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+              "field" : "user"
+            }
+        }
+    }
+}
+'
+```
--- a/api/internal/crawl/ui/package-lock.json
+++ b/api/internal/crawl/ui/package-lock.json
--- a/api/internal/crawl/ui/package.json
+++ b/api/internal/crawl/ui/package.json
@@ -12,7 +12,7 @@
  "private": true,
  "dependencies": {
    "@angular/animations": "^8.2.14",
-    "@angular/cdk": "~8.0.2",
+    "@angular/cdk": "~8.2.3",
    "@angular/common": "^8.2.14",
    "@angular/compiler": "^8.2.14",
    "@angular/core": "^8.2.14",
@@ -26,29 +26,30 @@
    "chart.js": "^2.9.3",
    "core-js": "^3.5.0",
    "hammerjs": "^2.0.8",
-    "rxjs": "~6.4.0",
-    "tslib": "^1.9.0",
-    "zone.js": "~0.9.1"
+    "rxjs": "~6.5.3",
+    "serialize-javascript": "^2.1.2",
+    "tslib": "^1.10.0",
+    "zone.js": "~0.10.2"
  },
  "devDependencies": {
-    "@angular-devkit/build-angular": "^0.801.3",
+    "@angular-devkit/build-angular": "^0.803.20",
    "@angular/cli": "^8.3.20",
    "@angular/compiler-cli": "^8.2.14",
    "@angular/language-service": "^8.2.14",
    "@types/jasmine": "^3.5.0",
    "@types/jasminewd2": "^2.0.8",
-    "@types/node": "~8.9.4",
+    "@types/node": "~12.12.17",
    "codelyzer": "^5.2.0",
-    "jasmine-core": "~3.4.0",
+    "jasmine-core": "~3.5.0",
    "jasmine-spec-reporter": "~4.2.1",
-    "karma": "~4.1.0",
-    "karma-chrome-launcher": "~2.2.0",
+    "karma": "~4.4.1",
+    "karma-chrome-launcher": "~3.1.0",
    "karma-coverage-istanbul-reporter": "^2.1.1",
    "karma-jasmine": "~2.0.1",
-    "karma-jasmine-html-reporter": "^1.4.0",
-    "protractor": "~5.4.0",
-    "ts-node": "~7.0.0",
-    "tslint": "~5.15.0",
-    "typescript": "~3.4.3"
+    "karma-jasmine-html-reporter": "^1.4.2",
+    "protractor": "~5.4.2",
+    "ts-node": "~8.5.4",
+    "tslint": "~5.20.1",
+    "typescript": "~3.7.3"
  }
 }
--- a/api/internal/crawl/utils/utils.go
+++ b/api/internal/crawl/utils/utils.go
@@ -0,0 +1,21 @@
+package utils
+
+type SeenMap map[string]string
+
+func (seen SeenMap) Seen(item string) bool {
+	_, ok := seen[item]
+	return ok
+}
+
+func (seen SeenMap) Set(k, v string) {
+	seen[k] = v
+}
+
+// The caller should make sure that key is in the map.
+func (seen SeenMap) Value(k string) string {
+	return seen[k]
+}
+
+func NewSeenMap() SeenMap {
+	return make(map[string]string)
+}
--- a/api/internal/git/cloner.go
+++ b/api/internal/git/cloner.go
@@ -27,90 +27,28 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 	if err != nil {
 		return err
 	}
+
+	if repoSpec.Ref == "" {
+		repoSpec.Ref = "master"
+	}
 	cmd := exec.Command(
 		gitProgram,
-		"init",
+		"clone",
+		"--depth=1",
+		repoSpec.CloneSpec(),
+		"-b",
+		repoSpec.Ref,
 		repoSpec.Dir.String())
 	var out bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &out
 	err = cmd.Run()
 	if err != nil {
-		log.Printf("Error initializing empty git repo: %s", out.String())
+		log.Printf("Error cloning git repo: %s", out.String())
 		return errors.Wrapf(
 			err,
-			"trouble initializing empty git repo in %s",
-			repoSpec.Dir.String())
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"remote",
-		"add",
-		"origin",
-		repoSpec.CloneSpec())
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error setting git remote: %s", out.String())
-		return errors.Wrapf(
-			err,
-			"trouble adding remote %s",
-			repoSpec.CloneSpec())
-	}
-	if repoSpec.Ref == "" {
-		repoSpec.Ref = "master"
-	}
-	cmd = exec.Command(
-		gitProgram,
-		"fetch",
-		"--depth=1",
-		"origin",
-		repoSpec.Ref)
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		cmd = exec.Command(
-			gitProgram,
-			"pull",
-			"origin",
-			"master")
-		var out bytes.Buffer
-		cmd.Stdout = &out
-		cmd.Dir = repoSpec.Dir.String()
-		err := cmd.Run()
-		if err != nil {
-			return errors.Wrapf(err, "trouble pulling %s", repoSpec.OrgRepo)
-		}
-		if repoSpec.Ref == "" {
-			repoSpec.Ref = "master"
-		}
-		cmd = exec.Command(gitProgram, "checkout", repoSpec.Ref)
-		cmd.Dir = repoSpec.Dir.String()
-		err = cmd.Run()
-		if err != nil {
-			return errors.Wrapf(
-				err, "trouble checking out href %s", repoSpec.Ref)
-		}
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"reset",
-		"--hard",
-		"FETCH_HEAD")
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error performing git reset: %s", out.String())
-		return errors.Wrapf(
-			err, "trouble hard resetting empty repository to %s", repoSpec.Ref)
+			"trouble cloning git repo %v in %s",
+			repoSpec.CloneSpec(), repoSpec.Dir.String())
 	}

 	cmd = exec.Command(
@@ -120,10 +58,11 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 		"--init",
 		"--recursive")
 	cmd.Stdout = &out
+	cmd.Stderr = &out
 	cmd.Dir = repoSpec.Dir.String()
 	err = cmd.Run()
 	if err != nil {
-		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.Ref)
+		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec())
 	}

 	return nil
--- a/api/internal/tools/tools.go
+++ b/api/internal/tools/tools.go
@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0

 // This file exists to automatically trigger installs
-// of the given tools, and is the offical 'unofficial'
+// of the given tools, and is the official 'unofficial'
 // way to declare a dependence on a Go binary until
 // some better technique comes along.

@@ -15,8 +15,6 @@ import (
 	_ "golang.org/x/tools/cmd/stringer"
 	// for lint checks
 	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
-	// for integration tests driven by the examples
-	_ "github.com/monopole/mdrip"
 	// REMOVED pluginator from this process, and leaving
 	// this note to discourage its reintroduction,
 	// because pluginator depends on the api, forcing
--- a/api/konfig/builtinpluginconsts/varreference.go
+++ b/api/konfig/builtinpluginconsts/varreference.go
@@ -190,5 +190,7 @@ varReference:
  kind: StatefulSet

 - path: metadata/labels
+
+- path: metadata/annotations
 `
 )
--- a/api/krusty/baseandoverlaymedium_test.go
+++ b/api/krusty/baseandoverlaymedium_test.go
@@ -154,8 +154,9 @@ LEGUME=chickpea
 `)
 	th.WriteF("/app/overlay/configmap/dummy.txt",
 		`Lorem ipsum dolor sit amet, consectetur
-adipiscing elit, sed do eiusmod tempor
-incididunt ut labore et dolore magna aliqua. 
+ 		 
+adipiscing elit, sed do eiusmod tempor		
+incididunt ut labore et dolore magna aliqua.   
 `)
 	th.WriteF("/app/overlay/deployment/deployment.yaml", `
 apiVersion: apps/v1
@@ -292,8 +293,11 @@ metadata:
 ---
 apiVersion: v1
 data:
-  nonsense: "Lorem ipsum dolor sit amet, consectetur\nadipiscing elit, sed do eiusmod
-    tempor\nincididunt ut labore et dolore magna aliqua. \n"
+  nonsense: |
+    Lorem ipsum dolor sit amet, consectetur
+
+    adipiscing elit, sed do eiusmod tempor
+    incididunt ut labore et dolore magna aliqua.
 kind: ConfigMap
 metadata:
  annotations:
@@ -302,6 +306,6 @@ metadata:
    app: mungebot
    org: kubernetes
    repo: test-infra
-  name: test-infra-app-config-f462h769f9
+  name: test-infra-app-config-hh272bg5d4
 `)
 }
--- a/api/krusty/variableref_test.go
+++ b/api/krusty/variableref_test.go
@@ -1157,6 +1157,8 @@ vars:
    name: my-deployment
    labels:
      my-label: $(NAMESPACE)
+    annotations:
+      my-annotation: $(NAMESPACE)
  spec:
    template:
      spec:
@@ -1176,6 +1178,8 @@ vars:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    my-annotation: my-namespace
  labels:
    my-label: my-namespace
  name: my-deployment
--- a/api/kv/kv.go
+++ b/api/kv/kv.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"regexp"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -85,11 +86,17 @@ func (kvl *loader) keyValuesFromFileSources(sources []string) ([]types.Pair, err
 		if err != nil {
 			return nil, err
 		}
-		kvs = append(kvs, types.Pair{Key: k, Value: string(content)})
+		kvs = append(kvs, types.Pair{Key: k, Value: trimTrailingSpacesInLines(string(content))})
 	}
 	return kvs, nil
 }

+// trimTrailingSpacesInLines takes string with multiple lines and trims the trailing white spaces and tabs from each line.
+func trimTrailingSpacesInLines(str string) string {
+	re := regexp.MustCompile(`[ \t]*\n`)
+	return re.ReplaceAllString(str, "\n")
+}
+
 func (kvl *loader) keyValuesFromEnvFiles(paths []string) ([]types.Pair, error) {
 	var kvs []types.Pair
 	for _, p := range paths {
--- a/api/kv/kv_test.go
+++ b/api/kv/kv_test.go
@@ -95,3 +95,12 @@ func TestKeyValuesFromFileSources(t *testing.T) {
 		}
 	}
 }
+
+func TestTrimTrailingSpacesInLines(t *testing.T) {
+	input := "\"fooKey\": \"fooValue\"   \t\n \t\t  \n\t\"barKey\": \"barValue\""
+	expected := "\"fooKey\": \"fooValue\"\n\n\t\"barKey\": \"barValue\""
+	res := trimTrailingSpacesInLines(input)
+	if !reflect.DeepEqual(res, expected) {
+		t.Errorf("Trim trailing spaces in lines should succeed, got: %s exptected: %s", res, expected)
+	}
+}
--- a/cmd/config/.golangci.yml
+++ b/cmd/config/.golangci.yml
@@ -23,7 +23,7 @@ linters:
    - gofmt
    - goimports
 #    - golint
-    - gosec
+#    - gosec
    - gosimple
    - govet
    - ineffassign
--- a/cmd/config/complete/complete.go
+++ b/cmd/config/complete/complete.go
@@ -52,6 +52,10 @@ func Complete(cmd *cobra.Command) *complete.Command {
 		Flags: map[string]complete.Predictor{},
 		Sub:   map[string]*complete.Command{},
 	}
+	if strings.Contains(cmd.Use, "DIR") {
+		// if usage contains directory, then use a file predictor
+		cc.Args = predict.Dirs("*")
+	}

 	// add completion for each subcommand
 	for i := range cmd.Commands() {
--- a/cmd/config/configcobra/cmds.go
+++ b/cmd/config/configcobra/cmds.go
@@ -43,6 +43,30 @@ Advanced Documentation Topics:
 `,
 }

+// Export commands publicly for composition
+var (
+	Annotate           = commands.AnnotateCommand
+	Cat                = commands.CatCommand
+	Count              = commands.CountCommand
+	CreateSetter       = commands.CreateSetterCommand
+	CreateSubstitution = commands.CreateSubstitutionCommand
+	Fmt                = commands.FmtCommand
+	Grep               = commands.GrepCommand
+	ListSetters        = commands.ListSettersCommand
+	Merge              = commands.MergeCommand
+	Merge3             = commands.Merge3Command
+	RunFn              = commands.RunFnCommand
+	Set                = commands.SetCommand
+	Sink               = commands.SinkCommand
+	Source             = commands.SourceCommand
+	Tree               = commands.TreeCommand
+	Wrap               = commands.WrapCommand
+	XArgs              = commands.XArgsCommand
+
+	StackOnError = &commands.StackOnError
+	ExitOnError  = &commands.ExitOnError
+)
+
 // NewConfigCommand returns a new *cobra.Command for the config command group.  This may
 // be embedded into other go binaries as a way of packaging the "config" command as part
 // of another binary.
@@ -69,13 +93,24 @@ func NewConfigCommand(name string) *cobra.Command {

 	name = strings.TrimSpace(name + " config")
 	commands.ExitOnError = true
+	root.AddCommand(commands.AnnotateCommand(name))
 	root.AddCommand(commands.GrepCommand(name))
 	root.AddCommand(commands.TreeCommand(name))
 	root.AddCommand(commands.CatCommand(name))
 	root.AddCommand(commands.FmtCommand(name))
 	root.AddCommand(commands.MergeCommand(name))
+	root.AddCommand(commands.Merge3Command(name))
 	root.AddCommand(commands.CountCommand(name))
 	root.AddCommand(commands.RunFnCommand(name))
+	root.AddCommand(commands.XArgsCommand())
+	root.AddCommand(commands.WrapCommand())
+
+	root.AddCommand(commands.SetCommand(name))
+	root.AddCommand(commands.ListSettersCommand(name))
+	root.AddCommand(commands.CreateSetterCommand(name))
+	root.AddCommand(commands.CreateSubstitutionCommand(name))
+	root.AddCommand(commands.SinkCommand(name))
+	root.AddCommand(commands.SourceCommand(name))

 	root.AddCommand(&cobra.Command{
 		Use:   "docs-merge",
@@ -89,8 +124,13 @@ func NewConfigCommand(name string) *cobra.Command {
 	})
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-fn",
-		Short: "[Alpha] Documentation for writing containerized functions run by run.",
-		Long:  api.ConfigFnLong,
+		Short: "[Alpha] Documentation for developing and invoking Configuration Functions.",
+		Long:  api.FunctionsImplLong,
+	})
+	root.AddCommand(&cobra.Command{
+		Use:   "docs-fn-spec",
+		Short: "[Alpha] Documentation for Configuration Functions Specification.",
+		Long:  api.FunctionsSpecLong,
 	})
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-io-annotations",
--- a/cmd/config/docs/api-conventions/config-fn.md
+++ b/cmd/config/docs/api-conventions/config-fn.md
@@ -1,281 +0,0 @@
-# Configuration Functions API Semantics
-
-  Configuration Functions are functions packaged as executables in containers which enable
-  **shift-left practices**.  They configure applications and infrastructure through
-  Kubernetes style Resource Configuration, but run locally pre-commit.
-
-  Configuration functions enable shift-left practices (client-side) through:
-
-  - Pre-commit / delivery validation and linting of configuration
-    - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
-  - Implementation of abstractions as client actuated APIs (e.g. templating)
-    - e.g. Create a client-side *"CRD"* for generating configuration checked into git
-  - Aspect Orient configuration / Injection of cross-cutting configuration
-    - e.g. T-Shirt size containers by annotating Resources with `small`, `medium`, `large`
-      and inject the cpu and memory resources into containers accordingly.
-    - e.g. Inject `init` and `side-car` containers into Resources based off of Resource
-      Type, annotations, etc.
-
-  Performing these on the client rather than the server enables:
-
-  - Configuration to be reviewed prior to being sent to the API server
-  - Configuration to be validated as part of the CD pipeline
-  - Configuration for Resources to validated holistically rather than individually
-    per-Resource -- e.g. ensure the `Service.selector` and `Deployment.spec.template` labels
-    match.
-    - MutatingWebHooks are scoped to a single Resource instance at a time.
-  - Low-level tweaks to the output of high-level abstractions -- e.g. add an `init container`
-    to a client *"CRD"* Resource after it was generated.
-  - Composition and layering of multiple functions together
-    - Compose generation, injection, validation together
-
-  Configuration Functions are implemented as executable programs published in containers which:
-
-  - Accept as input (stdin):
-    - A list of Resource Configuration
-    - A Function Configuration (to configure the function itself)
-  - Emit as output (stdout + exit):
-    - A list of Resource Configuration
-    - An exit code for success / failure
-
-### Function Specification
-
-  - Functions **SHOULD** be published as container images containing a `CMD` invoking an executable.
-  - Functions **MUST** accept input on STDIN a `ResourceList` containing the Resources and
-    `functionConfig`.
-  - Functions **MUST** emit output on STDOUT a `ResourceList` containing the modified
-    Resources.
-  - Functions **MUST** exit non-0 on failure, and exit 0 on success.
-  - Functions **MAY** emit output on STDERR with error messaging.
-  - Functions performing validation **SHOULD** exit failure and emit error messaging
-    on a validation failure.
-  - Functions generating Resources **SHOULD** retain non-conflicting changes on the
-    generated Resources -- e.g. 1. the function generates a Deployment, but doesn't
-    specify `cpu`, 2. the user sets the `cpu` on the generated Resource, 3. the
-    function should keep the `cpu` when regenerating the Resource a second time.
-  - Functions **SHOULD** be usable outside `kustomize config run` -- e.g. though pipeline
-    mechanisms such as Tekton.
-
-#### Input Format
-
-  Functions must accept on STDIN:
-
-  `ResourceList`:
-  - contains `items` field, same as `List.items`
-  - contains `functionConfig` field -- a single item with the configuration for the function itself
-
-  Example `ResourceList` Input:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    functionConfig:
-      apiVersion: example.com/v1beta1
-      kind: Nginx
-      metadata:
-        name: my-instance
-        annotations:
-          config.kubernetes.io/local-config: "true"
-      spec:
-        replicas: 5
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 3
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Output Format
-
-  Functions must emit on STDOUT:
-
-  `ResourceList`:
-  - contains `items` field, same as `List.items`
-
-  Example `ResourceList` Output:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 5
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Container Environment
-
-  When run by `kustomize config run`, functions are run in containers with the
-  following environment:
-
-  - Network: `none`
-  - User: `nobody`
-  - Security Options: `no-new-privileges`
-  - Volumes: the volume containing the `functionConfig` yaml is mounted under `/local` as `ro`
-
-### Example Function Implementation
-
-  Following is an example for implementing an nginx abstraction using a config
-  function.
-
-#### `nginx-template.sh`
-
-  `nginx-template.sh` is a simple bash script which uses a *heredoc* as a templating solution
-  for generating Resources from the functionConfig input fields.
-
-  The script wraps itself using `config run wrap -- $0` which will:
-
-  1. Parse the `ResourceList.functionConfig` (provided to the container stdin) into env vars
-  2. Merge the stdout into the original list of Resources
-  3. Defaults filenames for newly generated Resources (if they are not set as annotations)
-     to `config/NAME_KIND.yaml`
-  4. Format the output
-
-    #!/bin/bash
-    # script must run wrapped by `kustomize config run wrap`
-    # for parsing input the functionConfig into env vars
-    if [ -z ${WRAPPED} ]; then
-      export WRAPPED=true
-      config run wrap -- $0
-      exit $?
-    fi
-
-    cat <<End-of-message
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: ${NAME}
-    ---
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      replicas: ${REPLICAS}
-      selector:
-        matchLabels:
-          app: nginx
-          instance: ${NAME}
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: ${NAME}
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-    End-of-message
-
-#### `Dockerfile`
-
-  `Dockerfile` installs `kustomize config` and copies the script into the container image.
-
-    FROM golang:1.13-stretch
-    RUN go get sigs.k8s.io/kustomize/cmd/config
-    RUN mv /go/bin/config /usr/bin/config
-    COPY nginx-template.sh /usr/bin/nginx-template.sh
-    CMD ["nginx-template.sh]
-
-### Example Function Usage
-
-Following is an example of running the `kustomize config run` using the preceding API.
-
-#### `nginx.yaml` (Input)
-
-  `dir/nginx.yaml` contains a reference to the Function.  The contents of `nginx.yaml`
-  are passed to the Function through the `ResourceList.functionConfig` field.
-
-    apiVersion: example.com/v1beta1
-    kind: Nginx
-    metadata:
-      name: my-instance
-      annotations:
-        config.kubernetes.io/local-config: "true"
-      configFn:
-        container:
-          image: gcr.io/example-functions/nginx-template:v1.0.0
-    spec:
-      replicas: 5
-
-  - `configFn.container.image`: the image to use for this API
-  - `annotations[config.kubernetes.io/local-config]`: mark this as not a Resource that should
-    be applied
-
-#### `kustomize config run dir/` (Output)
-
-  `dir/my-instance_deployment.yaml` contains the Deployment:
-
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      replicas: 5
-      selector:
-        matchLabels:
-          app: nginx
-          instance: my-instance
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: my-instance
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-
-  `dir/my-instance_service.yaml` contains the Service:
-
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: my-instance
--- a/cmd/config/docs/api-conventions/config-io.md
+++ b/cmd/config/docs/api-conventions/config-io.md
@@ -4,16 +4,17 @@ Resource Configuration may be read / written from / to sources such as directori
 stdin|out or network. Tools may be composed using pipes such that the tools writing
 Resource Configuration may be a different tool from the one that read the configuration.
 In order for tools to be composed in this way, while preserving origin information --
-such as the original file, index, etc.
+such as the original file, index, etc.:

-Tools **SHOULD** write the following annotations when reading from sources,
-and **SHOULD** respect the annotations when writing to sinks.
+Tools **SHOULD** insert the following annotations when reading from sources,
+and **SHOULD** delete the annotations when writing to sinks.

 ### `config.kubernetes.io/path`

 Records the slash-delimited, OS-agnostic, relative file path to a Resource.

 This annotation **SHOULD** be set when reading Resources from files.
+It **SHOULD** be unset when writing Resources to files.
 When writing Resources to a directory, the Resource **SHOULD** be written to the corresponding
 path relative to that directory.

@@ -27,10 +28,11 @@ metadata:

 ### `config.kubernetes.io/index`

-Records the index of a Resource in file. In a multi-object files YAML file, Resources are separated
-by three dashes (`---`), and the index represents the positon of the Resource starting from zero.
+Records the index of a Resource in file. In a multi-object YAML file, Resources are separated
+by three dashes (`---`), and the index represents the position of the Resource starting from zero.

 This annotation **SHOULD** be set when reading Resources from files.
+It **SHOULD** be unset when writing Resources to files.
 When writing multiple Resources to the same file, the Resource **SHOULD** be written in the
 relative order matching the index.

--- a/cmd/config/docs/api-conventions/functions-impl.md
+++ b/cmd/config/docs/api-conventions/functions-impl.md
@@ -0,0 +1,181 @@
+# Running Configuration Functions using kustomize CLI
+
+Configuration functions can be implemented using any toolchain and invoked using any
+container workflow orchestrator including Tekton, Cloud Build, or run directly using `docker run`.
+
+Run `config help docs-fn-spec` to see the Configuration Functions Specification.
+
+`kustomize config run` is an example orchestrator for invoking Configuration Functions. This
+document describes how to implement and invoke an example function.
+
+## Example Function Implementation
+
+Following is an example for implementing an nginx abstraction using a configuration
+function.
+
+### `nginx-template.sh`
+
+`nginx-template.sh` is a simple bash script which uses a _heredoc_ as a templating solution
+for generating Resources from the functionConfig input fields.
+
+The script wraps itself using `config run wrap -- $0` which will:
+
+1. Parse the `ResourceList.functionConfig` (provided to the container stdin) into env vars
+2. Merge the stdout into the original list of Resources
+3. Defaults filenames for newly generated Resources (if they are not set as annotations)
+   to `config/NAME_KIND.yaml`
+4. Format the output
+
+```bash
+#!/bin/bash
+# script must run wrapped by "kustomize config run wrap"
+# for parsing input the functionConfig into env vars
+if [ -z ${WRAPPED} ]; then
+  export WRAPPED=true
+  config run wrap -- $0
+  exit $?
+fi
+
+cat <<End-of-message
+apiVersion: v1
+kind: Service
+metadata:
+  name: ${NAME}
+  labels:
+    app: nginx
+    instance: ${NAME}
+spec:
+  ports:
+  - port: 80
+    targetPort: 80
+    name: http
+  selector:
+    app: nginx
+    instance: ${NAME}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${NAME}
+  labels:
+    app: nginx
+    instance: ${NAME}
+spec:
+  replicas: ${REPLICAS}
+  selector:
+    matchLabels:
+      app: nginx
+      instance: ${NAME}
+  template:
+    metadata:
+      labels:
+        app: nginx
+        instance: ${NAME}
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+End-of-message
+```
+
+### Dockerfile
+
+`Dockerfile` installs `kustomize config` and copies the script into the container image.
+
+```
+FROM golang:1.13-stretch
+RUN go get sigs.k8s.io/kustomize/cmd/config
+RUN mv /go/bin/config /usr/bin/config
+COPY nginx-template.sh /usr/bin/nginx-template.sh
+CMD ["nginx-template.sh]
+```
+
+## Example Function Usage
+
+Following is an example of running the `kustomize config run` using the preceding API.
+
+When run by `kustomize config run`, functions are run in containers with the
+following environment:
+
+- Network: `none`
+- User: `nobody`
+- Security Options: `no-new-privileges`
+- Volumes: the volume containing the `functionConfig` yaml is mounted under `/local` as `ro`
+
+### Input
+
+`dir/nginx.yaml` contains a reference to the Function. The contents of `nginx.yaml`
+are passed to the Function through the `ResourceList.functionConfig` field.
+
+```yaml
+apiVersion: example.com/v1beta1
+kind: Nginx
+metadata:
+  name: my-instance
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    config.k8s.io/function: |
+      container:
+        image: gcr.io/example-functions/nginx-template:v1.0.0
+spec:
+  replicas: 5
+```
+
+- `annotations[config.k8s.io/function].container.image`: the image to use for this API
+- `annotations[config.kubernetes.io/local-config]`: mark this as not a Resource that should
+  be applied
+
+### Output
+
+The function is invoked using byrunning `kustomize config run dir/`.
+
+`dir/my-instance_deployment.yaml` contains the Deployment:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: my-instance
+  labels:
+    app: nginx
+    instance: my-instance
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: nginx
+      instance: my-instance
+  template:
+    metadata:
+      labels:
+        app: nginx
+        instance: my-instance
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:1.7.9
+          ports:
+            - containerPort: 80
+```
+
+`dir/my-instance_service.yaml` contains the Service:
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: my-instance
+  labels:
+    app: nginx
+    instance: my-instance
+spec:
+  ports:
+    - port: 80
+      targetPort: 80
+      name: http
+  selector:
+    app: nginx
+    instance: my-instance
+```
--- a/cmd/config/docs/api-conventions/functions-spec.md
+++ b/cmd/config/docs/api-conventions/functions-spec.md
@@ -0,0 +1,186 @@
+# Configuration Functions Specification
+
+This document specifies a standard for client-side functions that operate on
+Kubernetes declarative configurations. This standard enables creating
+small, interoperable, and language-independent executable programs packaged as
+containers that can be chained together as part of a configuration management pipeline.
+The end result of such a pipeline are fully rendered configurations that can then be
+applied to a control plane (e.g. Using ‘kubectl apply’ for Kubernetes control plane).
+As such, although this document references Kubernetes Resource Model and API conventions,
+it is completely decoupled from Kuberentes API machinery and does not depend on any
+in-cluster components.
+
+This document references terms described in [Kubernetes API Conventions][1].
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD",
+"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be
+interpreted as described in [RFC 2119][2].
+
+## Use Cases
+
+_Configuration functions_ enable shift-left practices (client-side) through:
+
+- Pre-commit / delivery validation and linting of configuration
+  - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
+- Implementation of abstractions as client actuated APIs (e.g. templating)
+  - e.g. Create a client-side _"CRD"_ for generating configuration checked into git
+- Aspect Orient configuration / Injection of cross-cutting configuration
+  - e.g. T-Shirt size containers by annotating Resources with `small`, `medium`, `large`
+    and inject the cpu and memory resources into containers accordingly.
+  - e.g. Inject `init` and `side-car` containers into Resources based off of Resource
+    Type, annotations, etc.
+
+Performing these on the client rather than the server enables:
+
+- Configuration to be reviewed prior to being sent to the API server
+- Configuration to be validated as part of the CI?CD pipeline
+- Configuration for Resources to validated holistically rather than individually
+  per-Resource
+  - e.g. ensure the `Service.selector` and `Deployment.spec.template` labels
+    match.
+  - e.g. MutatingWebHooks are scoped to a single Resource instance at a time.
+- Low-level tweaks to the output of high-level abstractions
+  - e.g. add an `init container` to a client _"CRD"_ Resource after it was generated.
+- Composition and layering of multiple functions together
+  - Compose generation, injection, validation together
+
+## Spec
+
+### Input Type
+
+A function MUST accept as input a single [Kubernetes List type][3].
+The `items` field in the input will contain a sequence of [Object types][3].
+A function MAY not support [Simple types][3] and List types.
+
+An example using `v1/ConfigMapList` as input:
+
+```yaml
+apiVersion: v1
+kind: ConfigMapList
+items:
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: config1
+    data:
+      p1: v1
+      p2: v2
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: config2
+```
+
+An example using `v1/List` as input:
+
+```yaml
+apiVersion: v1
+kind: List
+items:
+  spec:
+   - apiVersion: foo-corp.com/v1
+    kind: FulfillmentCenter
+    metadata:
+      name: staging
+    address: "100 Main St."
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: namespace-reader
+    rules:
+      - resources:
+          - namespaces
+        apiGroups:
+          - ""
+        verbs:
+          - get
+          - watch
+          - list
+```
+
+In addition, a function MUST accept as input a List of kind `ResourceList` where the
+`functionConfig` field, if present, will contain the invocation-specific configuration passed to the function
+by the orchestrator.
+Functions MAY consider this field optional so that they can be triggered in an ad-hoc fashion.
+
+An example using `config.kubernetes.io/v1beta1/ResourceList` as input:
+
+```yaml
+apiVersion: config.kubernetes.io/v1beta1
+kind: ResourceList
+functionConfig:
+  apiVersion: foo-corp.com/v1
+  kind: FulfillmentCenter
+  metadata:
+    name: staging
+    metadata:
+      annotations:
+        config.k8s.io/function: |
+          container:
+            image: gcr.io/example/foo:v1.0.0
+  spec:
+    address: "100 Main St."
+items:
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: namespace-reader
+    rules:
+      - resources:
+          - namespaces
+        apiGroups:
+          - ""
+        verbs:
+          - get
+          - watch
+          - list
+```
+
+Here `FulfillmentCenter` kind with name `staging` is passed as the invocation-specific configuration
+to the function.
+
+### Output Type
+
+A function’s output MUST be the same as the input specification above
+-- i.e. `ResourceList` or `List`.
+This is necessary to enable chaining two or more functions together in a pipeline.
+The serialization format of the output SHOULD match that of its input on each invocation
+-- e.g. if the input was a `ResourceList`, the output should also be a `ResourceList`.
+
+### Serialization Format
+
+A function MUST support YAML as a serialization format for the input and output.
+A function MUST use utf8 encoding (as YAML is a superset of JSON, JSON will also be supported
+by any conforming function).
+
+### Operations
+
+A function MAY Create, Update, or Delete any number of items in the `items` field and output the
+resultant list.
+
+A function MAY modify annotations with prefix `config.kubernetes.io`, but must be careful about
+doing so since they’re used for orchestration purposes and will likely impact subsequent functions
+in the pipeline.
+
+A function SHOULD preserve comments when input serialization format is YAML.
+This allows for human authoring of configuration to coexist with changes made by functions.
+
+### Containerization
+
+A function MUST be implemented as a container.
+
+A function container MUST be capable of running as a non-root user if it does not require
+access to host filesystem or makes network calls.
+
+### stdin/stdout/stderr and Exit Codes
+
+A function MUST accept input from stdin and emit output to stdout.
+
+Any error messages MUST be emitted to stderr.
+
+An exit code of zero indicates function execution was successful.
+A non-zero exit code indicates a failure.
+
+[1]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md
+[2]: https://tools.ietf.org/html/rfc2119
+[3]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds
--- a/cmd/config/docs/commands/annotate.md
+++ b/cmd/config/docs/commands/annotate.md
@@ -0,0 +1,18 @@
+## annotate
+
+[Alpha] Set an annotation on Resources.
+
+### Synopsis
+
+[Alpha]  Set an annotation on Resources.
+
+  DIR:
+    Path to local directory.
+
+### Examples
+
+    kustomize config annotate my-dir/ --kv foo=bar
+
+    kustomize config annotate my-dir/ --kv foo=bar --kv a=b
+
+    kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo
--- a/cmd/config/docs/commands/create-setter.md
+++ b/cmd/config/docs/commands/create-setter.md
@@ -0,0 +1,93 @@
+## create-setter
+
+[Alpha] Create a custom setter for a Resource field
+
+### Synopsis
+
+Create a custom setter for a Resource field by inlining OpenAPI as comments.
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    The name of the setter to create.
+
+  VALUE
+
+    The current value of the field, or a substring within the field.
+
+### Creating a Custom Setter
+
+**Given the YAML:**
+
+    # resource.yaml
+    apiVersion: v1
+    kind: Service
+    metadata:
+      ...
+    spec:
+      ...
+      ports:
+        ...
+      - name: http
+        port: 8080
+        ...
+
+**Create a new setter:**
+
+    # create a setter for ports
+    $ kustomize config set create DIR/ http-port 8080 --type "integer" --field "port"
+
+  Resources fields with a field name matching `--field` and field value matching `VALUE` will
+  have a line comment added marking this field as settable.
+
+**Newly modified YAML:**
+
+    # resource.yaml
+    apiVersion: v1
+    kind: Service
+    metadata:
+      ...
+    spec:
+      ...
+      ports:
+        ...
+      - name: http
+        port: 8080  # {"type":"integer","x-kustomize":{"partialFieldSetters":[{"name":"http-port","value":"8080"}]}}
+        ...
+
+  Setters may also be defined directly by editing the yaml and adding the comment.
+
+Users may not set the field value using the `set` command:
+
+    # change the http-port value to 8081
+    $ kustomize config set DIR/ http-port 8081
+
+### Using default values
+
+The default values for a setter may be:
+
+- valid field values (e.g. `8080` or `008080` for a port)
+- invalid values that adhere to the schema (e.g. `0000` for a port)
+- values that do not adhere to the schema (e.g. `[PORT]` for port)
+
+A setter may be for a substring of a full field:
+
+    $ kustomize config set create DIR/ image-tag v1.0.01 --type "string" --field "image"
+
+    image: gcr.io/example/app:v1.0.1 # # {"type":"string","x-kustomize":{"partialFieldSetters":[{"name":"image-tag","value":"v1.0.1"}]}}
+
+A single field value may have multiple setters applied to it for different parts of the field.
+
+### Examples
+
+    # create a setter for port fields matching "8080"
+    kustomize config create-setter DIR/ port 8080 --type "integer" --field port \
+         --description "default port used by the app"
+
+    # create a setter for a substring of a field rather than the full field -- e.g. only the
+    # image tag, not the full image
+    kustomize config create-setter DIR/ image-tag v1.0.1 --type "string" \
+        --field image --description "current stable release"
--- a/cmd/config/docs/commands/list-setters.md
+++ b/cmd/config/docs/commands/list-setters.md
@@ -0,0 +1,23 @@
+## set
+
+[Alpha] List setters for Resources.
+
+### Synopsis
+
+List setters for Resources.
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    Optional.  The name of the setter to display.
+
+### Examples
+
+  Show setters:
+
+    $ config set DIR/
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    name-prefix   ''            PREFIX    string   2
--- a/cmd/config/docs/commands/merge3.md
+++ b/cmd/config/docs/commands/merge3.md
@@ -0,0 +1,23 @@
+## merge3
+
+[Alpha] Merge diff of Resource configuration files into a destination (3-way)
+
+### Synopsis
+
+[Alpha] Merge diff of Resource configuration files into a destination (3-way)
+
+Merge3 performs a 3-way merge by applying the diff between 2 sets of Resources to a 3rd set.
+
+Merge3 may be for rebasing changes to a forked set of configuration -- e.g. compute the difference between the original
+set of Resources that was forked and an updated set of those Resources, then apply that difference to the fork.
+
+If a field value differs between the ORIGINAL_DIR and UPDATED_DIR, the value from the UPDATED_DIR is taken and applied
+to the Resource in the DEST_DIR.
+
+For information on merge rules, run:
+
+	kustomize config docs-merge3
+
+### Examples
+
+    kustomize config merge3 --ancestor a/ --from b/ --to c/
--- a/cmd/config/docs/commands/run-fns.md
+++ b/cmd/config/docs/commands/run-fns.md
@@ -22,8 +22,8 @@ order they appear in the file).

 #### Config Functions:

-  Config functions are specified as Kubernetes types containing a metadata.configFn.container.image
-  field.  This field tells run how to invoke the container.
+  Config functions are specified as Kubernetes types containing a metadata.annotations.[config.kubernetes.io/function]
+  field specifying an image for the container to run.  This image tells run how to invoke the container.

  Example config function:

@@ -31,17 +31,17 @@ order they appear in the file).
 	apiVersion: fn.example.com/v1beta1
 	kind: ExampleFunctionKind
 	metadata:
-	  configFn:
-	    container:
-	      # function is invoked as a container running this image
-	      image: gcr.io/example/examplefunction:v1.0.1
 	  annotations:
+	    config.kubernetes.io/function: |
+	      container:
+	        # function is invoked as a container running this image
+	        image: gcr.io/example/examplefunction:v1.0.1
 	    config.kubernetes.io/local-config: "true" # tools should ignore this
 	spec:
 	  configField: configValue

  In the preceding example, 'kustomize config run example/' would identify the function by
-  the metadata.configFn field.  It would then write all Resources in the directory to
+  the metadata.annotations.[config.kubernetes.io/function] field.  It would then write all Resources in the directory to
  a container stdin (running the gcr.io/example/examplefunction:v1.0.1 image).  It
  would then write the container stdout back to example/, replacing the directory
  file contents.
--- a/cmd/config/docs/commands/set.md
+++ b/cmd/config/docs/commands/set.md
@@ -0,0 +1,87 @@
+## set
+
+[Alpha] Set values on Resources fields values.
+
+### Synopsis
+
+Set values on Resources fields.  May set either the complete or partial field value.
+
+`set` identifies setters using field metadata published as OpenAPI extensions.
+`set` parses both the Kubernetes OpenAPI, as well OpenAPI published inline in
+the configuration as comments.
+
+`set` maybe be used to:
+
+- edit configuration programmatically from the cli
+- create reusable bundles of configuration with custom setters
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    Optional.  The name of the setter to perform or display.
+
+  VALUE
+
+    Optional.  The value to set on the field.
+
+
+To print the possible setters for the Resources in a directory, run `set` on
+a directory -- e.g. `kustomize config set DIR/`.
+
+#### Tips
+
+- A description of the value may be specified with `--description`.
+- The last setter for the field's value may be defined with `--set-by`.
+- Create custom setters on Resources, Kustomization.yaml's, patches, etc
+
+The description and setBy fields are left unmodified unless specified with flags.
+
+To create a custom setter for a field see: `kustomize help config create-setter`
+
+### Examples
+
+  Resource YAML: Name Prefix Setter
+
+    # DIR/resources.yaml
+    ...
+    metadata:
+        name: PREFIX-app1 # {"type":"string","x-kustomize":{"partialFieldSetters":[{"name":"name-prefix","value":"PREFIX"}]}}
+    ...
+    ---
+    ...
+    metadata:
+        name: PREFIX-app2 # {"type":"string","x-kustomize":{"partialFieldSetters":[{"name":"name-prefix","value":"PREFIX"}]}}
+    ...
+
+  List setters: Show the possible setters
+
+    $ config set DIR/
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    name-prefix   ''            PREFIX    string   2
+
+  Perform set: set a new value, owner and description
+
+    $ kustomize config set DIR/ name-prefix "test" --description "test environment" --set-by "dev"
+    set 2 values
+
+  List setters: Show the new values
+
+    $ config set DIR/
+        NAME      DESCRIPTION         VALUE     TYPE     COUNT     SETBY 
+    name-prefix   'test environment'   test     string   2          dev
+
+  New Resource YAML:
+
+    # DIR/resources.yaml
+    ...
+    metadata:
+        name: test-app1 # {"description":"test environment","type":"string","x-kustomize":{"setBy":"dev","partialFieldSetters":[{"name":"name-prefix","value":"test"}]}}
+    ...
+    ---
+    ...
+    metadata:
+        name: test-app2 # {"description":"test environment","type":"string","x-kustomize":{"setBy":"dev","partialFieldSetters":[{"name":"name-prefix","value":"test"}]}}
+    ...
--- a/cmd/config/docs/commands/sink.md
+++ b/cmd/config/docs/commands/sink.md
@@ -0,0 +1,18 @@
+## sink
+
+[Alpha] Implement a Sink by writing input to a local directory.
+
+### Synopsis
+
+[Alpha] Implement a Sink by writing input to a local directory.
+
+    kustomize config sink [DIR]
+
+  DIR:
+    Path to local directory.  If unspecified, sink will write to stdout as if it were a single file.
+
+`sink` writes its input to a directory
+
+### Examples
+
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/
--- a/cmd/config/docs/commands/source.md
+++ b/cmd/config/docs/commands/source.md
@@ -0,0 +1,22 @@
+## source
+
+[Alpha] Implement a Source by reading a local directory.
+
+### Synopsis
+
+[Alpha] Implement a Source by reading a local directory.
+
+    kustomize config source DIR...
+
+  DIR:
+    One or more paths to local directories.  Contents from directories will be concatenated.
+    If no directories are provided, source will read from stdin as if it were a single file.
+
+`source` emits configuration to act as input to a function
+
+### Examples
+
+    # emity configuration directory as input source to a function
+    kustomize config source DIR/
+
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/
--- a/cmd/config/docs/commands/tree.md
+++ b/cmd/config/docs/commands/tree.md
@@ -20,8 +20,9 @@ container names, etc.

 kustomize config tree supports printing arbitrary fields using the '--field' flag.

-By default, kustomize config tree uses the directory structure for the tree structure, however when printing
-from the cluster, the Resource graph structure may be used instead.
+By default, kustomize config tree uses Resource graph structure if any relationships between resources (ownerReferences)
+are detected, as is typically the case when printing from a cluster. Otherwise, directory graph structure is used. The
+graph structure can also be selected explicitly using the '--graph-structure' flag.

 ### Examples

@@ -42,8 +43,7 @@ from the cluster, the Resource graph structure may be used instead.
      --field="status.conditions[type=Completed].status"

    # print live Resources from a cluster using owners for graph structure
-    kubectl get all -o yaml | kustomize config tree --replicas --name --image \
-      --graph-structure=owners
+    kubectl get all -o yaml | kustomize config tree --replicas --name --image

    # print live Resources with status condition fields
    kubectl get all -o yaml | kustomize config tree \
--- a/cmd/config/docs/tutorials/function-basics.md
+++ b/cmd/config/docs/tutorials/function-basics.md
@@ -39,7 +39,7 @@
  #### 2. Modify the Generated Resources

  - modify the generated Resources by adding an annotation, sidecar container, etc.
-  - modify the `local-resources/example-use.yaml` by changing the replicas
+  - modify the `local-resource/example-use.yaml` by changing the replicas

  re-run `run`.  this will apply the updated replicas to the generated Resources,
  but keep the fields that you manually added to the generated Resource configuration.
@@ -86,7 +86,7 @@
  #### 2. Modify the Generated Resources

  - modify the generated Resources by adding an annotation, sidecar container, etc.
-  - modify the `local-resources/example-use.yaml` by changing the replicas
+  - modify the `local-resource/example-use.yaml` by changing the replicas

  re-run `run`.  this will apply the updated replicas to the generated Resources,
  but keep the fields that you manually added to the generated Resource configuration.
--- a/cmd/config/ext/ext.go
+++ b/cmd/config/ext/ext.go
@@ -0,0 +1,12 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package ext
+
+import "path/filepath"
+
+// GetOpenAPIFile returns the path to the file containing supplementary OpenAPI definitions.
+// Maybe be overridden to configure which file to read OpenAPI definitions from.
+var GetOpenAPIFile = func(args []string) (string, error) {
+	return filepath.Join(args[0], "kustomization"), nil
+}
--- a/cmd/config/fixgomod.sh
+++ b/cmd/config/fixgomod.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+: "${kyaml_major?Need to source VERSIONS}"
+: "${kyaml_minor?Need to source VERSIONS}"
+: "${kyaml_patch?Need to source VERSIONS}"
+
+go mod edit -dropreplace=sigs.k8s.io/kustomize/kyaml@v0.0.0
+go mod edit -require=sigs.k8s.io/kustomize/kyaml@v$kyaml_major.$kyaml_minor.$kyaml_patch
--- a/cmd/config/go.mod
+++ b/cmd/config/go.mod
@@ -4,11 +4,11 @@ go 1.13

 require (
 	github.com/go-errors/errors v1.0.1
+	github.com/olekukonko/tablewriter v0.0.4
 	github.com/posener/complete/v2 v2.0.1-alpha.12
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	k8s.io/apimachinery v0.17.0
 	sigs.k8s.io/kustomize/kyaml v0.0.0
 )
--- a/cmd/config/go.sum
+++ b/cmd/config/go.sum
@@ -1,8 +1,11 @@
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
@@ -16,16 +19,25 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw=
+github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -43,7 +55,6 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -51,19 +62,20 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
-github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -72,14 +84,14 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
+github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -89,12 +101,12 @@ github.com/posener/complete/v2 v2.0.1-alpha.12/go.mod h1://JlL91cS2JV7rOl6LVHrRq
 github.com/posener/script v1.0.4 h1:nSuXW5ZdmFnQIueLB2s0qvs4oNsUloM1Zydzh75v42w=
 github.com/posener/script v1.0.4/go.mod h1:Rg3ijooqulo05aGLyGsHoLmIOUzHUVK19WVgrYBPU/E=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -115,6 +127,9 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -124,12 +139,12 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
--- a/cmd/config/internal/commands/annotate.go
+++ b/cmd/config/internal/commands/annotate.go
@@ -0,0 +1,103 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"strings"
+
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// NewAnnotateRunner returns a command runner.
+func NewAnnotateRunner(parent string) *AnnotateRunner {
+	r := &AnnotateRunner{}
+	c := &cobra.Command{
+		Use:     "annotate [DIR]",
+		Args:    cobra.MaximumNArgs(1),
+		Short:   commands.AnnotateShort,
+		Long:    commands.AnnotateLong,
+		Example: commands.AnnotateExamples,
+		RunE:    r.runE,
+	}
+	fixDocs(parent, c)
+	r.Command = c
+	c.Flags().StringVar(&r.Kind, "kind", "", "Resource kind to annotate")
+	c.Flags().StringVar(&r.ApiVersion, "apiVersion", "", "Resource apiVersion to annotate")
+	c.Flags().StringVar(&r.Name, "name", "", "Resource name to annotate")
+	c.Flags().StringVar(&r.Namespace, "namespace", "", "Resource namespace to annotate")
+	c.Flags().StringSliceVar(&r.Values, "kv", []string{}, "annotation as KEY=VALUE")
+	return r
+}
+
+func AnnotateCommand(parent string) *cobra.Command {
+	return NewAnnotateRunner(parent).Command
+}
+
+type AnnotateRunner struct {
+	Command    *cobra.Command
+	Values     []string
+	Kind       string
+	Name       string
+	ApiVersion string
+	Namespace  string
+	Path       string
+}
+
+func (r *AnnotateRunner) runE(c *cobra.Command, args []string) error {
+	var input []kio.Reader
+	var output []kio.Writer
+	if len(args) == 0 {
+		rw := &kio.ByteReadWriter{Reader: c.InOrStdin(), Writer: c.OutOrStdout()}
+		input = []kio.Reader{rw}
+		output = []kio.Writer{rw}
+	} else {
+		rw := &kio.LocalPackageReadWriter{PackagePath: args[0], NoDeleteFiles: true}
+		input = []kio.Reader{rw}
+		output = []kio.Writer{rw}
+	}
+	return handleError(c, kio.Pipeline{
+		Inputs:  input,
+		Filters: []kio.Filter{r},
+		Outputs: output,
+	}.Execute())
+}
+
+func (r *AnnotateRunner) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	for i := range nodes {
+		n := nodes[i]
+		m, err := n.GetMeta()
+		if err != nil {
+			return nil, err
+		}
+		if r.Kind != "" && r.Kind != m.Kind {
+			continue
+		}
+		if r.ApiVersion != "" && r.ApiVersion != m.APIVersion {
+			continue
+		}
+		if r.Namespace != "" && r.Namespace != m.Namespace {
+			continue
+		}
+		if r.Name != "" && r.Name != m.Name {
+			continue
+		}
+
+		for i := range r.Values {
+			// split key, value pairs
+			kv := strings.SplitN(r.Values[i], "=", 2)
+			if len(kv) != 2 {
+				return nil, errors.Errorf("must specify --kv as KEY=VALUE: %s", r.Values[i])
+			}
+			if err := n.PipeE(yaml.SetAnnotation(kv[0], kv[1])); err != nil {
+				return nil, err
+			}
+		}
+
+	}
+	return nodes, nil
+}
--- a/cmd/config/internal/commands/annotate_test.go
+++ b/cmd/config/internal/commands/annotate_test.go
@@ -0,0 +1,488 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func TestAnnotateCommand(t *testing.T) {
+	var tests = []struct {
+		name     string
+		args     []string
+		expected string
+	}{
+		{
+			name:     "single value",
+			args:     []string{"--kv", "a=b"},
+			expected: expectedSingleValue,
+		},
+		{
+			name:     "multi value",
+			args:     []string{"--kv", "a=b", "--kv", "c=d"},
+			expected: expectedMultiValue,
+		},
+		{
+			name:     "filter kind",
+			args:     []string{"--kv", "a=b", "--kind", "Service"},
+			expected: expectedFilterKindService,
+		},
+		{
+			name:     "filter apiVersion",
+			args:     []string{"--kv", "a=b", "--apiVersion", "v1"},
+			expected: expectedFilterApiVersionV1,
+		},
+		{
+			name:     "filter name",
+			args:     []string{"--kv", "a=b", "--name", "bar"},
+			expected: expectedFilterNameBar,
+		},
+		{
+			name:     "filter namespace",
+			args:     []string{"--kv", "a=b", "--namespace", "bar"},
+			expected: expectedFilterNamespaceBar,
+		},
+	}
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			d := initTestDir(t)
+			defer os.RemoveAll(d)
+
+			a := NewAnnotateRunner("")
+			a.Command.SetArgs(append([]string{d}, tt.args...))
+			a.Command.SilenceUsage = true
+			a.Command.SilenceErrors = true
+
+			err := a.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			actual := &bytes.Buffer{}
+			err = kio.Pipeline{
+				Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: d}},
+				Outputs: []kio.Writer{kio.ByteWriter{Writer: actual, KeepReaderAnnotations: true}},
+			}.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(tt.expected),
+				strings.TrimSpace(actual.String())) {
+				t.FailNow()
+			}
+		})
+	}
+}
+
+func initTestDir(t *testing.T) string {
+	d, err := ioutil.TempDir("", "kustomize-annotate-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	err = ioutil.WriteFile(filepath.Join(d, "f1.yaml"), []byte(f1Input), 0600)
+	if !assert.NoError(t, err) {
+		defer os.RemoveAll(d)
+		t.FailNow()
+	}
+	err = ioutil.WriteFile(filepath.Join(d, "f2.yaml"), []byte(f2Input), 0600)
+	if !assert.NoError(t, err) {
+		defer os.RemoveAll(d)
+		t.FailNow()
+	}
+	return d
+}
+
+var (
+	f1Input = `
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`
+
+	f2Input = `
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedSingleValue = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedMultiValue = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterKindService = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterApiVersionV1 = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterNameBar = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterNamespaceBar = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+)
--- a/cmd/config/internal/commands/cat.go
+++ b/cmd/config/internal/commands/cat.go
@@ -121,6 +121,13 @@ func (r *CatRunner) runE(c *cobra.Command, args []string) error {
 		out = o
 	}

+	// remove this annotation explicitly, the ByteWriter won't clear it by
+	// default because it doesn't set it
+	clear := []string{"config.kubernetes.io/path"}
+	if r.KeepAnnotations {
+		clear = nil
+	}
+
 	var outputs []kio.Writer
 	outputs = append(outputs, kio.ByteWriter{
 		Writer:                out,
@@ -129,6 +136,7 @@ func (r *CatRunner) runE(c *cobra.Command, args []string) error {
 		WrappingAPIVersion:    r.WrapApiVersion,
 		FunctionConfig:        functionConfig,
 		Style:                 yaml.GetStyle(r.Styles...),
+		ClearAnnotations:      clear,
 	})

 	return handleError(c, kio.Pipeline{Inputs: inputs, Filters: fltr, Outputs: outputs}.Execute())
--- a/cmd/config/internal/commands/cat_test.go
+++ b/cmd/config/internal/commands/cat_test.go
@@ -90,8 +90,6 @@ metadata:
  name: foo
  annotations:
    app: nginx2
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
  replicas: 1
 ---
@@ -100,8 +98,6 @@ metadata:
  name: foo
  annotations:
    app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
  selector:
    app: nginx
@@ -114,8 +110,6 @@ metadata:
    app: nginx
  annotations:
    app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
 spec:
  replicas: 3
 `, b.String()) {
@@ -196,8 +190,6 @@ metadata:
  name: foo
  annotations:
    app: nginx2
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
  replicas: 1
 ---
@@ -206,8 +198,6 @@ metadata:
  name: foo
  annotations:
    app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
  selector:
    app: nginx
@@ -218,8 +208,6 @@ metadata:
  name: foo
  annotations:
    config.kubernetes.io/local-config: "true"
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
  configFn:
    container:
      image: gcr.io/example/image:version
@@ -233,8 +221,6 @@ metadata:
  name: bar
  annotations:
    app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
 spec:
  replicas: 3
 `, b.String()) {
@@ -314,8 +300,6 @@ metadata:
  name: foo
  annotations:
    config.kubernetes.io/local-config: "true"
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
  configFn:
    container:
      image: gcr.io/example/reconciler:v1
@@ -414,8 +398,6 @@ metadata:
  name: foo
  annotations:
    app: nginx2
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
  replicas: 1
 ---
@@ -424,8 +406,6 @@ metadata:
  name: foo
  annotations:
    app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
  selector:
    app: nginx
@@ -438,8 +418,6 @@ metadata:
    app: nginx
  annotations:
    app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
 spec:
  replicas: 3
 `, string(actual)) {
@@ -536,8 +514,6 @@ metadata:
  name: foo
  annotations:
    app: nginx2
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
  replicas: 1
 ---
@@ -546,8 +522,6 @@ metadata:
  name: foo
  annotations:
    app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
  selector:
    app: nginx
@@ -560,8 +534,6 @@ metadata:
    app: nginx
  annotations:
    app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
 spec:
  replicas: 3
 `, string(actual)) {
--- a/cmd/config/internal/commands/cmdcreatesetter.go
+++ b/cmd/config/internal/commands/cmdcreatesetter.go
@@ -0,0 +1,113 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/setters"
+	"sigs.k8s.io/kustomize/kyaml/setters2/settersutil"
+)
+
+// NewCreateSetterRunner returns a command runner.
+func NewCreateSetterRunner(parent string) *CreateSetterRunner {
+	r := &CreateSetterRunner{}
+	set := &cobra.Command{
+		Use:     "create-setter DIR NAME VALUE",
+		Args:    cobra.ExactArgs(3),
+		Short:   commands.CreateSetterShort,
+		Long:    commands.CreateSetterLong,
+		Example: commands.CreateSetterExamples,
+		PreRunE: r.preRunE,
+		RunE:    r.runE,
+	}
+	set.Flags().StringVar(&r.Set.SetPartialField.SetBy, "set-by", "",
+		"record who the field was default by.")
+	set.Flags().StringVar(&r.Set.SetPartialField.Description, "description", "",
+		"record a description for the current setter value.")
+	set.Flags().StringVar(&r.Set.SetPartialField.Field, "field", "",
+		"name of the field to set -- e.g. --field port.  defaults to all fields match"+
+			"VALUE.  maybe be the field name, field path, or partial field path (suffix)")
+	set.Flags().StringVar(&r.Set.ResourceMeta.Name, "name", "",
+		"name of the Resource on which to create the setter.")
+	set.Flags().MarkHidden("name")
+	set.Flags().StringVar(&r.Set.ResourceMeta.Kind, "kind", "",
+		"kind of the Resource on which to create the setter.")
+	set.Flags().MarkHidden("kind")
+	set.Flags().StringVar(&r.Set.SetPartialField.Type, "type", "",
+		"valid OpenAPI field type -- e.g. integer,boolean,string.")
+	set.Flags().MarkHidden("type")
+	set.Flags().BoolVar(&r.Set.SetPartialField.Partial, "partial", false,
+		"create a partial setter for only part of the field value.")
+	set.Flags().MarkHidden("partial")
+	set.Flags().StringVar(&setterVersion, "version", "",
+		"use this version of the setter format")
+	set.Flags().MarkHidden("version")
+	fixDocs(parent, set)
+	r.Command = set
+	return r
+}
+
+func CreateSetterCommand(parent string) *cobra.Command {
+	return NewCreateSetterRunner(parent).Command
+}
+
+type CreateSetterRunner struct {
+	Command      *cobra.Command
+	Set          setters.CreateSetter
+	CreateSetter settersutil.SetterCreator
+	OpenAPIFile  string
+}
+
+func (r *CreateSetterRunner) runE(c *cobra.Command, args []string) error {
+	return handleError(c, r.set(c, args))
+}
+
+func (r *CreateSetterRunner) preRunE(c *cobra.Command, args []string) error {
+	var err error
+	r.Set.SetPartialField.Setter.Name = args[1]
+	r.Set.SetPartialField.Setter.Value = args[2]
+	r.CreateSetter.Name = args[1]
+	r.CreateSetter.FieldValue = args[2]
+	r.CreateSetter.FieldName, err = c.Flags().GetString("field")
+	if err != nil {
+		return err
+	}
+
+	if setterVersion == "" {
+		if len(args) < 3 {
+			setterVersion = "v1"
+		} else if err := initSetterVersion(c, args); err != nil {
+			return err
+		}
+	}
+	if setterVersion == "v2" {
+		var err error
+		r.OpenAPIFile, err = ext.GetOpenAPIFile(args)
+		r.CreateSetter.Description = r.Set.SetPartialField.Description
+		r.CreateSetter.SetBy = r.Set.SetPartialField.SetBy
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *CreateSetterRunner) set(c *cobra.Command, args []string) error {
+	if setterVersion == "v2" {
+		return r.CreateSetter.Create(r.OpenAPIFile, args[0])
+	}
+
+	rw := &kio.LocalPackageReadWriter{PackagePath: args[0]}
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{rw},
+		Filters: []kio.Filter{&r.Set},
+		Outputs: []kio.Writer{rw}}.Execute()
+	if err != nil {
+		return err
+	}
+	return nil
+}
--- a/cmd/config/internal/commands/cmdcreatesetter_test.go
+++ b/cmd/config/internal/commands/cmdcreatesetter_test.go
@@ -0,0 +1,131 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
+)
+
+func TestCreateSetterCommand(t *testing.T) {
+	var tests = []struct {
+		name              string
+		input             string
+		args              []string
+		out               string
+		expectedOpenAPI   string
+		expectedResources string
+	}{
+		{
+			name: "add replicas",
+			args: []string{"replicas", "3", "--description", "hello world", "--set-by", "me"},
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hello world
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			// reset the openAPI afterward
+			openapi.ResetOpenAPI()
+			defer openapi.ResetOpenAPI()
+
+			f, err := ioutil.TempFile("", "k8s-cli-")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(f.Name())
+			err = ioutil.WriteFile(f.Name(), []byte(`
+apiVersion: v1alpha1
+kind: Example
+`), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			old := ext.GetOpenAPIFile
+			defer func() { ext.GetOpenAPIFile = old }()
+			ext.GetOpenAPIFile = func(args []string) (s string, err error) {
+				return f.Name(), nil
+			}
+
+			r, err := ioutil.TempFile("", "k8s-cli-*.yaml")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(r.Name())
+			err = ioutil.WriteFile(r.Name(), []byte(test.input), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			runner := commands.NewCreateSetterRunner("")
+			out := &bytes.Buffer{}
+			runner.Command.SetOut(out)
+			runner.Command.SetArgs(append([]string{r.Name()}, test.args...))
+			err = runner.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			if !assert.Equal(t, test.out, out.String()) {
+				t.FailNow()
+			}
+
+			actualResources, err := ioutil.ReadFile(r.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedResources),
+				strings.TrimSpace(string(actualResources))) {
+				t.FailNow()
+			}
+
+			actualOpenAPI, err := ioutil.ReadFile(f.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedOpenAPI),
+				strings.TrimSpace(string(actualOpenAPI))) {
+				t.FailNow()
+			}
+		})
+	}
+}
--- a/cmd/config/internal/commands/cmdcreatesubstitution.go
+++ b/cmd/config/internal/commands/cmdcreatesubstitution.go
@@ -0,0 +1,81 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"strings"
+
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/setters2"
+	"sigs.k8s.io/kustomize/kyaml/setters2/settersutil"
+)
+
+// NewCreateSubstitutionRunner returns a command runner.
+func NewCreateSubstitutionRunner(parent string) *CreateSubstitutionRunner {
+	r := &CreateSubstitutionRunner{}
+	cs := &cobra.Command{
+		Use:     "create-subst DIR NAME VALUE",
+		Args:    cobra.ExactArgs(3),
+		PreRunE: r.preRunE,
+		RunE:    r.runE,
+	}
+	cs.Flags().StringVar(&r.CreateSubstitution.FieldName, "field", "",
+		"name of the field to set -- e.g. --field port")
+	cs.Flags().StringVar(&r.CreateSubstitution.Pattern, "pattern", "",
+		"substitution pattern")
+	cs.Flags().StringSliceVar(&r.Values, "value", []string{""},
+		"substitution values for the pattern.  format is PATTERN_MARKER=SETTER_NAME"+
+			"where PATTERN_MARKER is the pattern substring to replace, and SETTER_NAME is the"+
+			"setter from which to take the replacement value.")
+	_ = cs.MarkFlagRequired("pattern")
+	fixDocs(parent, cs)
+	r.Command = cs
+	return r
+}
+
+func CreateSubstitutionCommand(parent string) *cobra.Command {
+	return NewCreateSubstitutionRunner(parent).Command
+}
+
+type CreateSubstitutionRunner struct {
+	Command            *cobra.Command
+	CreateSubstitution settersutil.SubstitutionCreator
+	OpenAPIFile        string
+	Values             []string
+}
+
+func (r *CreateSubstitutionRunner) runE(c *cobra.Command, args []string) error {
+	return handleError(c, r.CreateSubstitution.Create(r.OpenAPIFile, args[0]))
+}
+
+func (r *CreateSubstitutionRunner) preRunE(c *cobra.Command, args []string) error {
+	var err error
+	r.CreateSubstitution.Name = args[1]
+	r.CreateSubstitution.FieldValue = args[2]
+	if err != nil {
+		return err
+	}
+
+	r.OpenAPIFile, err = ext.GetOpenAPIFile(args)
+	if err != nil {
+		return err
+	}
+
+	// parse the marker values
+	for i := range r.Values {
+		parts := strings.SplitN(r.Values[i], "=", 2)
+		if len(parts) < 2 {
+			return errors.Errorf("values must be specified as PATTERN_MARKER=SETTER_NAME")
+		}
+		ref := setters2.DefinitionsPrefix + setters2.SetterDefinitionPrefix + parts[1]
+		r.CreateSubstitution.Values = append(
+			r.CreateSubstitution.Values,
+			setters2.Value{Marker: parts[0], Ref: ref},
+		)
+	}
+
+	return nil
+}
--- a/cmd/config/internal/commands/cmdcreatesubstitution_test.go
+++ b/cmd/config/internal/commands/cmdcreatesubstitution_test.go
@@ -0,0 +1,174 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
+)
+
+func TestCreateSubstitutionCommand(t *testing.T) {
+	var tests = []struct {
+		name              string
+		inputOpenAPI      string
+		input             string
+		args              []string
+		out               string
+		expectedOpenAPI   string
+		expectedResources string
+	}{
+		{
+			name: "substitution replicas",
+			args: []string{
+				"image", "nginx:1.7.9", "--pattern", "IMAGE:TAG",
+				"--value", "IMAGE=image", "--value", "TAG=tag"},
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+      - name: sidecar
+        image: sidecar:1.7.9
+ `,
+			inputOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.image:
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+    io.k8s.cli.setters.tag:
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.image:
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+    io.k8s.cli.setters.tag:
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref":"#/definitions/io.k8s.cli.substitutions.image"}
+      - name: sidecar
+        image: sidecar:1.7.9
+ `,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			// reset the openAPI afterward
+			openapi.ResetOpenAPI()
+			defer openapi.ResetOpenAPI()
+
+			f, err := ioutil.TempFile("", "k8s-cli-")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(f.Name())
+			err = ioutil.WriteFile(f.Name(), []byte(test.inputOpenAPI), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			old := ext.GetOpenAPIFile
+			defer func() { ext.GetOpenAPIFile = old }()
+			ext.GetOpenAPIFile = func(args []string) (s string, err error) {
+				return f.Name(), nil
+			}
+
+			r, err := ioutil.TempFile("", "k8s-cli-*.yaml")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(r.Name())
+			err = ioutil.WriteFile(r.Name(), []byte(test.input), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			runner := commands.NewCreateSubstitutionRunner("")
+			out := &bytes.Buffer{}
+			runner.Command.SetOut(out)
+			runner.Command.SetArgs(append([]string{r.Name()}, test.args...))
+			err = runner.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			if !assert.Equal(t, test.out, out.String()) {
+				t.FailNow()
+			}
+
+			actualResources, err := ioutil.ReadFile(r.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedResources),
+				strings.TrimSpace(string(actualResources))) {
+				t.FailNow()
+			}
+
+			actualOpenAPI, err := ioutil.ReadFile(f.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedOpenAPI),
+				strings.TrimSpace(string(actualOpenAPI))) {
+				t.FailNow()
+			}
+		})
+	}
+}
--- a/cmd/config/internal/commands/cmdlistsetters.go
+++ b/cmd/config/internal/commands/cmdlistsetters.go
@@ -0,0 +1,96 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/olekukonko/tablewriter"
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/setters"
+	"sigs.k8s.io/kustomize/kyaml/setters2"
+)
+
+// NewListSettersRunner returns a command runner.
+func NewListSettersRunner(parent string) *ListSettersRunner {
+	r := &ListSettersRunner{}
+	c := &cobra.Command{
+		Use:     "list-setters DIR [NAME]",
+		Args:    cobra.RangeArgs(1, 2),
+		Short:   commands.ListSettersShort,
+		Long:    commands.ListSettersLong,
+		Example: commands.ListSettersExamples,
+		PreRunE: r.preRunE,
+		RunE:    r.runE,
+	}
+	fixDocs(parent, c)
+	r.Command = c
+	return r
+}
+
+func ListSettersCommand(parent string) *cobra.Command {
+	return NewListSettersRunner(parent).Command
+}
+
+type ListSettersRunner struct {
+	Command *cobra.Command
+	Lookup  setters.LookupSetters
+	List    setters2.List
+}
+
+func (r *ListSettersRunner) preRunE(c *cobra.Command, args []string) error {
+	if len(args) > 1 {
+		r.Lookup.Name = args[1]
+		r.List.Name = args[1]
+	}
+
+	initSetterVersion(c, args)
+	return nil
+}
+
+func (r *ListSettersRunner) runE(c *cobra.Command, args []string) error {
+	if setterVersion == "v2" {
+		// use setters v2
+		path, err := ext.GetOpenAPIFile(args)
+		if err != nil {
+			return err
+		}
+		if err := r.List.List(path, args[0]); err != nil {
+			return err
+		}
+		table := newTable(c.OutOrStdout())
+		table.SetHeader([]string{"NAME", "VALUE", "SET BY", "DESCRIPTION", "COUNT"})
+		for i := range r.List.Setters {
+			s := r.List.Setters[i]
+			table.Append([]string{
+				s.Name, s.Value, s.SetBy, s.Description, fmt.Sprintf("%d", s.Count)})
+		}
+		table.Render()
+
+		if len(r.List.Setters) == 0 {
+			// exit non-0 if no matching setters are found
+			if ExitOnError {
+				os.Exit(1)
+			}
+		}
+		return nil
+	}
+
+	return handleError(c, lookup(r.Lookup, c, args))
+}
+
+func newTable(o io.Writer) *tablewriter.Table {
+	table := tablewriter.NewWriter(o)
+	table.SetRowLine(false)
+	table.SetBorder(false)
+	table.SetHeaderLine(false)
+	table.SetColumnSeparator(" ")
+	table.SetCenterSeparator(" ")
+	table.SetAlignment(tablewriter.ALIGN_LEFT)
+	return table
+}
--- a/cmd/config/internal/commands/cmdlistsetters_test.go
+++ b/cmd/config/internal/commands/cmdlistsetters_test.go
@@ -0,0 +1,299 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
+)
+
+func TestListSettersCommand(t *testing.T) {
+	var tests = []struct {
+		name     string
+		openapi  string
+		input    string
+		args     []string
+		expected string
+	}{
+		{
+			name: "list-replicas",
+			openapi: `
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me
+      description: "hello world"
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+			expected: `    NAME     VALUE   SET BY   DESCRIPTION   COUNT  
+  replicas   3       me       hello world   1      
+`,
+		},
+		{
+			name: "list-multiple",
+			openapi: `
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: "hello world 1"
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me1
+    io.k8s.cli.setters.image:
+      description: "hello world 2"
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+          setBy: me2
+    io.k8s.cli.setters.tag:
+      description: "hello world 3"
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+          setBy: me3
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx # {"$ref": "#/definitions/io.k8s.cli.setters.image"}
+ `,
+			expected: `    NAME     VALUE   SET BY    DESCRIPTION    COUNT  
+  image      nginx   me2      hello world 2   2      
+  replicas   3       me1      hello world 1   1      
+  tag        1.7.9   me3      hello world 3   1      
+`,
+		},
+		{
+			name: "list-multiple-resources",
+			openapi: `
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: "hello world 1"
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me1
+    io.k8s.cli.setters.image:
+      description: "hello world 2"
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+          setBy: me2
+    io.k8s.cli.setters.tag:
+      description: "hello world 3"
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+          setBy: me3
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-1
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx # {"$ref": "#/definitions/io.k8s.cli.setters.image"}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-2
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx
+`,
+			expected: `    NAME     VALUE   SET BY    DESCRIPTION    COUNT  
+  image      nginx   me2      hello world 2   3      
+  replicas   3       me1      hello world 1   2      
+  tag        1.7.9   me3      hello world 3   2      
+`,
+		},
+		{
+			name: "list-name",
+			args: []string{"image"},
+			openapi: `
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: "hello world 1"
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me1
+    io.k8s.cli.setters.image:
+      description: "hello world 2"
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+          setBy: me2
+    io.k8s.cli.setters.tag:
+      description: "hello world 3"
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+          setBy: me3
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-1
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx # {"$ref": "#/definitions/io.k8s.cli.setters.image"}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-2
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx
+`,
+			expected: `  NAME    VALUE   SET BY    DESCRIPTION    COUNT  
+  image   nginx   me2      hello world 2   3      
+`,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			// reset the openAPI afterward
+			openapi.ResetOpenAPI()
+			defer openapi.ResetOpenAPI()
+
+			f, err := ioutil.TempFile("", "k8s-cli-")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(f.Name())
+			old := ext.GetOpenAPIFile
+			defer func() { ext.GetOpenAPIFile = old }()
+			ext.GetOpenAPIFile = func(args []string) (s string, err error) {
+				err = ioutil.WriteFile(f.Name(), []byte(test.openapi), 0600)
+				if !assert.NoError(t, err) {
+					t.FailNow()
+				}
+				return f.Name(), nil
+			}
+
+			r, err := ioutil.TempFile("", "k8s-cli-*.yaml")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(r.Name())
+			err = ioutil.WriteFile(r.Name(), []byte(test.input), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			runner := commands.NewListSettersRunner("")
+			actual := &bytes.Buffer{}
+			runner.Command.SetOut(actual)
+			runner.Command.SetArgs(append([]string{r.Name()}, test.args...))
+			err = runner.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			if !assert.Equal(t, test.expected, actual.String()) {
+				t.FailNow()
+			}
+		})
+	}
+}
--- a/cmd/config/internal/commands/cmdset.go
+++ b/cmd/config/internal/commands/cmdset.go
@@ -0,0 +1,180 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/olekukonko/tablewriter"
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/setters"
+	"sigs.k8s.io/kustomize/kyaml/setters2/settersutil"
+)
+
+// NewSetRunner returns a command runner.
+func NewSetRunner(parent string) *SetRunner {
+	r := &SetRunner{}
+	c := &cobra.Command{
+		Use:     "set DIR NAME [VALUE]",
+		Args:    cobra.RangeArgs(1, 3),
+		Short:   commands.SetShort,
+		Long:    commands.SetLong,
+		Example: commands.SetExamples,
+		PreRunE: r.preRunE,
+		RunE:    r.runE,
+	}
+	fixDocs(parent, c)
+	r.Command = c
+	c.Flags().StringVar(&r.Perform.SetBy, "set-by", "",
+		"annotate the field with who set it")
+	c.Flags().StringVar(&r.Perform.Description, "description", "",
+		"annotate the field with a description of its value")
+	c.Flags().StringVar(&setterVersion, "version", "",
+		"use this version of the setter format")
+	c.Flags().MarkHidden("version")
+
+	return r
+}
+
+var setterVersion string
+
+func SetCommand(parent string) *cobra.Command {
+	return NewSetRunner(parent).Command
+}
+
+type SetRunner struct {
+	Command     *cobra.Command
+	Lookup      setters.LookupSetters
+	Perform     setters.PerformSetters
+	Set         settersutil.FieldSetter
+	OpenAPIFile string
+}
+
+func initSetterVersion(c *cobra.Command, args []string) error {
+	setterVersion = "v2"
+	l := setters.LookupSetters{}
+
+	// backwards compatibility for resources with setter v1
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.LocalPackageReader{PackagePath: args[0]}},
+		Filters: []kio.Filter{&l},
+	}.Execute()
+	if err != nil {
+		return err
+	}
+	if len(l.SetterCounts) > 0 {
+		setterVersion = "v1"
+	}
+
+	return nil
+}
+
+func (r *SetRunner) preRunE(c *cobra.Command, args []string) error {
+	if len(args) > 1 {
+		r.Perform.Name = args[1]
+		r.Lookup.Name = args[1]
+	}
+	if len(args) > 2 {
+		r.Perform.Value = args[2]
+	}
+
+	if setterVersion == "" {
+		if len(args) < 3 {
+			setterVersion = "v1"
+		} else if err := initSetterVersion(c, args); err != nil {
+			return err
+		}
+	}
+	if setterVersion == "v2" {
+		var err error
+		r.Set.Name = args[1]
+		r.Set.Value = args[2]
+		r.Set.Description = r.Perform.Description
+		r.Set.SetBy = r.Perform.SetBy
+		r.OpenAPIFile, err = ext.GetOpenAPIFile(args)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *SetRunner) runE(c *cobra.Command, args []string) error {
+	if setterVersion == "v2" {
+		count, err := r.Set.Set(r.OpenAPIFile, args[0])
+		fmt.Fprintf(c.OutOrStdout(), "set %d fields\n", count)
+		return handleError(c, err)
+	}
+	if len(args) == 3 {
+		return handleError(c, r.perform(c, args))
+	}
+	return handleError(c, lookup(r.Lookup, c, args))
+}
+
+func lookup(l setters.LookupSetters, c *cobra.Command, args []string) error {
+	// lookup the setters
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.LocalPackageReader{PackagePath: args[0]}},
+		Filters: []kio.Filter{&l},
+	}.Execute()
+	if err != nil {
+		return err
+	}
+
+	table := tablewriter.NewWriter(c.OutOrStdout())
+	table.SetRowLine(false)
+	table.SetBorder(false)
+	table.SetHeaderLine(false)
+	table.SetColumnSeparator(" ")
+	table.SetCenterSeparator(" ")
+	table.SetAlignment(tablewriter.ALIGN_LEFT)
+	table.SetHeader([]string{
+		"NAME", "DESCRIPTION", "VALUE", "TYPE", "COUNT", "SETBY",
+	})
+	for i := range l.SetterCounts {
+		s := l.SetterCounts[i]
+		v := s.Value
+		if s.Value == "" {
+			v = s.Value
+		}
+		table.Append([]string{
+			s.Name,
+			"'" + s.Description + "'",
+			v,
+			fmt.Sprintf("%v", s.Type),
+			fmt.Sprintf("%d", s.Count),
+			s.SetBy,
+		})
+	}
+	table.Render()
+
+	if len(l.SetterCounts) == 0 {
+		// exit non-0 if no matching setters are found
+		os.Exit(1)
+	}
+	return nil
+}
+
+// perform the setters
+func (r *SetRunner) perform(c *cobra.Command, args []string) error {
+	rw := &kio.LocalPackageReadWriter{
+		PackagePath: args[0],
+	}
+	// perform the setters in the package
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{rw},
+		Filters: []kio.Filter{&r.Perform},
+		Outputs: []kio.Writer{rw},
+	}.Execute()
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(c.OutOrStdout(), "set %d fields\n", r.Perform.Count)
+	return nil
+}
--- a/cmd/config/internal/commands/cmdset_test.go
+++ b/cmd/config/internal/commands/cmdset_test.go
@@ -0,0 +1,278 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
+)
+
+func TestSetCommand(t *testing.T) {
+	var tests = []struct {
+		name              string
+		inputOpenAPI      string
+		input             string
+		args              []string
+		out               string
+		expectedOpenAPI   string
+		expectedResources string
+	}{
+		{
+			name: "set replicas",
+			args: []string{"replicas", "4", "--description", "hi there", "--set-by", "pw"},
+			out:  "set 1 fields\n",
+			inputOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hello world
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hi there
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "4"
+          setBy: pw
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 4 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+		},
+		{
+			name: "set replicas no description",
+			args: []string{"replicas", "4"},
+			out:  "set 1 fields\n",
+			inputOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hello world
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hello world
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "4"
+          setBy: me
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 4 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+		},
+		{
+			name: "set image",
+			args: []string{"tag", "1.8.1"},
+			out:  "set 1 fields\n",
+			inputOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.image:
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+    io.k8s.cli.setters.tag:
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref":"#/definitions/io.k8s.cli.substitutions.image"}
+      - name: sidecar
+        image: sidecar:1.7.9
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.image:
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+    io.k8s.cli.setters.tag:
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.8.1"
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.8.1 # {"$ref":"#/definitions/io.k8s.cli.substitutions.image"}
+      - name: sidecar
+        image: sidecar:1.7.9
+`,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			// reset the openAPI afterward
+			openapi.ResetOpenAPI()
+			defer openapi.ResetOpenAPI()
+
+			f, err := ioutil.TempFile("", "k8s-cli-")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(f.Name())
+			err = ioutil.WriteFile(f.Name(), []byte(test.inputOpenAPI), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			old := ext.GetOpenAPIFile
+			defer func() { ext.GetOpenAPIFile = old }()
+			ext.GetOpenAPIFile = func(args []string) (s string, err error) {
+				return f.Name(), nil
+			}
+
+			r, err := ioutil.TempFile("", "k8s-cli-*.yaml")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(r.Name())
+			err = ioutil.WriteFile(r.Name(), []byte(test.input), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			runner := commands.NewSetRunner("")
+			out := &bytes.Buffer{}
+			runner.Command.SetOut(out)
+			runner.Command.SetArgs(append([]string{r.Name()}, test.args...))
+			err = runner.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			if !assert.Equal(t, test.out, out.String()) {
+				t.FailNow()
+			}
+
+			actualResources, err := ioutil.ReadFile(r.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedResources),
+				strings.TrimSpace(string(actualResources))) {
+				t.FailNow()
+			}
+
+			actualOpenAPI, err := ioutil.ReadFile(f.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedOpenAPI),
+				strings.TrimSpace(string(actualOpenAPI))) {
+				t.FailNow()
+			}
+		})
+	}
+}
--- a/cmd/config/internal/commands/cmdwrap_test.go
+++ b/cmd/config/internal/commands/cmdwrap_test.go
@@ -78,8 +78,8 @@ items:
      name: test
      app: nginx
    annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_deployment.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_deployment.yaml'
  spec:
    replicas: 11
    selector:
@@ -109,8 +109,8 @@ items:
      name: test
      app: nginx
    annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_service.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_service.yaml'
  spec:
    selector:
      name: test
@@ -133,8 +133,8 @@ items:
      name: test
      app: nginx
    annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_deployment.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_deployment.yaml'
  spec:
    replicas: 11
    selector:
@@ -161,8 +161,8 @@ items:
      name: test
      app: nginx
    annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_service.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_service.yaml'
  spec:
    selector:
      name: test
@@ -185,8 +185,8 @@ items:
      name: test
      app: nginx
    annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_deployment.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_deployment.yaml'
  spec:
    replicas: 11
    selector:
@@ -216,8 +216,8 @@ items:
      name: test
      app: nginx
    annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_service.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_service.yaml'
  spec:
    selector:
      name: test
--- a/cmd/config/internal/commands/fmt.go
+++ b/cmd/config/internal/commands/fmt.go
@@ -14,7 +14,7 @@ import (
 func GetFmtRunner(name string) *FmtRunner {
 	r := &FmtRunner{}
 	c := &cobra.Command{
-		Use:     "fmt",
+		Use:     "fmt DIR...",
 		Short:   commands.FmtShort,
 		Long:    commands.FmtLong,
 		Example: commands.FmtExamples,
@@ -31,6 +31,8 @@ formatting substitution verbs {'%n': 'metadata.name', '%s': 'metadata.namespace'
 		`if true, keep index and filename annotations set on Resources.`)
 	c.Flags().BoolVar(&r.Override, "override", false,
 		`if true, override existing filepath annotations.`)
+	c.Flags().BoolVar(&r.UseSchema, "use-schema", false,
+		`if true, uses openapi resource schema to format resources.`)
 	r.Command = c
 	return r
 }
@@ -46,6 +48,7 @@ type FmtRunner struct {
 	SetFilenames    bool
 	KeepAnnotations bool
 	Override        bool
+	UseSchema       bool
 }

 func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
@@ -56,7 +59,9 @@ func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
 }

 func (r *FmtRunner) runE(c *cobra.Command, args []string) error {
-	f := []kio.Filter{filters.FormatFilter{}}
+	f := []kio.Filter{filters.FormatFilter{
+		UseSchema: r.UseSchema,
+	}}

 	// format with file names
 	if r.SetFilenames {
--- a/cmd/config/internal/commands/fmt_test.go
+++ b/cmd/config/internal/commands/fmt_test.go
@@ -143,6 +143,8 @@ func TestCmd_failFiles(t *testing.T) {
 	// fmt the files
 	r := commands.GetFmtRunner("")
 	r.Command.SetArgs([]string{"notrealfile"})
+	r.Command.SilenceUsage = true
+	r.Command.SilenceErrors = true
 	err := r.Command.Execute()
 	assert.EqualError(t, err, "lstat notrealfile: no such file or directory")
 }
--- a/Show More
+++ b/Show More