update go.mod for release

drop short-hand flags from `config run` command

.travis.yml

@@ -36,6 +36,7 @@ install: true
 script:
   - make verify-kustomize
   - ./travis/kyaml-pre-commit.sh
+  - ./travis/check-go-mod.sh
 
 # TBD. Suppressing for now.
 notifications:

Makefile

@@ -17,6 +17,9 @@ verify-kustomize: \
 	test-examples-kustomize-against-HEAD \
 	test-examples-kustomize-against-latest
 
+.PHONY: verify-kustomize-e2e
+verify-kustomize-e2e: test-examples-e2e-kustomize-against-HEAD
+
 # Other builds in this repo might want a different linter version.
 # Without one Makefile to rule them all, the different makes
 # cannot assume that golanci-lint is at the version they want
@@ -48,6 +51,11 @@ $(MYGOBIN)/goimports:
 	cd api; \
 	go install golang.org/x/tools/cmd/goimports
 
+# Install resource from whatever is checked out.
+$(MYGOBIN)/resource:
+	cd cmd/resource; \
+	go install .
+
 # To pin pluginator, use this recipe instead:
 #   cd api;
 #   go install sigs.k8s.io/kustomize/pluginator/v2
@@ -193,6 +201,10 @@ test-unit-kustomize-all: \
 test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
 	./hack/testExamplesAgainstKustomize.sh HEAD
 
+.PHONY:
+test-examples-e2e-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip $(MYGOBIN)/resource
+	./hack/testExamplesE2EAgainstKustomize.sh HEAD
+
 .PHONY:
 test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
 	( \

api/builtins/NamespaceTransformer.go

@@ -52,7 +52,7 @@ func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
 
 		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
 		if len(matches) != 1 {
-			return fmt.Errorf("namespace tranformation produces ID conflict: %#v", matches)
+			return fmt.Errorf("namespace tranformation produces ID conflict: %+v", matches)
 		}
 	}
 	return nil

api/go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/go-openapi/spec v0.19.4
 	github.com/golangci/golangci-lint v1.21.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
-	github.com/monopole/mdrip v1.0.1
 	github.com/pkg/errors v0.8.1
 	golang.org/x/tools v0.0.0-20191010075000-0337d82405ff
 	gopkg.in/yaml.v2 v2.2.4

api/go.sum

@@ -2,14 +2,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -49,14 +45,9 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w=
-github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -119,13 +110,10 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -145,15 +133,12 @@ github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgO
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps=
 github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI=
 github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
@@ -169,9 +154,8 @@ github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -184,31 +168,16 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
-github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
-github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0 h1:UykbtMB/w5No2LmE16gINgLj+r/vbziTgaoERQv6U+0=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a h1:YH0IojQwndMQdeRWdw1aPT8bkbiWaYR3WD+Zf5e09DU=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741 h1:OuuPl66BpF1q3OEkaPpp+VfzxrBBY62ATGdWqql/XX8=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -248,10 +217,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
 github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
 github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
@@ -270,10 +237,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.0 h1:RFDBa+tab6mW+gX4Ww2SZDc4kS6p01FwnLtgz64Il+I=
-github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM=
-github.com/monopole/mdrip v1.0.1 h1:LzwNWb8ge3+4iBZmxIE6VfUR5KIxhF7DKdf85t8Yvos=
-github.com/monopole/mdrip v1.0.1/go.mod h1:/7E04hlzRG9Jrp6WILZfYYm/REoJWL2l+MlsCO1eH74=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -308,12 +271,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
@@ -322,8 +281,6 @@ github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -356,7 +313,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -364,7 +320,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
 github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs=
@@ -387,7 +342,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -410,8 +364,6 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -437,7 +389,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -447,7 +398,6 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -462,12 +412,11 @@ golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs=
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -492,8 +441,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
-gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
@@ -518,7 +465,6 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
@@ -526,10 +472,6 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphD
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4=
-sigs.k8s.io/kustomize/pluginator/v2 v2.0.0 h1:sES7e09G19Q0VjRp4ATSYKpTXoWaX8WMSHfw6u3G2Ok=
-sigs.k8s.io/kustomize/pluginator/v2 v2.0.0/go.mod h1:zrXhTv8BAKt0egmZX/8AtMOSFUSWM9YuoHvvqz8/eHE=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/internal/crawl/cmd/crawler/crawler.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"time"
@@ -23,71 +25,182 @@ const (
 	retryCount           = 3
 )
 
+type CrawlMode int
+const (
+	CrawlUnknown CrawlMode = iota
+	// Crawl all the kustomization files in all the repositories of a Github user
+	CrawlUser
+	// Crawl all the kustomization files in a Github repo
+	CrawlRepo
+	// Crawl all the documents in the index
+	CrawlIndex
+	// Crawl all the kustomization files on Github
+	CrawlGithub
+	// Crawl all the documents in the index and crawling all the kustomization files on Github
+	CrawlIndexAndGithub
+)
+
+func NewCrawlMode(s string) CrawlMode {
+	switch s {
+	case "github-user":
+		return CrawlUser
+	case "github-repo":
+		return CrawlRepo
+	case "index+github":
+		return CrawlIndexAndGithub
+	case "index":
+		return CrawlIndex
+	case "github":
+		return CrawlGithub
+	default:
+		return CrawlUnknown
+	}
+}
+
 func main() {
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	modePtr := flag.String("mode", "index+github",
+		`The crawling mode, which can be one of [github-user, github-repo, index, github, index+github].
+  * github-user: crawl all the kustomization files in all the repositories of a Github user (--github-user must be specified for this mode).
+  * github-repo: crawl all the kustomization files in a Github repository (--github-repo must be specified for this mode).
+  * index: crawl all the documents in the index.
+  * gihub: crawl all the kustomization files on Github.
+  * index+github: crawl all the documents in the index and crawling all the kustomization files on Github.`)
+	githubUserPtr := flag.String("github-user", "",
+		"A github user name (e.g., kubernetes-sigs). This flag is required for the `github-user` mode.")
+	githubRepoPtr := flag.String("github-repo", "",
+		"A github repository name (e.g., kubernetes-sigs/kustomize). This flag is required for the `github-repo` mode.")
+	flag.Parse()
+
 	githubToken := os.Getenv(githubAccessTokenVar)
 	if githubToken == "" {
-		fmt.Printf("Must set the variable '%s' to make github requests.\n",
+		log.Printf("Must set the variable '%s' to make github requests.\n",
 			githubAccessTokenVar)
 		return
 	}
 
 	ctx := context.Background()
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
 	if err != nil {
-		fmt.Printf("Could not create an index: %v\n", err)
+		log.Printf("Could not create an index: %v\n", err)
 		return
 	}
 
 	cacheURL := os.Getenv(redisCacheURL)
-
-	query := []byte(`{ "query":{ "match_all":{} } }`)
-	it := idx.IterateQuery(query, 10000, 60*time.Second)
-	docs := make(crawler.CrawlSeed, 0)
-	for it.Next() {
-		for _, hit := range it.Value().Hits.Hits {
-			docs = append(docs, hit.Document.Copy())
-		}
-	}
-
-	if err := it.Err(); err != nil {
-		fmt.Printf("Error iterating: %v\n", err)
-	}
-
 	cache, err := redis.DialURL(cacheURL)
 	clientCache := &http.Client{}
 	if err != nil {
-		fmt.Printf("Error: redis could not make a connection: %v\n", err)
+		log.Printf("Error: redis could not make a connection: %v\n", err)
 	} else {
 		clientCache = httpclient.NewClient(cache)
 	}
 
-	ghCrawler := github.NewCrawler(githubToken, retryCount, clientCache,
-		github.QueryWith(
-			github.Filename("kustomization.yaml"),
-			github.Filename("kustomization.yml")),
-	)
+	// docConverter takes in a plain document and processes it for the index.
+	docConverter := func(d *doc.Document) (crawler.CrawledDocument, error) {
+		kdoc := doc.KustomizationDocument{
+			Document: *d,
+		}
 
-	crawler.CrawlFromSeed(ctx, docs, []crawler.Crawler{ghCrawler},
-		// Converter takes in a plain document and processes it for the
-		// index.
-		func(d *doc.Document) (crawler.CrawledDocument, error) {
-			kdoc := doc.KustomizationDocument{
-				Document: *d,
-			}
+		err := kdoc.ParseYAML()
+		return &kdoc, err
+	}
 
-			err := kdoc.ParseYAML()
-			return &kdoc, err
-		},
-		// IndexFunc updates the value in the index.
-		func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler) error {
-			switch d := cdoc.(type) {
-			case *doc.KustomizationDocument:
-				fmt.Println("Inserting: ", d.ID(), d)
-				_, err := idx.Put(d.ID(), d)
-				return err
+	// Index updates the value in the index.
+	indexFunc := func(cdoc crawler.CrawledDocument, mode index.Mode) error {
+		switch d := cdoc.(type) {
+		case *doc.KustomizationDocument:
+			switch mode {
+			case index.Delete:
+				log.Printf("Deleting: %v", d)
+				return idx.Delete(d.ID())
 			default:
-				return fmt.Errorf("type %T not supported", d)
+				log.Printf("Inserting: %v", d)
+				return idx.Put(d.ID(), d)
 			}
-		},
-	)
+		default:
+			return fmt.Errorf("type %T not supported", d)
+		}
+	}
+
+	// seen tracks the IDs of all the documents in the index.
+	// This helps avoid indexing a given document multiple times.
+	seen := crawler.NewSeenMap()
+
+	mode := NewCrawlMode(*modePtr)
+
+	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
+		if user != "" {
+			return 	github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.User(user)),
+			)
+		} else if repo != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.Repo(repo)),
+			)
+		} else {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization")),
+			)
+		}
+	}
+
+	seedDocs := make(crawler.CrawlSeed, 0)
+
+	// get all the documents in the index
+	getSeedDocsFunc := func() {
+		query := []byte(`{ "query":{ "match_all":{} } }`)
+		it := idx.IterateQuery(query, 10000, 60*time.Second)
+		for it.Next() {
+			for _, hit := range it.Value().Hits.Hits {
+				seedDocs = append(seedDocs, hit.Document.Copy())
+			}
+		}
+		if err := it.Err(); err != nil {
+			log.Fatalf("getSeedDocsFunc Error iterating: %v\n", err)
+		}
+	}
+
+	switch mode {
+	case CrawlIndexAndGithub:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlIndex:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+	case CrawlGithub:
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUser:
+		if *githubUserPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github user with the github-user flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(*githubUserPtr, "")}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlRepo:
+		if *githubRepoPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github repository with the github-repo flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", *githubRepoPtr)}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUnknown:
+		flag.Usage()
+		log.Fatalf("The --mode flag must be one of [github-user, github-repo, index, github, index+github].")
+	}
 }

api/internal/crawl/cmd/kustomize_stats/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install ./cmd/kustomize_stats
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/kustomize_stats /
+ENTRYPOINT ["/kustomize_stats"]

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -0,0 +1,150 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+	"sigs.k8s.io/kustomize/api/konfig"
+)
+
+// iterateArr adds each item in arr into countMap.
+func iterateArr(arr []string, countMap map[string]int) {
+	for _, item := range arr {
+		if _, ok := countMap[item]; !ok {
+			countMap[item] = 1
+		} else {
+			countMap[item]++
+		}
+	}
+
+}
+
+// isKustomizationFile determines whether a file path is a kustomization file
+func isKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
+		}
+	}
+	return false
+}
+
+// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValue(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool {return m[keys[i]] > m[keys[j]]})
+	return keys
+}
+
+func main() {
+	topKindsPtr := flag.Int(
+		"kinds", -1,
+		`the number of kubernetes object kinds to be listed according to their popularities.
+By default, all the kinds will be listed.
+If you only want to list the 10 most popular kinds, set the flag to 10.`)
+	topIdentifiersPtr := flag.Int(
+		"identifiers", -1,
+		`the number of identifiers to be listed according to their popularities.
+By default, all the identifiers will be listed.
+If you only want to list the 10 most popular identifiers, set the flag to 10.`)
+	topKustomizeFeaturesPtr := flag.Int(
+		"kustomize-features", -1,
+		`the number of kustomize features to be listed according to their popularities.
+By default, all the features will be listed.
+If you only want to list the 10 most popular features, set the flag to 10.`)
+	flag.Parse()
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx)
+	if err != nil {
+		log.Fatalf("Could not create an index: %v\n", err)
+	}
+
+	// count tracks the number of documents in the index
+	count := 0
+
+	// kustomizationFilecount tracks the number of kustomization files in the index
+	kustomizationFilecount := 0
+
+	kindsMap := make(map[string]int)
+	identifiersMap := make(map[string]int)
+	kustomizeIdentifiersMap := make(map[string]int)
+
+	// ids tracks the unique IDs of the documents in the index
+	ids := make(map[string]struct{})
+
+	// get all the documents in the index
+	query := []byte(`{ "query":{ "match_all":{} } }`)
+	it := idx.IterateQuery(query, 10000, 60*time.Second)
+	for it.Next() {
+		for _, hit := range it.Value().Hits.Hits {
+			// check whether there is any duplicate IDs in the index
+			if _, ok := ids[hit.ID]; !ok {
+				ids[hit.ID] = struct{}{}
+			} else {
+				fmt.Printf("Found duplicate ID (%s)\n", hit.ID)
+			}
+
+			count++
+			iterateArr(hit.Document.Kinds, kindsMap)
+			iterateArr(hit.Document.Identifiers, identifiersMap)
+
+			if isKustomizationFile(hit.Document.FilePath) {
+				kustomizationFilecount++
+				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
+			}
+		}
+	}
+	if err := it.Err(); err != nil {
+		fmt.Printf("Error iterating: %v\n", err)
+	}
+
+	sortedKindsMapKeys := SortMapKeyByValue(kindsMap)
+	sortedIdentifiersMapKeys := SortMapKeyByValue(identifiersMap)
+	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValue(kustomizeIdentifiersMap)
+
+	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
+There are %d documents in the kustomize index.
+%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
+	fmt.Printf("\n")
+
+	kindCount := 0
+	for _, key := range sortedKindsMapKeys {
+		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
+			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
+			kindCount++
+		}
+	}
+
+	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
+	identifierCount := 0
+	for _, key := range sortedIdentifiersMapKeys {
+		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
+			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
+			identifierCount++
+		}
+	}
+
+	fmt.Printf(`There are %d kustomization files in the kustomize index.
+%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
+	fmt.Printf("\n")
+	kustomizeFeatureCount := 0
+	for _, key := range sortedKustomizeIdentifiersMapKeys {
+		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
+			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
+			kustomizeFeatureCount++
+		}
+	}
+}

api/internal/crawl/cmd/log-parser/README.md

@@ -0,0 +1,8 @@
+This binary takes as its input a json file including GKE logs (which can be
+[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into 
+[Cloud Storage](https://cloud.google.com/storage/docs/)), 
+and extracts the `textPayload` field of each log entry.
+
+Here is an log entry example:
+
+{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"}

api/internal/crawl/cmd/log-parser/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatalf("The usage of the command is: \n\t%s <log-file.json>", os.Args[0])
+	}
+
+	file, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	closeFile := func(file *os.File) {
+		if err := file.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}
+	defer closeFile(file)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		var entry interface{}
+		if err := json.Unmarshal([]byte(line), &entry); err != nil {
+			log.Printf("failed to unmarshal a log entry: %s\n", line)
+		}
+
+		m := entry.(map[string]interface{})
+		if payload, ok := m["textPayload"]; ok {
+			// use fmt.Printf here instead of log.Printf to avoid the time and code location info the log package provides
+			fmt.Printf("%s", payload)
+		} else {
+			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}

api/internal/crawl/config/base/kustomization.yaml

@@ -2,5 +2,4 @@ configmapGenerator:
 - name: elasticsearch-config
   literals:
   - es-url="http://esbasic-master:9200"
-  - kustomize-index-name="kustomize"
   - plugin-index-name="plugin"

api/internal/crawl/config/crawler/job/README.md

@@ -0,0 +1,53 @@
+The crawler job can run in one of the following mode:
+
+# Crawling all the documents in the index and crawling all the kustomization files on Github
+
+This is the default setting of the crawler job. The `command` and `args` field
+of the container should be:
+
+```
+        command: ["/crawler"]
+```
+
+Or
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index+github"]
+```
+
+# Crawling all the documents in the index
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index"]
+```
+
+# Crawling all the kustomization files on Github
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github"]
+```
+
+# Crawling all the kustomization files in a Github repo
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize"]
+```
+
+# Crawling all the kustomization files in all the repositories of a Github user
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--github-user", "--github-user=kubernetes-sigs"]
+```

api/internal/crawl/config/crawler/job/job.yaml

@@ -8,8 +8,10 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: crawler
-        image: gcr.io/kustomize-search/crawler:latest
+        image: gcr.io/haiyanmeng-gke-dev/crawler:v1
         imagePullPolicy: Always
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
         env:
         - name: GITHUB_ACCESS_TOKEN
           valueFrom:

api/internal/crawl/config/crawler/kustomize_stats/job.yaml

@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kustomize-stats
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: kustomize-stats
+        image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
+        imagePullPolicy: Always
+        command: ["/kustomize_stats"]
+        args: ["--kinds=50", "--identifiers=50", "--kustomize-features=50"]
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url

api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml

api/internal/crawl/crawler/crawler.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"sync"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
 	_ "github.com/gomodule/redigo/redis"
 
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -27,7 +29,7 @@ type Crawler interface {
 	// Crawl returns when it is done processing. This method does not take
 	// ownership of the channel. The channel is write only, and it
 	// designates where the crawler should forward the documents.
-	Crawl(ctx context.Context, output chan<- CrawledDocument) error
+	Crawl(ctx context.Context, output chan<- CrawledDocument, seen SeenMap) error
 
 	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
 	FetchDocument(context.Context, *doc.Document) error
@@ -40,144 +42,194 @@ type Crawler interface {
 type CrawledDocument interface {
 	ID() string
 	GetDocument() *doc.Document
+	// Get all the Documents directly referred in a Document.
 	GetResources() ([]*doc.Document, error)
 	WasCached() bool
 }
 
+type SeenMap map[string]struct{}
+
+func (seen SeenMap) Seen(item string) bool {
+	_, ok := seen[item]
+	return ok
+}
+
+func (seen SeenMap) Add(item string) {
+	seen[item] = struct{}{}
+}
+
+func NewSeenMap() SeenMap {
+	return make(map[string]struct{})
+}
+
 type CrawlSeed []*doc.Document
 
-type IndexFunc func(CrawledDocument, Crawler) error
+type IndexFunc func(CrawledDocument, index.Mode) error
 type Converter func(*doc.Document) (CrawledDocument, error)
 
-// Cleaner, more efficient, and more extensible crawler implementation.
-// The seed must include the ids of each document in the index.
-func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
-	crawlers []Crawler, conv Converter, indx IndexFunc) {
+func logIfErr(err error) {
+	if err == nil {
+		return
+	}
+	logger.Println("error: ", err)
+}
 
-	seen := make(map[string]struct{})
-
-	logIfErr := func(err error) {
-		if err == nil {
-			return
+func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
+	for _, crawl := range crawlers {
+		if crawl.Match(d) {
+			return crawl
 		}
-		logger.Println("error: ", err)
+	}
+	return nil
+}
+
+func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
+	seen SeenMap, stack *CrawlSeed) {
+
+	seen.Add(cdoc.ID())
+
+	// Insert into index
+	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
+		logger.Printf("Failed to insert or update %s %s: %v",
+			cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err)
+		return
 	}
 
+	deps, err := cdoc.GetResources()
+	if err != nil {
+		logger.Println(err)
+		return
+	}
+
+	for _, dep := range deps {
+		if seen.Seen(dep.ID()) {
+			continue
+		}
+		*stack = append(*stack, dep)
+	}
+}
+
+func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
+	seen SeenMap, stack *CrawlSeed) {
+
+	UpdatedDocCount := 0
+	seenDocCount := 0
+	cachedDocCount := 0
+	findMatchErrCount := 0
+	FetchDocumentErrCount := 0
+	SetCreatedErrCount := 0
+	convErrCount := 0
+	deleteDocCount := 0
+	crawledDocCount := 0
+
+	// During the execution of the for loop, more Documents may be added into (*docsPtr).
+	for len(*docsPtr) > 0 {
+		// get the last Document in (*docPtr), which will be crawled in this iteration.
+		tail := (*docsPtr)[len(*docsPtr)-1]
+
+		// remove the last Document in (*docPtr)
+		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
+
+		crawledDocCount++
+		logger.Printf("Crawling doc %d: %s %s", crawledDocCount, tail.RepositoryURL, tail.FilePath)
+
+		if seen.Seen(tail.ID()) {
+			logger.Printf("this doc has been seen before")
+			seenDocCount++
+			continue
+		}
+
+		if tail.WasCached() {
+			logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath)
+			cachedDocCount++
+			continue
+		}
+
+		match := findMatch(tail, crawlers)
+		if match == nil {
+			logIfErr(fmt.Errorf("%v could not match any crawler", tail))
+			findMatchErrCount++
+			continue
+		}
+
+		// If the Document represents a kustomization root, FetchDcoument will change
+		// the `filePath` field of the Document by adding `kustomization.yaml` or
+		// `kustomization.yml` or `kustomization` into the the field.
+		// Therefore, it is necessary to add the ID of the Document into seen before
+		// calling FetchDocument. Otherwise, the binary may enter into an infinite loop
+		// if a kustomization file points to its kustmozation root in its `resources` or
+		// `bases` field.
+		seen.Add(tail.ID())
+
+
+		if err := match.FetchDocument(ctx, tail); err != nil {
+			logger.Printf("FetchDocument failed on %s %s: %v",
+				tail.RepositoryURL, tail.FilePath, err)
+			FetchDocumentErrCount++
+			// delete the document from the index
+			cdoc := &doc.KustomizationDocument{
+				Document: *tail,
+			}
+			seen.Add(cdoc.ID())
+			if err := indx(cdoc, index.Delete); err != nil {
+				logger.Printf("Failed to delete %s %s: %v",
+					cdoc.RepositoryURL, cdoc.FilePath, err)
+			}
+			deleteDocCount++
+			continue
+		}
+
+		if err := match.SetCreated(ctx, tail); err != nil {
+			logger.Printf("SetCreated failed on %s %s: %v",
+				tail.RepositoryURL, tail.FilePath, err)
+			SetCreatedErrCount++
+		}
+
+		cdoc, err := conv(tail)
+		// If conv returns an error, cdoc can still be added into the index so that
+		// cdoc.Document can be searched.
+		if err != nil {
+			logger.Printf("conv failed on %s %s: %v",
+				tail.RepositoryURL, tail.FilePath, err)
+			convErrCount++
+		}
+
+		UpdatedDocCount++
+		addBranches(cdoc, match, indx, seen, stack)
+	}
+	logger.Printf("Summary of doCrawl:\n")
+	logger.Printf("\t%d documents were updated\n", UpdatedDocCount)
+	logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount)
+	logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount)
+	logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount)
+	logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n",
+		FetchDocumentErrCount, deleteDocCount)
+	logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount)
+	logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount)
+}
+
+// CrawlFromSeed updates all the documents in seed, and crawls all the new
+// documents referred in the seed.
+func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
+	conv Converter, indx IndexFunc, seen SeenMap) {
+
+	// stack tracks the documents directly referred in other documents.
 	stack := make(CrawlSeed, 0)
 
-	findMatch := func(d *doc.Document) Crawler {
-		for _, crawl := range crawlers {
-			if crawl.Match(d) {
-				return crawl
-			}
-		}
-
-		return nil
-	}
-
-	addBranches := func(cdoc CrawledDocument, match Crawler) {
-		if _, ok := seen[cdoc.ID()]; ok {
-			return
-		}
-
-		seen[cdoc.ID()] = struct{}{}
-		// Insert into index
-		err := indx(cdoc, match)
-		logIfErr(err)
-		if err != nil {
-			return
-		}
-
-		deps, err := cdoc.GetResources()
-		logIfErr(err)
-		if err != nil {
-			return
-		}
-		for _, dep := range deps {
-			if _, ok := seen[dep.ID()]; ok {
-				continue
-			}
-			stack = append(stack, dep)
-		}
-	}
-
-	doCrawl := func(docsPtr *CrawlSeed) {
-		n := len(*docsPtr)
-		for i := 0; i < n; i++ {
-			next := (*docsPtr)[i]
-			match := findMatch(next)
-			if match == nil {
-				logIfErr(fmt.Errorf(
-					"%v could not match any crawler", next))
-				continue
-			}
-
-			logger.Println("Crawling ", next.RepositoryURL, next.FilePath)
-			err := match.FetchDocument(ctx, next)
-			logIfErr(err)
-			// If there was no change or there is an error, we don't have
-			// to branch out, since the dependencies are already in the
-			// index, or we cannot find the document.
-			if err != nil || next.WasCached() {
-				if next.WasCached() {
-					logger.Println(next.RepositoryURL, next.FilePath, "is cached already")
-				}
-				continue
-			}
-
-			logIfErr(match.SetCreated(ctx, next))
-
-			cdoc, err := conv(next)
-			logIfErr(err)
-
-			addBranches(cdoc, match)
-		}
-	}
-
 	// Exploit seed to update bulk of corpus.
 	logger.Printf("updating %d documents from seed\n", len(seed))
-	doCrawl(&seed)
-	// Traverse any new links added while updating corpus.
+	// each unique document in seed will be crawled once.
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack)
+
+	// Traverse any new documents added while updating corpus.
 	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
-	doCrawl(&stack)
-
-	ch := make(chan CrawledDocument, 1<<10)
-	wg := sync.WaitGroup{}
-
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for cdoc := range ch {
-			if _, ok := seen[cdoc.ID()]; ok {
-				continue
-			}
-			match := findMatch(cdoc.GetDocument())
-			if match == nil {
-				logIfErr(fmt.Errorf(
-					"%v could not match any crawler", cdoc))
-				continue
-			}
-			addBranches(cdoc, match)
-		}
-	}()
-
-	// Exploration through APIs.
-	errs := CRunner(ctx, ch, crawlers)
-	if errs != nil {
-		for _, err := range errs {
-			logIfErr(err)
-		}
-	}
-	close(ch)
-	logger.Println("Processing the new documents from the crawlers' exploration.")
-	wg.Wait()
-	// Handle deps of newly discovered documents.
-	logger.Printf("crawling the %d new documents from the crawlers' exploration.",
-		len(stack))
-	doCrawl(&stack)
+	// While crawling each document in stack, the documents directly referred in the document
+	// will be added into stack.
+	// After this statement is done, stack will become empty.
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
 }
 
-// CRunner is a blocking function and only returns once all of the
+// CrawlGithubRunner is a blocking function and only returns once all of the
 // crawlers are finished with execution.
 //
 // This function uses the output channel to forward kustomization documents
@@ -188,14 +240,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
 // index of the crawler that emitted the error. Although the errors themselves
 // can be nil, the array will always be exactly the size of the crawlers array.
 //
-// CRunner takes in a seed, which represents the documents stored in an
+// CrawlGithubRunner takes in a seed, which represents the documents stored in an
 // index somewhere. The document data is not required to be populated. If there
 // are many documents, this is preferable. The order of iteration over the seed
-// is not guaranteed, but the CRunner does guarantee that every element
+// is not guaranteed, but the CrawlGithub does guarantee that every element
 // from the seed will be processed before any other documents from the
 // crawlers.
-func CRunner(ctx context.Context,
-	output chan<- CrawledDocument, crawlers []Crawler) []error {
+func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
+	crawlers []Crawler, seen SeenMap) []error {
 
 	errs := make([]error, len(crawlers))
 	wg := sync.WaitGroup{}
@@ -229,10 +281,57 @@ func CRunner(ctx context.Context,
 				}
 			}()
 			defer close(docs)
-			errs[idx] = crawler.Crawl(ctx, docs)
+			errs[idx] = crawler.Crawl(ctx, docs, seen)
 		}(i, crawler, docs) // Copies the index and the crawler
 	}
 
 	wg.Wait()
 	return errs
 }
+
+// CrawlGithub crawls all the kustomization files on Github.
+func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
+	indx IndexFunc, seen SeenMap) {
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// ch is channel where all the crawlers sends the crawled documents to.
+	ch := make(chan CrawledDocument, 1<<10)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		docCount := 0
+		for cdoc := range ch {
+			docCount++
+			logger.Printf("Processing doc %d found on Github", docCount)
+			if seen.Seen(cdoc.ID()) {
+				logger.Printf("the doc has been seen before")
+				continue
+			}
+			match := findMatch(cdoc.GetDocument(), crawlers)
+			if match == nil {
+				logIfErr(fmt.Errorf(
+					"%v could not match any crawler", cdoc))
+				continue
+			}
+			addBranches(cdoc, match, indx, seen, &stack)
+		}
+	}()
+
+	logger.Println("processing the documents found from crawling github")
+	if errs := CrawlGithubRunner(ctx, ch, crawlers, seen); errs != nil {
+		for _, err := range errs {
+			logIfErr(err)
+		}
+	}
+	close(ch)
+	wg.Wait()
+
+	// Handle deps of newly discovered documents.
+	logger.Printf("crawling the %d new documents referred by other documents",
+		len(stack))
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
+}

api/internal/crawl/crawler/crawler_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"reflect"
 	"sort"
 	"strings"
@@ -11,6 +12,8 @@ import (
 	"testing"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
 	"sigs.k8s.io/kustomize/api/konfig"
 )
@@ -38,12 +41,13 @@ func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 		return nil
 	}
 	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
-		fmt.Println(d.ID(), "/", suffix)
-		i, ok := c.lukp[d.ID()+"/"+suffix]
+		savedFilePath := d.FilePath
+		d.FilePath += "/" + suffix
+		i, ok := c.lukp[d.ID()]
 		if !ok {
+			d.FilePath = savedFilePath
 			continue
 		}
-		d.FilePath += "/" + suffix
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
@@ -72,7 +76,7 @@ func newCrawler(matchPrefix string, err error,
 
 // Crawl implements the Crawler interface for testing.
 func (c testCrawler) Crawl(_ context.Context,
-	output chan<- CrawledDocument) error {
+	output chan<- CrawledDocument, _ SeenMap) error {
 
 	for i, d := range c.docs {
 		isResource := true
@@ -106,8 +110,8 @@ func (s sortableDocs) Len() int {
 	return len(s)
 }
 
-func TestCrawlerRunner(t *testing.T) {
-	fmt.Println("testing CRunner")
+func TestCrawlGithubRunner(t *testing.T) {
+	log.Println("testing CrawlGithubRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -178,8 +182,9 @@ func TestCrawlerRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()
 
-			errs := CRunner(context.Background(),
-				output, test.tc)
+			seen := NewSeenMap()
+			errs := CrawlGithubRunner(context.Background(),
+				output, test.tc, seen)
 
 			// Check that errors are returned as they should be.
 			if !reflect.DeepEqual(errs, test.errs) {
@@ -212,7 +217,7 @@ func TestCrawlerRunner(t *testing.T) {
 }
 
 func TestCrawlFromSeed(t *testing.T) {
-	fmt.Println("testing CrawlFromSeed")
+	log.Println("testing CrawlFromSeed")
 
 	tests := []struct {
 		seed    CrawlSeed
@@ -302,29 +307,6 @@ resources:
 					RepositoryURL: kustomizeRepo,
 					FilePath:      "examples/seedcrawl2/job.yaml",
 				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/base/kustomization.yaml",
-					DocumentData: `
-resources:
-- ../app
-`,
-				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/kustomization.yaml",
-					DocumentData: `
-resources:
-- resource.yaml
-`,
-				}},
-				// Visited from crawling runner imported as resource.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/resource.yaml",
-				}},
 			},
 		},
 	}
@@ -338,10 +320,11 @@ resources:
 					Document: *d,
 				}, nil
 			},
-			func(d CrawledDocument, cr Crawler) error {
+			func(d CrawledDocument, mode index.Mode) error {
 				visited[d.ID()]++
 				return nil
 			},
+			NewSeenMap(),
 		)
 		if lv, lc := len(visited), len(tc.corpus); lv != lc {
 			t.Errorf("error: %d of %d documents visited.", lv, lc)

api/internal/crawl/crawler/github/crawler.go

@@ -30,6 +30,8 @@ var logger = log.New(os.Stdout, "Github Crawler: ",
 type githubCrawler struct {
 	client GhClient
 	query  Query
+	// branchMap maps github repositories to their default branches
+	branchMap map[string]string
 }
 
 type GhClient struct {
@@ -51,13 +53,22 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 			},
 			accessToken: accessToken,
 		},
-		query: query,
+		query:     query,
+		branchMap: map[string]string{},
 	}
 }
 
+func (gc githubCrawler) SetDefaultBranch(repo, branch string) {
+	gc.branchMap[repo] = branch
+}
+
+func (gc githubCrawler) DefaultBranch(repo string) string {
+	return gc.branchMap[repo]
+}
+
 // Implements crawler.Crawler.
-func (gc githubCrawler) Crawl(
-	ctx context.Context, output chan<- crawler.CrawledDocument) error {
+func (gc githubCrawler) Crawl(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen crawler.SeenMap) error {
 
 	noETagClient := GhClient{
 		RequestConfig: gc.client.RequestConfig,
@@ -79,13 +90,17 @@ func (gc githubCrawler) Crawl(
 
 	// Query each range for files.
 	errs := make(multiError, 0)
+	queryResult := RangeQueryResult{}
 	for _, query := range ranges {
-		err := processQuery(ctx, gc.client, query, output)
+		rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap)
 		if err != nil {
 			errs = append(errs, err)
 		}
+		queryResult.Add(rangeResult)
 	}
 
+	logger.Printf("Summary of Crawl: %s", queryResult.String())
+
 	if len(errs) > 0 {
 		return errs
 	}
@@ -93,7 +108,23 @@ func (gc githubCrawler) Crawl(
 	return nil
 }
 
+// FetchDocument first tries to fetch the document with d.FilePath. If it fails,
+// it will try to add each string in konfig.RecognizedKustomizationFileNames() to
+// d.FilePath, and try to fetch the document again.
 func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
+	// set the default branch if it is empty
+	if d.DefaultBranch == "" {
+		url := gc.client.ReposRequest(d.RepositoryFullName())
+		defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap)
+		if err != nil {
+			logger.Printf(
+				"(error: %v) setting default_branch to master\n", err)
+			defaultBranch = "master"
+		}
+		d.DefaultBranch = defaultBranch
+	}
+	gc.SetDefaultBranch(d.RepositoryURL, d.DefaultBranch)
+
 	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
 	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
 	if err != nil {
@@ -104,9 +135,13 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 		"/" + repoSpec.Ref + "/" + repoSpec.Path
 
 	handle := func(resp *http.Response, err error, path string) error {
+		if resp == nil {
+			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
+				url, path, err)
+		}
 		if err == nil && resp.StatusCode == http.StatusOK {
 			d.IsSame = httpclient.FromCache(resp.Header)
-			defer resp.Body.Close()
+			defer CloseResponseBody(resp)
 			data, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				return err
@@ -115,18 +150,18 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 			d.FilePath = d.FilePath + path
 			return nil
 		}
+
 		return err
 	}
-	resp, err := gc.client.GetRawUserContent(url)
-	if err := handle(resp, err, ""); err == nil {
+	resp, errGetRawUserContent := gc.client.GetRawUserContent(url)
+	if err := handle(resp, errGetRawUserContent, ""); err == nil {
 		return nil
 	}
 
 	for _, file := range konfig.RecognizedKustomizationFileNames() {
-		resp, err = gc.client.GetRawUserContent(url + "/" + file)
-		err := handle(resp, err, "/"+file)
-		if err != nil {
-			continue
+		resp, errGetRawUserContent = gc.client.GetRawUserContent(url + "/" + file)
+		if err = handle(resp, errGetRawUserContent, "/"+file); err == nil {
+			return nil
 		}
 	}
 	return fmt.Errorf("file not found: %s, error: %v", url, err)
@@ -158,10 +193,32 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
 	return strings.Contains(repoSpec.Host, "github.com")
 }
 
+type RangeQueryResult struct {
+	totalDocCnt uint64
+	seenDocCnt   uint64
+	newDocCnt uint64
+	errorCnt  uint64
+}
+
+func (r *RangeQueryResult) Add(other RangeQueryResult) {
+	r.totalDocCnt += other.totalDocCnt
+	r.newDocCnt += other.newDocCnt
+	r.seenDocCnt += other.seenDocCnt
+	r.errorCnt += other.errorCnt
+}
+
+func (r *RangeQueryResult) String() string {
+	return fmt.Sprintf("got %d files from API. "+
+		"%d have been seen before. %d are new and sent to the output channel." +
+		" %d have kustomizationResultAdapter errors.",
+		r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
+}
+
 // processQuery follows all of the pages in a query, and updates/adds the
 // documents from the crawl to the datastore/index.
 func processQuery(ctx context.Context, gcl GhClient, query string,
-	output chan<- crawler.CrawledDocument) error {
+	output chan<- crawler.CrawledDocument, seen crawler.SeenMap,
+	branchMap map[string]string) (RangeQueryResult, error) {
 
 	queryPages := make(chan GhResponseInfo)
 
@@ -178,50 +235,67 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
 	}()
 
 	errs := make(multiError, 0)
-	errorCnt := 0
-	totalCnt := 0
+	result := RangeQueryResult{}
+	pageID := 1
 	for page := range queryPages {
 		if page.Error != nil {
 			errs = append(errs, page.Error)
 			continue
 		}
-
+		pageResult := RangeQueryResult{}
 		for _, file := range page.Parsed.Items {
-			k, err := kustomizationResultAdapter(gcl, file)
+			k, err := kustomizationResultAdapter(gcl, file, seen, branchMap)
 			if err != nil {
 				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
-				errorCnt++
+				pageResult.errorCnt++
 			}
 			if k != nil {
+				pageResult.newDocCnt++
 				output <- k
+			} else {
+				pageResult.seenDocCnt++
 			}
-			totalCnt++
+			pageResult.totalDocCnt++
 		}
 
-		logger.Printf("got %d files out of %d from API. %d of %d had errors\n",
-			totalCnt, page.Parsed.TotalCount, errorCnt, totalCnt)
+		logger.Printf("processQuery [TotalCount %d - page %d]: %s",
+			page.Parsed.TotalCount, pageID, pageResult.String())
+		result.Add(pageResult)
+
+		pageID++
 	}
 
-	return errs
+	logger.Printf("Summary of processQuery: %s", result.String())
+
+	return result, errs
 }
 
-func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
-	crawler.CrawledDocument, error) {
-
-	data, err := gcl.GetFileData(k)
-	if err != nil {
-		return nil, err
-	}
-
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen crawler.SeenMap,
+	branchMap map[string]string) (crawler.CrawledDocument, error) {
 	url := gcl.ReposRequest(k.Repository.FullName)
-	defaultBranch, err := gcl.GetDefaultBranch(url)
+	defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
 	if err != nil {
 		logger.Printf(
 			"(error: %v) setting default_branch to master\n", err)
 		defaultBranch = "master"
 	}
 
+	document := doc.Document{
+		FilePath:      k.Path,
+		DefaultBranch: defaultBranch,
+		RepositoryURL: k.Repository.URL,
+	}
+
+	if seen.Seen(document.ID()) {
+		return nil, nil
+	}
+
+	data, err := gcl.GetFileData(k)
+	if err != nil {
+		return nil, err
+	}
+
 	d := doc.KustomizationDocument{
 		Document: doc.Document{
 			DocumentData:  string(data),
@@ -230,7 +304,6 @@ func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
 			RepositoryURL: k.Repository.URL,
 		},
 	}
-	logger.Printf("Set the creationTime field")
 	creationTime, err := gcl.GetFileCreationTime(k)
 	if err != nil {
 		logger.Printf("GetFileCreationTime failed: %v", err)
@@ -294,7 +367,10 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
 			k, url, err)
 	}
-	resp.Body.Close()
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}
 
 	type githubContentRawURL struct {
 		DownloadURL string `json:"download_url,omitempty"`
@@ -313,18 +389,32 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 			k, rawURL.DownloadURL, err)
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err = ioutil.ReadAll(resp.Body)
 	return data, err
 }
 
-func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
+func CloseResponseBody(resp *http.Response) {
+	if err := resp.Body.Close(); err != nil {
+		log.Printf("failed to close response body: %v", err)
+	}
+}
+
+// GetDefaultBranch gets the default branch of a github repository.
+// m is a map which maps a github repository to its default branch.
+// If repo is already in m, the default branch for url will be obtained from m;
+// otherwise, a query will be made to github to obtain the default branch.
+func (gcl GhClient) GetDefaultBranch(url, repo string, m map[string]string) (string, error) {
+	if v, ok := m[repo]; ok {
+		return v, nil
+	}
+
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
 			"'%s' could not get default_branch: %v", url, err)
 	}
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -376,7 +466,7 @@ func (gcl GhClient) GetFileCreationTime(
 		}
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return defaultTime, fmt.Errorf(
@@ -499,7 +589,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	}
 
 	var data []byte
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
 	if requestInfo.Error != nil {
 		return requestInfo
@@ -533,7 +623,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 }
 
 // SearchGithubAPI performs a search query and handles rate limitting for
-// the 'code/search?' endpoint as well as timed retries in the case of abuse
+// the 'search/code?' endpoint as well as timed retries in the case of abuse
 // prevention.
 func (gcl GhClient) SearchGithubAPI(query string) (*http.Response, error) {
 	throttleSearchAPI()
@@ -560,7 +650,15 @@ func (gcl GhClient) Do(query string) (*http.Response, error) {
 		return nil, err
 	}
 	req.Header.Add("Authorization", fmt.Sprintf("token %s", gcl.accessToken))
-	return gcl.client.Do(req)
+
+	// gcl.client.Do: a non-2xx status code doesn't cause an error.
+	// See https://golang.org/pkg/net/http/#Client.Do for more info.
+	resp, err := gcl.client.Do(req)
+	if resp != nil && resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
+			query, resp.StatusCode)
+	}
+	return resp, err
 }
 
 func (gcl GhClient) getWithRetry(
@@ -570,13 +668,10 @@ func (gcl GhClient) getWithRetry(
 
 	retryCount := gcl.retryCount
 
-	for err == nil &&
-		resp.StatusCode == http.StatusForbidden &&
-		retryCount > 0 {
-
+	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
 		retryTime := resp.Header.Get("Retry-After")
-		i, err := strconv.Atoi(retryTime)
-		if err != nil {
+		i, errAtoi := strconv.Atoi(retryTime)
+		if errAtoi != nil {
 			return resp, fmt.Errorf(
 				"query '%s' forbidden without 'Retry-After'", query)
 		}

api/internal/crawl/crawler/github/queries.go

@@ -90,6 +90,17 @@ func Path(p string) queryField {
 	return queryField{name: "path", value: p}
 }
 
+// Repo takes a repository (i.e., kubernetes-sigs/kustomize) and formats
+// it according to the Github API.
+func Repo(r string) queryField {
+	return queryField{name: "repo", value: r}
+}
+
+// Path takes a github username and formats it according to the Github API.
+func User(u string) queryField {
+	return queryField{name: "user", value: u}
+}
+
 // RequestConfig stores common variables that must be present for the queries.
 // - CodeSearchRequests: ask Github to check the code indices given a query.
 // - ContentsRequests: ask Github where to download a resource given a repo and a
@@ -105,9 +116,11 @@ type RequestConfig struct {
 // the URL method to get the string value of the URL. See request.CopyWith, to
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
-	req := rc.makeRequest("search/code", query)
-	req.vals.Set("sort", "indexed")
-	req.vals.Set("order", "desc")
+	vals := url.Values{
+		"sort": []string{"indexed"},
+		"order": []string{"desc"},
+	}
+	req := rc.makeRequest("search/code", query, vals)
 	return req
 }
 
@@ -115,23 +128,25 @@ func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 // query for the Github API to find the dowload information of this filepath.
 func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 func (rc RequestConfig) ReposRequest(fullRepoName string) string {
 	uri := fmt.Sprintf("repos/%s", fullRepoName)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 // CommitsRequest given the repo name, and a filepath returns a formatted query
 // for the Github API to find the commits that affect this file.
 func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
-	return rc.makeRequest(uri, Query{Path(path)}).URL()
+	vals := url.Values{
+		"path": []string{path},
+	}
+	return rc.makeRequest(uri, Query{}, vals).URL()
 }
 
-func (rc RequestConfig) makeRequest(path string, query Query) request {
-	vals := url.Values{}
+func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
 	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
 
 	return request{

api/internal/crawl/crawler/github/queries_test.go

@@ -53,8 +53,11 @@ func TestQueryType(t *testing.T) {
 				Filename("kustomization.yaml"),
 				Keyword("keyword1"),
 				Keyword("keyword2"),
+				Repo("user1/repo1"),
+				User("user1"),
 			),
-			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2",
+			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2+" +
+				"repo:user1/repo1+user:user1",
 		},
 	}
 
@@ -98,7 +101,27 @@ func TestGithubSearchQuery(t *testing.T) {
 				"examples/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"q=path:examples/helloWorld/kustomization.yaml&per_page=100",
+				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+		},
+		{
+			rc: RequestConfig{
+				perPage: perPage,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples 1/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples%201/helloWorld/kustomization.yaml?per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
 		},
 	}
 

api/internal/crawl/crawler/github/split_search_ranges.go

@@ -243,7 +243,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	logger.Println("total files: ", totalFiles)
+	logger.Println("total kustomization files: ", totalFiles)
 
 	if githubMaxResultsPerQuery >= totalFiles {
 		return []string{

api/internal/crawl/crawler/github/split_search_ranges_test.go

@@ -2,6 +2,7 @@ package github
 
 import (
 	"fmt"
+	"log"
 	"reflect"
 	"testing"
 )
@@ -11,7 +12,7 @@ type testCachedSearch struct {
 }
 
 func (c testCachedSearch) CountResults(upperBound uint64) (uint64, error) {
-	fmt.Printf("CountResults(%05x)\n", upperBound)
+	log.Printf("CountResults(%05x)\n", upperBound)
 	count, ok := c.cache[upperBound]
 	if !ok {
 		return count, fmt.Errorf("cache not set at %x", upperBound)

api/internal/crawl/doc/doc.go

@@ -2,6 +2,8 @@ package doc
 
 import (
 	"fmt"
+	"log"
+	"sort"
 	"strings"
 
 	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
@@ -46,7 +48,7 @@ type set map[string]struct{}
 func (doc *KustomizationDocument) String() string {
 	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
 		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
-		doc.IsSame,	doc.Kinds, len(doc.Identifiers), len(doc.Values))
+		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
 }
 
 // Implements the CrawlerDocument interface.
@@ -77,9 +79,12 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 
 	res := make([]*Document, 0, len(k.Resources))
 	for _, r := range k.Resources {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
 		next, err := doc.Document.FromRelativePath(r)
 		if err != nil {
-			fmt.Printf("GetResources error: %v\n", err)
+			log.Printf("GetResources error: %v\n", err)
 			continue
 		}
 		res = append(res, &next)
@@ -116,6 +121,8 @@ func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error)
 	return configs, nil
 }
 
+// ParseYAML parses doc.Document and sets the following fields of doc:
+// Kinds, Values, Identifiers.
 func (doc *KustomizationDocument) ParseYAML() error {
 	doc.Identifiers = make([]string, 0)
 	doc.Values = make([]string, 0)
@@ -159,6 +166,13 @@ func (doc *KustomizationDocument) ParseYAML() error {
 		doc.Identifiers = append(doc.Identifiers, key)
 	}
 
+	// Without sorting these fields, every time when the string order in these fields changes,
+	// the document in the index will be updated.
+	// Sorting these fields are necessary to avoid a document being updated unnecessarily.
+	sort.Strings(doc.Kinds)
+	sort.Strings(doc.Values)
+	sort.Strings(doc.Identifiers)
+
 	return nil
 }
 

api/internal/crawl/doc/docname.go

@@ -78,11 +78,17 @@ func (doc *Document) ID() string {
 }
 
 func (doc *Document) RepositoryFullName() string {
-	doc.RepositoryURL = strings.TrimRight(doc.RepositoryURL, "/")
-	sections := strings.Split(doc.RepositoryURL, "/")
+	url := strings.TrimRight(doc.RepositoryURL, "/")
+
+	gitPrefix := "git@github.com:"
+	if strings.HasPrefix(url, gitPrefix) {
+		url = url[len(gitPrefix):]
+	}
+
+	sections := strings.Split(url, "/")
 	l := len(sections)
 	if l < 2 {
-		return doc.RepositoryURL
+		return url
 	}
 	return path.Join(sections[l-2], sections[l-1])
 }

api/internal/crawl/doc/docname_test.go

@@ -92,6 +92,12 @@ func TestDocument_RepositoryFullName(t *testing.T) {
 			},
 			expectedRepositoryFullName: "",
 		},
+		{
+			doc: Document{
+				RepositoryURL: "git@github.com:user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
 	}
 
 	for _, tc := range testCases {

api/internal/crawl/go.mod

@@ -1,13 +1,13 @@
 module sigs.k8s.io/kustomize/api/internal/crawl
 
-go 1.13
+go 1.12
 
 require (
-	github.com/elastic/go-elasticsearch/v6 v6.8.2
+	github.com/elastic/go-elasticsearch/v6 v6.8.5
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/gorilla/mux v1.7.3
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/rs/cors v1.7.0
-	sigs.k8s.io/kustomize/api v0.2.0
+	sigs.k8s.io/kustomize/api v0.3.1
 	sigs.k8s.io/yaml v1.1.0
 )

api/internal/crawl/go.sum

@@ -1,14 +1,11 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -25,6 +22,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -41,13 +39,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/elastic/go-elasticsearch/v6 v6.8.2 h1:rp5DGrd63V5c6nHLjF6QEXUpZSvs0+QM3ld7m9VhV2g=
-github.com/elastic/go-elasticsearch/v6 v6.8.2/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/elastic/go-elasticsearch/v6 v6.8.5 h1:U2HtkBseC1FNBmDr0TR2tKltL6FxoY+niDAlj5M8TK8=
+github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -87,14 +84,16 @@ github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Il
 github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
 github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -106,10 +105,10 @@ github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvL
 github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
 github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
@@ -117,34 +116,35 @@ github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunE
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
 github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
-github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -154,6 +154,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -177,8 +178,8 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -192,7 +193,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -225,18 +225,15 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -261,12 +258,13 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
@@ -274,16 +272,22 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -294,19 +298,23 @@ golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -315,19 +323,22 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -336,20 +347,26 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -371,7 +388,14 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
+k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
+k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
+k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
+k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
@@ -379,14 +403,12 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.2.0 h1:e++6JpysnnlUbHmFrv6jvfF5rFlgQ103bS1DO7r5bWA=
-sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0 h1:otg4dLFc03c3gzl+2CV8GPGcd1kk8wjXwD+UhhcCn5I=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
+sigs.k8s.io/kustomize/api v0.3.1 h1:oqMIXvS6tFEUVuKIRUKDa05eC4Hh+cb9JYg8Zhp2d24=
+sigs.k8s.io/kustomize/api v0.3.1/go.mod h1:A+ATnlHqzictQfQC1q3KB/T6MSr0UWQsrrLxMWkge2E=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/internal/crawl/index/elasticsearch.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"time"
 
 	es "github.com/elastic/go-elasticsearch/v6"
@@ -88,7 +87,7 @@ func (idx *index) responseErrorOrNil(info string, res *esapi.Response,
 
 	defer res.Body.Close()
 	if res.IsError() {
-		return fmt.Errorf("%s: %s", messageStart, res.String())
+		return fmt.Errorf("%s: %s [%d]", messageStart, res.String(), res.StatusCode)
 	}
 
 	if reader != nil {
@@ -179,44 +178,47 @@ func (idx *index) DeleteIndex() error {
 }
 
 // Insert or update the document by ID.
-func (idx *index) Put(uniqueID string, doc interface{}) (string, error) {
-	body, err := json.Marshal(doc)
+func (idx *index) Put(uniqueID string, doc interface{}) error {
+	exists, err := idx.Exists(uniqueID)
 	if err != nil {
-		return "", err
+		return err
 	}
 
-	req := esapi.IndexRequest{
-		Index:      idx.name,
-		Body:       bytes.NewReader(body),
-		DocumentID: uniqueID,
-	}
-	res, err := req.Do(idx.ctx, idx.client)
-
-	var id string
-	readId := func(reader io.Reader) error {
-		type InsertResult struct {
-			ID string `json:"_id,omitempty"`
+	if exists {
+		docBytes, err := json.Marshal(doc)
+		if err != nil {
+			return err
 		}
-		var ir InsertResult
-		data, err := ioutil.ReadAll(reader)
+		body := byteJoin(`{"doc":`, docBytes, `}`)
+
+		// For a document with a given id, every call of IndexRequest.Do will increase the version of a document.
+		// To avoid increasing the document version unnecessarily, use UpdateRequest here.
+		req := esapi.UpdateRequest{
+			Index:      idx.name,
+			Body:       bytes.NewReader(body),
+			DocumentID: uniqueID,
+		}
+		res, err := req.Do(idx.ctx, idx.client)
+
+		err = idx.responseErrorOrNil("could not update document",
+			res, err, ignoreResponseBody)
+	} else {
+		body, err := json.Marshal(doc)
 		if err != nil {
 			return err
 		}
 
-		err = json.Unmarshal(data, &ir)
-		if err != nil {
-			return err
+		req := esapi.IndexRequest{
+			Index:      idx.name,
+			Body:       bytes.NewReader(body),
+			DocumentID: uniqueID,
 		}
-		id = ir.ID
+		res, err := req.Do(idx.ctx, idx.client)
 
-		return nil
+		err = idx.responseErrorOrNil("could not insert document",
+			res, err, ignoreResponseBody)
 	}
-
-	// populates the id field.
-	err = idx.responseErrorOrNil("could not insert document",
-		res, err, readId)
-
-	return id, err
+	return err
 }
 
 type scrollUpdater func(string, readerFunc) error
@@ -296,3 +298,24 @@ func (idx *index) Delete(id string) error {
 		fmt.Sprintf("could not delete id(%s) from index(%s)", id, idx.name),
 		res, err, ignoreResponseBody)
 }
+
+// Check whether a given document id is in the index
+func (idx *index) Exists(id string) (bool, error) {
+	op := idx.client.Exists
+	res, err := op(
+		idx.name,
+		id,
+		op.WithContext(idx.ctx),
+		op.WithPretty(),
+	)
+
+	if res != nil && !res.IsError() {
+		return true, nil
+	} else if res != nil && res.StatusCode == 404 {
+		return false, nil
+	} else {
+		return false, idx.responseErrorOrNil(
+			fmt.Sprintf("could not check the existence of id(%s) from index(%s)", id, idx.name),
+			res, err, ignoreResponseBody)
+	}
+}

api/internal/crawl/index/kustomize.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"strings"
 	"time"
 
@@ -16,6 +17,12 @@ const (
 	AggregationKeyword = "aggs"
 )
 
+type Mode int
+const (
+	InsertOrUpdate = iota
+	Delete
+)
+
 // Redefinition of Hits structure. Must match the json string of
 // KustomizeResult.Hits.Hits. Declared as a convenience for iteration.
 type KustomizeHits []struct {
@@ -91,14 +98,14 @@ type KustomizeIndex struct {
 }
 
 // Create index reference to the index containing the kustomize documents.
-func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
-	idx, err := newIndex(ctx, "kustomize")
+func NewKustomizeIndex(ctx context.Context, indexName string) (*KustomizeIndex, error) {
+	idx, err := newIndex(ctx, indexName)
 	if err != nil {
 		return nil, err
 	}
 
 	indicesExistsOp := idx.client.Indices.Exists
-	resp, err := indicesExistsOp([]string{"kustomize"},
+	resp, err := indicesExistsOp([]string{indexName},
 		indicesExistsOp.WithContext(idx.ctx),
 		indicesExistsOp.WithPretty())
 	if err != nil {
@@ -106,9 +113,9 @@ func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
 	}
 
 	if resp.StatusCode == 200 {
-		fmt.Printf("The kustomize index already exists\n")
+		log.Printf("The %s index already exists", indexName)
 	} else {
-		fmt.Printf("Creating the kustomize index\n")
+		log.Printf("Creating the %s index\n", indexName)
 		if err := idx.CreateIndex([]byte(IndexConfig)); err != nil {
 			return nil, err
 		}
@@ -246,7 +253,7 @@ func (it *KustomizeIterator) Next() bool {
 	}
 
 	if it.err == nil {
-		fmt.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
+		log.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
 		it.err = it.update(*it.scrollImpl.ScrollID, reader)
 	}
 
@@ -293,12 +300,13 @@ func (ki *KustomizeIndex) IterateQuery(query []byte, batchSize int,
 }
 
 // type specific Put for inserting structured kustomization documents.
-func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) (string, error) {
-	id, err := ki.index.Put(id, doc)
-	if err != nil {
-		return id, fmt.Errorf("could not insert in elastic: %v", err)
-	}
-	return id, nil
+func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) error {
+	return ki.index.Put(id, doc)
+}
+
+// Delete a document with a given id from the kustomize index.
+func (ki *KustomizeIndex) Delete(id string) error {
+	return ki.index.Delete(id)
 }
 
 // Kustomize search options: What metrics should be returned? Kind Aggregation,
@@ -334,7 +342,7 @@ func (ki *KustomizeIndex) Search(query string,
 	if err != nil {
 		return nil, fmt.Errorf("failed to format query %s", query)
 	}
-	fmt.Printf("formated query: %s\n", data)
+	log.Printf("formated query: %s\n", data)
 
 	var kr ElasticKustomizeResult
 	err = ki.index.Search(data, opts.SearchOptions, func(results io.Reader) error {

api/internal/crawl/search_cmds/creationTime.md

@@ -0,0 +1,180 @@
+Find out the largest value of the `creationTime` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "max_creationTime" : { "max" : { "field" : "creationTime" } }
+    }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "min_creationTime" : { "min" : { "field" : "creationTime" } }
+    }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all the kustomization files:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+        { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize resource files:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }
+      }
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Query all the documents whose `creationTime` <= `2016-07-29T17:38:26.000Z`:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "range": {
+      "creationTime": {
+       "lte": "2016-07-29T17:38:26.000Z"
+      }
+    }
+  }
+}
+'
+```
+
+Query all the documents whose `creationTime` falls within the specific range:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "range": {
+      "creationTime": {
+       "gte": "2016-07-29T17:38:26.000Z",
+       "lte": "2016-08-29T17:38:26.000Z"
+      }
+    }
+  }
+}
+'
+```
+
+Aggregate how many new kustomization files were added into Github each month:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize resource files were added into Github each month:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomization files were added into Github each year:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize resource files were added into Github each year:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/defaultBranch.md

@@ -0,0 +1,32 @@
+Count distinct values of the `defaultBranch` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "defaultBranch_count" : {
+            "cardinality" : {
+                "field" : "defaultBranch",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the github branches where kustomization files and kustomize resource files live, 
+and how many kustomization files and kustomize resource files live in each branch:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "defaultBranch" : {
+            "terms" : {
+              "field" : "defaultBranch",
+              "size": 41
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/fieldExistence.md

@@ -0,0 +1,55 @@
+Count the documents whose `document` field is empty (The reason why the `document` field
+of a document is empty is because of empty documents):
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "document"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Find all the documents having the `creationTime` field set:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "creationTime"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `creationTime` field is not set:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "creationTime"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+The following fields of a document in the kustomize index are always non-empty:
+`repositoryUrl`, `filePath`, `defaultBranch`.
+
+The following fields of a document in the kustomize index may be empty:
+`kinds`, `identifiers`, `values`.

api/internal/crawl/search_cmds/keyword_search.md

@@ -0,0 +1,82 @@
+Count the documents in the index whose `repositoryUrl` field starts with
+`https://github.com/`:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Count the documents in the index whose `repositoryUrl` field does not start with
+`https://github.com/`:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": [
+       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents matching the given `repositoryUrl` and `filePath`, and return 
+a version for each search hit:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "version": true,
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "repositoryUrl": "git@github.com:talos-systems/talos-controller-manager" }},
+       { "regexp": { "filePath": "hack/config.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents whose filePath ends with one of these following three filenames:
+`kustomization.yaml`, `kustomization.yml`, `kustomization`:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents whose filePath does not end with any of these following
+three filenames: `kustomization.yaml`, `kustomization.yml`, `kustomization`:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": [
+       { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+      ]
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/misc.md

@@ -0,0 +1,19 @@
+Check the health status of an ElasticSearch cluster:
+```
+curl -X GET "${ElasticSearchURL}:9200/_cat/health?v&pretty"
+```
+
+Check the indices in an ElasticSearch cluster:
+```
+curl "${ElasticSearchURL}:9200/_cat/indices?v"
+```
+
+Get the mapping of the index:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_mapping?pretty"
+```
+
+Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**):
+```
+curl -X DELETE "${ElasticSearchURL}:9200/kustomize?pretty"
+```

api/internal/crawl/search_cmds/repositoryUrl.md

@@ -0,0 +1,125 @@
+Count distinct values of the `repositoryUrl` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count how many Github repositories include kustomization files:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+             ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count how many Github repositories include kustomize resource files:
+```
+curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }
+            }
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the github repositories including kustomization files and kustomize resource files,
+and how many kustomization files and kustomize resource files each github repository includes
+(the github repository including the most kustomization files is listed first):
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 2082
+            }
+        }
+    }
+}
+'
+```
+
+List the top 20 Github repositories including the most amount of kustomization files:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+             ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+List the top 20 Github repositories including the most amount of kustomize resource files:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }
+            }
+        }
+    },
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 20
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/text_search.md

@@ -0,0 +1,148 @@
+Search for all the kustomize resource files including a Deployment object:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files including a Deployment object, but only
+including the `kinds` field in the result:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "_source": {
+    "includes": ["kinds"]
+  },
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files including both a Deployment object and 
+a Service object:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment Service",
+        "operator" : "and"
+      }
+    }
+  }
+}
+'
+```
+
+Count the number of documents including Deployment and the number of documents 
+including Service:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 0,
+  "aggs" : {
+    "messages" : {
+      "filters" : {
+        "filters" : {
+          "Deployment" : { "match" : { "kinds" : "Deployment"   }},
+          "Service" : { "match" : { "kinds" : "Service" }}
+        }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files involving CRDs:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "crds"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files defining configMapGenerator:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "configMapGenerator"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents having a `kind` field:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "match" : { "identifiers" : { "query" : "kind" }}}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kuostmization files having a `kind` field:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)" }},
+       { "match" : { "identifiers" : { "query" : "kind" }}}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files defining the `generatorOptions:disableNameSuffixHash` feature:
+```
+curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "generatorOptions:disableNameSuffixHash"
+      }
+    }
+  }
+}
+'
+```

api/internal/crawl/ui/package.json

@@ -12,7 +12,7 @@
   "private": true,
   "dependencies": {
     "@angular/animations": "^8.2.14",
-    "@angular/cdk": "~8.0.2",
+    "@angular/cdk": "~8.2.3",
     "@angular/common": "^8.2.14",
     "@angular/compiler": "^8.2.14",
     "@angular/core": "^8.2.14",
@@ -26,29 +26,30 @@
     "chart.js": "^2.9.3",
     "core-js": "^3.5.0",
     "hammerjs": "^2.0.8",
-    "rxjs": "~6.4.0",
-    "tslib": "^1.9.0",
-    "zone.js": "~0.9.1"
+    "rxjs": "~6.5.3",
+    "serialize-javascript": "^2.1.2",
+    "tslib": "^1.10.0",
+    "zone.js": "~0.10.2"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^0.801.3",
+    "@angular-devkit/build-angular": "^0.803.20",
     "@angular/cli": "^8.3.20",
     "@angular/compiler-cli": "^8.2.14",
     "@angular/language-service": "^8.2.14",
     "@types/jasmine": "^3.5.0",
     "@types/jasminewd2": "^2.0.8",
-    "@types/node": "~8.9.4",
+    "@types/node": "~12.12.17",
     "codelyzer": "^5.2.0",
-    "jasmine-core": "~3.4.0",
+    "jasmine-core": "~3.5.0",
     "jasmine-spec-reporter": "~4.2.1",
-    "karma": "~4.1.0",
-    "karma-chrome-launcher": "~2.2.0",
+    "karma": "~4.4.1",
+    "karma-chrome-launcher": "~3.1.0",
     "karma-coverage-istanbul-reporter": "^2.1.1",
     "karma-jasmine": "~2.0.1",
-    "karma-jasmine-html-reporter": "^1.4.0",
-    "protractor": "~5.4.0",
-    "ts-node": "~7.0.0",
-    "tslint": "~5.15.0",
-    "typescript": "~3.4.3"
+    "karma-jasmine-html-reporter": "^1.4.2",
+    "protractor": "~5.4.2",
+    "ts-node": "~8.5.4",
+    "tslint": "~5.20.1",
+    "typescript": "~3.7.3"
   }
 }

api/internal/git/cloner.go

@@ -27,90 +27,28 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 	if err != nil {
 		return err
 	}
+
+	if repoSpec.Ref == "" {
+		repoSpec.Ref = "master"
+	}
 	cmd := exec.Command(
 		gitProgram,
-		"init",
+		"clone",
+		"--depth=1",
+		repoSpec.CloneSpec(),
+		"-b",
+		repoSpec.Ref,
 		repoSpec.Dir.String())
 	var out bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &out
 	err = cmd.Run()
 	if err != nil {
-		log.Printf("Error initializing empty git repo: %s", out.String())
+		log.Printf("Error cloning git repo: %s", out.String())
 		return errors.Wrapf(
 			err,
-			"trouble initializing empty git repo in %s",
-			repoSpec.Dir.String())
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"remote",
-		"add",
-		"origin",
-		repoSpec.CloneSpec())
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error setting git remote: %s", out.String())
-		return errors.Wrapf(
-			err,
-			"trouble adding remote %s",
-			repoSpec.CloneSpec())
-	}
-	if repoSpec.Ref == "" {
-		repoSpec.Ref = "master"
-	}
-	cmd = exec.Command(
-		gitProgram,
-		"fetch",
-		"--depth=1",
-		"origin",
-		repoSpec.Ref)
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		cmd = exec.Command(
-			gitProgram,
-			"pull",
-			"origin",
-			"master")
-		var out bytes.Buffer
-		cmd.Stdout = &out
-		cmd.Dir = repoSpec.Dir.String()
-		err := cmd.Run()
-		if err != nil {
-			return errors.Wrapf(err, "trouble pulling %s", repoSpec.OrgRepo)
-		}
-		if repoSpec.Ref == "" {
-			repoSpec.Ref = "master"
-		}
-		cmd = exec.Command(gitProgram, "checkout", repoSpec.Ref)
-		cmd.Dir = repoSpec.Dir.String()
-		err = cmd.Run()
-		if err != nil {
-			return errors.Wrapf(
-				err, "trouble checking out href %s", repoSpec.Ref)
-		}
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"reset",
-		"--hard",
-		"FETCH_HEAD")
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error performing git reset: %s", out.String())
-		return errors.Wrapf(
-			err, "trouble hard resetting empty repository to %s", repoSpec.Ref)
+			"trouble cloning git repo %v in %s",
+			repoSpec.CloneSpec(), repoSpec.Dir.String())
 	}
 
 	cmd = exec.Command(
@@ -120,10 +58,11 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 		"--init",
 		"--recursive")
 	cmd.Stdout = &out
+	cmd.Stderr = &out
 	cmd.Dir = repoSpec.Dir.String()
 	err = cmd.Run()
 	if err != nil {
-		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.Ref)
+		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec())
 	}
 
 	return nil

api/internal/tools/tools.go

@@ -15,8 +15,6 @@ import (
 	_ "golang.org/x/tools/cmd/stringer"
 	// for lint checks
 	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
-	// for integration tests driven by the examples
-	_ "github.com/monopole/mdrip"
 	// REMOVED pluginator from this process, and leaving
 	// this note to discourage its reintroduction,
 	// because pluginator depends on the api, forcing

api/konfig/builtinpluginconsts/varreference.go

@@ -190,5 +190,7 @@ varReference:
   kind: StatefulSet
 
 - path: metadata/labels
+
+- path: metadata/annotations
 `
 )

api/krusty/baseandoverlaymedium_test.go

@@ -153,9 +153,9 @@ FRUIT=banana
 LEGUME=chickpea
 `)
 	th.WriteF("/app/overlay/configmap/dummy.txt",
-		`Lorem ipsum dolor sit amet, consectetur
-adipiscing elit, sed do eiusmod tempor
-incididunt ut labore et dolore magna aliqua. 
+		`Lorem ipsum dolor sit amet, consectetur  
+adipiscing elit, sed do eiusmod tempor		
+incididunt ut labore et dolore magna aliqua.   
 `)
 	th.WriteF("/app/overlay/deployment/deployment.yaml", `
 apiVersion: apps/v1
@@ -292,8 +292,10 @@ metadata:
 ---
 apiVersion: v1
 data:
-  nonsense: "Lorem ipsum dolor sit amet, consectetur\nadipiscing elit, sed do eiusmod
-    tempor\nincididunt ut labore et dolore magna aliqua. \n"
+  nonsense: |
+    Lorem ipsum dolor sit amet, consectetur
+    adipiscing elit, sed do eiusmod tempor
+    incididunt ut labore et dolore magna aliqua.
 kind: ConfigMap
 metadata:
   annotations:
@@ -302,6 +304,6 @@ metadata:
     app: mungebot
     org: kubernetes
     repo: test-infra
-  name: test-infra-app-config-f462h769f9
+  name: test-infra-app-config-4mt28b5bg2
 `)
 }

api/krusty/variableref_test.go

@@ -1157,6 +1157,8 @@ vars:
     name: my-deployment
     labels:
       my-label: $(NAMESPACE)
+    annotations:
+      my-annotation: $(NAMESPACE)
   spec:
     template:
       spec:
@@ -1176,6 +1178,8 @@ vars:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    my-annotation: my-namespace
   labels:
     my-label: my-namespace
   name: my-deployment

api/kv/kv.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"regexp"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -85,11 +86,17 @@ func (kvl *loader) keyValuesFromFileSources(sources []string) ([]types.Pair, err
 		if err != nil {
 			return nil, err
 		}
-		kvs = append(kvs, types.Pair{Key: k, Value: string(content)})
+		kvs = append(kvs, types.Pair{Key: k, Value: trimTrailingSpacesInLines(string(content))})
 	}
 	return kvs, nil
 }
 
+// trimTrailingSpacesInLines takes string with multiple lines and trims the trailing white spaces and tabs from each line.
+func trimTrailingSpacesInLines(str string) string {
+	re := regexp.MustCompile(`\s*\n`)
+	return re.ReplaceAllString(str, "\n")
+}
+
 func (kvl *loader) keyValuesFromEnvFiles(paths []string) ([]types.Pair, error) {
 	var kvs []types.Pair
 	for _, p := range paths {

api/kv/kv_test.go

@@ -95,3 +95,12 @@ func TestKeyValuesFromFileSources(t *testing.T) {
 		}
 	}
 }
+
+func TestTrimTrailingSpacesInLines(t *testing.T) {
+	input := "\"fooKey\": \"fooValue\"   \t\n\t\"barKey\": \"barValue\""
+	expected := "\"fooKey\": \"fooValue\"\n\t\"barKey\": \"barValue\""
+	res := trimTrailingSpacesInLines(input)
+	if !reflect.DeepEqual(res, expected) {
+		t.Errorf("Trim trailing spaces in lines should succeed, got: %s exptected: %s", res, expected)
+	}
+}

cmd/config/.golangci.yml

@@ -23,7 +23,7 @@ linters:
     - gofmt
     - goimports
 #    - golint
-    - gosec
+#    - gosec
     - gosimple
     - govet
     - ineffassign

cmd/config/complete/complete.go

@@ -52,6 +52,10 @@ func Complete(cmd *cobra.Command) *complete.Command {
 		Flags: map[string]complete.Predictor{},
 		Sub:   map[string]*complete.Command{},
 	}
+	if strings.Contains(cmd.Use, "DIR") {
+		// if usage contains directory, then use a file predictor
+		cc.Args = predict.Dirs("*")
+	}
 
 	// add completion for each subcommand
 	for i := range cmd.Commands() {

cmd/config/configcobra/cmds.go

@@ -43,6 +43,26 @@ Advanced Documentation Topics:
 `,
 }
 
+// Export commands publicly for composition
+var (
+	Cat          = commands.CatCommand
+	Count        = commands.CountCommand
+	CreateSetter = commands.CreateSetterCommand
+	Fmt          = commands.FmtCommand
+	Grep         = commands.GrepCommand
+	ListSetters  = commands.ListSettersCommand
+	Merge        = commands.MergeCommand
+	Merge3       = commands.Merge3Command
+	RunFn        = commands.RunFnCommand
+	Set          = commands.SetCommand
+	Sink         = commands.SinkCommand
+	Source       = commands.SourceCommand
+	Tree         = commands.TreeCommand
+
+	StackOnError = &commands.StackOnError
+	ExitOnError  = &commands.ExitOnError
+)
+
 // NewConfigCommand returns a new *cobra.Command for the config command group.  This may
 // be embedded into other go binaries as a way of packaging the "config" command as part
 // of another binary.
@@ -74,8 +94,17 @@ func NewConfigCommand(name string) *cobra.Command {
 	root.AddCommand(commands.CatCommand(name))
 	root.AddCommand(commands.FmtCommand(name))
 	root.AddCommand(commands.MergeCommand(name))
+	root.AddCommand(commands.Merge3Command(name))
 	root.AddCommand(commands.CountCommand(name))
 	root.AddCommand(commands.RunFnCommand(name))
+	root.AddCommand(commands.XArgsCommand())
+	root.AddCommand(commands.WrapCommand())
+
+	root.AddCommand(commands.SetCommand(name))
+	root.AddCommand(commands.ListSettersCommand(name))
+	root.AddCommand(commands.CreateSetterCommand(name))
+	root.AddCommand(commands.SinkCommand(name))
+	root.AddCommand(commands.SourceCommand(name))
 
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-merge",
@@ -89,8 +118,13 @@ func NewConfigCommand(name string) *cobra.Command {
 	})
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-fn",
-		Short: "[Alpha] Documentation for writing containerized functions run by run.",
-		Long:  api.ConfigFnLong,
+		Short: "[Alpha] Documentation for developing and invoking Configuration Functions.",
+		Long:  api.FunctionsImplLong,
+	})
+	root.AddCommand(&cobra.Command{
+		Use:   "docs-fn-spec",
+		Short: "[Alpha] Documentation for Configuration Functions Specification.",
+		Long:  api.FunctionsSpecLong,
 	})
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-io-annotations",

cmd/config/docs/api-conventions/config-fn.md

@@ -1,281 +0,0 @@
-# Configuration Functions API Semantics
-
-  Configuration Functions are functions packaged as executables in containers which enable
-  **shift-left practices**.  They configure applications and infrastructure through
-  Kubernetes style Resource Configuration, but run locally pre-commit.
-
-  Configuration functions enable shift-left practices (client-side) through:
-
-  - Pre-commit / delivery validation and linting of configuration
-    - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
-  - Implementation of abstractions as client actuated APIs (e.g. templating)
-    - e.g. Create a client-side *"CRD"* for generating configuration checked into git
-  - Aspect Orient configuration / Injection of cross-cutting configuration
-    - e.g. T-Shirt size containers by annotating Resources with `small`, `medium`, `large`
-      and inject the cpu and memory resources into containers accordingly.
-    - e.g. Inject `init` and `side-car` containers into Resources based off of Resource
-      Type, annotations, etc.
-
-  Performing these on the client rather than the server enables:
-
-  - Configuration to be reviewed prior to being sent to the API server
-  - Configuration to be validated as part of the CD pipeline
-  - Configuration for Resources to validated holistically rather than individually
-    per-Resource -- e.g. ensure the `Service.selector` and `Deployment.spec.template` labels
-    match.
-    - MutatingWebHooks are scoped to a single Resource instance at a time.
-  - Low-level tweaks to the output of high-level abstractions -- e.g. add an `init container`
-    to a client *"CRD"* Resource after it was generated.
-  - Composition and layering of multiple functions together
-    - Compose generation, injection, validation together
-
-  Configuration Functions are implemented as executable programs published in containers which:
-
-  - Accept as input (stdin):
-    - A list of Resource Configuration
-    - A Function Configuration (to configure the function itself)
-  - Emit as output (stdout + exit):
-    - A list of Resource Configuration
-    - An exit code for success / failure
-
-### Function Specification
-
-  - Functions **SHOULD** be published as container images containing a `CMD` invoking an executable.
-  - Functions **MUST** accept input on STDIN a `ResourceList` containing the Resources and
-    `functionConfig`.
-  - Functions **MUST** emit output on STDOUT a `ResourceList` containing the modified
-    Resources.
-  - Functions **MUST** exit non-0 on failure, and exit 0 on success.
-  - Functions **MAY** emit output on STDERR with error messaging.
-  - Functions performing validation **SHOULD** exit failure and emit error messaging
-    on a validation failure.
-  - Functions generating Resources **SHOULD** retain non-conflicting changes on the
-    generated Resources -- e.g. 1. the function generates a Deployment, but doesn't
-    specify `cpu`, 2. the user sets the `cpu` on the generated Resource, 3. the
-    function should keep the `cpu` when regenerating the Resource a second time.
-  - Functions **SHOULD** be usable outside `kustomize config run` -- e.g. though pipeline
-    mechanisms such as Tekton.
-
-#### Input Format
-
-  Functions must accept on STDIN:
-
-  `ResourceList`:
-  - contains `items` field, same as `List.items`
-  - contains `functionConfig` field -- a single item with the configuration for the function itself
-
-  Example `ResourceList` Input:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    functionConfig:
-      apiVersion: example.com/v1beta1
-      kind: Nginx
-      metadata:
-        name: my-instance
-        annotations:
-          config.kubernetes.io/local-config: "true"
-      spec:
-        replicas: 5
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 3
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Output Format
-
-  Functions must emit on STDOUT:
-
-  `ResourceList`:
-  - contains `items` field, same as `List.items`
-
-  Example `ResourceList` Output:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 5
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Container Environment
-
-  When run by `kustomize config run`, functions are run in containers with the
-  following environment:
-
-  - Network: `none`
-  - User: `nobody`
-  - Security Options: `no-new-privileges`
-  - Volumes: the volume containing the `functionConfig` yaml is mounted under `/local` as `ro`
-
-### Example Function Implementation
-
-  Following is an example for implementing an nginx abstraction using a config
-  function.
-
-#### `nginx-template.sh`
-
-  `nginx-template.sh` is a simple bash script which uses a *heredoc* as a templating solution
-  for generating Resources from the functionConfig input fields.
-
-  The script wraps itself using `config run wrap -- $0` which will:
-
-  1. Parse the `ResourceList.functionConfig` (provided to the container stdin) into env vars
-  2. Merge the stdout into the original list of Resources
-  3. Defaults filenames for newly generated Resources (if they are not set as annotations)
-     to `config/NAME_KIND.yaml`
-  4. Format the output
-
-    #!/bin/bash
-    # script must run wrapped by `kustomize config run wrap`
-    # for parsing input the functionConfig into env vars
-    if [ -z ${WRAPPED} ]; then
-      export WRAPPED=true
-      config run wrap -- $0
-      exit $?
-    fi
-
-    cat <<End-of-message
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: ${NAME}
-    ---
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      replicas: ${REPLICAS}
-      selector:
-        matchLabels:
-          app: nginx
-          instance: ${NAME}
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: ${NAME}
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-    End-of-message
-
-#### `Dockerfile`
-
-  `Dockerfile` installs `kustomize config` and copies the script into the container image.
-
-    FROM golang:1.13-stretch
-    RUN go get sigs.k8s.io/kustomize/cmd/config
-    RUN mv /go/bin/config /usr/bin/config
-    COPY nginx-template.sh /usr/bin/nginx-template.sh
-    CMD ["nginx-template.sh]
-
-### Example Function Usage
-
-Following is an example of running the `kustomize config run` using the preceding API.
-
-#### `nginx.yaml` (Input)
-
-  `dir/nginx.yaml` contains a reference to the Function.  The contents of `nginx.yaml`
-  are passed to the Function through the `ResourceList.functionConfig` field.
-
-    apiVersion: example.com/v1beta1
-    kind: Nginx
-    metadata:
-      name: my-instance
-      annotations:
-        config.kubernetes.io/local-config: "true"
-      configFn:
-        container:
-          image: gcr.io/example-functions/nginx-template:v1.0.0
-    spec:
-      replicas: 5
-
-  - `configFn.container.image`: the image to use for this API
-  - `annotations[config.kubernetes.io/local-config]`: mark this as not a Resource that should
-    be applied
-
-#### `kustomize config run dir/` (Output)
-
-  `dir/my-instance_deployment.yaml` contains the Deployment:
-
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      replicas: 5
-      selector:
-        matchLabels:
-          app: nginx
-          instance: my-instance
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: my-instance
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-
-  `dir/my-instance_service.yaml` contains the Service:
-
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: my-instance

cmd/config/docs/api-conventions/config-io.md

@@ -4,16 +4,17 @@ Resource Configuration may be read / written from / to sources such as directori
 stdin|out or network. Tools may be composed using pipes such that the tools writing
 Resource Configuration may be a different tool from the one that read the configuration.
 In order for tools to be composed in this way, while preserving origin information --
-such as the original file, index, etc.
+such as the original file, index, etc.:
 
-Tools **SHOULD** write the following annotations when reading from sources,
-and **SHOULD** respect the annotations when writing to sinks.
+Tools **SHOULD** insert the following annotations when reading from sources,
+and **SHOULD** delete the annotations when writing to sinks.
 
 ### `config.kubernetes.io/path`
 
 Records the slash-delimited, OS-agnostic, relative file path to a Resource.
 
 This annotation **SHOULD** be set when reading Resources from files.
+It **SHOULD** be unset when writing Resources to files.
 When writing Resources to a directory, the Resource **SHOULD** be written to the corresponding
 path relative to that directory.
 
@@ -27,10 +28,11 @@ metadata:
 
 ### `config.kubernetes.io/index`
 
-Records the index of a Resource in file. In a multi-object files YAML file, Resources are separated
+Records the index of a Resource in file. In a multi-object YAML file, Resources are separated
 by three dashes (`---`), and the index represents the positon of the Resource starting from zero.
 
 This annotation **SHOULD** be set when reading Resources from files.
+It **SHOULD** be unset when writing Resources to files.
 When writing multiple Resources to the same file, the Resource **SHOULD** be written in the
 relative order matching the index.
 

cmd/config/docs/api-conventions/functions-impl.md

@@ -0,0 +1,181 @@
+# Running Configuration Functions using kustomize CLI
+
+Configuration functions can be implemented using any toolchain and invoked using any
+container workflow orchestrator including Tekton, Cloud Build, or run directly using `docker run`.
+
+Run `config help docs-fn-spec` to see the Configuration Functions Specification.
+
+`kustomize config run` is an example orchestrator for invoking Configuration Functions. This
+document describes how to implement and invoke an example function.
+
+## Example Function Implementation
+
+Following is an example for implementing an nginx abstraction using a configuration
+function.
+
+### `nginx-template.sh`
+
+`nginx-template.sh` is a simple bash script which uses a _heredoc_ as a templating solution
+for generating Resources from the functionConfig input fields.
+
+The script wraps itself using `config run wrap -- $0` which will:
+
+1. Parse the `ResourceList.functionConfig` (provided to the container stdin) into env vars
+2. Merge the stdout into the original list of Resources
+3. Defaults filenames for newly generated Resources (if they are not set as annotations)
+   to `config/NAME_KIND.yaml`
+4. Format the output
+
+```bash
+#!/bin/bash
+# script must run wrapped by "kustomize config run wrap"
+# for parsing input the functionConfig into env vars
+if [ -z ${WRAPPED} ]; then
+  export WRAPPED=true
+  config run wrap -- $0
+  exit $?
+fi
+
+cat <<End-of-message
+apiVersion: v1
+kind: Service
+metadata:
+  name: ${NAME}
+  labels:
+    app: nginx
+    instance: ${NAME}
+spec:
+  ports:
+  - port: 80
+    targetPort: 80
+    name: http
+  selector:
+    app: nginx
+    instance: ${NAME}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${NAME}
+  labels:
+    app: nginx
+    instance: ${NAME}
+spec:
+  replicas: ${REPLICAS}
+  selector:
+    matchLabels:
+      app: nginx
+      instance: ${NAME}
+  template:
+    metadata:
+      labels:
+        app: nginx
+        instance: ${NAME}
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+End-of-message
+```
+
+### Dockerfile
+
+`Dockerfile` installs `kustomize config` and copies the script into the container image.
+
+```
+FROM golang:1.13-stretch
+RUN go get sigs.k8s.io/kustomize/cmd/config
+RUN mv /go/bin/config /usr/bin/config
+COPY nginx-template.sh /usr/bin/nginx-template.sh
+CMD ["nginx-template.sh]
+```
+
+## Example Function Usage
+
+Following is an example of running the `kustomize config run` using the preceding API.
+
+When run by `kustomize config run`, functions are run in containers with the
+following environment:
+
+- Network: `none`
+- User: `nobody`
+- Security Options: `no-new-privileges`
+- Volumes: the volume containing the `functionConfig` yaml is mounted under `/local` as `ro`
+
+### Input
+
+`dir/nginx.yaml` contains a reference to the Function. The contents of `nginx.yaml`
+are passed to the Function through the `ResourceList.functionConfig` field.
+
+```yaml
+apiVersion: example.com/v1beta1
+kind: Nginx
+metadata:
+  name: my-instance
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    config.k8s.io/function: |
+      container:
+        image: gcr.io/example-functions/nginx-template:v1.0.0
+spec:
+  replicas: 5
+```
+
+- `annotations[config.k8s.io/function].container.image`: the image to use for this API
+- `annotations[config.kubernetes.io/local-config]`: mark this as not a Resource that should
+  be applied
+
+### Output
+
+The function is invoked using by runing `kustomize config run dir/`.
+
+`dir/my-instance_deployment.yaml` contains the Deployment:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: my-instance
+  labels:
+    app: nginx
+    instance: my-instance
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: nginx
+      instance: my-instance
+  template:
+    metadata:
+      labels:
+        app: nginx
+        instance: my-instance
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:1.7.9
+          ports:
+            - containerPort: 80
+```
+
+`dir/my-instance_service.yaml` contains the Service:
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: my-instance
+  labels:
+    app: nginx
+    instance: my-instance
+spec:
+  ports:
+    - port: 80
+      targetPort: 80
+      name: http
+  selector:
+    app: nginx
+    instance: my-instance
+```

cmd/config/docs/api-conventions/functions-spec.md

@@ -0,0 +1,186 @@
+# Configuration Functions Specification
+
+This document specifies a standard for client-side functions that operate on
+Kubernetes declarative configurations. This standard enables creating
+small, interoperable, and language-independent executable programs packaged as
+containers that can be chained together as part of a configuration management pipeline.
+The end result of such a pipeline are fully rendered configurations that can then be
+applied to a control plane (e.g. Using ‘kubectl apply’ for Kubernetes control plane).
+As such, although this document references Kubernetes Resource Model and API conventions,
+it is completely decoupled from Kuberentes API machinery and does not depend on any
+in-cluster components.
+
+This document references terms described in [Kubernetes API Conventions][1].
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD",
+"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be
+interpreted as described in [RFC 2119][2].
+
+## Use Cases
+
+_Configuration functions_ enable shift-left practices (client-side) through:
+
+- Pre-commit / delivery validation and linting of configuration
+  - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
+- Implementation of abstractions as client actuated APIs (e.g. templating)
+  - e.g. Create a client-side _"CRD"_ for generating configuration checked into git
+- Aspect Orient configuration / Injection of cross-cutting configuration
+  - e.g. T-Shirt size containers by annotating Resources with `small`, `medium`, `large`
+    and inject the cpu and memory resources into containers accordingly.
+  - e.g. Inject `init` and `side-car` containers into Resources based off of Resource
+    Type, annotations, etc.
+
+Performing these on the client rather than the server enables:
+
+- Configuration to be reviewed prior to being sent to the API server
+- Configuration to be validated as part of the CI?CD pipeline
+- Configuration for Resources to validated holistically rather than individually
+  per-Resource
+  - e.g. ensure the `Service.selector` and `Deployment.spec.template` labels
+    match.
+  - e.g. MutatingWebHooks are scoped to a single Resource instance at a time.
+- Low-level tweaks to the output of high-level abstractions
+  - e.g. add an `init container` to a client _"CRD"_ Resource after it was generated.
+- Composition and layering of multiple functions together
+  - Compose generation, injection, validation together
+
+## Spec
+
+### Input Type
+
+A function MUST accept as input a single [Kubernetes List type][3].
+The `items` field in the input will contain a sequence of [Object types][3].
+A function MAY not support [Simple types][3] and List types.
+
+An example using `v1/ConfigMapList` as input:
+
+```yaml
+apiVersion: v1
+kind: ConfigMapList
+items:
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: config1
+    data:
+      p1: v1
+      p2: v2
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: config2
+```
+
+An example using `v1/List` as input:
+
+```yaml
+apiVersion: v1
+kind: List
+items:
+  spec:
+   - apiVersion: foo-corp.com/v1
+    kind: FulfillmentCenter
+    metadata:
+      name: staging
+    address: "100 Main St."
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: namespace-reader
+    rules:
+      - resources:
+          - namespaces
+        apiGroups:
+          - ""
+        verbs:
+          - get
+          - watch
+          - list
+```
+
+In addition, a function MUST accept as input a List of kind `ResourceList` where the
+`functionConfig` field, if present, will contain the invocation-specific configuration passed to the function
+by the orchestrator.
+Functions MAY consider this field optional so that they can be triggered in an ad-hoc fashion.
+
+An example using `config.kubernetes.io/v1beta1/ResourceList` as input:
+
+```yaml
+apiVersion: config.kubernetes.io/v1beta1
+kind: ResourceList
+functionConfig:
+  apiVersion: foo-corp.com/v1
+  kind: FulfillmentCenter
+  metadata:
+    name: staging
+    metadata:
+      annotations:
+        config.k8s.io/function: |
+          container:
+            image: gcr.io/example/foo:v1.0.0
+  spec:
+    address: "100 Main St."
+items:
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: namespace-reader
+    rules:
+      - resources:
+          - namespaces
+        apiGroups:
+          - ""
+        verbs:
+          - get
+          - watch
+          - list
+```
+
+Here `FulfillmentCenter` kind with name `staging` is passed as the invocation-specific configuration
+to the function.
+
+### Output Type
+
+A function’s output MUST be the same as the input specification above
+-- i.e. `ResourceList` or `List`.
+This is necessary to enable chaining two or more functions together in a pipeline.
+The serialization format of the output SHOULD match that of its input on each invocation
+-- e.g. if the input was a `ResourceList`, the output should also be a `ResourceList`.
+
+### Serialization Format
+
+A function MUST support YAML as a serialization format for the input and output.
+A function MUST use utf8 encoding (as YAML is a superset of JSON, JSON will also be supported
+by any conforming function).
+
+### Operations
+
+A function MAY Create, Update, or Delete any number of items in the `items` field and output the
+resultant list.
+
+A function MAY modify annotations with prefix `config.kubernetes.io`, but must be careful about
+doing so since they’re used for orchestration purposes and will likely impact subsequent functions
+in the pipeline.
+
+A function SHOULD preserve comments when input serialization format is YAML.
+This allows for human authoring of configuration to coexist with changes made by functions.
+
+### Containerization
+
+A function MUST be implemented as a container.
+
+A function container MUST be capable of running as a non-root user if it does not require
+access to host filesystem or makes network calls.
+
+### stdin/stdout/stderr and Exit Codes
+
+A function MUST accept input from stdin and emit output to stdout.
+
+Any error messages MUST be emitted to stderr.
+
+An exit code of zero indicates function execution was successful.
+A non-zero exit code indicates a failure.
+
+[1]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md
+[2]: https://tools.ietf.org/html/rfc2119
+[3]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds

cmd/config/docs/commands/create-setter.md

@@ -0,0 +1,93 @@
+## create-setter
+
+[Alpha] Create a custom setter for a Resource field
+
+### Synopsis
+
+Create a custom setter for a Resource field by inlining OpenAPI as comments.
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    The name of the setter to create.
+
+  VALUE
+
+    The current value of the field, or a substring within the field.
+
+### Creating a Custom Setter
+
+**Given the YAML:**
+
+    # resource.yaml
+    apiVersion: v1
+    kind: Service
+    metadata:
+      ...
+    spec:
+      ...
+      ports:
+        ...
+      - name: http
+        port: 8080
+        ...
+
+**Create a new setter:**
+
+    # create a setter for ports
+    $ kustomize config set create DIR/ http-port 8080 --type "integer" --field "port"
+
+  Resources fields with a field name matching `--field` and field value matching `VALUE` will
+  have a line comment added marking this field as settable.
+
+**Newly modified YAML:**
+
+    # resource.yaml
+    apiVersion: v1
+    kind: Service
+    metadata:
+      ...
+    spec:
+      ...
+      ports:
+        ...
+      - name: http
+        port: 8080  # {"type":"integer","x-kustomize":{"partialFieldSetters":[{"name":"http-port","value":"8080"}]}}
+        ...
+
+  Setters may also be defined directly by editing the yaml and adding the comment.
+
+Users may not set the field value using the `set` command:
+
+    # change the http-port value to 8081
+    $ kustomize config set DIR/ http-port 8081
+
+### Using default values
+
+The default values for a setter may be:
+
+- valid field values (e.g. `8080` or `008080` for a port)
+- invalid values that adhere to the schema (e.g. `0000` for a port)
+- values that do not adhere to the schema (e.g. `[PORT]` for port)
+
+A setter may be for a substring of a full field:
+
+    $ kustomize config set create DIR/ image-tag v1.0.01 --type "string" --field "image"
+
+    image: gcr.io/example/app:v1.0.1 # # {"type":"string","x-kustomize":{"partialFieldSetters":[{"name":"image-tag","value":"v1.0.1"}]}}
+
+A single field value may have multiple setters applied to it for different parts of the field.
+
+### Examples
+
+    # create a setter for port fields matching "8080"
+    kustomize config create-setter DIR/ port 8080 --type "integer" --field port \
+         --description "default port used by the app"
+
+    # create a setter for a substring of a field rather than the full field -- e.g. only the
+    # image tag, not the full image
+    kustomize config create-setter DIR/ image-tag v1.0.1 --type "string" \
+        --field image --description "current stable release"

cmd/config/docs/commands/list-setters.md

@@ -0,0 +1,23 @@
+## set
+
+[Alpha] List setters for Resources.
+
+### Synopsis
+
+List setters for Resources.
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    Optional.  The name of the setter to display.
+
+### Examples
+
+  Show setters:
+
+    $ config set DIR/
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    name-prefix   ''            PREFIX    string   2

cmd/config/docs/commands/merge3.md

@@ -0,0 +1,23 @@
+## merge3
+
+[Alpha] Merge diff of Resource configuration files into a destination (3-way)
+
+### Synopsis
+
+[Alpha] Merge diff of Resource configuration files into a destination (3-way)
+
+Merge3 performs a 3-way merge by applying the diff between 2 sets of Resources to a 3rd set.
+
+Merge3 may be for rebasing changes to a forked set of configuration -- e.g. compute the difference between the original
+set of Resources that was forked and an updated set of those Resources, then apply that difference to the fork.
+
+If a field value differs between the ORIGINAL_DIR and UPDATED_DIR, the value from the UPDATED_DIR is taken and applied
+to the Resource in the DEST_DIR.
+
+For information on merge rules, run:
+
+	kustomize config docs-merge3
+
+### Examples
+
+    kustomize config merge3 --ancestor a/ --from b/ --to c/

cmd/config/docs/commands/run-fns.md

@@ -22,8 +22,8 @@ order they appear in the file).
 
 #### Config Functions:
 
-  Config functions are specified as Kubernetes types containing a metadata.configFn.container.image
-  field.  This field tells run how to invoke the container.
+  Config functions are specified as Kubernetes types containing a metadata.annotations.[config.kubernetes.io/function]
+  field specifying an image for the container to run.  This image tells run how to invoke the container.
 
   Example config function:
 
@@ -31,17 +31,17 @@ order they appear in the file).
 	apiVersion: fn.example.com/v1beta1
 	kind: ExampleFunctionKind
 	metadata:
-	  configFn:
-	    container:
-	      # function is invoked as a container running this image
-	      image: gcr.io/example/examplefunction:v1.0.1
 	  annotations:
+	    config.kubernetes.io/function: |
+	      container:
+	        # function is invoked as a container running this image
+	        image: gcr.io/example/examplefunction:v1.0.1
 	    config.kubernetes.io/local-config: "true" # tools should ignore this
 	spec:
 	  configField: configValue
 
   In the preceding example, 'kustomize config run example/' would identify the function by
-  the metadata.configFn field.  It would then write all Resources in the directory to
+  the metadata.annotations.[config.kubernetes.io/function] field.  It would then write all Resources in the directory to
   a container stdin (running the gcr.io/example/examplefunction:v1.0.1 image).  It
   would then write the container stdout back to example/, replacing the directory
   file contents.

cmd/config/docs/commands/set.md

@@ -0,0 +1,87 @@
+## set
+
+[Alpha] Set values on Resources fields values.
+
+### Synopsis
+
+Set values on Resources fields.  May set either the complete or partial field value.
+
+`set` identifies setters using field metadata published as OpenAPI extensions.
+`set` parses both the Kubernetes OpenAPI, as well OpenAPI published inline in
+the configuration as comments.
+
+`set` maybe be used to:
+
+- edit configuration programmatically from the cli
+- create reusable bundles of configuration with custom setters
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    Optional.  The name of the setter to perform or display.
+
+  VALUE
+
+    Optional.  The value to set on the field.
+
+
+To print the possible setters for the Resources in a directory, run `set` on
+a directory -- e.g. `kustomize config set DIR/`.
+
+#### Tips
+
+- A description of the value may be specified with `--description`.
+- The last setter for the field's value may be defined with `--set-by`.
+- Create custom setters on Resources, Kustomization.yaml's, patches, etc
+
+The description and setBy fields are left unmodified unless specified with flags.
+
+To create a custom setter for a field see: `kustomize help config create-setter`
+
+### Examples
+
+  Resource YAML: Name Prefix Setter
+
+    # DIR/resources.yaml
+    ...
+    metadata:
+        name: PREFIX-app1 # {"type":"string","x-kustomize":{"partialFieldSetters":[{"name":"name-prefix","value":"PREFIX"}]}}
+    ...
+    ---
+    ...
+    metadata:
+        name: PREFIX-app2 # {"type":"string","x-kustomize":{"partialFieldSetters":[{"name":"name-prefix","value":"PREFIX"}]}}
+    ...
+
+  List setters: Show the possible setters
+
+    $ config set DIR/
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    name-prefix   ''            PREFIX    string   2
+
+  Perform set: set a new value, owner and description
+
+    $ kustomize config set DIR/ name-prefix "test" --description "test environment" --set-by "dev"
+    set 2 values
+
+  List setters: Show the new values
+
+    $ config set DIR/
+        NAME      DESCRIPTION         VALUE     TYPE     COUNT     SETBY 
+    name-prefix   'test environment'   test     string   2          dev
+
+  New Resource YAML:
+
+    # DIR/resources.yaml
+    ...
+    metadata:
+        name: test-app1 # {"description":"test environment","type":"string","x-kustomize":{"setBy":"dev","partialFieldSetters":[{"name":"name-prefix","value":"test"}]}}
+    ...
+    ---
+    ...
+    metadata:
+        name: test-app2 # {"description":"test environment","type":"string","x-kustomize":{"setBy":"dev","partialFieldSetters":[{"name":"name-prefix","value":"test"}]}}
+    ...

cmd/config/docs/commands/sink.md

@@ -0,0 +1,18 @@
+## sink
+
+[Alpha] Implement a Sink by writing input to a local directory.
+
+### Synopsis
+
+[Alpha] Implement a Sink by writing input to a local directory.
+
+    kustomize config sink DIR
+
+  DIR:
+    Path to local directory.
+
+`sink` writes its input to a directory
+
+### Examples
+
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/

cmd/config/docs/commands/source.md

@@ -0,0 +1,21 @@
+## source
+
+[Alpha] Implement a Source by reading a local directory.
+
+### Synopsis
+
+[Alpha] Implement a Source by reading a local directory.
+
+    kustomize config source DIR
+
+  DIR:
+    Path to local directory.
+
+`source` emits configuration to act as input to a function
+
+### Examples
+
+    # emity configuration directory as input source to a function
+    kustomize config source DIR/
+
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/

cmd/config/docs/commands/tree.md

@@ -20,8 +20,9 @@ container names, etc.
 
 kustomize config tree supports printing arbitrary fields using the '--field' flag.
 
-By default, kustomize config tree uses the directory structure for the tree structure, however when printing
-from the cluster, the Resource graph structure may be used instead.
+By default, kustomize config tree uses Resource graph structure if any relationships between resources (ownerReferences)
+are detected, as is typically the case when printing from a cluster. Otherwise, directory graph structure is used. The
+graph structure can also be selected explicitly using the '--graph-structure' flag.
 
 ### Examples
 
@@ -42,8 +43,7 @@ from the cluster, the Resource graph structure may be used instead.
       --field="status.conditions[type=Completed].status"
 
     # print live Resources from a cluster using owners for graph structure
-    kubectl get all -o yaml | kustomize config tree --replicas --name --image \
-      --graph-structure=owners
+    kubectl get all -o yaml | kustomize config tree --replicas --name --image
 
     # print live Resources with status condition fields
     kubectl get all -o yaml | kustomize config tree \

cmd/config/docs/tutorials/function-basics.md

@@ -39,7 +39,7 @@
   #### 2. Modify the Generated Resources
 
   - modify the generated Resources by adding an annotation, sidecar container, etc.
-  - modify the `local-resources/example-use.yaml` by changing the replicas
+  - modify the `local-resource/example-use.yaml` by changing the replicas
 
   re-run `run`.  this will apply the updated replicas to the generated Resources,
   but keep the fields that you manually added to the generated Resource configuration.
@@ -86,7 +86,7 @@
   #### 2. Modify the Generated Resources
 
   - modify the generated Resources by adding an annotation, sidecar container, etc.
-  - modify the `local-resources/example-use.yaml` by changing the replicas
+  - modify the `local-resource/example-use.yaml` by changing the replicas
 
   re-run `run`.  this will apply the updated replicas to the generated Resources,
   but keep the fields that you manually added to the generated Resource configuration.

cmd/config/fixgomod.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+: "${kyaml_major?Need to source VERSIONS}"
+: "${kyaml_minor?Need to source VERSIONS}"
+: "${kyaml_patch?Need to source VERSIONS}"
+
+go mod edit -dropreplace=sigs.k8s.io/kustomize/kyaml@v0.0.0
+go mod edit -require=sigs.k8s.io/kustomize/kyaml@v$kyaml_major.$kyaml_minor.$kyaml_patch

cmd/config/go.mod

@@ -4,13 +4,11 @@ go 1.13
 
 require (
 	github.com/go-errors/errors v1.0.1
+	github.com/olekukonko/tablewriter v0.0.4
 	github.com/posener/complete/v2 v2.0.1-alpha.12
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	k8s.io/apimachinery v0.17.0
-	sigs.k8s.io/kustomize/kyaml v0.0.0
+	sigs.k8s.io/kustomize/kyaml v0.0.8
 )
-
-replace sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml

cmd/config/go.sum

@@ -1,8 +1,11 @@
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
@@ -16,16 +19,25 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw=
+github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -43,7 +55,6 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -51,19 +62,20 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
-github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -72,14 +84,14 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
+github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -89,12 +101,12 @@ github.com/posener/complete/v2 v2.0.1-alpha.12/go.mod h1://JlL91cS2JV7rOl6LVHrRq
 github.com/posener/script v1.0.4 h1:nSuXW5ZdmFnQIueLB2s0qvs4oNsUloM1Zydzh75v42w=
 github.com/posener/script v1.0.4/go.mod h1:Rg3ijooqulo05aGLyGsHoLmIOUzHUVK19WVgrYBPU/E=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -115,6 +127,9 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -124,12 +139,12 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -149,5 +164,7 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
+sigs.k8s.io/kustomize/kyaml v0.0.8 h1:RJeaGsiSo1X9qa2pH9NeYZEeK+5/7nIilwfDCIoW3YU=
+sigs.k8s.io/kustomize/kyaml v0.0.8/go.mod h1:tDOfJjL6slQVBLHJ76XfXAFgAOEdfm04AW2HehYOp8k=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

cmd/config/internal/commands/cat.go

@@ -121,6 +121,13 @@ func (r *CatRunner) runE(c *cobra.Command, args []string) error {
 		out = o
 	}
 
+	// remove this annotation explicitly, the ByteWriter won't clear it by
+	// default because it doesn't set it
+	clear := []string{"config.kubernetes.io/path"}
+	if r.KeepAnnotations {
+		clear = nil
+	}
+
 	var outputs []kio.Writer
 	outputs = append(outputs, kio.ByteWriter{
 		Writer:                out,
@@ -129,6 +136,7 @@ func (r *CatRunner) runE(c *cobra.Command, args []string) error {
 		WrappingAPIVersion:    r.WrapApiVersion,
 		FunctionConfig:        functionConfig,
 		Style:                 yaml.GetStyle(r.Styles...),
+		ClearAnnotations:      clear,
 	})
 
 	return handleError(c, kio.Pipeline{Inputs: inputs, Filters: fltr, Outputs: outputs}.Execute())

cmd/config/internal/commands/cat_test.go

@@ -90,8 +90,6 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
 ---
@@ -100,8 +98,6 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
   selector:
     app: nginx
@@ -114,8 +110,6 @@ metadata:
     app: nginx
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
 spec:
   replicas: 3
 `, b.String()) {
@@ -196,8 +190,6 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
 ---
@@ -206,8 +198,6 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
   selector:
     app: nginx
@@ -218,8 +208,6 @@ metadata:
   name: foo
   annotations:
     config.kubernetes.io/local-config: "true"
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
   configFn:
     container:
       image: gcr.io/example/image:version
@@ -233,8 +221,6 @@ metadata:
   name: bar
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
 spec:
   replicas: 3
 `, b.String()) {
@@ -314,8 +300,6 @@ metadata:
   name: foo
   annotations:
     config.kubernetes.io/local-config: "true"
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
   configFn:
     container:
       image: gcr.io/example/reconciler:v1
@@ -414,8 +398,6 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
 ---
@@ -424,8 +406,6 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
   selector:
     app: nginx
@@ -438,8 +418,6 @@ metadata:
     app: nginx
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
 spec:
   replicas: 3
 `, string(actual)) {
@@ -536,8 +514,6 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
 ---
@@ -546,8 +522,6 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
 spec:
   selector:
     app: nginx
@@ -560,8 +534,6 @@ metadata:
     app: nginx
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f2.yaml
 spec:
   replicas: 3
 `, string(actual)) {

cmd/config/internal/commands/cmdcreatesetter.go

@@ -0,0 +1,75 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/setters"
+)
+
+// NewCreateSetterRunner returns a command runner.
+func NewCreateSetterRunner(parent string) *CreateSetterRunner {
+	r := &CreateSetterRunner{}
+	set := &cobra.Command{
+		Use:     "create-setter DIR NAME VALUE",
+		Args:    cobra.ExactArgs(3),
+		Short:   commands.CreateSetterShort,
+		Long:    commands.CreateSetterLong,
+		Example: commands.CreateSetterExamples,
+		PreRunE: r.preRunE,
+		RunE:    r.runE,
+	}
+	set.Flags().StringVar(&r.Set.SetPartialField.SetBy, "set-by", "",
+		"set the setBy annotation.")
+	set.Flags().StringVar(&r.Set.SetPartialField.Description, "description", "",
+		"set the description of the field value.")
+	set.Flags().StringVar(&r.Set.SetPartialField.Field, "field", "",
+		"name of the field to set -- e.g. --field port")
+	set.Flags().StringVar(&r.Set.ResourceMeta.Name, "name", "",
+		"name of the Resource on which to create the setter.")
+	set.Flags().StringVar(&r.Set.ResourceMeta.Kind, "kind", "",
+		"kind of the Resource on which to create the setter.")
+	set.Flags().StringVar(&r.Set.SetPartialField.Type, "type", "",
+		"valid OpenAPI field type -- e.g. integer,boolean,string.")
+	set.Flags().BoolVar(&r.Set.SetPartialField.Partial, "partial", false,
+		"create a partial setter for only part of the field value.")
+	fixDocs(parent, set)
+	set.MarkFlagRequired("type")
+	set.MarkFlagRequired("field")
+	r.Command = set
+	return r
+}
+
+func CreateSetterCommand(parent string) *cobra.Command {
+	return NewCreateSetterRunner(parent).Command
+}
+
+type CreateSetterRunner struct {
+	Command *cobra.Command
+	Set     setters.CreateSetter
+}
+
+func (r *CreateSetterRunner) runE(c *cobra.Command, args []string) error {
+	return handleError(c, r.set(c, args))
+}
+
+func (r *CreateSetterRunner) preRunE(c *cobra.Command, args []string) error {
+	r.Set.SetPartialField.Setter.Name = args[1]
+	r.Set.SetPartialField.Setter.Value = args[2]
+	return nil
+}
+
+func (r *CreateSetterRunner) set(c *cobra.Command, args []string) error {
+	rw := &kio.LocalPackageReadWriter{PackagePath: args[0]}
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{rw},
+		Filters: []kio.Filter{&r.Set},
+		Outputs: []kio.Writer{rw}}.Execute()
+	if err != nil {
+		return err
+	}
+	return nil
+}

cmd/config/internal/commands/cmdlistsetters.go

@@ -0,0 +1,47 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/setters"
+)
+
+// NewListSettersRunner returns a command runner.
+func NewListSettersRunner(parent string) *ListSettersRunner {
+	r := &ListSettersRunner{}
+	c := &cobra.Command{
+		Use:     "list-setters DIR [NAME]",
+		Args:    cobra.RangeArgs(1, 2),
+		Short:   commands.ListSettersShort,
+		Long:    commands.ListSettersLong,
+		Example: commands.ListSettersExamples,
+		PreRunE: r.preRunE,
+		RunE:    r.runE,
+	}
+	fixDocs(parent, c)
+	r.Command = c
+	return r
+}
+
+func ListSettersCommand(parent string) *cobra.Command {
+	return NewListSettersRunner(parent).Command
+}
+
+type ListSettersRunner struct {
+	Command *cobra.Command
+	Lookup  setters.LookupSetters
+}
+
+func (r *ListSettersRunner) preRunE(c *cobra.Command, args []string) error {
+	if len(args) > 1 {
+		r.Lookup.Name = args[1]
+	}
+	return nil
+}
+
+func (r *ListSettersRunner) runE(c *cobra.Command, args []string) error {
+	return handleError(c, lookup(r.Lookup, c, args))
+}

cmd/config/internal/commands/cmdset.go

@@ -0,0 +1,130 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/olekukonko/tablewriter"
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/setters"
+)
+
+// NewSetRunner returns a command runner.
+func NewSetRunner(parent string) *SetRunner {
+	r := &SetRunner{}
+	c := &cobra.Command{
+		Use:     "set DIR [NAME] [VALUE]",
+		Args:    cobra.RangeArgs(1, 3),
+		Short:   commands.SetShort,
+		Long:    commands.SetLong,
+		Example: commands.SetExamples,
+		PreRunE: r.preRunE,
+		RunE:    r.runE,
+	}
+	fixDocs(parent, c)
+	r.Command = c
+	c.Flags().StringVar(&r.Perform.SetBy, "set-by", "",
+		"annotate the field with who set it")
+	c.Flags().StringVar(&r.Perform.Description, "description", "",
+		"annotate the field with a description of its value")
+
+	return r
+}
+
+func SetCommand(parent string) *cobra.Command {
+	return NewSetRunner(parent).Command
+}
+
+type SetRunner struct {
+	Command *cobra.Command
+	Lookup  setters.LookupSetters
+	Perform setters.PerformSetters
+}
+
+func (r *SetRunner) preRunE(c *cobra.Command, args []string) error {
+	if len(args) > 1 {
+		r.Perform.Name = args[1]
+		r.Lookup.Name = args[1]
+	}
+	if len(args) > 2 {
+		r.Perform.Value = args[2]
+	}
+
+	return nil
+}
+
+func (r *SetRunner) runE(c *cobra.Command, args []string) error {
+
+	if len(args) == 3 {
+		return handleError(c, r.perform(c, args))
+	}
+
+	return handleError(c, lookup(r.Lookup, c, args))
+}
+
+func lookup(l setters.LookupSetters, c *cobra.Command, args []string) error {
+	// lookup the setters
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.LocalPackageReader{PackagePath: args[0]}},
+		Filters: []kio.Filter{&l},
+	}.Execute()
+	if err != nil {
+		return err
+	}
+
+	table := tablewriter.NewWriter(c.OutOrStdout())
+	table.SetRowLine(false)
+	table.SetBorder(false)
+	table.SetHeaderLine(false)
+	table.SetColumnSeparator(" ")
+	table.SetCenterSeparator(" ")
+	table.SetAlignment(tablewriter.ALIGN_LEFT)
+	table.SetHeader([]string{
+		"NAME", "DESCRIPTION", "VALUE", "TYPE", "COUNT", "SETBY",
+	})
+	for i := range l.SetterCounts {
+		s := l.SetterCounts[i]
+		v := s.Value
+		if s.Value == "" {
+			v = s.Value
+		}
+		table.Append([]string{
+			s.Name,
+			"'" + s.Description + "'",
+			v,
+			fmt.Sprintf("%v", s.Type),
+			fmt.Sprintf("%d", s.Count),
+			s.SetBy,
+		})
+	}
+	table.Render()
+
+	if len(l.SetterCounts) == 0 {
+		// exit non-0 if no matching setters are found
+		os.Exit(1)
+	}
+	return nil
+}
+
+// perform the setters
+func (r *SetRunner) perform(c *cobra.Command, args []string) error {
+	rw := &kio.LocalPackageReadWriter{
+		PackagePath: args[0],
+	}
+	// perform the setters in the package
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{rw},
+		Filters: []kio.Filter{&r.Perform},
+		Outputs: []kio.Writer{rw},
+	}.Execute()
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(c.OutOrStdout(), "set %d fields\n", r.Perform.Count)
+	return nil
+}

cmd/config/internal/commands/cmdwrap_test.go

@@ -78,8 +78,8 @@ items:
       name: test
       app: nginx
     annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_deployment.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_deployment.yaml'
   spec:
     replicas: 11
     selector:
@@ -109,8 +109,8 @@ items:
       name: test
       app: nginx
     annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_service.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_service.yaml'
   spec:
     selector:
       name: test
@@ -133,8 +133,8 @@ items:
       name: test
       app: nginx
     annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_deployment.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_deployment.yaml'
   spec:
     replicas: 11
     selector:
@@ -161,8 +161,8 @@ items:
       name: test
       app: nginx
     annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_service.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_service.yaml'
   spec:
     selector:
       name: test
@@ -185,8 +185,8 @@ items:
       name: test
       app: nginx
     annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_deployment.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_deployment.yaml'
   spec:
     replicas: 11
     selector:
@@ -216,8 +216,8 @@ items:
       name: test
       app: nginx
     annotations:
-      config.kubernetes.io/index: 0
-      config.kubernetes.io/path: config/test_service.yaml
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'config/test_service.yaml'
   spec:
     selector:
       name: test

cmd/config/internal/commands/fmt.go

@@ -14,7 +14,7 @@ import (
 func GetFmtRunner(name string) *FmtRunner {
 	r := &FmtRunner{}
 	c := &cobra.Command{
-		Use:     "fmt",
+		Use:     "fmt DIR...",
 		Short:   commands.FmtShort,
 		Long:    commands.FmtLong,
 		Example: commands.FmtExamples,

cmd/config/internal/commands/grep_test.go

@@ -75,9 +75,8 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/index: 0
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
 ---
@@ -86,9 +85,8 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/index: 1
-    config.kubernetes.io/package: .
-    config.kubernetes.io/path: f1.yaml
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:
     app: nginx
@@ -145,7 +143,7 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/index: 0
+    config.kubernetes.io/index: '0'
 spec:
   replicas: 1
 ---
@@ -154,7 +152,7 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/index: 1
+    config.kubernetes.io/index: '1'
 spec:
   selector:
     app: nginx

cmd/config/internal/commands/merge3.go

@@ -0,0 +1,55 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+)
+
+func GetMerge3Runner(name string) *Merge3Runner {
+	r := &Merge3Runner{}
+	c := &cobra.Command{
+		Use:     "merge3 --ancestor [ORIGINAL_DIR] --from [UPDATED_DIR] --to [DESTINATION_DIR]",
+		Short:   commands.Merge3Short,
+		Long:    commands.Merge3Long,
+		Example: commands.Merge3Examples,
+		RunE:    r.runE,
+	}
+	fixDocs(name, c)
+	c.Flags().StringVar(&r.ancestor, "ancestor", "",
+		"Path to original package")
+	c.Flags().StringVar(&r.fromDir, "from", "",
+		"Path to updated package")
+	c.Flags().StringVar(&r.toDir, "to", "",
+		"Path to destination package")
+
+	r.Command = c
+	return r
+}
+
+func Merge3Command(name string) *cobra.Command {
+	return GetMerge3Runner(name).Command
+}
+
+// Merge3Runner contains the run function
+type Merge3Runner struct {
+	Command  *cobra.Command
+	ancestor string
+	fromDir  string
+	toDir    string
+}
+
+func (r *Merge3Runner) runE(c *cobra.Command, args []string) error {
+	err := filters.Merge3{
+		OriginalPath: r.ancestor,
+		UpdatedPath:  r.fromDir,
+		DestPath:     r.toDir,
+	}.Merge()
+	if err != nil {
+		return err
+	}
+	return nil
+}

cmd/config/internal/commands/merge3_test.go

@@ -0,0 +1,236 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+	"sigs.k8s.io/kustomize/kyaml/copyutil"
+)
+
+// TestMerge3Command verifies the merge3 correctly applies the diff between 2 sets of resources into another
+func TestMerge3Command(t *testing.T) {
+	datadir, err := ioutil.TempDir("", "test-data")
+	defer os.RemoveAll(datadir)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	err = ioutil.WriteFile(filepath.Join(datadir, "java-deployment.resource.yaml"), []byte(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5
+`), 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	expected_dir, err := ioutil.TempDir("", "test-data-expected")
+	defer os.RemoveAll(expected_dir)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	err = ioutil.WriteFile(filepath.Join(expected_dir, "java-deployment.resource.yaml"), []byte(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+    new-local: label
+    new-remote: label
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        - otherstuff
+        args:
+        - foo
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 20
+`), 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	updated_dir, err := ioutil.TempDir("", "test-data-updated")
+	defer os.RemoveAll(updated_dir)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	err = ioutil.WriteFile(filepath.Join(updated_dir, "java-deployment.resource.yaml"), []byte(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+    new-remote: label
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        - otherstuff
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5
+`), 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	dest_dir, err := ioutil.TempDir("", "test-data-dest")
+	defer os.RemoveAll(dest_dir)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	err = ioutil.WriteFile(filepath.Join(dest_dir, "java-deployment.resource.yaml"), []byte(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+    new-local: label
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        args:
+        - foo
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 20
+`), 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// Perform merge3 with newly created sets
+	r := commands.GetMerge3Runner("")
+	r.Command.SetArgs([]string{
+		"--ancestor",
+		datadir,
+		"--from",
+		updated_dir,
+		"--to",
+		dest_dir,
+	})
+	if !assert.NoError(t, r.Command.Execute()) {
+		return
+	}
+
+	diffs, err := copyutil.Diff(dest_dir, expected_dir)
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	// Verify there are no diffs
+	if !assert.Empty(t, diffs.List()) {
+		t.FailNow()
+	}
+}

cmd/config/internal/commands/run-fns.go

@@ -4,22 +4,27 @@
 package commands
 
 import (
+	"fmt"
+	"io"
+	"strings"
+
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/runfn"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
 // GetCatRunner returns a RunFnRunner.
 func GetRunFnRunner(name string) *RunFnRunner {
 	r := &RunFnRunner{}
 	c := &cobra.Command{
-		Use:     "run DIR",
-		Aliases: []string{"run-fns"},
+		Use:     "run [DIR]",
 		Short:   commands.RunFnsShort,
 		Long:    commands.RunFnsLong,
 		Example: commands.RunFnsExamples,
 		RunE:    r.runE,
-		Args:    cobra.ExactArgs(1),
+		PreRunE: r.preRunE,
 	}
 	fixDocs(name, c)
 	c.Flags().BoolVar(&r.IncludeSubpackages, "include-subpackages", true,
@@ -27,11 +32,14 @@ func GetRunFnRunner(name string) *RunFnRunner {
 	r.Command = c
 	r.Command.Flags().BoolVar(
 		&r.DryRun, "dry-run", false, "print results to stdout")
+	r.Command.Flags().BoolVar(
+		&r.GlobalScope, "global-scope", false, "set global scope for functions.")
 	r.Command.Flags().StringSliceVar(
 		&r.FnPaths, "fn-path", []string{},
-		"directories containing functions without configuration")
-	r.Command.AddCommand(XArgsCommand())
-	r.Command.AddCommand(WrapCommand())
+		"read functions from these directories instead of the configuration directory.")
+	r.Command.Flags().StringVar(
+		&r.Image, "image", "",
+		"run this image as a function instead of discovering them.")
 	return r
 }
 
@@ -44,13 +52,144 @@ type RunFnRunner struct {
 	IncludeSubpackages bool
 	Command            *cobra.Command
 	DryRun             bool
+	GlobalScope        bool
 	FnPaths            []string
+	Image              string
+	RunFns             runfn.RunFns
 }
 
 func (r *RunFnRunner) runE(c *cobra.Command, args []string) error {
-	rec := runfn.RunFns{Path: args[0], FunctionPaths: r.FnPaths}
-	if r.DryRun {
-		rec.Output = c.OutOrStdout()
-	}
-	return handleError(c, rec.Execute())
+	return handleError(c, r.RunFns.Execute())
+}
+
+// getFunctions parses the commandline flags and arguments into explicit
+// Functions to run.
+func (r *RunFnRunner) getFunctions(c *cobra.Command, args, dataItems []string) (
+	[]*yaml.RNode, error) {
+	// if image isn't specified, then Functions is empty
+	if r.Image == "" {
+		return nil, nil
+	}
+
+	// create the function spec to set as an annotation
+	fn, err := yaml.Parse(`container: {}`)
+	if err != nil {
+		return nil, err
+	}
+	// TODO: add support network, volumes, etc based on flag values
+	err = fn.PipeE(
+		yaml.Lookup("container"),
+		yaml.SetField("image", yaml.NewScalarRNode(r.Image)))
+	if err != nil {
+		return nil, err
+	}
+
+	// create the function config
+	rc, err := yaml.Parse(`
+metadata:
+  name: function-input
+data: {}
+`)
+	if err != nil {
+		return nil, err
+	}
+
+	// set the function annotation on the function config so it
+	// is parsed by RunFns
+	value, err := fn.String()
+	if err != nil {
+		return nil, err
+	}
+	err = rc.PipeE(
+		yaml.LookupCreate(yaml.MappingNode, "metadata", "annotations"),
+		yaml.SetField("config.kubernetes.io/function", yaml.NewScalarRNode(value)))
+	if err != nil {
+		return nil, err
+	}
+
+	// default the function config kind to ConfigMap, this may be overridden
+	var kind = "ConfigMap"
+	var version = "v1"
+
+	// populate the function config with data.  this is a convention for functions
+	// to be more commandline friendly
+	if len(dataItems) > 0 {
+		dataField, err := rc.Pipe(yaml.Lookup("data"))
+		if err != nil {
+			return nil, err
+		}
+		for i, s := range dataItems {
+			kv := strings.SplitN(s, "=", 2)
+			if i == 0 && len(kv) == 1 {
+				// first argument may be the kind
+				kind = s
+				continue
+			}
+			if len(kv) != 2 {
+				return nil, fmt.Errorf("args must have keys and values separated by =")
+			}
+			err := dataField.PipeE(yaml.SetField(kv[0], yaml.NewScalarRNode(kv[1])))
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	err = rc.PipeE(yaml.SetField("kind", yaml.NewScalarRNode(kind)))
+	if err != nil {
+		return nil, err
+	}
+	err = rc.PipeE(yaml.SetField("apiVersion", yaml.NewScalarRNode(version)))
+	if err != nil {
+		return nil, err
+	}
+	return []*yaml.RNode{rc}, nil
+}
+
+func (r *RunFnRunner) preRunE(c *cobra.Command, args []string) error {
+	if c.ArgsLenAtDash() >= 0 && r.Image == "" {
+		return errors.Errorf("must specify --image")
+	}
+
+	var dataItems []string
+	if c.ArgsLenAtDash() >= 0 {
+		dataItems = args[c.ArgsLenAtDash():]
+		args = args[:c.ArgsLenAtDash()]
+	}
+	if len(args) > 1 {
+		return errors.Errorf("0 or 1 arguments supported, function arguments go after '--'")
+	}
+
+	fns, err := r.getFunctions(c, args, dataItems)
+	if err != nil {
+		return err
+	}
+
+	// set the output to stdout if in dry-run mode or no arguments are specified
+	var output io.Writer
+	var input io.Reader
+	if len(args) == 0 {
+		output = c.OutOrStdout()
+		input = c.InOrStdin()
+	} else if r.DryRun {
+		output = c.OutOrStdout()
+	}
+
+	// set the path if specified as an argument
+	var path string
+	if len(args) == 1 {
+		// argument is the directory
+		path = args[0]
+	}
+
+	r.RunFns = runfn.RunFns{
+		FunctionPaths: r.FnPaths,
+		GlobalScope:   r.GlobalScope,
+		Functions:     fns,
+		Output:        output,
+		Input:         input,
+		Path:          path,
+	}
+
+	// don't consider args for the function
+	return nil
 }

cmd/config/internal/commands/run_test.go

@@ -0,0 +1,232 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestRunFnCommand_preRunE verifies that preRunE correctly parses the commandline
+// flags and arguments into the RunFns structure to be executed.
+func TestRunFnCommand_preRunE(t *testing.T) {
+	tests := []struct {
+		name          string
+		args          []string
+		expected      string
+		err           string
+		path          string
+		input         io.Reader
+		output        io.Writer
+		functionPaths []string
+	}{
+		{
+			name: "config map",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {a: b, c: d, e: f}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name:   "config map stdin / stdout",
+			args:   []string{"run", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
+			input:  os.Stdin,
+			output: os.Stdout,
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {a: b, c: d, e: f}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name:   "config map dry-run",
+			args:   []string{"run", "dir", "--image", "foo:bar", "--dry-run", "--", "a=b", "c=d", "e=f"},
+			output: os.Stdout,
+			path:   "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {a: b, c: d, e: f}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name: "config map no args",
+			args: []string{"run", "dir", "--image", "foo:bar"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name: "custom kind",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "Foo", "g=h"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {g: h}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name: "custom kind '=' in data",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "Foo", "g=h", "i=j=k"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {g: h, i: j=k}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name:          "function paths",
+			args:          []string{"run", "dir", "--fn-path", "path1", "--fn-path", "path2"},
+			path:          "dir",
+			functionPaths: []string{"path1", "path2"},
+		},
+		{
+			name: "custom kind with function paths",
+			args: []string{
+				"run", "dir", "--fn-path", "path", "--image", "foo:bar", "--", "Foo", "g=h", "i=j=k"},
+			path:          "dir",
+			functionPaths: []string{"path"},
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {g: h, i: j=k}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name: "config map multi args",
+			args: []string{"run", "dir", "dir2", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
+			err:  "0 or 1 arguments supported",
+		},
+		{
+			name: "config map not image",
+			args: []string{"run", "dir", "--", "a=b", "c=d", "e=f"},
+			err:  "must specify --image",
+		},
+		{
+			name: "config map bad data",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "a=b", "c", "e=f"},
+			err:  "must have keys and values separated by",
+		},
+	}
+
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			r := GetRunFnRunner("kustomize")
+			// Don't run the actual command
+			r.Command.Run = nil
+			r.Command.RunE = func(cmd *cobra.Command, args []string) error { return nil }
+			r.Command.SilenceErrors = true
+			r.Command.SilenceUsage = true
+
+			// hack due to https://github.com/spf13/cobra/issues/42
+			root := &cobra.Command{Use: "root"}
+			root.AddCommand(r.Command)
+			root.SetArgs(tt.args)
+
+			// error case
+			err := r.Command.Execute()
+			if tt.err != "" {
+				if !assert.Error(t, err) {
+					t.FailNow()
+				}
+				if !assert.Contains(t, err.Error(), tt.err) {
+					t.FailNow()
+				}
+				// don't check anything else in error case
+				return
+			}
+
+			// non-error case
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			// check if Input was set
+			if !assert.Equal(t, tt.input, r.RunFns.Input) {
+				t.FailNow()
+			}
+
+			// check if Output was set
+			if !assert.Equal(t, tt.output, r.RunFns.Output) {
+				t.FailNow()
+			}
+
+			// check if Path was set
+			if !assert.Equal(t, tt.path, r.RunFns.Path) {
+				t.FailNow()
+			}
+
+			// check if FunctionPaths were set
+			if tt.functionPaths == nil {
+				// make Equal work against flag default
+				tt.functionPaths = []string{}
+			}
+			if !assert.Equal(t, tt.functionPaths, r.RunFns.FunctionPaths) {
+				t.FailNow()
+			}
+
+			// check if Functions were set
+			if tt.expected != "" {
+				if !assert.Len(t, r.RunFns.Functions, 1) {
+					t.FailNow()
+				}
+				actual := strings.TrimSpace(r.RunFns.Functions[0].MustString())
+				if !assert.Equal(t, strings.TrimSpace(tt.expected), actual) {
+					t.FailNow()
+				}
+			}
+
+		})
+	}
+
+}

cmd/config/internal/commands/sink.go

@@ -0,0 +1,48 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+// GetSinkRunner returns a command for Sink.
+func GetSinkRunner(name string) *SinkRunner {
+	r := &SinkRunner{}
+	c := &cobra.Command{
+		Use:     "sink DIR",
+		Short:   commands.SinkShort,
+		Long:    commands.SinkLong,
+		Example: commands.SinkExamples,
+		RunE:    r.runE,
+		Args:    cobra.ExactArgs(1),
+	}
+	fixDocs(name, c)
+	r.Command = c
+	return r
+}
+
+func SinkCommand(name string) *cobra.Command {
+	return GetSinkRunner(name).Command
+}
+
+// SinkRunner contains the run function
+type SinkRunner struct {
+	Command *cobra.Command
+}
+
+func (r *SinkRunner) runE(c *cobra.Command, args []string) error {
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}},
+		Outputs: []kio.Writer{
+			&kio.LocalPackageWriter{
+				PackagePath:      args[0],
+				ClearAnnotations: []string{"config.kubernetes.io/path"},
+			},
+		},
+	}.Execute()
+	return handleError(c, err)
+}

cmd/config/internal/commands/sink_test.go

@@ -0,0 +1,140 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+)
+
+func TestSinkCommand(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(d)
+
+	// fmt the files
+	b := &bytes.Buffer{}
+	r := commands.GetSinkRunner("")
+	r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    selector:
+      app: nginx
+- apiVersion: v1
+  kind: Abstraction
+  metadata:
+    name: foo
+    annotations:
+      config.kubernetes.io/function: |
+        container:
+          image: gcr.io/example/reconciler:v1
+      config.kubernetes.io/local-config: "true"
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    labels:
+      app: nginx
+    name: bar
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+`))
+	r.Command.SetArgs([]string{d})
+	r.Command.SetOut(b)
+	if !assert.NoError(t, r.Command.Execute()) {
+		t.FailNow()
+	}
+
+	actual, err := ioutil.ReadFile(filepath.Join(d, "f1.yaml"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	expected := `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`
+	if !assert.Equal(t, expected, string(actual)) {
+		t.FailNow()
+	}
+
+	actual, err = ioutil.ReadFile(filepath.Join(d, "f2.yaml"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	expected = `apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: gcr.io/example/reconciler:v1
+    config.kubernetes.io/local-config: "true"
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+spec:
+  replicas: 3
+`
+	if !assert.Equal(t, expected, string(actual)) {
+		t.FailNow()
+	}
+}

cmd/config/internal/commands/source.go

@@ -0,0 +1,77 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// GetSourceRunner returns a command for Source.
+func GetSourceRunner(name string) *SourceRunner {
+	r := &SourceRunner{}
+	c := &cobra.Command{
+		Use:     "source DIR",
+		Short:   commands.SourceShort,
+		Long:    commands.SourceLong,
+		Example: commands.SourceExamples,
+		RunE:    r.runE,
+		Args:    cobra.ExactArgs(1),
+	}
+	fixDocs(name, c)
+	c.Flags().StringVar(&r.WrapKind, "wrap-kind", kio.ResourceListKind,
+		"output using this format.")
+	c.Flags().StringVar(&r.WrapApiVersion, "wrap-version", kio.ResourceListAPIVersion,
+		"output using this format.")
+	c.Flags().StringVar(&r.FunctionConfig, "function-config", "",
+		"path to function config.")
+	r.Command = c
+	_ = c.MarkFlagFilename("function-config", "yaml", "json", "yml")
+	return r
+}
+
+func SourceCommand(name string) *cobra.Command {
+	return GetSourceRunner(name).Command
+}
+
+// SourceRunner contains the run function
+type SourceRunner struct {
+	WrapKind       string
+	WrapApiVersion string
+	FunctionConfig string
+	Command        *cobra.Command
+}
+
+func (r *SourceRunner) runE(c *cobra.Command, args []string) error {
+	// if there is a function-config specified, emit it
+	var functionConfig *yaml.RNode
+	if r.FunctionConfig != "" {
+		configs, err := kio.LocalPackageReader{PackagePath: r.FunctionConfig}.Read()
+		if err != nil {
+			return err
+		}
+		if len(configs) != 1 {
+			return fmt.Errorf("expected exactly 1 functionConfig, found %d", len(configs))
+		}
+		functionConfig = configs[0]
+	}
+
+	var outputs []kio.Writer
+	outputs = append(outputs, kio.ByteWriter{
+		Writer:                c.OutOrStdout(),
+		KeepReaderAnnotations: true,
+		WrappingKind:          r.WrapKind,
+		WrappingAPIVersion:    r.WrapApiVersion,
+		FunctionConfig:        functionConfig,
+	})
+
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: args[0]}},
+		Outputs: outputs}.Execute()
+	return handleError(c, err)
+}

cmd/config/internal/commands/source_test.go

@@ -0,0 +1,136 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+)
+
+func TestSourceCommand(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer os.RemoveAll(d)
+
+	err = ioutil.WriteFile(filepath.Join(d, "f1.yaml"), []byte(`
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`), 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+	err = ioutil.WriteFile(filepath.Join(d, "f2.yaml"), []byte(`
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: gcr.io/example/reconciler:v1
+    config.kubernetes.io/local-config: "true"
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+spec:
+  replicas: 3
+`), 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// fmt the files
+	b := &bytes.Buffer{}
+	r := commands.GetSourceRunner("")
+	r.Command.SetArgs([]string{d})
+	r.Command.SetOut(b)
+	if !assert.NoError(t, r.Command.Execute()) {
+		return
+	}
+
+	if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    selector:
+      app: nginx
+- apiVersion: v1
+  kind: Abstraction
+  metadata:
+    name: foo
+    annotations:
+      config.kubernetes.io/function: |
+        container:
+          image: gcr.io/example/reconciler:v1
+      config.kubernetes.io/local-config: "true"
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    labels:
+      app: nginx
+    name: bar
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+`, b.String()) {
+		return
+	}
+}

cmd/config/internal/commands/tree.go

@@ -45,7 +45,7 @@ func GetTreeRunner(name string) *TreeRunner {
 		"if true, include local-config in the output.")
 	c.Flags().BoolVar(&r.excludeNonLocal, "exclude-non-local", false,
 		"if true, exclude non-local-config in the output.")
-	c.Flags().StringVar(&r.structure, "graph-structure", "directory",
+	c.Flags().StringVar(&r.structure, "graph-structure", "",
 		"Graph structure to use for printing the tree.  may be any of: "+
 			strings.Join(kio.GraphStructures, ","))
 

cmd/config/internal/commands/tree_test.go

@@ -104,7 +104,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -118,7 +117,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -132,7 +130,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -146,7 +143,6 @@ metadata:
   namespace: default2
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -160,7 +156,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx3
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -171,8 +166,7 @@ metadata:
     app: nginx
   annotations:
     app: nginx
-    config.kubernetes.io/package: bar-package
-    config.kubernetes.io/path: f2.yaml
+    config.kubernetes.io/path: bar-package/f2.yaml
   name: bar
 spec:
   replicas: 3
@@ -183,7 +177,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   selector:

cmd/config/internal/generateddocs/api/docs.go

@@ -4,304 +4,23 @@
 // Code generated by "mdtogo"; DO NOT EDIT.
 package api
 
-var ConfigFnLong = `# Configuration Functions API Semantics
-
-  Configuration Functions are functions packaged as executables in containers which enable
-  **shift-left practices**.  They configure applications and infrastructure through
-  Kubernetes style Resource Configuration, but run locally pre-commit.
-
-  Configuration functions enable shift-left practices (client-side) through:
-
-  - Pre-commit / delivery validation and linting of configuration
-    - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
-  - Implementation of abstractions as client actuated APIs (e.g. templating)
-    - e.g. Create a client-side *"CRD"* for generating configuration checked into git
-  - Aspect Orient configuration / Injection of cross-cutting configuration
-    - e.g. T-Shirt size containers by annotating Resources with ` + "`" + `small` + "`" + `, ` + "`" + `medium` + "`" + `, ` + "`" + `large` + "`" + `
-      and inject the cpu and memory resources into containers accordingly.
-    - e.g. Inject ` + "`" + `init` + "`" + ` and ` + "`" + `side-car` + "`" + ` containers into Resources based off of Resource
-      Type, annotations, etc.
-
-  Performing these on the client rather than the server enables:
-
-  - Configuration to be reviewed prior to being sent to the API server
-  - Configuration to be validated as part of the CD pipeline
-  - Configuration for Resources to validated holistically rather than individually
-    per-Resource -- e.g. ensure the ` + "`" + `Service.selector` + "`" + ` and ` + "`" + `Deployment.spec.template` + "`" + ` labels
-    match.
-    - MutatingWebHooks are scoped to a single Resource instance at a time.
-  - Low-level tweaks to the output of high-level abstractions -- e.g. add an ` + "`" + `init container` + "`" + `
-    to a client *"CRD"* Resource after it was generated.
-  - Composition and layering of multiple functions together
-    - Compose generation, injection, validation together
-
-  Configuration Functions are implemented as executable programs published in containers which:
-
-  - Accept as input (stdin):
-    - A list of Resource Configuration
-    - A Function Configuration (to configure the function itself)
-  - Emit as output (stdout + exit):
-    - A list of Resource Configuration
-    - An exit code for success / failure
-
-### Function Specification
-
-  - Functions **SHOULD** be published as container images containing a ` + "`" + `CMD` + "`" + ` invoking an executable.
-  - Functions **MUST** accept input on STDIN a ` + "`" + `ResourceList` + "`" + ` containing the Resources and
-    ` + "`" + `functionConfig` + "`" + `.
-  - Functions **MUST** emit output on STDOUT a ` + "`" + `ResourceList` + "`" + ` containing the modified
-    Resources.
-  - Functions **MUST** exit non-0 on failure, and exit 0 on success.
-  - Functions **MAY** emit output on STDERR with error messaging.
-  - Functions performing validation **SHOULD** exit failure and emit error messaging
-    on a validation failure.
-  - Functions generating Resources **SHOULD** retain non-conflicting changes on the
-    generated Resources -- e.g. 1. the function generates a Deployment, but doesn't
-    specify ` + "`" + `cpu` + "`" + `, 2. the user sets the ` + "`" + `cpu` + "`" + ` on the generated Resource, 3. the
-    function should keep the ` + "`" + `cpu` + "`" + ` when regenerating the Resource a second time.
-  - Functions **SHOULD** be usable outside ` + "`" + `kustomize config run` + "`" + ` -- e.g. though pipeline
-    mechanisms such as Tekton.
-
-#### Input Format
-
-  Functions must accept on STDIN:
-
-  ` + "`" + `ResourceList` + "`" + `:
-  - contains ` + "`" + `items` + "`" + ` field, same as ` + "`" + `List.items` + "`" + `
-  - contains ` + "`" + `functionConfig` + "`" + ` field -- a single item with the configuration for the function itself
-
-  Example ` + "`" + `ResourceList` + "`" + ` Input:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    functionConfig:
-      apiVersion: example.com/v1beta1
-      kind: Nginx
-      metadata:
-        name: my-instance
-        annotations:
-          config.kubernetes.io/local-config: "true"
-      spec:
-        replicas: 5
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 3
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Output Format
-
-  Functions must emit on STDOUT:
-
-  ` + "`" + `ResourceList` + "`" + `:
-  - contains ` + "`" + `items` + "`" + ` field, same as ` + "`" + `List.items` + "`" + `
-
-  Example ` + "`" + `ResourceList` + "`" + ` Output:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 5
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Container Environment
-
-  When run by ` + "`" + `kustomize config run` + "`" + `, functions are run in containers with the
-  following environment:
-
-  - Network: ` + "`" + `none` + "`" + `
-  - User: ` + "`" + `nobody` + "`" + `
-  - Security Options: ` + "`" + `no-new-privileges` + "`" + `
-  - Volumes: the volume containing the ` + "`" + `functionConfig` + "`" + ` yaml is mounted under ` + "`" + `/local` + "`" + ` as ` + "`" + `ro` + "`" + `
-
-### Example Function Implementation
-
-  Following is an example for implementing an nginx abstraction using a config
-  function.
-
-#### ` + "`" + `nginx-template.sh` + "`" + `
-
-  ` + "`" + `nginx-template.sh` + "`" + ` is a simple bash script which uses a *heredoc* as a templating solution
-  for generating Resources from the functionConfig input fields.
-
-  The script wraps itself using ` + "`" + `config run wrap -- $0` + "`" + ` which will:
-
-  1. Parse the ` + "`" + `ResourceList.functionConfig` + "`" + ` (provided to the container stdin) into env vars
-  2. Merge the stdout into the original list of Resources
-  3. Defaults filenames for newly generated Resources (if they are not set as annotations)
-     to ` + "`" + `config/NAME_KIND.yaml` + "`" + `
-  4. Format the output
-
-    #!/bin/bash
-    # script must run wrapped by ` + "`" + `kustomize config run wrap` + "`" + `
-    # for parsing input the functionConfig into env vars
-    if [ -z ${WRAPPED} ]; then
-      export WRAPPED=true
-      config run wrap -- $0
-      exit $?
-    fi
-
-    cat <<End-of-message
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: ${NAME}
-    ---
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      replicas: ${REPLICAS}
-      selector:
-        matchLabels:
-          app: nginx
-          instance: ${NAME}
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: ${NAME}
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-    End-of-message
-
-#### ` + "`" + `Dockerfile` + "`" + `
-
-  ` + "`" + `Dockerfile` + "`" + ` installs ` + "`" + `kustomize config` + "`" + ` and copies the script into the container image.
-
-    FROM golang:1.13-stretch
-    RUN go get sigs.k8s.io/kustomize/cmd/config
-    RUN mv /go/bin/config /usr/bin/config
-    COPY nginx-template.sh /usr/bin/nginx-template.sh
-    CMD ["nginx-template.sh]
-
-### Example Function Usage
-
-Following is an example of running the ` + "`" + `kustomize config run` + "`" + ` using the preceding API.
-
-#### ` + "`" + `nginx.yaml` + "`" + ` (Input)
-
-  ` + "`" + `dir/nginx.yaml` + "`" + ` contains a reference to the Function.  The contents of ` + "`" + `nginx.yaml` + "`" + `
-  are passed to the Function through the ` + "`" + `ResourceList.functionConfig` + "`" + ` field.
-
-    apiVersion: example.com/v1beta1
-    kind: Nginx
-    metadata:
-      name: my-instance
-      annotations:
-        config.kubernetes.io/local-config: "true"
-      configFn:
-        container:
-          image: gcr.io/example-functions/nginx-template:v1.0.0
-    spec:
-      replicas: 5
-
-  - ` + "`" + `configFn.container.image` + "`" + `: the image to use for this API
-  - ` + "`" + `annotations[config.kubernetes.io/local-config]` + "`" + `: mark this as not a Resource that should
-    be applied
-
-#### ` + "`" + `kustomize config run dir/` + "`" + ` (Output)
-
-  ` + "`" + `dir/my-instance_deployment.yaml` + "`" + ` contains the Deployment:
-
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      replicas: 5
-      selector:
-        matchLabels:
-          app: nginx
-          instance: my-instance
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: my-instance
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-
-  ` + "`" + `dir/my-instance_service.yaml` + "`" + ` contains the Service:
-
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: my-instance`
-
 var ConfigIoLong = `# Configuration IO API Semantics
 
 Resource Configuration may be read / written from / to sources such as directories,
 stdin|out or network. Tools may be composed using pipes such that the tools writing
 Resource Configuration may be a different tool from the one that read the configuration.
 In order for tools to be composed in this way, while preserving origin information --
-such as the original file, index, etc.
+such as the original file, index, etc.:
 
-Tools **SHOULD** write the following annotations when reading from sources,
-and **SHOULD** respect the annotations when writing to sinks.
+Tools **SHOULD** insert the following annotations when reading from sources,
+and **SHOULD** delete the annotations when writing to sinks.
 
 ### ` + "`" + `config.kubernetes.io/path` + "`" + `
 
 Records the slash-delimited, OS-agnostic, relative file path to a Resource.
 
 This annotation **SHOULD** be set when reading Resources from files.
+It **SHOULD** be unset when writing Resources to files.
 When writing Resources to a directory, the Resource **SHOULD** be written to the corresponding
 path relative to that directory.
 
@@ -313,10 +32,11 @@ Example:
 
 ### ` + "`" + `config.kubernetes.io/index` + "`" + `
 
-Records the index of a Resource in file. In a multi-object files YAML file, Resources are separated
+Records the index of a Resource in file. In a multi-object YAML file, Resources are separated
 by three dashes (` + "`" + `---` + "`" + `), and the index represents the positon of the Resource starting from zero.
 
 This annotation **SHOULD** be set when reading Resources from files.
+It **SHOULD** be unset when writing Resources to files.
 When writing multiple Resources to the same file, the Resource **SHOULD** be written in the
 relative order matching the index.
 
@@ -344,6 +64,355 @@ Example:
 	  annotations:
 	    config.kubernetes.io/local-config: "true"`
 
+var FunctionsImplShort = `Following is an example for implementing an nginx abstraction using a configuration`
+var FunctionsImplLong = `# Running Configuration Functions using kustomize CLI
+
+Configuration functions can be implemented using any toolchain and invoked using any
+container workflow orchestrator including Tekton, Cloud Build, or run directly using ` + "`" + `docker run` + "`" + `.
+
+Run ` + "`" + `config help docs-fn-spec` + "`" + ` to see the Configuration Functions Specification.
+
+` + "`" + `kustomize config run` + "`" + ` is an example orchestrator for invoking Configuration Functions. This
+document describes how to implement and invoke an example function.
+
+function.
+
+### ` + "`" + `nginx-template.sh` + "`" + `
+
+` + "`" + `nginx-template.sh` + "`" + ` is a simple bash script which uses a _heredoc_ as a templating solution
+for generating Resources from the functionConfig input fields.
+
+The script wraps itself using ` + "`" + `config run wrap -- $0` + "`" + ` which will:
+
+1. Parse the ` + "`" + `ResourceList.functionConfig` + "`" + ` (provided to the container stdin) into env vars
+2. Merge the stdout into the original list of Resources
+3. Defaults filenames for newly generated Resources (if they are not set as annotations)
+   to ` + "`" + `config/NAME_KIND.yaml` + "`" + `
+4. Format the output
+
+	#!/bin/bash
+	# script must run wrapped by "kustomize config run wrap"
+	# for parsing input the functionConfig into env vars
+	if [ -z ${WRAPPED} ]; then
+	  export WRAPPED=true
+	  config run wrap -- $0
+	  exit $?
+	fi
+	
+	cat <<End-of-message
+	apiVersion: v1
+	kind: Service
+	metadata:
+	  name: ${NAME}
+	  labels:
+	    app: nginx
+	    instance: ${NAME}
+	spec:
+	  ports:
+	  - port: 80
+	    targetPort: 80
+	    name: http
+	  selector:
+	    app: nginx
+	    instance: ${NAME}
+	---
+	apiVersion: apps/v1
+	kind: Deployment
+	metadata:
+	  name: ${NAME}
+	  labels:
+	    app: nginx
+	    instance: ${NAME}
+	spec:
+	  replicas: ${REPLICAS}
+	  selector:
+	    matchLabels:
+	      app: nginx
+	      instance: ${NAME}
+	  template:
+	    metadata:
+	      labels:
+	        app: nginx
+	        instance: ${NAME}
+	    spec:
+	      containers:
+	      - name: nginx
+	        image: nginx:1.7.9
+	        ports:
+	        - containerPort: 80
+	End-of-message
+
+### Dockerfile
+
+` + "`" + `Dockerfile` + "`" + ` installs ` + "`" + `kustomize config` + "`" + ` and copies the script into the container image.
+
+	FROM golang:1.13-stretch
+	RUN go get sigs.k8s.io/kustomize/cmd/config
+	RUN mv /go/bin/config /usr/bin/config
+	COPY nginx-template.sh /usr/bin/nginx-template.sh
+	CMD ["nginx-template.sh]
+
+## Example Function Usage
+
+Following is an example of running the ` + "`" + `kustomize config run` + "`" + ` using the preceding API.
+
+When run by ` + "`" + `kustomize config run` + "`" + `, functions are run in containers with the
+following environment:
+
+- Network: ` + "`" + `none` + "`" + `
+- User: ` + "`" + `nobody` + "`" + `
+- Security Options: ` + "`" + `no-new-privileges` + "`" + `
+- Volumes: the volume containing the ` + "`" + `functionConfig` + "`" + ` yaml is mounted under ` + "`" + `/local` + "`" + ` as ` + "`" + `ro` + "`" + `
+
+### Input
+
+` + "`" + `dir/nginx.yaml` + "`" + ` contains a reference to the Function. The contents of ` + "`" + `nginx.yaml` + "`" + `
+are passed to the Function through the ` + "`" + `ResourceList.functionConfig` + "`" + ` field.
+
+	apiVersion: example.com/v1beta1
+	kind: Nginx
+	metadata:
+	  name: my-instance
+	  annotations:
+	    config.kubernetes.io/local-config: "true"
+	    config.k8s.io/function: |
+	      container:
+	        image: gcr.io/example-functions/nginx-template:v1.0.0
+	spec:
+	  replicas: 5
+
+- ` + "`" + `annotations[config.k8s.io/function].container.image` + "`" + `: the image to use for this API
+- ` + "`" + `annotations[config.kubernetes.io/local-config]` + "`" + `: mark this as not a Resource that should
+  be applied
+
+### Output
+
+The function is invoked using by runing ` + "`" + `kustomize config run dir/` + "`" + `.
+
+` + "`" + `dir/my-instance_deployment.yaml` + "`" + ` contains the Deployment:
+
+	apiVersion: apps/v1
+	kind: Deployment
+	metadata:
+	  name: my-instance
+	  labels:
+	    app: nginx
+	    instance: my-instance
+	spec:
+	  replicas: 5
+	  selector:
+	    matchLabels:
+	      app: nginx
+	      instance: my-instance
+	  template:
+	    metadata:
+	      labels:
+	        app: nginx
+	        instance: my-instance
+	    spec:
+	      containers:
+	        - name: nginx
+	          image: nginx:1.7.9
+	          ports:
+	            - containerPort: 80
+
+` + "`" + `dir/my-instance_service.yaml` + "`" + ` contains the Service:
+
+	apiVersion: v1
+	kind: Service
+	metadata:
+	  name: my-instance
+	  labels:
+	    app: nginx
+	    instance: my-instance
+	spec:
+	  ports:
+	    - port: 80
+	      targetPort: 80
+	      name: http
+	  selector:
+	    app: nginx
+	    instance: my-instance`
+
+var FunctionsSpecShort = `_Configuration functions_ enable shift-left practices (client-side) through:`
+var FunctionsSpecLong = `# Configuration Functions Specification
+
+This document specifies a standard for client-side functions that operate on
+Kubernetes declarative configurations. This standard enables creating
+small, interoperable, and language-independent executable programs packaged as
+containers that can be chained together as part of a configuration management pipeline.
+The end result of such a pipeline are fully rendered configurations that can then be
+applied to a control plane (e.g. Using ‘kubectl apply’ for Kubernetes control plane).
+As such, although this document references Kubernetes Resource Model and API conventions,
+it is completely decoupled from Kuberentes API machinery and does not depend on any
+in-cluster components.
+
+This document references terms described in [Kubernetes API Conventions][1].
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD",
+"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be
+interpreted as described in [RFC 2119][2].
+
+
+- Pre-commit / delivery validation and linting of configuration
+  - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
+- Implementation of abstractions as client actuated APIs (e.g. templating)
+  - e.g. Create a client-side _"CRD"_ for generating configuration checked into git
+- Aspect Orient configuration / Injection of cross-cutting configuration
+  - e.g. T-Shirt size containers by annotating Resources with ` + "`" + `small` + "`" + `, ` + "`" + `medium` + "`" + `, ` + "`" + `large` + "`" + `
+    and inject the cpu and memory resources into containers accordingly.
+  - e.g. Inject ` + "`" + `init` + "`" + ` and ` + "`" + `side-car` + "`" + ` containers into Resources based off of Resource
+    Type, annotations, etc.
+
+Performing these on the client rather than the server enables:
+
+- Configuration to be reviewed prior to being sent to the API server
+- Configuration to be validated as part of the CI?CD pipeline
+- Configuration for Resources to validated holistically rather than individually
+  per-Resource
+  - e.g. ensure the ` + "`" + `Service.selector` + "`" + ` and ` + "`" + `Deployment.spec.template` + "`" + ` labels
+    match.
+  - e.g. MutatingWebHooks are scoped to a single Resource instance at a time.
+- Low-level tweaks to the output of high-level abstractions
+  - e.g. add an ` + "`" + `init container` + "`" + ` to a client _"CRD"_ Resource after it was generated.
+- Composition and layering of multiple functions together
+  - Compose generation, injection, validation together
+
+## Spec
+
+### Input Type
+
+A function MUST accept as input a single [Kubernetes List type][3].
+The ` + "`" + `items` + "`" + ` field in the input will contain a sequence of [Object types][3].
+A function MAY not support [Simple types][3] and List types.
+
+An example using ` + "`" + `v1/ConfigMapList` + "`" + ` as input:
+
+	apiVersion: v1
+	kind: ConfigMapList
+	items:
+	  - apiVersion: v1
+	    kind: ConfigMap
+	    metadata:
+	      name: config1
+	    data:
+	      p1: v1
+	      p2: v2
+	  - apiVersion: v1
+	    kind: ConfigMap
+	    metadata:
+	      name: config2
+
+An example using ` + "`" + `v1/List` + "`" + ` as input:
+
+	apiVersion: v1
+	kind: List
+	items:
+	  spec:
+	   - apiVersion: foo-corp.com/v1
+	    kind: FulfillmentCenter
+	    metadata:
+	      name: staging
+	    address: "100 Main St."
+	  - apiVersion: rbac.authorization.k8s.io/v1
+	    kind: ClusterRole
+	    metadata:
+	      name: namespace-reader
+	    rules:
+	      - resources:
+	          - namespaces
+	        apiGroups:
+	          - ""
+	        verbs:
+	          - get
+	          - watch
+	          - list
+
+In addition, a function MUST accept as input a List of kind ` + "`" + `ResourceList` + "`" + ` where the
+` + "`" + `functionConfig` + "`" + ` field, if present, will contain the invocation-specific configuration passed to the function
+by the orchestrator.
+Functions MAY consider this field optional so that they can be triggered in an ad-hoc fashion.
+
+An example using ` + "`" + `config.kubernetes.io/v1beta1/ResourceList` + "`" + ` as input:
+
+	apiVersion: config.kubernetes.io/v1beta1
+	kind: ResourceList
+	functionConfig:
+	  apiVersion: foo-corp.com/v1
+	  kind: FulfillmentCenter
+	  metadata:
+	    name: staging
+	    metadata:
+	      annotations:
+	        config.k8s.io/function: |
+	          container:
+	            image: gcr.io/example/foo:v1.0.0
+	  spec:
+	    address: "100 Main St."
+	items:
+	  - apiVersion: rbac.authorization.k8s.io/v1
+	    kind: ClusterRole
+	    metadata:
+	      name: namespace-reader
+	    rules:
+	      - resources:
+	          - namespaces
+	        apiGroups:
+	          - ""
+	        verbs:
+	          - get
+	          - watch
+	          - list
+
+Here ` + "`" + `FulfillmentCenter` + "`" + ` kind with name ` + "`" + `staging` + "`" + ` is passed as the invocation-specific configuration
+to the function.
+
+### Output Type
+
+A function’s output MUST be the same as the input specification above
+-- i.e. ` + "`" + `ResourceList` + "`" + ` or ` + "`" + `List` + "`" + `.
+This is necessary to enable chaining two or more functions together in a pipeline.
+The serialization format of the output SHOULD match that of its input on each invocation
+-- e.g. if the input was a ` + "`" + `ResourceList` + "`" + `, the output should also be a ` + "`" + `ResourceList` + "`" + `.
+
+### Serialization Format
+
+A function MUST support YAML as a serialization format for the input and output.
+A function MUST use utf8 encoding (as YAML is a superset of JSON, JSON will also be supported
+by any conforming function).
+
+### Operations
+
+A function MAY Create, Update, or Delete any number of items in the ` + "`" + `items` + "`" + ` field and output the
+resultant list.
+
+A function MAY modify annotations with prefix ` + "`" + `config.kubernetes.io` + "`" + `, but must be careful about
+doing so since they’re used for orchestration purposes and will likely impact subsequent functions
+in the pipeline.
+
+A function SHOULD preserve comments when input serialization format is YAML.
+This allows for human authoring of configuration to coexist with changes made by functions.
+
+### Containerization
+
+A function MUST be implemented as a container.
+
+A function container MUST be capable of running as a non-root user if it does not require
+access to host filesystem or makes network calls.
+
+### stdin/stdout/stderr and Exit Codes
+
+A function MUST accept input from stdin and emit output to stdout.
+
+Any error messages MUST be emitted to stderr.
+
+An exit code of zero indicates function execution was successful.
+A non-zero exit code indicates a failure.
+
+[1]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md
+[2]: https://tools.ietf.org/html/rfc2119
+[3]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds`
+
 var Merge2Long = `# Merge (2-way)
 
   2-way merges fields from a source to a destination, overriding the destination fields

cmd/config/internal/generateddocs/commands/docs.go

@@ -49,6 +49,32 @@ var CountExamples = `
     # print Resource counts from a directory
     kustomize config count my-dir/`
 
+var CreateSetterShort = `[Alpha] Create a custom setter for a Resource field`
+var CreateSetterLong = `
+Create a custom setter for a Resource field by inlining OpenAPI as comments.
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    The name of the setter to create.
+
+  VALUE
+
+    The current value of the field, or a substring within the field.
+`
+var CreateSetterExamples = `
+    # create a setter for port fields matching "8080"
+    kustomize config create-setter DIR/ port 8080 --type "integer" --field port \
+         --description "default port used by the app"
+
+    # create a setter for a substring of a field rather than the full field -- e.g. only the
+    # image tag, not the full image
+    kustomize config create-setter DIR/ image-tag v1.0.1 --type "string" \
+        --field image --description "current stable release"`
+
 var FmtShort = `[Alpha] Format yaml configuration files.`
 var FmtLong = `
 [Alpha] Format yaml configuration files.
@@ -116,6 +142,25 @@ var GrepExamples = `
     # look for Resources matching a specific container image
     kustomize config grep "spec.template.spec.containers[name=nginx].image=nginx:1\.7\.9" my-dir/ | kustomize config tree`
 
+var ListSettersShort = `[Alpha] List setters for Resources.`
+var ListSettersLong = `
+List setters for Resources.
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    Optional.  The name of the setter to display.
+`
+var ListSettersExamples = `
+  Show setters:
+
+    $ config set DIR/
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    name-prefix   ''            PREFIX    string   2`
+
 var MergeShort = `[Alpha] Merge Resource configuration files`
 var MergeLong = `
 [Alpha] Merge Resource configuration files
@@ -136,6 +181,25 @@ For information on merge rules, run:
 var MergeExamples = `
     cat resources_and_patches.yaml | kustomize config merge > merged_resources.yaml`
 
+var Merge3Short = `[Alpha] Merge diff of Resource configuration files into a destination (3-way)`
+var Merge3Long = `
+[Alpha] Merge diff of Resource configuration files into a destination (3-way)
+
+Merge3 performs a 3-way merge by applying the diff between 2 sets of Resources to a 3rd set.
+
+Merge3 may be for rebasing changes to a forked set of configuration -- e.g. compute the difference between the original
+set of Resources that was forked and an updated set of those Resources, then apply that difference to the fork.
+
+If a field value differs between the ORIGINAL_DIR and UPDATED_DIR, the value from the UPDATED_DIR is taken and applied
+to the Resource in the DEST_DIR.
+
+For information on merge rules, run:
+
+	kustomize config docs-merge3
+`
+var Merge3Examples = `
+    kustomize config merge3 --ancestor a/ --from b/ --to c/`
+
 var RunFnsShort = `[Alpha] Reoncile config functions to Resources.`
 var RunFnsLong = `
 [Alpha] Reconcile config functions to Resources.
@@ -156,8 +220,8 @@ order they appear in the file).
 
 #### Config Functions:
 
-  Config functions are specified as Kubernetes types containing a metadata.configFn.container.image
-  field.  This field tells run how to invoke the container.
+  Config functions are specified as Kubernetes types containing a metadata.annotations.[config.kubernetes.io/function]
+  field specifying an image for the container to run.  This image tells run how to invoke the container.
 
   Example config function:
 
@@ -165,17 +229,17 @@ order they appear in the file).
 	apiVersion: fn.example.com/v1beta1
 	kind: ExampleFunctionKind
 	metadata:
-	  configFn:
-	    container:
-	      # function is invoked as a container running this image
-	      image: gcr.io/example/examplefunction:v1.0.1
 	  annotations:
+	    config.kubernetes.io/function: |
+	      container:
+	        # function is invoked as a container running this image
+	        image: gcr.io/example/examplefunction:v1.0.1
 	    config.kubernetes.io/local-config: "true" # tools should ignore this
 	spec:
 	  configField: configValue
 
   In the preceding example, 'kustomize config run example/' would identify the function by
-  the metadata.configFn field.  It would then write all Resources in the directory to
+  the metadata.annotations.[config.kubernetes.io/function] field.  It would then write all Resources in the directory to
   a container stdin (running the gcr.io/example/examplefunction:v1.0.1 image).  It
   would then write the container stdout back to example/, replacing the directory
   file contents.
@@ -185,6 +249,120 @@ order they appear in the file).
 var RunFnsExamples = `
 kustomize config run example/`
 
+var SetShort = `[Alpha] Set values on Resources fields values.`
+var SetLong = `
+Set values on Resources fields.  May set either the complete or partial field value.
+
+` + "`" + `set` + "`" + ` identifies setters using field metadata published as OpenAPI extensions.
+` + "`" + `set` + "`" + ` parses both the Kubernetes OpenAPI, as well OpenAPI published inline in
+the configuration as comments.
+
+` + "`" + `set` + "`" + ` maybe be used to:
+
+- edit configuration programmatically from the cli
+- create reusable bundles of configuration with custom setters
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    Optional.  The name of the setter to perform or display.
+
+  VALUE
+
+    Optional.  The value to set on the field.
+
+
+To print the possible setters for the Resources in a directory, run ` + "`" + `set` + "`" + ` on
+a directory -- e.g. ` + "`" + `kustomize config set DIR/` + "`" + `.
+
+#### Tips
+
+- A description of the value may be specified with ` + "`" + `--description` + "`" + `.
+- The last setter for the field's value may be defined with ` + "`" + `--set-by` + "`" + `.
+- Create custom setters on Resources, Kustomization.yaml's, patches, etc
+
+The description and setBy fields are left unmodified unless specified with flags.
+
+To create a custom setter for a field see: ` + "`" + `kustomize help config create-setter` + "`" + `
+`
+var SetExamples = `
+  Resource YAML: Name Prefix Setter
+
+    # DIR/resources.yaml
+    ...
+    metadata:
+        name: PREFIX-app1 # {"type":"string","x-kustomize":{"partialFieldSetters":[{"name":"name-prefix","value":"PREFIX"}]}}
+    ...
+    ---
+    ...
+    metadata:
+        name: PREFIX-app2 # {"type":"string","x-kustomize":{"partialFieldSetters":[{"name":"name-prefix","value":"PREFIX"}]}}
+    ...
+
+  List setters: Show the possible setters
+
+    $ config set DIR/
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    name-prefix   ''            PREFIX    string   2
+
+  Perform set: set a new value, owner and description
+
+    $ kustomize config set DIR/ name-prefix "test" --description "test environment" --set-by "dev"
+    set 2 values
+
+  List setters: Show the new values
+
+    $ config set DIR/
+        NAME      DESCRIPTION         VALUE     TYPE     COUNT     SETBY 
+    name-prefix   'test environment'   test     string   2          dev
+
+  New Resource YAML:
+
+    # DIR/resources.yaml
+    ...
+    metadata:
+        name: test-app1 # {"description":"test environment","type":"string","x-kustomize":{"setBy":"dev","partialFieldSetters":[{"name":"name-prefix","value":"test"}]}}
+    ...
+    ---
+    ...
+    metadata:
+        name: test-app2 # {"description":"test environment","type":"string","x-kustomize":{"setBy":"dev","partialFieldSetters":[{"name":"name-prefix","value":"test"}]}}
+    ...`
+
+var SinkShort = `[Alpha] Implement a Sink by writing input to a local directory.`
+var SinkLong = `
+[Alpha] Implement a Sink by writing input to a local directory.
+
+    kustomize config sink DIR
+
+  DIR:
+    Path to local directory.
+
+` + "`" + `sink` + "`" + ` writes its input to a directory
+`
+var SinkExamples = `
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/`
+
+var SourceShort = `[Alpha] Implement a Source by reading a local directory.`
+var SourceLong = `
+[Alpha] Implement a Source by reading a local directory.
+
+    kustomize config source DIR
+
+  DIR:
+    Path to local directory.
+
+` + "`" + `source` + "`" + ` emits configuration to act as input to a function
+`
+var SourceExamples = `
+    # emity configuration directory as input source to a function
+    kustomize config source DIR/
+
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/`
+
 var TreeShort = `[Alpha] Display Resource structure from a directory or stdin.`
 var TreeLong = `
 [Alpha] Display Resource structure from a directory or stdin.
@@ -203,8 +381,9 @@ container names, etc.
 
 kustomize config tree supports printing arbitrary fields using the '--field' flag.
 
-By default, kustomize config tree uses the directory structure for the tree structure, however when printing
-from the cluster, the Resource graph structure may be used instead.
+By default, kustomize config tree uses Resource graph structure if any relationships between resources (ownerReferences)
+are detected, as is typically the case when printing from a cluster. Otherwise, directory graph structure is used. The
+graph structure can also be selected explicitly using the '--graph-structure' flag.
 `
 var TreeExamples = `
     # print Resources using directory structure
@@ -224,8 +403,7 @@ var TreeExamples = `
       --field="status.conditions[type=Completed].status"
 
     # print live Resources from a cluster using owners for graph structure
-    kubectl get all -o yaml | kustomize config tree --replicas --name --image \
-      --graph-structure=owners
+    kubectl get all -o yaml | kustomize config tree --replicas --name --image
 
     # print live Resources with status condition fields
     kubectl get all -o yaml | kustomize config tree \

cmd/config/internal/generateddocs/tutorials/docs.go

@@ -316,7 +316,7 @@ var FunctionBasicsLong = `
   #### 2. Modify the Generated Resources
 
   - modify the generated Resources by adding an annotation, sidecar container, etc.
-  - modify the ` + "`" + `local-resources/example-use.yaml` + "`" + ` by changing the replicas
+  - modify the ` + "`" + `local-resource/example-use.yaml` + "`" + ` by changing the replicas
 
   re-run ` + "`" + `run` + "`" + `.  this will apply the updated replicas to the generated Resources,
   but keep the fields that you manually added to the generated Resource configuration.
@@ -363,7 +363,7 @@ var FunctionBasicsLong = `
   #### 2. Modify the Generated Resources
 
   - modify the generated Resources by adding an annotation, sidecar container, etc.
-  - modify the ` + "`" + `local-resources/example-use.yaml` + "`" + ` by changing the replicas
+  - modify the ` + "`" + `local-resource/example-use.yaml` + "`" + ` by changing the replicas
 
   re-run ` + "`" + `run` + "`" + `.  this will apply the updated replicas to the generated Resources,
   but keep the fields that you manually added to the generated Resource configuration.

cmd/config/main.go

@@ -9,6 +9,7 @@ package main
 import (
 	"os"
 
+	"sigs.k8s.io/kustomize/cmd/config/complete"
 	"sigs.k8s.io/kustomize/cmd/config/configcobra"
 	"sigs.k8s.io/kustomize/kyaml/commandutil"
 )
@@ -16,7 +17,10 @@ import (
 func main() {
 	// enable the config commands
 	os.Setenv(commandutil.EnableAlphaCommmandsEnvName, "true")
-	if err := configcobra.NewConfigCommand("").Execute(); err != nil {
+	cmd := configcobra.NewConfigCommand("")
+	complete.Complete(cmd).Complete("config")
+
+	if err := cmd.Execute(); err != nil {
 		os.Exit(1)
 	}
 }

cmd/kubectl/fixgomod.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+: "${kyaml_major?Need to source VERSIONS}"
+: "${kyaml_minor?Need to source VERSIONS}"
+: "${kyaml_patch?Need to source VERSIONS}"
+
+go mod edit -dropreplace=sigs.k8s.io/kustomize/kyaml@v0.0.0
+go mod edit -require=sigs.k8s.io/kustomize/kyaml@v$kyaml_major.$kyaml_minor.$kyaml_patch

cmd/kubectl/go.mod

@@ -4,10 +4,18 @@ go 1.13
 
 require (
 	github.com/spf13/cobra v0.0.5
+	k8s.io/api v0.17.0
+	k8s.io/apimachinery v0.17.0
 	k8s.io/cli-runtime v0.17.0
 	k8s.io/client-go v0.17.0
-	k8s.io/kubectl v0.17.0
-	sigs.k8s.io/kustomize/kyaml v0.0.0
+	k8s.io/component-base v0.17.0 // indirect
+	k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd
+	sigs.k8s.io/controller-runtime v0.4.0
+	sigs.k8s.io/kustomize/kstatus v0.0.0-20200109211150-9555095de939
+	sigs.k8s.io/kustomize/kyaml v0.0.2
 )
 
-replace sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml
+replace (
+	sigs.k8s.io/kustomize/kstatus v0.0.0-20200109211150-9555095de939 => ../../kstatus
+	sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml
+)

cmd/kubectl/go.sum

@@ -23,6 +23,7 @@ github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -31,15 +32,23 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -51,8 +60,10 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -60,6 +71,8 @@ github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
+github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
@@ -68,38 +81,75 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54=
+github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
+github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
+github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
+github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
+github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
+github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
 github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
+github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 h1:u4bArs140e9+AfE52mFHOXVFnOSBJBRlzTHrOPLOIhE=
+github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -115,20 +165,30 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
+github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -137,12 +197,15 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
+github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -163,6 +226,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -184,13 +249,18 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -201,12 +271,16 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
@@ -214,6 +288,7 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -234,19 +309,35 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -254,19 +345,26 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMx
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -278,7 +376,9 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -287,14 +387,18 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
+golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -302,17 +406,24 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuA
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
 gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
@@ -324,17 +435,25 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
@@ -344,28 +463,48 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48=
+k8s.io/api v0.0.0-20191214185829-ca1d04f8b0d3/go.mod h1:itOjKREfmUTvcjantxOsyYU5mbFsU7qUnyUuRfF5+5M=
 k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
 k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
+k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783 h1:V6ndwCPoao1yZ52agqOKaUAl7DYWVGiXjV7ePA2i610=
+k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY=
+k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4=
+k8s.io/apimachinery v0.0.0-20191214185652-442f8fb2f03a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY=
+k8s.io/apimachinery v0.0.0-20191216025728-0ee8b4573e3a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY=
 k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
 k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg=
+k8s.io/cli-runtime v0.0.0-20191214191754-e6dc6d5c8724/go.mod h1:wzlq80lvjgHW9if6MlE4OIGC86MDKsy5jtl9nxz/IYY=
 k8s.io/cli-runtime v0.17.0 h1:XEuStbJBHCQlEKFyTQmceDKEWOSYHZkcYWKp3SsQ9Hk=
 k8s.io/cli-runtime v0.17.0/go.mod h1:1E5iQpMODZq2lMWLUJELwRu2MLWIzwvMgDBpn3Y81Qo=
+k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk=
+k8s.io/client-go v0.0.0-20191214190045-a32a6f7a3052/go.mod h1:tAaoc/sYuIL0+njJefSAmE28CIcxyaFV4kbIujBlY2s=
+k8s.io/client-go v0.0.0-20191219150334-0b8da7416048/go.mod h1:ZEe8ZASDUAuqVGJ+UN0ka0PfaR+b6a6E1PGsSNZRui8=
 k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
 k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
-k8s.io/code-generator v0.17.0/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s=
+k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE=
+k8s.io/code-generator v0.0.0-20191214185510-0b9b3c99f9f2/go.mod h1:BjGKcoq1MRUmcssvHiSxodCco1T6nVIt4YeCT5CMSao=
+k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA=
+k8s.io/component-base v0.0.0-20191214190519-d868452632e2/go.mod h1:wupxkh1T/oUDqyTtcIjiEfpbmIHGm8By/vqpSKC6z8c=
 k8s.io/component-base v0.17.0 h1:BnDFcmBDq+RPpxXjmuYnZXb59XNN9CaFrX8ba9+3xrA=
 k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/kubectl v0.17.0 h1:xD4EWlL+epc/JTO1gvSjmV9yiYF0Z2wiHK2DIek6URY=
-k8s.io/kubectl v0.17.0/go.mod h1:jIPrUAW656Vzn9wZCCe0PC+oTcu56u2HgFD21Xbfk1s=
-k8s.io/metrics v0.17.0/go.mod h1:EH1D3YAwN6d7bMelrElnLhLg72l/ERStyv2SIQVt6Do=
+k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd h1:nZX5+wEqTu/EBIYjrZlFOA63z4+Zcy96lDkCZPU9a9c=
+k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd/go.mod h1:9ehGcuUGjXVZh0qbYSB0vvofQw2JQe6c6cO0k4wu/Oo=
+k8s.io/metrics v0.0.0-20191214191643-6b1944c9f765/go.mod h1:5V7rewilItwK0cz4nomU0b3XCcees2Ka5EBYWS1HBeM=
+k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
@@ -373,9 +512,16 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+sigs.k8s.io/controller-runtime v0.4.0 h1:wATM6/m+3w8lj8FXNaO6Fs/rq/vqoOjO1Q116Z9NPsg=
+sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns=
 sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
+sigs.k8s.io/kustomize/kyaml v0.0.2 h1:Rl/wMrnpZzZjsVeFIIOAb92Kz/UfLrTUEXjiHW6oS0o=
+sigs.k8s.io/kustomize/kyaml v0.0.2/go.mod h1:rywm/rcR5LmCBghz9956tE45OdUPChFoXVVs+WmhMTI=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
+sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA=
+sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM=
+sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI=

cmd/kubectl/kubectlcobra/commands.go

@@ -6,6 +6,7 @@ package kubectlcobra
 
 import (
 	"flag"
+	"fmt"
 	"os"
 	"strings"
 
@@ -17,9 +18,6 @@ import (
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
 	"k8s.io/kubectl/pkg/util/i18n"
 	"sigs.k8s.io/kustomize/kyaml/commandutil"
-
-	// initialize auth
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
 )
 
 // GetCommand returns a command from kubectl to install
@@ -82,6 +80,8 @@ func updateHelp(names []string, c *cobra.Command) {
 // NewCmdApply creates the `apply` command
 func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command {
 	o := apply.NewApplyOptions(ioStreams)
+	so := newStatusOptions(f, ioStreams)
+	o.PreProcessorFn = PrependGroupingObject(o)
 
 	cmd := &cobra.Command{
 		Use:                   "apply (-f FILENAME | -k DIRECTORY)",
@@ -98,6 +98,10 @@ func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IO
 
 			cmdutil.CheckErr(o.Complete(f, cmd))
 			cmdutil.CheckErr(o.Run())
+			infos, _ := o.GetObjects()
+			if so.wait {
+				cmdutil.CheckErr(so.waitForStatus(infos))
+			}
 		},
 	}
 
@@ -105,6 +109,7 @@ func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IO
 	o.DeleteFlags.AddFlags(cmd)
 	o.RecordFlags.AddFlags(cmd)
 	o.PrintFlags.AddFlags(cmd)
+	so.AddFlags(cmd)
 
 	o.Overwrite = true
 
@@ -115,3 +120,27 @@ func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IO
 
 	return cmd
 }
+
+// PrependGroupingObject orders the objects to apply so the "grouping"
+// object stores the inventory, and it is first to be applied.
+func PrependGroupingObject(o *apply.ApplyOptions) func() error {
+	return func() error {
+		if o == nil {
+			return fmt.Errorf("ApplyOptions are nil")
+		}
+		infos, err := o.GetObjects()
+		if err != nil {
+			return err
+		}
+		_, exists := findGroupingObject(infos)
+		if exists {
+			if err := addInventoryToGroupingObj(infos); err != nil {
+				return err
+			}
+			if !sortGroupingObject(infos) {
+				return err
+			}
+		}
+		return nil
+	}
+}

cmd/kubectl/kubectlcobra/commands_test.go

@@ -0,0 +1,58 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"testing"
+
+	"k8s.io/cli-runtime/pkg/resource"
+	"k8s.io/kubectl/pkg/cmd/apply"
+)
+
+func TestPrependGroupingObject(t *testing.T) {
+	tests := []struct {
+		infos []*resource.Info
+	}{
+		{
+			infos: []*resource.Info{copyGroupingInfo()},
+		},
+		{
+			infos: []*resource.Info{pod1Info, pod3Info, copyGroupingInfo()},
+		},
+		{
+			infos: []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
+		},
+	}
+
+	for _, test := range tests {
+		applyOptions := createApplyOptions(test.infos)
+		f := PrependGroupingObject(applyOptions)
+		err := f()
+		if err != nil {
+			t.Errorf("Error running pre-processor callback: %s", err)
+		}
+		infos, _ := applyOptions.GetObjects()
+		if len(test.infos) != len(infos) {
+			t.Fatalf("Wrong number of objects after prepending grouping object")
+		}
+		groupingInfo := infos[0]
+		if !isGroupingObject(groupingInfo.Object) {
+			t.Fatalf("First object is not the grouping object")
+		}
+		inventory, _ := retrieveInventoryFromGroupingObj(infos)
+		if len(inventory) != (len(infos) - 1) {
+			t.Errorf("Wrong number of inventory items stored in grouping object")
+		}
+	}
+
+}
+
+// createApplyOptions is a helper function to assemble the ApplyOptions
+// with the passed objects (infos).
+func createApplyOptions(infos []*resource.Info) *apply.ApplyOptions {
+	applyOptions := &apply.ApplyOptions{}
+	applyOptions.SetObjects(infos)
+	return applyOptions
+}

cmd/kubectl/kubectlcobra/grouping.go

@@ -0,0 +1,252 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strconv"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/cli-runtime/pkg/resource"
+)
+
+const (
+	GroupingLabel = "kustomize.config.k8s.io/inventory-id"
+	GroupingHash  = "kustomize.config.k8s.io/inventory-hash"
+)
+
+// isGroupingObject returns true if the passed object has the
+// grouping label.
+// TODO(seans3): Check type is ConfigMap.
+func isGroupingObject(obj runtime.Object) bool {
+	if obj == nil {
+		return false
+	}
+	accessor, err := meta.Accessor(obj)
+	if err == nil {
+		labels := accessor.GetLabels()
+		_, exists := labels[GroupingLabel]
+		if exists {
+			return true
+		}
+	}
+	return false
+}
+
+// findGroupingObject returns the "Grouping" object (ConfigMap with
+// grouping label) if it exists, and a boolean describing if it was found.
+func findGroupingObject(infos []*resource.Info) (*resource.Info, bool) {
+	for _, info := range infos {
+		if info != nil && isGroupingObject(info.Object) {
+			return info, true
+		}
+	}
+	return nil, false
+}
+
+// sortGroupingObject reorders the infos slice to place the grouping
+// object in the first position. Returns true if grouping object found,
+// false otherwise.
+func sortGroupingObject(infos []*resource.Info) bool {
+	for i, info := range infos {
+		if info != nil && isGroupingObject(info.Object) {
+			// If the grouping object is not already in the first position,
+			// swap the grouping object with the first object.
+			if i > 0 {
+				infos[0], infos[i] = infos[i], infos[0]
+			}
+			return true
+		}
+	}
+	return false
+}
+
+// Adds the inventory of all objects (passed as infos) to the
+// grouping object. Returns an error if a grouping object does not
+// exist, or we are unable to successfully add the inventory to
+// the grouping object; nil otherwise. Each object is in
+// unstructured.Unstructured format.
+func addInventoryToGroupingObj(infos []*resource.Info) error {
+
+	// Iterate through the objects (infos), creating an Inventory struct
+	// as metadata for each object, or if it's the grouping object, store it.
+	var groupingInfo *resource.Info
+	var groupingObj *unstructured.Unstructured
+	inventoryMap := map[string]string{}
+	for _, info := range infos {
+		obj := info.Object
+		if isGroupingObject(obj) {
+			// If we have more than one grouping object--error.
+			if groupingObj != nil {
+				return fmt.Errorf("Error--applying more than one grouping object.")
+			}
+			var ok bool
+			groupingObj, ok = obj.(*unstructured.Unstructured)
+			if !ok {
+				return fmt.Errorf("Grouping object is not an Unstructured: %#v", groupingObj)
+			}
+			groupingInfo = info
+		} else {
+			if obj == nil {
+				return fmt.Errorf("Creating inventory; object is nil")
+			}
+			gk := obj.GetObjectKind().GroupVersionKind().GroupKind()
+			inventory, err := createInventory(info.Namespace, info.Name, gk)
+			if err != nil {
+				return err
+			}
+			inventoryMap[inventory.String()] = ""
+		}
+	}
+
+	// If we've found the grouping object, store the object metadata inventory
+	// in the grouping config map.
+	if groupingObj == nil {
+		return fmt.Errorf("Grouping object not found")
+	}
+
+	if len(inventoryMap) > 0 {
+		// Adds the inventory map to the ConfigMap "data" section.
+		err := unstructured.SetNestedStringMap(groupingObj.UnstructuredContent(),
+			inventoryMap, "data")
+		if err != nil {
+			return err
+		}
+		// Adds the hash of the inventory strings as an annotation to the
+		// grouping object. Inventory strings must be sorted to make hash
+		// deterministic.
+		inventoryList := mapKeysToSlice(inventoryMap)
+		sort.Strings(inventoryList)
+		invHash, err := calcInventoryHash(inventoryList)
+		if err != nil {
+			return err
+		}
+		// Add the hash as a suffix to the grouping object's name.
+		invHashStr := strconv.FormatUint(uint64(invHash), 16)
+		if err := addSuffixToName(groupingInfo, invHashStr); err != nil {
+			return err
+		}
+		annotations := groupingObj.GetAnnotations()
+		if annotations == nil {
+			annotations = map[string]string{}
+		}
+		annotations[GroupingHash] = invHashStr
+		groupingObj.SetAnnotations(annotations)
+	}
+	return nil
+}
+
+// retrieveInventoryFromGroupingObj returns a slice of pointers to the
+// inventory metadata. This function finds the grouping object, then
+// parses the stored resource metadata into Inventory structs. Returns
+// an error if there is a problem parsing the data into Inventory
+// structs, or if the grouping object is not in Unstructured format; nil
+// otherwise. If a grouping object does not exist, or it does not have a
+// "data" map, then returns an empty slice and no error.
+func retrieveInventoryFromGroupingObj(infos []*resource.Info) ([]*Inventory, error) {
+	inventory := []*Inventory{}
+	groupingInfo, exists := findGroupingObject(infos)
+	if exists {
+		groupingObj, ok := groupingInfo.Object.(*unstructured.Unstructured)
+		if !ok {
+			err := fmt.Errorf("Grouping object is not an Unstructured: %#v", groupingObj)
+			return inventory, err
+		}
+		invMap, exists, err := unstructured.NestedStringMap(groupingObj.Object, "data")
+		if err != nil {
+			err := fmt.Errorf("Error retrieving inventory from grouping object.")
+			return inventory, err
+		}
+		if exists {
+			for invStr := range invMap {
+				inv, err := parseInventory(invStr)
+				if err != nil {
+					return inventory, err
+				}
+				inventory = append(inventory, inv)
+			}
+		}
+	}
+	return inventory, nil
+}
+
+// calcInventoryHash returns an unsigned int32 representing the hash
+// of the inventory strings. If there is an error writing bytes to
+// the hash, then the error is returned; nil is returned otherwise.
+// Used to quickly identify the set of resources in the grouping object.
+func calcInventoryHash(inv []string) (uint32, error) {
+	h := fnv.New32a()
+	for _, is := range inv {
+		_, err := h.Write([]byte(is))
+		if err != nil {
+			return uint32(0), err
+		}
+	}
+	return h.Sum32(), nil
+}
+
+// retrieveInventoryHash takes a grouping object (encapsulated by
+// a resource.Info), and returns the string representing the hash
+// of the grouping inventory; returns empty string if the grouping
+// object is not in Unstructured format, or if the hash annotation
+// does not exist.
+func retrieveInventoryHash(groupingInfo *resource.Info) string {
+	var invHash = ""
+	groupingObj, ok := groupingInfo.Object.(*unstructured.Unstructured)
+	if ok {
+		annotations := groupingObj.GetAnnotations()
+		if annotations != nil {
+			invHash = annotations[GroupingHash]
+		}
+	}
+	return invHash
+}
+
+// mapKeysToSlice returns the map keys as a slice of strings.
+func mapKeysToSlice(m map[string]string) []string {
+	s := make([]string, len(m))
+	i := 0
+	for k := range m {
+		s[i] = k
+		i++
+	}
+	return s
+}
+
+// addSuffixToName adds the passed suffix (usually a hash) as a suffix
+// to the name of the passed object stored in the Info struct. Returns
+// an error if the object is not "*unstructured.Unstructured" or if the
+// name stored in the object differs from the name in the Info struct.
+func addSuffixToName(info *resource.Info, suffix string) error {
+
+	if info == nil {
+		return fmt.Errorf("Nil resource.Info")
+	}
+	suffix = strings.TrimSpace(suffix)
+	if len(suffix) == 0 {
+		return fmt.Errorf("Passed empty suffix")
+	}
+
+	accessor, _ := meta.Accessor(info.Object)
+	name := accessor.GetName()
+	if name != info.Name {
+		return fmt.Errorf("Grouping object (%s) and resource.Info (%s) have different names\n", name, info.Name)
+	}
+	// Error if name alread has suffix.
+	suffix = "-" + suffix
+	if strings.HasSuffix(name, suffix) {
+		return fmt.Errorf("Name already has suffix: %s\n", name)
+	}
+	name += suffix
+	accessor.SetName(name)
+	info.Name = name
+
+	return nil
+}

cmd/kubectl/kubectlcobra/grouping_test.go

@@ -0,0 +1,537 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"fmt"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/cli-runtime/pkg/resource"
+)
+
+var testNamespace = "test-grouping-namespace"
+var groupingObjName = "test-grouping-obj"
+var pod1Name = "pod-1"
+var pod2Name = "pod-2"
+var pod3Name = "pod-3"
+
+var groupingObj = unstructured.Unstructured{
+	Object: map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "ConfigMap",
+		"metadata": map[string]interface{}{
+			"name":      groupingObjName,
+			"namespace": testNamespace,
+			"labels": map[string]interface{}{
+				GroupingLabel: "true",
+			},
+		},
+	},
+}
+
+var pod1 = unstructured.Unstructured{
+	Object: map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "Pod",
+		"metadata": map[string]interface{}{
+			"name":      pod1Name,
+			"namespace": testNamespace,
+		},
+	},
+}
+
+var pod1Info = &resource.Info{
+	Namespace: testNamespace,
+	Name:      pod1Name,
+	Object:    &pod1,
+}
+
+var pod2 = unstructured.Unstructured{
+	Object: map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "Pod",
+		"metadata": map[string]interface{}{
+			"name":      pod2Name,
+			"namespace": testNamespace,
+		},
+	},
+}
+
+var pod2Info = &resource.Info{
+	Namespace: testNamespace,
+	Name:      pod2Name,
+	Object:    &pod2,
+}
+
+var pod3 = unstructured.Unstructured{
+	Object: map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "Pod",
+		"metadata": map[string]interface{}{
+			"name":      pod3Name,
+			"namespace": testNamespace,
+		},
+	},
+}
+
+var pod3Info = &resource.Info{
+	Namespace: testNamespace,
+	Name:      pod3Name,
+	Object:    &pod3,
+}
+
+var nonUnstructuredGroupingObj = &corev1.ConfigMap{
+	ObjectMeta: metav1.ObjectMeta{
+		Namespace: testNamespace,
+		Name:      groupingObjName,
+		Labels: map[string]string{
+			GroupingLabel: "true",
+		},
+	},
+}
+
+var nonUnstructuredGroupingInfo = &resource.Info{
+	Namespace: testNamespace,
+	Name:      groupingObjName,
+	Object:    nonUnstructuredGroupingObj,
+}
+
+var nilInfo = &resource.Info{
+	Namespace: testNamespace,
+	Name:      groupingObjName,
+	Object:    nil,
+}
+
+func TestIsGroupingObject(t *testing.T) {
+	tests := []struct {
+		obj        runtime.Object
+		isGrouping bool
+	}{
+		{
+			obj:        nil,
+			isGrouping: false,
+		},
+		{
+			obj:        &groupingObj,
+			isGrouping: true,
+		},
+		{
+			obj:        &pod2,
+			isGrouping: false,
+		},
+	}
+
+	for _, test := range tests {
+		grouping := isGroupingObject(test.obj)
+		if test.isGrouping && !grouping {
+			t.Errorf("Grouping object not identified: %#v", test.obj)
+		}
+		if !test.isGrouping && grouping {
+			t.Errorf("Non-grouping object identifed as grouping obj: %#v", test.obj)
+		}
+	}
+}
+
+func TestFindGroupingObject(t *testing.T) {
+	tests := []struct {
+		infos  []*resource.Info
+		exists bool
+		name   string
+	}{
+		{
+			infos:  []*resource.Info{},
+			exists: false,
+			name:   "",
+		},
+		{
+			infos:  []*resource.Info{nil},
+			exists: false,
+			name:   "",
+		},
+		{
+			infos:  []*resource.Info{copyGroupingInfo()},
+			exists: true,
+			name:   groupingObjName,
+		},
+		{
+			infos:  []*resource.Info{pod1Info},
+			exists: false,
+			name:   "",
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info, pod3Info},
+			exists: false,
+			name:   "",
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
+			exists: true,
+			name:   groupingObjName,
+		},
+	}
+
+	for _, test := range tests {
+		groupingObj, found := findGroupingObject(test.infos)
+		if test.exists && !found {
+			t.Errorf("Should have found grouping object")
+		}
+		if !test.exists && found {
+			t.Errorf("Grouping object found, but it does not exist: %#v", groupingObj)
+		}
+		if test.exists && found && test.name != groupingObj.Name {
+			t.Errorf("Grouping object name does not match: %s/%s", test.name, groupingObj.Name)
+		}
+	}
+}
+
+func TestSortGroupingObject(t *testing.T) {
+	tests := []struct {
+		infos  []*resource.Info
+		sorted bool
+	}{
+		{
+			infos:  []*resource.Info{},
+			sorted: false,
+		},
+		{
+			infos:  []*resource.Info{copyGroupingInfo()},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{pod1Info},
+			sorted: false,
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info},
+			sorted: false,
+		},
+		{
+			infos:  []*resource.Info{copyGroupingInfo(), pod1Info},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{pod1Info, copyGroupingInfo()},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info, pod3Info, copyGroupingInfo()},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{copyGroupingInfo(), pod1Info, pod2Info, pod3Info},
+			sorted: true,
+		},
+	}
+
+	for _, test := range tests {
+		wasSorted := sortGroupingObject(test.infos)
+		if wasSorted && !test.sorted {
+			t.Errorf("Grouping object was sorted, but it shouldn't have been")
+		}
+		if !wasSorted && test.sorted {
+			t.Errorf("Grouping object was NOT sorted, but it should have been")
+		}
+		if wasSorted {
+			first := test.infos[0]
+			if !isGroupingObject(first.Object) {
+				t.Errorf("Grouping object was not sorted into first position")
+			}
+		}
+	}
+}
+
+func TestAddRetrieveInventoryToFromGroupingObject(t *testing.T) {
+	tests := []struct {
+		infos    []*resource.Info
+		expected []*Inventory
+		isError  bool
+	}{
+		// No grouping object is an error.
+		{
+			infos:   []*resource.Info{},
+			isError: true,
+		},
+		// No grouping object is an error.
+		{
+			infos:   []*resource.Info{pod1Info, pod2Info},
+			isError: true,
+		},
+		// Grouping object without other objects is OK.
+		{
+			infos:   []*resource.Info{copyGroupingInfo(), nilInfo},
+			isError: true,
+		},
+		{
+			infos:   []*resource.Info{nonUnstructuredGroupingInfo},
+			isError: true,
+		},
+		{
+			infos:    []*resource.Info{copyGroupingInfo()},
+			expected: []*Inventory{},
+			isError:  false,
+		},
+		// More than one grouping object is an error.
+		{
+			infos:    []*resource.Info{copyGroupingInfo(), copyGroupingInfo()},
+			expected: []*Inventory{},
+			isError:  true,
+		},
+		// More than one grouping object is an error.
+		{
+			infos:    []*resource.Info{copyGroupingInfo(), pod1Info, copyGroupingInfo()},
+			expected: []*Inventory{},
+			isError:  true,
+		},
+		// Basic test case: one grouping object, one pod.
+		{
+			infos: []*resource.Info{copyGroupingInfo(), pod1Info},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+		{
+			infos: []*resource.Info{pod1Info, copyGroupingInfo()},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+		{
+			infos: []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod2Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod3Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+		{
+			infos: []*resource.Info{pod1Info, pod2Info, pod3Info, copyGroupingInfo()},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod2Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod3Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+		{
+			infos: []*resource.Info{copyGroupingInfo(), pod1Info, pod2Info, pod3Info},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod2Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod3Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+	}
+
+	for _, test := range tests {
+		err := addInventoryToGroupingObj(test.infos)
+		if test.isError && err == nil {
+			t.Errorf("Should have produced an error, but returned none.")
+		}
+		if !test.isError {
+			if err != nil {
+				t.Fatalf("Received error when expecting none (%s)\n", err)
+			}
+			retrieved, err := retrieveInventoryFromGroupingObj(test.infos)
+			if err != nil {
+				t.Fatalf("Error retrieving inventory: %s\n", err)
+			}
+			if len(test.expected) != len(retrieved) {
+				t.Errorf("Expected inventory for %d resources, actual %d",
+					len(test.expected), len(retrieved))
+			}
+			for _, expected := range test.expected {
+				found := false
+				for _, actual := range retrieved {
+					if expected.Equals(actual) {
+						found = true
+						continue
+					}
+				}
+				if !found {
+					t.Errorf("Expected inventory (%s) not found", expected)
+				}
+			}
+			// If the grouping object has an inventory, check the
+			// grouping object has an inventory hash.
+			groupingInfo, exists := findGroupingObject(test.infos)
+			if exists && len(test.expected) > 0 {
+				invHash := retrieveInventoryHash(groupingInfo)
+				if len(invHash) == 0 {
+					t.Errorf("Grouping object missing inventory hash")
+				}
+			}
+		}
+	}
+}
+
+func TestAddSuffixToName(t *testing.T) {
+	tests := []struct {
+		info     *resource.Info
+		suffix   string
+		expected string
+		isError  bool
+	}{
+		// Nil info should return error.
+		{
+			info:     nil,
+			suffix:   "",
+			expected: "",
+			isError:  true,
+		},
+		// Empty suffix should return error.
+		{
+			info:     copyGroupingInfo(),
+			suffix:   "",
+			expected: "",
+			isError:  true,
+		},
+		// Empty suffix should return error.
+		{
+			info:     copyGroupingInfo(),
+			suffix:   " \t",
+			expected: "",
+			isError:  true,
+		},
+		{
+			info:     copyGroupingInfo(),
+			suffix:   "hashsuffix",
+			expected: groupingObjName + "-hashsuffix",
+			isError:  false,
+		},
+	}
+
+	for _, test := range tests {
+		//t.Errorf("%#v [%s]", test.info, test.suffix)
+		err := addSuffixToName(test.info, test.suffix)
+		if test.isError {
+			if err == nil {
+				t.Errorf("Should have produced an error, but returned none.")
+			}
+		}
+		if !test.isError {
+			if err != nil {
+				t.Fatalf("Received error when expecting none (%s)\n", err)
+			}
+			actualName, err := getObjectName(test.info.Object)
+			if err != nil {
+				t.Fatalf("Error getting object name: %s", err)
+			}
+			if actualName != test.info.Name {
+				t.Errorf("Object name (%s) does not match info name (%s)\n", actualName, test.info.Name)
+			}
+			if test.expected != actualName {
+				t.Errorf("Expected name (%s), got (%s)\n", test.expected, actualName)
+			}
+		}
+	}
+}
+
+func getObjectName(obj runtime.Object) (string, error) {
+	u, ok := obj.(*unstructured.Unstructured)
+	if !ok {
+		return "", fmt.Errorf("Grouping object is not Unstructured format")
+	}
+	return u.GetName(), nil
+}
+
+func copyGroupingInfo() *resource.Info {
+	groupingObjCopy := groupingObj.DeepCopy()
+	var groupingInfo = &resource.Info{
+		Namespace: testNamespace,
+		Name:      groupingObjName,
+		Object:    groupingObjCopy,
+	}
+	return groupingInfo
+}

cmd/kubectl/kubectlcobra/inventory.go

@@ -0,0 +1,83 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// Separates inventory fields. This string is allowable as a
+// ConfigMap key, but it is not allowed as a character in
+// resource name.
+const fieldSeparator = "_"
+
+// Inventory organizes and stores the indentifying information
+// for an object. This struct (as a string) is stored in a
+// grouping object to keep track of sets of applied objects.
+type Inventory struct {
+	Namespace string
+	Name      string
+	GroupKind schema.GroupKind
+}
+
+// createInventory returns a pointer to an Inventory struct filled
+// with the passed values. This function validates the passed fields
+// and returns an error for bad parameters.
+func createInventory(namespace string,
+	name string, gk schema.GroupKind) (*Inventory, error) {
+
+	// Namespace can be empty, but name cannot.
+	name = strings.TrimSpace(name)
+	if name == "" {
+		return nil, fmt.Errorf("Empty name for inventory object")
+	}
+	if gk.Empty() {
+		return nil, fmt.Errorf("Empty GroupKind for inventory object")
+	}
+
+	return &Inventory{
+		Namespace: strings.TrimSpace(namespace),
+		Name:      name,
+		GroupKind: gk,
+	}, nil
+}
+
+// parseInventory takes a string, splits it into its five fields,
+// and returns a pointer to an Inventory struct storing the
+// five fields. Example inventory string:
+//
+//   test-namespace/test-name/apps/v1/ReplicaSet
+//
+// Returns an error if unable to parse and create the Inventory
+// struct.
+func parseInventory(inv string) (*Inventory, error) {
+	parts := strings.Split(inv, fieldSeparator)
+	if len(parts) == 4 {
+		gk := schema.GroupKind{
+			Group: strings.TrimSpace(parts[2]),
+			Kind:  strings.TrimSpace(parts[3]),
+		}
+		return createInventory(parts[0], parts[1], gk)
+	}
+	return nil, fmt.Errorf("Unable to decode inventory: %s\n", inv)
+}
+
+// Equals returns true if the Inventory structs are identical;
+// false otherwise.
+func (i *Inventory) Equals(other *Inventory) bool {
+	return i.String() == other.String()
+}
+
+// String create a string version of the Inventory struct.
+func (i *Inventory) String() string {
+	return fmt.Sprintf("%s%s%s%s%s%s%s",
+		i.Namespace, fieldSeparator,
+		i.Name, fieldSeparator,
+		i.GroupKind.Group, fieldSeparator,
+		i.GroupKind.Kind)
+}

cmd/kubectl/kubectlcobra/inventory_test.go

@@ -0,0 +1,218 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"testing"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func TestCreateInventory(t *testing.T) {
+	tests := []struct {
+		namespace string
+		name      string
+		gk        schema.GroupKind
+		expected  string
+		isError   bool
+	}{
+		{
+			namespace: "  \n",
+			name:      " test-name\t",
+			gk: schema.GroupKind{
+				Group: "apps",
+				Kind:  "ReplicaSet",
+			},
+			expected: "_test-name_apps_ReplicaSet",
+			isError:  false,
+		},
+		{
+			namespace: "test-namespace ",
+			name:      " test-name\t",
+			gk: schema.GroupKind{
+				Group: "apps",
+				Kind:  "ReplicaSet",
+			},
+			expected: "test-namespace_test-name_apps_ReplicaSet",
+			isError:  false,
+		},
+		// Error with empty name.
+		{
+			namespace: "test-namespace ",
+			name:      " \t",
+			gk: schema.GroupKind{
+				Group: "apps",
+				Kind:  "ReplicaSet",
+			},
+			expected: "",
+			isError:  true,
+		},
+		// Error with empty GroupKind.
+		{
+			namespace: "test-namespace",
+			name:      "test-name",
+			gk:        schema.GroupKind{},
+			expected:  "",
+			isError:   true,
+		},
+	}
+
+	for _, test := range tests {
+		inv, err := createInventory(test.namespace, test.name, test.gk)
+		if !test.isError {
+			if err != nil {
+				t.Errorf("Error creating inventory when it should have worked.")
+			} else if test.expected != inv.String() {
+				t.Errorf("Expected inventory (%s) != created inventory(%s)\n", test.expected, inv.String())
+			}
+		}
+		if test.isError && err == nil {
+			t.Errorf("Should have returned an error in createInventory()")
+		}
+	}
+}
+
+func TestInventoryEqual(t *testing.T) {
+	tests := []struct {
+		inventory1 *Inventory
+		inventory2 *Inventory
+		isEqual    bool
+	}{
+		// Two equal inventories without a namespace
+		{
+			inventory1: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			inventory2: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			isEqual: true,
+		},
+		// Two equal inventories with a namespace
+		{
+			inventory1: &Inventory{
+				Namespace: "test-namespace",
+				Name:      "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			inventory2: &Inventory{
+				Namespace: "test-namespace",
+				Name:      "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			isEqual: true,
+		},
+		// One inventory with a namespace, one without -- not equal.
+		{
+			inventory1: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			inventory2: &Inventory{
+				Namespace: "test-namespace",
+				Name:      "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			isEqual: false,
+		},
+		// One inventory with a Deployment, one with a ReplicaSet -- not equal.
+		{
+			inventory1: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			inventory2: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "ReplicaSet",
+				},
+			},
+			isEqual: false,
+		},
+	}
+
+	for _, test := range tests {
+		actual := test.inventory1.Equals(test.inventory2)
+		if test.isEqual && !actual {
+			t.Errorf("Expected inventories equal, but actual is not: (%s)/(%s)\n", test.inventory1, test.inventory2)
+		}
+	}
+}
+
+func TestParseInventory(t *testing.T) {
+	tests := []struct {
+		invStr    string
+		inventory *Inventory
+		isError   bool
+	}{
+		{
+			invStr: "_test-name_apps_ReplicaSet\t",
+			inventory: &Inventory{
+				Name: "test-name",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "ReplicaSet",
+				},
+			},
+			isError: false,
+		},
+		{
+			invStr: "test-namespace_test-name_apps_Deployment",
+			inventory: &Inventory{
+				Namespace: "test-namespace",
+				Name:      "test-name",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			isError: false,
+		},
+		// Not enough fields -- error
+		{
+			invStr:    "_test-name_apps",
+			inventory: &Inventory{},
+			isError:   true,
+		},
+	}
+
+	for _, test := range tests {
+		actual, err := parseInventory(test.invStr)
+		if !test.isError {
+			if err != nil {
+				t.Errorf("Error parsing inventory when it should have worked.")
+			} else if !test.inventory.Equals(actual) {
+				t.Errorf("Expected inventory (%s) != parsed inventory (%s)\n", test.inventory, actual)
+			}
+		}
+		if test.isError && err == nil {
+			t.Errorf("Should have returned an error in parseInventory()")
+		}
+	}
+}

cmd/kubectl/kubectlcobra/status.go

@@ -0,0 +1,105 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kubectlcobra
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/cli-runtime/pkg/resource"
+	"k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kustomize/kstatus/wait"
+)
+
+type StatusOptions struct {
+	factory   util.Factory
+	ioStreams genericclioptions.IOStreams
+
+	wait    bool
+	period  time.Duration
+	timeout time.Duration
+}
+
+func newStatusOptions(factory util.Factory, ioStreams genericclioptions.IOStreams) *StatusOptions {
+	return &StatusOptions{
+		factory:   factory,
+		ioStreams: ioStreams,
+
+		wait:    false,
+		period:  2 * time.Second,
+		timeout: 1 * time.Minute,
+	}
+}
+
+func (s *StatusOptions) AddFlags(c *cobra.Command) {
+	c.Flags().BoolVar(&s.wait, "status", s.wait, "Wait for all applied resources to reach the Current status.")
+	c.Flags().DurationVar(&s.period, "status-period", s.period, "Polling period for resource statuses.")
+	c.Flags().DurationVar(&s.timeout, "status-timeout", s.timeout, "Timeout threshold for waiting for all resources to reach the Current status.")
+}
+
+func (s *StatusOptions) waitForStatus(infos []*resource.Info) error {
+	mapper, err := getRESTMapper(s.factory)
+	if err != nil {
+		return err
+	}
+
+	c, err := getClient(s.factory, mapper)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
+	defer cancel()
+
+	resolver := wait.NewResolver(c, mapper, s.period)
+	ch := resolver.WaitForStatus(ctx, infosToResourceIdentifiers(infos))
+
+	for msg := range ch {
+		switch msg.Type {
+		case wait.ResourceUpdate:
+			id := msg.EventResource.ResourceIdentifier
+			gk := id.GroupKind
+			fmt.Fprintf(s.ioStreams.Out, "%s/%s is %s: %s\n", strings.ToLower(gk.String()), id.Name, msg.EventResource.Status.String(), msg.EventResource.Message)
+		case wait.Completed:
+			fmt.Fprint(s.ioStreams.Out, "all resources has reached the Current status\n")
+		case wait.Aborted:
+			fmt.Fprintf(s.ioStreams.Out, "resources failed to the reached Current status after %s\n", s.timeout.String())
+		}
+	}
+	return nil
+}
+
+func infosToResourceIdentifiers(infos []*resource.Info) []wait.ResourceIdentifier {
+	var resources []wait.ResourceIdentifier
+	for _, info := range infos {
+		u := info.Object.(*unstructured.Unstructured)
+		resources = append(resources, wait.ResourceIdentifier{
+			GroupKind: u.GroupVersionKind().GroupKind(),
+			Namespace: u.GetNamespace(),
+			Name:      u.GetName(),
+		})
+	}
+	return resources
+}
+
+func getRESTMapper(f util.Factory) (meta.RESTMapper, error) {
+	return f.ToRESTMapper()
+}
+
+func getClient(f util.Factory, mapper meta.RESTMapper) (client.Reader, error) {
+	config, err := f.ToRESTConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	return client.New(config, client.Options{Scheme: scheme.Scheme, Mapper: mapper})
+}

cmd/kubectl/main.go

@@ -8,6 +8,10 @@ import (
 
 	"sigs.k8s.io/kustomize/cmd/kubectl/kubectlcobra"
 	"sigs.k8s.io/kustomize/kyaml/commandutil"
+
+	// This is here rather than in the libraries because of
+	// https://github.com/kubernetes-sigs/kustomize/issues/2060
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
 )
 
 func main() {

cmd/resource/.golangci.yml

@@ -15,7 +15,7 @@ linters:
     - dogsled
     - dupl
     - errcheck
-    - funlen
+    # - funlen
     # - gochecknoinits
     - goconst
     - gocritic

cmd/resource/Makefile

@@ -17,8 +17,8 @@ fmt:
 	go fmt ./...
 
 generate:
-	#(which $(GOBIN)/mdtogo || go get sigs.k8s.io/kustomize/cmd/resource)
-	#GOBIN=$(GOBIN) go generate ./...
+	(which $(GOBIN)/mdtogo || go get sigs.k8s.io/kustomize/cmd/mdtogo)
+	GOBIN=$(GOBIN) go generate ./...
 
 license:
 	(which $(GOBIN)/addlicense || go get github.com/google/addlicense)

cmd/resource/fixgomod.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+: "${kyaml_major?Need to source VERSIONS}"
+: "${kyaml_minor?Need to source VERSIONS}"
+: "${kyaml_patch?Need to source VERSIONS}"
+
+: "${kstatus_major?Need to source VERSIONS}"
+: "${kstatus_minor?Need to source VERSIONS}"
+: "${kstatus_patch?Need to source VERSIONS}"
+
+
+go mod edit -dropreplace=sigs.k8s.io/kustomize/kyaml@v0.0.0
+go mod edit -require=sigs.k8s.io/kustomize/kyaml@v$kyaml_major.$kyaml_minor.$kyaml_patch
+
+go mod edit -dropreplace=sigs.k8s.io/kustomize/kstatus@v0.0.0
+go mod edit -require=sigs.k8s.io/kustomize/kstatus@v$kstatus_major.$kstatus_minor.$kstatus_patch