Merge pull request #2185 from pwittrock/master

cmd/config: support for stdin/stdout in source/sink

.golangci-kustomize.yml

@@ -7,8 +7,13 @@ linters:
     - bodyclose
     - deadcode
     - depguard
+    # - dogsled
     - dupl
+    # - errcheck
+    # - funlen
+    # - gochecknoinits
     - goconst
+    # - gocritic
     - gocyclo
     - gofmt
     - goimports
@@ -21,6 +26,7 @@ linters:
     - lll
     - misspell
     - nakedret
+    # - scopelint
     - staticcheck
     - structcheck
     # stylecheck demands that acronyms not be treated as words
@@ -28,9 +34,10 @@ linters:
     # - stylecheck
     - typecheck
     - unconvert
-    - unused
     - unparam
+    - unused
     - varcheck
+    # - whitespace
 
 linters-settings:
   dupl:

.travis.yml

@@ -36,6 +36,7 @@ install: true
 script:
   - make verify-kustomize
   - ./travis/kyaml-pre-commit.sh
+  - ./travis/check-go-mod.sh
 
 # TBD. Suppressing for now.
 notifications:

Makefile

@@ -17,6 +17,9 @@ verify-kustomize: \
 	test-examples-kustomize-against-HEAD \
 	test-examples-kustomize-against-latest
 
+.PHONY: verify-kustomize-e2e
+verify-kustomize-e2e: test-examples-e2e-kustomize
+
 # Other builds in this repo might want a different linter version.
 # Without one Makefile to rule them all, the different makes
 # cannot assume that golanci-lint is at the version they want
@@ -48,6 +51,11 @@ $(MYGOBIN)/goimports:
 	cd api; \
 	go install golang.org/x/tools/cmd/goimports
 
+# Install resource from whatever is checked out.
+$(MYGOBIN)/resource:
+	cd cmd/resource; \
+	go install .
+
 # To pin pluginator, use this recipe instead:
 #   cd api;
 #   go install sigs.k8s.io/kustomize/pluginator/v2
@@ -193,6 +201,16 @@ test-unit-kustomize-all: \
 test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
 	./hack/testExamplesAgainstKustomize.sh HEAD
 
+.PHONY:
+test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from ."; \
+		cd kustomize; go install .; cd ..; \
+		./hack/testExamplesE2EAgainstKustomize.sh .; \
+	)
+
 .PHONY:
 test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
 	( \
@@ -237,6 +255,16 @@ $(MYGOBIN)/helm:
 		rm -rf $$d \
 	)
 
+$(MYGOBIN)/kind:
+	( \
+        set -e; \
+        d=$(shell mktemp -d); cd $$d; \
+        wget -O ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(shell uname)-amd64; \
+        chmod +x ./kind; \
+        mv ./kind $(MYGOBIN); \
+        rm -rf $$d; \
+	)
+
 .PHONY: clean
 clean: kustomize-external-go-plugin-clean
 	go clean --cache

api/builtins/NamespaceTransformer.go

@@ -52,7 +52,7 @@ func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
 
 		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
 		if len(matches) != 1 {
-			return fmt.Errorf("namespace tranformation produces ID conflict: %#v", matches)
+			return fmt.Errorf("namespace tranformation produces ID conflict: %+v", matches)
 		}
 	}
 	return nil

api/internal/crawl/backend/search_backend.go

@@ -44,7 +44,7 @@ type kustomizeSearch struct {
 // /register: not implemented, but meant as an endpoint for adding new
 // kustomization files to the corpus.
 func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, "kustomize")
 	if err != nil {
 		return nil, err
 	}

api/internal/crawl/cmd/crawler/crawler.go

@@ -2,12 +2,15 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -25,6 +28,7 @@ const (
 )
 
 type CrawlMode int
+
 const (
 	CrawlUnknown CrawlMode = iota
 	// Crawl all the kustomization files in all the repositories of a Github user
@@ -45,7 +49,7 @@ func NewCrawlMode(s string) CrawlMode {
 		return CrawlUser
 	case "github-repo":
 		return CrawlRepo
-	case "":
+	case "index+github":
 		return CrawlIndexAndGithub
 	case "index":
 		return CrawlIndex
@@ -56,30 +60,33 @@ func NewCrawlMode(s string) CrawlMode {
 	}
 }
 
-func Usage() {
-	fmt.Printf("Usage: %s [mode] [githubUser|githubRepo]\n", os.Args[0])
-	fmt.Printf("\tmode can be one of [github-user, github-repo, index, github]\n")
-	fmt.Printf("%s: crawl all the documents in the index and crawling all the kustomization files on Github\n", os.Args[0])
-	fmt.Printf("%s index: crawl all the documents in the index\n", os.Args[0])
-	fmt.Printf("%s gihub: crawl all the kustomization files on Github\n", os.Args[0])
-	fmt.Printf("%s github-user <github-user>: Crawl all the kustomization files in all the repositories of a Github user\n", os.Args[0])
-	fmt.Printf("\tFor example, %s github-user kubernetes-sigs\n", os.Args[0])
-	fmt.Printf("%s github-repo <github-repo>: Crawl all the kustomization files in a Github repo\n", os.Args[0])
-	fmt.Printf("\tFor example, %s github-repo kubernetes-sigs/kustomize\n", os.Args[0])
-}
-
 func main() {
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	modePtr := flag.String("mode", "index+github",
+		`The crawling mode, which can be one of [github-user, github-repo, index, github, index+github].
+  * github-user: crawl all the kustomization files in all the repositories of a Github user (--github-user must be specified for this mode).
+  * github-repo: crawl all the kustomization files in a Github repository (--github-repo must be specified for this mode).
+  * index: crawl all the documents in the index.
+  * gihub: crawl all the kustomization files on Github.
+  * index+github: crawl all the documents in the index and crawling all the kustomization files on Github.`)
+	githubUserPtr := flag.String("github-user", "",
+		"A github user name (e.g., kubernetes-sigs). This flag is required for the `github-user` mode.")
+	githubRepoPtr := flag.String("github-repo", "",
+		"A github repository name (e.g., kubernetes-sigs/kustomize). This flag is required for the `github-repo` mode.")
+	flag.Parse()
+
 	githubToken := os.Getenv(githubAccessTokenVar)
 	if githubToken == "" {
-		fmt.Printf("Must set the variable '%s' to make github requests.\n",
+		log.Printf("Must set the variable '%s' to make github requests.\n",
 			githubAccessTokenVar)
 		return
 	}
 
 	ctx := context.Background()
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
 	if err != nil {
-		fmt.Printf("Could not create an index: %v\n", err)
+		log.Printf("Could not create an index: %v\n", err)
 		return
 	}
 
@@ -87,7 +94,7 @@ func main() {
 	cache, err := redis.DialURL(cacheURL)
 	clientCache := &http.Client{}
 	if err != nil {
-		fmt.Printf("Error: redis could not make a connection: %v\n", err)
+		log.Printf("Error: redis could not make a connection: %v\n", err)
 	} else {
 		clientCache = httpclient.NewClient(cache)
 	}
@@ -108,10 +115,10 @@ func main() {
 		case *doc.KustomizationDocument:
 			switch mode {
 			case index.Delete:
-				fmt.Println("Deleting: ", d)
+				log.Printf("Deleting: %v", d)
 				return idx.Delete(d.ID())
 			default:
-				fmt.Println("Inserting: ", d)
+				log.Printf("Inserting: %v", d)
 				return idx.Put(d.ID(), d)
 			}
 		default:
@@ -119,23 +126,19 @@ func main() {
 		}
 	}
 
-	// seen tracks the IDs of all the documents in the index.
+	// seen tracks the IDs of all the documents in the index and their corresponding file types.
 	// This helps avoid indexing a given document multiple times.
-	seen := make(map[string]struct{})
+	seen := utils.NewSeenMap()
 
-	var mode CrawlMode
-	if len(os.Args) == 1 {
-		mode = CrawlIndexAndGithub
-	} else {
-		mode = NewCrawlMode(os.Args[1])
-	}
+	mode := NewCrawlMode(*modePtr)
 
 	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
 		if user != "" {
-			return 	github.NewCrawler(githubToken, retryCount, clientCache,
+			return github.NewCrawler(githubToken, retryCount, clientCache,
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
 					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
 					github.User(user)),
 			)
 		} else if repo != "" {
@@ -143,13 +146,15 @@ func main() {
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
 					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
 					github.Repo(repo)),
 			)
 		} else {
 			return github.NewCrawler(githubToken, retryCount, clientCache,
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
-					github.Filename("kustomization.yml")),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization")),
 			)
 		}
 	}
@@ -162,11 +167,11 @@ func main() {
 		it := idx.IterateQuery(query, 10000, 60*time.Second)
 		for it.Next() {
 			for _, hit := range it.Value().Hits.Hits {
-				seedDocs = append(seedDocs, hit.Document.Copy())
+				seedDocs = append(seedDocs, hit.Document.Document.Copy())
 			}
 		}
 		if err := it.Err(); err != nil {
-			fmt.Printf("Error iterating: %v\n", err)
+			log.Fatalf("getSeedDocsFunc Error iterating: %v\n", err)
 		}
 	}
 
@@ -182,23 +187,29 @@ func main() {
 		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
 	case CrawlGithub:
 		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		// add all the documents in the index into seen.
+		// this greatly reduces the time overhead of CrawlGithub.
+		getSeedDocsFunc()
+		for _, d := range seedDocs {
+			seen[d.ID()] = d.FileType
+		}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlUser:
-		if len(os.Args) < 3 {
-			Usage()
-			log.Fatalf("Please specify a github user!")
+		if *githubUserPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github user with the github-user flag!")
 		}
-		crawlers := []crawler.Crawler{ghCrawlerConstructor(os.Args[2], "")}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(*githubUserPtr, "")}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlRepo:
-		if len(os.Args) < 3 {
-			Usage()
-			log.Fatalf("Please specify a github repo!")
+		if *githubRepoPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github repository with the github-repo flag!")
 		}
-		crawlers := []crawler.Crawler{ghCrawlerConstructor("", os.Args[2])}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", *githubRepoPtr)}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlUnknown:
-		Usage()
-		log.Fatalf("The crawler mode must be one of [github-user, github-repo, index, github]")
+		flag.Usage()
+		log.Fatalf("The --mode flag must be one of [github-user, github-repo, index, github, index+github].")
 	}
 }

api/internal/crawl/cmd/kustomize_stats/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install ./cmd/kustomize_stats
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/kustomize_stats /
+ENTRYPOINT ["/kustomize_stats"]

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -0,0 +1,254 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"flag"
+	"fmt"
+	"log"
+	"sort"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+)
+
+const (
+	githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
+	retryCount           = 3
+)
+
+// iterateArr adds each item in arr into countMap.
+func iterateArr(arr []string, countMap map[string]int) {
+	for _, item := range arr {
+		if _, ok := countMap[item]; !ok {
+			countMap[item] = 1
+		} else {
+			countMap[item]++
+		}
+	}
+
+}
+
+// SortMapKeyByValueInt takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueInt(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
+	return keys
+}
+
+// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueLen(m map[string][]string) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return len(m[keys[i]]) > len(m[keys[j]]) })
+	return keys
+}
+
+func GeneratorOrTransformerStats(docs []*doc.KustomizationDocument) {
+	n := len(docs)
+	if n == 0 {
+		return
+	}
+
+	fileType := docs[0].FileType
+	fmt.Printf("There are totally %d %s files.\n", n, fileType)
+
+	GitRepositorySummary(docs, fileType)
+
+	// key of kindToUrls: a string in the KustomizationDocument.Kinds field
+	// value of kindToUrls: a slice of string urls defining a given kind.
+	kindToUrls := make(map[string][]string)
+
+	for _, d := range docs {
+		url := fmt.Sprintf("%s/blob/%s/%s", d.RepositoryURL, d.DefaultBranch, d.FilePath)
+		for _, kind := range d.Kinds {
+			if _, ok := kindToUrls[kind]; !ok {
+				kindToUrls[kind] = []string{url}
+			} else {
+				kindToUrls[kind] = append(kindToUrls[kind], url)
+			}
+		}
+	}
+	fmt.Printf("There are totally %d kinds of %s\n", len(kindToUrls), fileType)
+	sortedKeys := SortMapKeyByValueLen(kindToUrls)
+	for _, k := range sortedKeys {
+		sort.Strings(kindToUrls[k])
+		fmt.Printf("%s kind %s appears %d times\n", fileType, k, len(kindToUrls[k]))
+		for _, url := range kindToUrls[k] {
+			fmt.Printf("%s\n", url)
+		}
+	}
+}
+
+// GitRepositorySummary counts the distribution of docs:
+// 1) how many git repositories are these docs from?
+// 2) how many docs are from each git repository?
+func GitRepositorySummary(docs []*doc.KustomizationDocument, fileType string) {
+	m := make(map[string]int)
+	for _, d := range docs {
+		if _, ok := m[d.RepositoryURL]; ok {
+			m[d.RepositoryURL]++
+		} else {
+			m[d.RepositoryURL] = 1
+		}
+	}
+	sortedKeys := SortMapKeyByValueInt(m)
+	topN := 10
+	i := 0
+	for _, k := range sortedKeys {
+		if i >= topN {
+			break
+		}
+		fmt.Printf("%d %s are from %s\n", m[k], fileType, k)
+		i++
+	}
+}
+
+func main() {
+	topKindsPtr := flag.Int(
+		"kinds", -1,
+		`the number of kubernetes object kinds to be listed according to their popularities.
+By default, all the kinds will be listed.
+If you only want to list the 10 most popular kinds, set the flag to 10.`)
+	topIdentifiersPtr := flag.Int(
+		"identifiers", -1,
+		`the number of identifiers to be listed according to their popularities.
+By default, all the identifiers will be listed.
+If you only want to list the 10 most popular identifiers, set the flag to 10.`)
+	topKustomizeFeaturesPtr := flag.Int(
+		"kustomize-features", -1,
+		`the number of kustomize features to be listed according to their popularities.
+By default, all the features will be listed.
+If you only want to list the 10 most popular features, set the flag to 10.`)
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	flag.Parse()
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	if err != nil {
+		log.Fatalf("Could not create an index: %v\n", err)
+	}
+
+	// count tracks the number of documents in the index
+	count := 0
+
+	// kustomizationFilecount tracks the number of kustomization files in the index
+	kustomizationFilecount := 0
+
+	kindsMap := make(map[string]int)
+	identifiersMap := make(map[string]int)
+	kustomizeIdentifiersMap := make(map[string]int)
+
+	// ids tracks the unique IDs of the documents in the index
+	ids := make(map[string]struct{})
+
+	// generatorFiles include all the non-kustomization files whose FileType is generator
+	generatorFiles := make([]*doc.KustomizationDocument, 0)
+
+	// transformersFiles include all the non-kustomization files whose FileType is transformer
+	transformersFiles := make([]*doc.KustomizationDocument, 0)
+
+	checksums := make(map[string]int)
+
+	// get all the documents in the index
+	query := []byte(`{ "query":{ "match_all":{} } }`)
+	it := idx.IterateQuery(query, 10000, 60*time.Second)
+	for it.Next() {
+		for _, hit := range it.Value().Hits.Hits {
+			sum := fmt.Sprintf("%x", sha256.Sum256([]byte(hit.Document.DocumentData)))
+			if _, ok := checksums[sum]; ok {
+				checksums[sum]++
+			} else {
+				checksums[sum] = 1
+			}
+
+			// check whether there is any duplicate IDs in the index
+			if _, ok := ids[hit.ID]; !ok {
+				ids[hit.ID] = struct{}{}
+			} else {
+				log.Printf("Found duplicate ID (%s)\n", hit.ID)
+			}
+
+			count++
+			iterateArr(hit.Document.Kinds, kindsMap)
+			iterateArr(hit.Document.Identifiers, identifiersMap)
+
+			if doc.IsKustomizationFile(hit.Document.FilePath) {
+				kustomizationFilecount++
+				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
+
+			} else {
+				switch hit.Document.FileType {
+				case "generator":
+					generatorFiles = append(generatorFiles, hit.Document.Copy())
+				case "transformer":
+					transformersFiles = append(transformersFiles, hit.Document.Copy())
+				}
+			}
+		}
+	}
+
+	if err := it.Err(); err != nil {
+		log.Fatalf("Error iterating: %v\n", err)
+	}
+
+	sortedKindsMapKeys := SortMapKeyByValueInt(kindsMap)
+	sortedIdentifiersMapKeys := SortMapKeyByValueInt(identifiersMap)
+	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValueInt(kustomizeIdentifiersMap)
+
+	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
+There are %d documents in the kustomize index.
+%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
+	fmt.Printf("\n")
+
+	kindCount := 0
+	for _, key := range sortedKindsMapKeys {
+		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
+			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
+			kindCount++
+		}
+	}
+
+	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
+	identifierCount := 0
+	for _, key := range sortedIdentifiersMapKeys {
+		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
+			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
+			identifierCount++
+		}
+	}
+
+	fmt.Printf(`There are %d kustomization files in the kustomize index.
+%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
+	fmt.Printf("\n")
+	kustomizeFeatureCount := 0
+	for _, key := range sortedKustomizeIdentifiersMapKeys {
+		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
+			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
+			kustomizeFeatureCount++
+		}
+	}
+
+	GeneratorOrTransformerStats(generatorFiles)
+	GeneratorOrTransformerStats(transformersFiles)
+
+	fmt.Printf("There are total %d checksums of document contents\n", len(checksums))
+	sortedChecksums := SortMapKeyByValueInt(checksums)
+	sortedChecksums = sortedChecksums[:20]
+	fmt.Printf("The top 20 checksums are:\n")
+	for _, key := range sortedChecksums {
+		fmt.Printf("checksum %s apprears %d\n", key, checksums[key])
+	}
+}

api/internal/crawl/cmd/log-parser/README.md

@@ -0,0 +1,8 @@
+This binary takes as its input a json file including GKE logs (which can be
+[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into 
+[Cloud Storage](https://cloud.google.com/storage/docs/)), 
+and extracts the `textPayload` field of each log entry.
+
+Here is an log entry example:
+
+{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"}

api/internal/crawl/cmd/log-parser/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatalf("The usage of the command is: \n\t%s <log-file.json>", os.Args[0])
+	}
+
+	file, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	closeFile := func(file *os.File) {
+		if err := file.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}
+	defer closeFile(file)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		var entry interface{}
+		if err := json.Unmarshal([]byte(line), &entry); err != nil {
+			log.Printf("failed to unmarshal a log entry: %s\n", line)
+		}
+
+		m := entry.(map[string]interface{})
+		if payload, ok := m["textPayload"]; ok {
+			// use fmt.Printf here instead of log.Printf to avoid the time and code location info the log package provides
+			fmt.Printf("%s", payload)
+		} else {
+			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}

api/internal/crawl/config/base/kustomization.yaml

@@ -2,5 +2,4 @@ configmapGenerator:
 - name: elasticsearch-config
   literals:
   - es-url="http://esbasic-master:9200"
-  - kustomize-index-name="kustomize"
   - plugin-index-name="plugin"

api/internal/crawl/config/crawler/cronjob/cronjob.yaml

@@ -1,9 +1,10 @@
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: crawler
+  name: crawler-cronjob
 spec:
-  schedule: "5 0 * * */1"
+  # run the cronjob at 00:00 every 7 days
+  schedule: "0 0 */7 * *"
   jobTemplate:
     spec:
       template:
@@ -11,7 +12,9 @@ spec:
           restartPolicy: OnFailure
           containers:
           - name: crawler
-            image: gcr.io/kustomize-search/crawler:latest
+            image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+            command: ["/crawler"]
+            args: ["--mode=index+github", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
             imagePullPolicy: Always
             env:
             - name: GITHUB_ACCESS_TOKEN

api/internal/crawl/config/crawler/job/README.md

@@ -1,4 +1,4 @@
-There are three ways of running the crawler job.
+The crawler job can run in one of the following mode:
 
 # Crawling all the documents in the index and crawling all the kustomization files on Github
 
@@ -7,14 +7,13 @@ of the container should be:
 
 ```
         command: ["/crawler"]
-        args: []
 ```
 
 Or
 
 ```
         command: ["/crawler"]
-        args: [""]
+        args: ["--mode=index+github"]
 ```
 
 # Crawling all the documents in the index
@@ -23,7 +22,7 @@ The `command` and `args` field of the container should be:
 
 ```
         command: ["/crawler"]
-        args: ["index"]
+        args: ["--mode=index"]
 ```
 
 # Crawling all the kustomization files on Github
@@ -32,7 +31,7 @@ The `command` and `args` field of the container should be:
 
 ```
         command: ["/crawler"]
-        args: ["github"]
+        args: ["--mode=github"]
 ```
 
 # Crawling all the kustomization files in a Github repo
@@ -41,7 +40,7 @@ The `command` and `args` field of the container should be like:
 
 ```
         command: ["/crawler"]
-        args: ["github-repo", "kubernetes-sigs/kustomize"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize"]
 ```
 
 # Crawling all the kustomization files in all the repositories of a Github user
@@ -50,5 +49,5 @@ The `command` and `args` field of the container should be like:
 
 ```
         command: ["/crawler"]
-        args: ["github-user", "kubernetes-sigs"]
+        args: ["--github-user", "--github-user=kubernetes-sigs"]
 ```

api/internal/crawl/config/crawler/job/job.yaml

@@ -11,7 +11,7 @@ spec:
         image: gcr.io/haiyanmeng-gke-dev/crawler:v1
         imagePullPolicy: Always
         command: ["/crawler"]
-        args: ["github-repo", "kubernetes-sigs/kustomize"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
         env:
         - name: GITHUB_ACCESS_TOKEN
           valueFrom:

api/internal/crawl/config/crawler/kustomize_stats/job.yaml

@@ -0,0 +1,25 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kustomize-stats
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: kustomize-stats
+        image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
+        imagePullPolicy: Always
+        command: ["/kustomize_stats"]
+        args: ["--index=kustomize", "--kinds=50", "--identifiers=50", "--kustomize-features=50"]
+        env:
+        - name: GITHUB_ACCESS_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: github-access-token
+              key: token
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url

api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml

api/internal/crawl/config/elastic/esbackup.yaml

@@ -0,0 +1,16 @@
+# Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository and all the snapshots in the repository.
+# Deleting `esbackup/kustomize-backbup` will NOT delete essnapshot and esrestore objects.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESBackup
+metadata:
+  name: kustomize-backup
+spec:
+  storage:
+    gcs:
+      bucket: kustomize-backup
+      path: kustomize
+      secret:
+        name: kustomizesa
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/escluster.yaml

@@ -8,6 +8,13 @@ spec:
     - repository-gcs
     - ingest-user-agent
     - ingest-geoip
+  # To set `gcpserviceaccount`,
+  # First, create and download a GCP service account into a json file, named `sakey.json` following the instruction:
+  #     https://www.elastic.co/guide/en/elasticsearch/plugins/6.5/repository-gcs-usage.html#repository-gcs-using-service-account
+  # Second, create a secret for the service account using the following command:
+  #     $ kubectl create secret generic kustomizesa --from-file=./sakey.json
+  gcpserviceaccount:
+    name: kustomizesa
   config:
     env:
       example: test

api/internal/crawl/config/elastic/esrestore.yaml

@@ -0,0 +1,16 @@
+# Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`.
+# Deleting `esrestore/kustomize-restore` will not delete the restored index.
+# Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESRestore
+metadata:
+  name: kustomize-restore
+spec:
+  include_global_state: true
+  ignore_unavailable: true
+  rename_pattern: kustomize
+  rename_replacement: kustomize-restore
+  essnapshot:
+    name: kustomize-snapshot
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/essnaptshot.yaml

@@ -0,0 +1,15 @@
+# Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository.
+# Deleting `essnapshot/kustomize-snapshot` will delete the snapshot.
+# Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESSnapshot
+metadata:
+  name: kustomize-snapshot
+spec:
+  # indices are optional. If not specified all indices are selected.
+  indices:
+  - kustomize
+  include_global_state: true
+  ignore_unavailable: true
+  esbackup:
+    name: kustomize-backup

api/internal/crawl/crawler/crawler.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"sync"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/index"
 
 	_ "github.com/gomodule/redigo/redis"
@@ -29,13 +31,15 @@ type Crawler interface {
 	// Crawl returns when it is done processing. This method does not take
 	// ownership of the channel. The channel is write only, and it
 	// designates where the crawler should forward the documents.
-	Crawl(ctx context.Context, output chan<- CrawledDocument) error
+	Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error
 
 	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
 	FetchDocument(context.Context, *doc.Document) error
 	// Write to the document what the created time is.
 	SetCreated(context.Context, *doc.Document) error
 
+	SetDefaultBranch(*doc.Document)
+
 	Match(*doc.Document) bool
 }
 
@@ -43,7 +47,12 @@ type CrawledDocument interface {
 	ID() string
 	GetDocument() *doc.Document
 	// Get all the Documents directly referred in a Document.
-	GetResources() ([]*doc.Document, error)
+	// For a Document representing a non-kustomization file, an empty slice will be returned.
+	// For a Document representing a kustomization file:
+	// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
+	// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
+	// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
+	GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
 	WasCached() bool
 }
 
@@ -69,25 +78,27 @@ func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
 }
 
 func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
-	seen map[string]struct{}, stack *CrawlSeed) {
+	seen utils.SeenMap, stack *CrawlSeed) {
 
-	seen[cdoc.ID()] = struct{}{}
+	seen.Set(cdoc.ID(), cdoc.GetDocument().FileType)
+
+	match.SetDefaultBranch(cdoc.GetDocument())
 
 	// Insert into index
 	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
-		logger.Printf("Failed to insert or update %s %s: %v",
-			cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err)
+		logger.Printf("Failed to insert or update doc(%s): %v",
+			cdoc.GetDocument().Path(), err)
 		return
 	}
 
-	deps, err := cdoc.GetResources()
+	deps, err := cdoc.GetResources(true, true, true)
 	if err != nil {
 		logger.Println(err)
 		return
 	}
 
 	for _, dep := range deps {
-		if _, ok := seen[dep.ID()]; ok {
+		if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType {
 			continue
 		}
 		*stack = append(*stack, dep)
@@ -95,7 +106,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
 }
 
 func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
-	seen map[string]struct{}, stack *CrawlSeed) {
+	seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) {
 
 	UpdatedDocCount := 0
 	seenDocCount := 0
@@ -105,6 +116,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 	SetCreatedErrCount := 0
 	convErrCount := 0
 	deleteDocCount := 0
+	crawledDocCount := 0
 
 	// During the execution of the for loop, more Documents may be added into (*docsPtr).
 	for len(*docsPtr) > 0 {
@@ -114,13 +126,19 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 		// remove the last Document in (*docPtr)
 		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
 
-		if _, ok := seen[tail.ID()]; ok {
-			seenDocCount++
-			continue
+		crawledDocCount++
+		logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
+
+		if seen.Seen(tail.ID()) {
+			if !updateFileType || seen.Value(tail.ID()) == tail.FileType {
+				logger.Printf("this doc has been seen before")
+				seenDocCount++
+				continue
+			}
 		}
 
 		if tail.WasCached() {
-			logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath)
+			logger.Printf("doc(%s) is cached already", tail.Path())
 			cachedDocCount++
 			continue
 		}
@@ -132,36 +150,52 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 			continue
 		}
 
-		logger.Println("Crawling ", tail.RepositoryURL, tail.FilePath)
-		if err := match.FetchDocument(ctx, tail); err != nil {
-			logger.Printf("FetchDocument failed on %s %s: %v",
-				tail.RepositoryURL, tail.FilePath, err)
-			FetchDocumentErrCount++
-			// delete the document from the index
-			cdoc := &doc.KustomizationDocument{
-				Document: *tail,
-			}
-			seen[cdoc.ID()] = struct{}{}
-			if err := indx(cdoc, index.Delete); err != nil {
-				logger.Printf("Failed to delete %s %s: %v",
-					cdoc.RepositoryURL, cdoc.FilePath, err)
-			}
-			deleteDocCount++
-			continue
+		if tail.User == "" {
+			tail.User = doc.UserName(tail.RepositoryURL)
 		}
 
-		if err := match.SetCreated(ctx, tail); err != nil {
-			logger.Printf("SetCreated failed on %s %s: %v",
-				tail.RepositoryURL, tail.FilePath, err)
-			SetCreatedErrCount++
+		// If the Document represents a kustomization root, FetchDcoument will change
+		// the `filePath` field of the Document by adding `kustomization.yaml` or
+		// `kustomization.yml` or `kustomization` into the the field.
+		// Therefore, it is necessary to add the ID of the Document into seen before
+		// calling FetchDocument. Otherwise, the binary may enter into an infinite loop
+		// if a kustomization file points to its kustmozation root in its `resources` or
+		// `bases` field.
+		seen.Set(tail.ID(), tail.FileType)
+
+		if refreshDoc || tail.DefaultBranch == "" {
+			match.SetDefaultBranch(tail)
+		}
+
+		if refreshDoc || tail.DocumentData == "" {
+			if err := match.FetchDocument(ctx, tail); err != nil {
+				logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
+				FetchDocumentErrCount++
+				// delete the document from the index
+				cdoc := &doc.KustomizationDocument{
+					Document: *tail,
+				}
+				seen.Set(cdoc.ID(), tail.FileType)
+				if err := indx(cdoc, index.Delete); err != nil {
+					logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
+				}
+				deleteDocCount++
+				continue
+			}
+		}
+
+		if refreshDoc || tail.CreationTime == nil {
+			if err := match.SetCreated(ctx, tail); err != nil {
+				logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
+				SetCreatedErrCount++
+			}
 		}
 
 		cdoc, err := conv(tail)
 		// If conv returns an error, cdoc can still be added into the index so that
 		// cdoc.Document can be searched.
 		if err != nil {
-			logger.Printf("conv failed on %s %s: %v",
-				tail.RepositoryURL, tail.FilePath, err)
+			logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
 			convErrCount++
 		}
 
@@ -182,7 +216,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 // CrawlFromSeed updates all the documents in seed, and crawls all the new
 // documents referred in the seed.
 func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
-	conv Converter, indx IndexFunc, seen map[string]struct{}) {
+	conv Converter, indx IndexFunc, seen utils.SeenMap) {
 
 	// stack tracks the documents directly referred in other documents.
 	stack := make(CrawlSeed, 0)
@@ -190,14 +224,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
 	// Exploit seed to update bulk of corpus.
 	logger.Printf("updating %d documents from seed\n", len(seed))
 	// each unique document in seed will be crawled once.
-	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack)
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false)
 
 	// Traverse any new documents added while updating corpus.
 	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
 	// While crawling each document in stack, the documents directly referred in the document
 	// will be added into stack.
 	// After this statement is done, stack will become empty.
-	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
 }
 
 // CrawlGithubRunner is a blocking function and only returns once all of the
@@ -218,7 +252,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
 // from the seed will be processed before any other documents from the
 // crawlers.
 func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
-	crawlers []Crawler) []error {
+	crawlers []Crawler, seen utils.SeenMap) []error {
 
 	errs := make([]error, len(crawlers))
 	wg := sync.WaitGroup{}
@@ -252,7 +286,7 @@ func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
 				}
 			}()
 			defer close(docs)
-			errs[idx] = crawler.Crawl(ctx, docs)
+			errs[idx] = crawler.Crawl(ctx, docs, seen)
 		}(i, crawler, docs) // Copies the index and the crawler
 	}
 
@@ -262,7 +296,7 @@ func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
 
 // CrawlGithub crawls all the kustomization files on Github.
 func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
-	indx IndexFunc, seen map[string]struct{}) {
+	indx IndexFunc, seen utils.SeenMap) {
 	// stack tracks the documents directly referred in other documents.
 	stack := make(CrawlSeed, 0)
 
@@ -274,8 +308,14 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+		docCount := 0
 		for cdoc := range ch {
-			if _, ok := seen[cdoc.ID()]; ok {
+			docCount++
+			logger.Printf("Processing doc %d found on Github", docCount)
+			// all the docs here are kustomization files found by querying Github, and
+			// their `FileType` fields all should be empty.
+			if seen.Seen(cdoc.ID()) {
+				logger.Printf("the doc has been seen before")
 				continue
 			}
 			match := findMatch(cdoc.GetDocument(), crawlers)
@@ -289,7 +329,7 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
 	}()
 
 	logger.Println("processing the documents found from crawling github")
-	if errs := CrawlGithubRunner(ctx, ch, crawlers); errs != nil {
+	if errs := CrawlGithubRunner(ctx, ch, crawlers, seen); errs != nil {
 		for _, err := range errs {
 			logIfErr(err)
 		}
@@ -300,5 +340,5 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
 	// Handle deps of newly discovered documents.
 	logger.Printf("crawling the %d new documents referred by other documents",
 		len(stack))
-	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
 }

api/internal/crawl/crawler/crawler_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"reflect"
 	"sort"
 	"strings"
@@ -11,6 +12,8 @@ import (
 	"testing"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/index"
 
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -34,6 +37,8 @@ func (c testCrawler) Match(d *doc.Document) bool {
 	return d != nil
 }
 
+func (c testCrawler) SetDefaultBranch(d *doc.Document) {}
+
 func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	if i, ok := c.lukp[d.ID()]; ok {
 		d.DocumentData = c.docs[i].DocumentData
@@ -75,7 +80,7 @@ func newCrawler(matchPrefix string, err error,
 
 // Crawl implements the Crawler interface for testing.
 func (c testCrawler) Crawl(_ context.Context,
-	output chan<- CrawledDocument) error {
+	output chan<- CrawledDocument, _ utils.SeenMap) error {
 
 	for i, d := range c.docs {
 		isResource := true
@@ -110,7 +115,7 @@ func (s sortableDocs) Len() int {
 }
 
 func TestCrawlGithubRunner(t *testing.T) {
-	fmt.Println("testing CrawlGithubRunner")
+	log.Println("testing CrawlGithubRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -181,8 +186,9 @@ func TestCrawlGithubRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()
 
+			seen := utils.NewSeenMap()
 			errs := CrawlGithubRunner(context.Background(),
-				output, test.tc)
+				output, test.tc, seen)
 
 			// Check that errors are returned as they should be.
 			if !reflect.DeepEqual(errs, test.errs) {
@@ -215,7 +221,7 @@ func TestCrawlGithubRunner(t *testing.T) {
 }
 
 func TestCrawlFromSeed(t *testing.T) {
-	fmt.Println("testing CrawlFromSeed")
+	log.Println("testing CrawlFromSeed")
 
 	tests := []struct {
 		seed    CrawlSeed
@@ -322,7 +328,7 @@ resources:
 				visited[d.ID()]++
 				return nil
 			},
-			make(map[string]struct{}),
+			utils.NewSeenMap(),
 		)
 		if lv, lc := len(visited), len(tc.corpus); lv != lc {
 			t.Errorf("error: %d of %d documents visited.", lv, lc)

api/internal/crawl/crawler/github/crawler.go

@@ -16,6 +16,8 @@ import (
 	"strings"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
 	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
@@ -30,6 +32,8 @@ var logger = log.New(os.Stdout, "Github Crawler: ",
 type githubCrawler struct {
 	client GhClient
 	query  Query
+	// branchMap maps github repositories to their default branches
+	branchMap map[string]string
 }
 
 type GhClient struct {
@@ -51,13 +55,60 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 			},
 			accessToken: accessToken,
 		},
-		query: query,
+		query:     query,
+		branchMap: map[string]string{},
 	}
 }
 
+func (gc githubCrawler) SetDefaultBranch(d *doc.Document) {
+	url := gc.client.ReposRequest(d.RepositoryFullName())
+	defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+	d.DefaultBranch = defaultBranch
+	gc.branchMap[d.RepositoryURL] = d.DefaultBranch
+}
+
+func (gc githubCrawler) DefaultBranch(repo string) string {
+	return gc.branchMap[repo]
+}
+
 // Implements crawler.Crawler.
-func (gc githubCrawler) Crawl(
-	ctx context.Context, output chan<- crawler.CrawledDocument) error {
+func (gc githubCrawler) Crawl(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
+
+	ranges := []RangeWithin{
+		RangeWithin{
+		start: uint64(0),
+		end:   githubMaxFileSize,
+	},
+	}
+
+	errs := make(multiError, 0)
+	for len(ranges) > 0 {
+		tailRange := ranges[len(ranges) - 1]
+		ranges = ranges[:(len(ranges) - 1)]
+		reProcessQueryRanges, err := gc.CrawlSingleRange(ctx, output, seen, tailRange.start, tailRange.end)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		ranges = append(ranges, reProcessQueryRanges...)
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	lowerBound, upperBound uint64) ([]RangeWithin, error) {
+
+	log.Printf("CrawlSingleRange [%d, %d]", lowerBound, upperBound)
 
 	noETagClient := GhClient{
 		RequestConfig: gc.client.RequestConfig,
@@ -66,12 +117,25 @@ func (gc githubCrawler) Crawl(
 		accessToken:   gc.client.accessToken,
 	}
 
+	var reProcessQueryRanges []RangeWithin
+
+	var ranges []string
+	var err error
 	// Since Github returns a max of 1000 results per query, we can use
 	// multiple queries that split the search space into chunks of at most
 	// 1000 files to get all of the data.
-	ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query))
+	for i := 0; i < 5; i++ {
+		ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query),
+			lowerBound, upperBound)
+		if err == nil {
+			logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i)
+			break
+		} else {
+			time.Sleep(time.Minute)
+		}
+	}
 	if err != nil {
-		return fmt.Errorf("could not split %v into ranges, %v\n",
+		return reProcessQueryRanges, fmt.Errorf("could not split %v into ranges, %v\n",
 			gc.query, err)
 	}
 
@@ -79,18 +143,25 @@ func (gc githubCrawler) Crawl(
 
 	// Query each range for files.
 	errs := make(multiError, 0)
+	queryResult := RangeQueryResult{}
 	for _, query := range ranges {
-		err := processQuery(ctx, gc.client, query, output)
+		reProcessQuery, rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap)
 		if err != nil {
 			errs = append(errs, err)
 		}
+		queryResult.Add(rangeResult)
+		if reProcessQuery {
+			reProcessQueryRanges = append(reProcessQueryRanges, RangeSizes(query))
+		}
 	}
 
+	logger.Printf("Summary of Crawl: %s", queryResult.String())
+
 	if len(errs) > 0 {
-		return errs
+		return reProcessQueryRanges, errs
 	}
 
-	return nil
+	return reProcessQueryRanges, nil
 }
 
 // FetchDocument first tries to fetch the document with d.FilePath. If it fails,
@@ -107,9 +178,13 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 		"/" + repoSpec.Ref + "/" + repoSpec.Path
 
 	handle := func(resp *http.Response, err error, path string) error {
+		if resp == nil {
+			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
+				url, path, err)
+		}
 		if err == nil && resp.StatusCode == http.StatusOK {
 			d.IsSame = httpclient.FromCache(resp.Header)
-			defer resp.Body.Close()
+			defer CloseResponseBody(resp)
 			data, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				return err
@@ -161,10 +236,32 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
 	return strings.Contains(repoSpec.Host, "github.com")
 }
 
+type RangeQueryResult struct {
+	totalDocCnt uint64
+	seenDocCnt  uint64
+	newDocCnt   uint64
+	errorCnt    uint64
+}
+
+func (r *RangeQueryResult) Add(other RangeQueryResult) {
+	r.totalDocCnt += other.totalDocCnt
+	r.newDocCnt += other.newDocCnt
+	r.seenDocCnt += other.seenDocCnt
+	r.errorCnt += other.errorCnt
+}
+
+func (r *RangeQueryResult) String() string {
+	return fmt.Sprintf("got %d files from API. "+
+		"%d have been seen before. %d are new and sent to the output channel."+
+		" %d have kustomizationResultAdapter errors.",
+		r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
+}
+
 // processQuery follows all of the pages in a query, and updates/adds the
 // documents from the crawl to the datastore/index.
 func processQuery(ctx context.Context, gcl GhClient, query string,
-	output chan<- crawler.CrawledDocument) error {
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	branchMap map[string]string) (bool, RangeQueryResult, error) {
 
 	queryPages := make(chan GhResponseInfo)
 
@@ -180,57 +277,84 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
 		close(queryPages)
 	}()
 
+	reProcessQuery := false
+
 	errs := make(multiError, 0)
-	errorCnt := 0
-	totalCnt := 0
+	result := RangeQueryResult{}
+	pageID := 1
 	for page := range queryPages {
 		if page.Error != nil {
 			errs = append(errs, page.Error)
 			continue
 		}
-
+		pageResult := RangeQueryResult{}
 		for _, file := range page.Parsed.Items {
-			k, err := kustomizationResultAdapter(gcl, file)
+			k, err := kustomizationResultAdapter(gcl, file, seen, branchMap)
 			if err != nil {
 				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
-				errorCnt++
+				pageResult.errorCnt++
 			}
 			if k != nil {
+				pageResult.newDocCnt++
 				output <- k
+			} else {
+				pageResult.seenDocCnt++
 			}
-			totalCnt++
+			pageResult.totalDocCnt++
 		}
 
-		logger.Printf("got %d files out of %d from API. %d of %d had errors\n",
-			totalCnt, page.Parsed.TotalCount, errorCnt, totalCnt)
+		logger.Printf("processQuery [TotalCount %d - page %d]: %s",
+			page.Parsed.TotalCount, pageID, pageResult.String())
+		result.Add(pageResult)
+
+		pageID++
+
+		if page.Parsed.TotalCount > githubMaxResultsPerQuery {
+			reProcessQuery = true
+		}
 	}
 
-	return errs
+	logger.Printf("Summary of processQuery: %s", result.String())
+
+	return reProcessQuery, result, errs
 }
 
-func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
-	crawler.CrawledDocument, error) {
-
-	data, err := gcl.GetFileData(k)
-	if err != nil {
-		return nil, err
-	}
-
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
+	branchMap map[string]string) (crawler.CrawledDocument, error) {
 	url := gcl.ReposRequest(k.Repository.FullName)
-	defaultBranch, err := gcl.GetDefaultBranch(url)
+	defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
 	if err != nil {
 		logger.Printf(
 			"(error: %v) setting default_branch to master\n", err)
 		defaultBranch = "master"
 	}
 
+	// document here is a kustomization file found by querying Github, whose
+	// `FileType` field should be empty.
+	document := doc.Document{
+		FilePath:      k.Path,
+		DefaultBranch: defaultBranch,
+		RepositoryURL: k.Repository.URL,
+		User:          doc.UserName(k.Repository.URL),
+	}
+
+	if seen.Seen(document.ID()) {
+		return nil, nil
+	}
+
+	data, err := gcl.GetFileData(k)
+	if err != nil {
+		return nil, err
+	}
+
 	d := doc.KustomizationDocument{
 		Document: doc.Document{
 			DocumentData:  string(data),
 			FilePath:      k.Path,
 			DefaultBranch: defaultBranch,
 			RepositoryURL: k.Repository.URL,
+			User:          doc.UserName(k.Repository.URL),
 		},
 	}
 	creationTime, err := gcl.GetFileCreationTime(k)
@@ -255,7 +379,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 	output chan<- GhResponseInfo) error {
 
 	logger.Println("querying: ", query)
-	response := gcl.parseGithubResponse(query)
+	response := gcl.parseGithubResponseWithRetry(query)
 
 	if response.Error != nil {
 		return response.Error
@@ -268,7 +392,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 		case <-ctx.Done():
 			return nil
 		default:
-			response = gcl.parseGithubResponse(response.NextURL)
+			response = gcl.parseGithubResponseWithRetry(response.NextURL)
 			if response.Error != nil {
 				return response.Error
 			}
@@ -296,7 +420,10 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
 			k, url, err)
 	}
-	resp.Body.Close()
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}
 
 	type githubContentRawURL struct {
 		DownloadURL string `json:"download_url,omitempty"`
@@ -315,18 +442,32 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 			k, rawURL.DownloadURL, err)
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err = ioutil.ReadAll(resp.Body)
 	return data, err
 }
 
-func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
+func CloseResponseBody(resp *http.Response) {
+	if err := resp.Body.Close(); err != nil {
+		log.Printf("failed to close response body: %v", err)
+	}
+}
+
+// GetDefaultBranch gets the default branch of a github repository.
+// m is a map which maps a github repository to its default branch.
+// If repo is already in m, the default branch for url will be obtained from m;
+// otherwise, a query will be made to github to obtain the default branch.
+func (gcl GhClient) GetDefaultBranch(url, repo string, m map[string]string) (string, error) {
+	if v, ok := m[repo]; ok {
+		return v, nil
+	}
+
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
 			"'%s' could not get default_branch: %v", url, err)
 	}
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -378,7 +519,7 @@ func (gcl GhClient) GetFileCreationTime(
 		}
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return defaultTime, fmt.Errorf(
@@ -446,6 +587,8 @@ type githubResponse struct {
 	// This is the number of files that match the query.
 	TotalCount uint64 `json:"total_count,omitempty"`
 
+	IncompleteResults bool `json:"incomplete_results,omitempty"`
+
 	// Github representation of a file.
 	Items []GhFileSpec `json:"items,omitempty"`
 }
@@ -488,6 +631,17 @@ func parseGithubLinkFormat(links string) (string, string) {
 	return next, last
 }
 
+func (gcl GhClient) parseGithubResponseWithRetry(getRequest string) GhResponseInfo {
+	resp := gcl.parseGithubResponse(getRequest)
+	retries := 0
+	for resp.Parsed.IncompleteResults {
+		resp = gcl.parseGithubResponse(getRequest)
+		retries++
+	}
+	log.Printf("The result of query(%s) is complete after %d retries", getRequest, retries)
+	return resp
+}
+
 func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	resp, err := gcl.SearchGithubAPI(getRequest)
 	requestInfo := GhResponseInfo{
@@ -501,7 +655,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	}
 
 	var data []byte
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
 	if requestInfo.Error != nil {
 		return requestInfo
@@ -565,8 +719,8 @@ func (gcl GhClient) Do(query string) (*http.Response, error) {
 
 	// gcl.client.Do: a non-2xx status code doesn't cause an error.
 	// See https://golang.org/pkg/net/http/#Client.Do for more info.
-	resp, err :=  gcl.client.Do(req)
-	if resp.StatusCode != http.StatusOK {
+	resp, err := gcl.client.Do(req)
+	if resp != nil && resp.StatusCode != http.StatusOK {
 		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
 			query, resp.StatusCode)
 	}
@@ -580,7 +734,7 @@ func (gcl GhClient) getWithRetry(
 
 	retryCount := gcl.retryCount
 
-	for resp.StatusCode == http.StatusForbidden && retryCount > 0 {
+	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
 		retryTime := resp.Header.Get("Retry-After")
 		i, errAtoi := strconv.Atoi(retryTime)
 		if errAtoi != nil {

api/internal/crawl/crawler/github/queries.go

@@ -116,9 +116,11 @@ type RequestConfig struct {
 // the URL method to get the string value of the URL. See request.CopyWith, to
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
-	req := rc.makeRequest("search/code", query)
-	req.vals.Set("sort", "indexed")
-	req.vals.Set("order", "desc")
+	vals := url.Values{
+		"sort":  []string{"indexed"},
+		"order": []string{"desc"},
+	}
+	req := rc.makeRequest("search/code", query, vals)
 	return req
 }
 
@@ -126,27 +128,25 @@ func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 // query for the Github API to find the dowload information of this filepath.
 func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 func (rc RequestConfig) ReposRequest(fullRepoName string) string {
 	uri := fmt.Sprintf("repos/%s", fullRepoName)
-	return rc.makeRequest(uri, Query{}).URL()
-}
-
-func escapeSpace(s string) string {
-	return strings.Replace(s, " ", "%20", -1)
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 // CommitsRequest given the repo name, and a filepath returns a formatted query
 // for the Github API to find the commits that affect this file.
 func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
-	return rc.makeRequest(uri, Query{Path(escapeSpace(path))}).URL()
+	vals := url.Values{
+		"path": []string{path},
+	}
+	return rc.makeRequest(uri, Query{}, vals).URL()
 }
 
-func (rc RequestConfig) makeRequest(path string, query Query) request {
-	vals := url.Values{}
+func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
 	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
 
 	return request{
@@ -166,7 +166,7 @@ type request struct {
 	query Query
 }
 
-// CopyWith copies the requests and adds the extra query parameters. Usefull
+// CopyWith copies the requests and adds the extra query parameters. It is useful
 // for dynamically adding sizes to a filename only query without modifying it.
 func (r request) CopyWith(queryParams ...queryField) request {
 	cpy := r

api/internal/crawl/crawler/github/queries_test.go

@@ -101,7 +101,7 @@ func TestGithubSearchQuery(t *testing.T) {
 				"examples/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"q=path:examples/helloWorld/kustomization.yaml&per_page=100",
+				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
 		},
 		{
 			rc: RequestConfig{
@@ -121,7 +121,7 @@ func TestGithubSearchQuery(t *testing.T) {
 				"examples%201/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"q=path:examples%201/helloWorld/kustomization.yaml&per_page=100",
+				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
 		},
 	}
 

api/internal/crawl/crawler/github/split_search_ranges.go

@@ -93,13 +93,15 @@ package github
 //   apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000
 //   = apiCallsPerResult * totalResults.
 //
-// So it could very well take apiCallsPerResult * 50 times longer to acutally
+// So it could very well take apiCallsPerResult * 50 times longer to actually
 // fetch the results (assuming the quotas for the API calls are the same as the
 // search API), than it does to perform these range searches.
 
 import (
 	"fmt"
 	"math/bits"
+	"strconv"
+	"strings"
 )
 
 // Files cannot be more than 2^19 bytes, according to
@@ -112,7 +114,7 @@ const (
 // Interface instead of struct for testing purposes.
 // Not expecting to have multiple implementations.
 type cachedSearch interface {
-	CountResults(uint64) (uint64, error)
+	CountResults(uint64, uint64) (uint64, error)
 	RequestString(filesize rangeFormatter) string
 }
 
@@ -139,7 +141,7 @@ type cachedSearch interface {
 //    problematic). The current cache implementation looks at the
 //    predecessor entry to find out if the current value is monotonic.
 //    This is where the bit trick is used, since each step in the binary
-//    search is adding or ommiting to add a decreasing power of 2 to the query
+//    search is adding or omitting to add a decreasing power of 2 to the query
 //    value, we can remove the least significant set bit to find the
 //    predecessor in constant time. Ultimately since the search is rate
 //    limited, we could also easily afford to compute this in linear time
@@ -161,16 +163,16 @@ func newCache(client GhClient, query Query) githubCachedSearch {
 	}
 }
 
-func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
+func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
 	count, cached := c.cache[upperBound]
 	if cached {
 		return count, nil
 	}
 
-	sizeRange := RangeWithin{0, upperBound}
+	sizeRange := RangeWithin{lowerBound, upperBound}
 	rangeRequest := c.RequestString(sizeRange)
 
-	result := c.gcl.parseGithubResponse(rangeRequest)
+	result := c.gcl.parseGithubResponseWithRetry(rangeRequest)
 	if result.Error != nil {
 		return count, result.Error
 	}
@@ -204,7 +206,7 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 			"Retrying query... current lower bound: %d, got: %d\n",
 			c.cache[prev], result.Parsed.TotalCount)
 
-		result = c.gcl.parseGithubResponse(rangeRequest)
+		result = c.gcl.parseGithubResponseWithRetry(rangeRequest)
 		if result.Error != nil {
 			return count, result.Error
 		}
@@ -219,8 +221,8 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 	}
 
 	count = result.Parsed.TotalCount
-	logger.Printf("Caching new query %s, with count %d\n",
-		sizeRange.RangeString(), count)
+	logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n",
+		sizeRange.RangeString(), count, result.Parsed.IncompleteResults)
 	c.cache[upperBound] = count
 	return count, nil
 }
@@ -238,8 +240,8 @@ func (c githubCachedSearch) RequestString(filesize rangeFormatter) string {
 // This would mean that the search as it is could not find all files. If queries
 // are sorted by last indexed, and retrieved on regular intervals, it should be
 // sufficient to get most if not all documents.
-func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
-	totalFiles, err := cache.CountResults(githubMaxFileSize)
+func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64) ([]string, error) {
+	totalFiles, err := cache.CountResults(lowerBound, upperBound)
 	if err != nil {
 		return nil, err
 	}
@@ -247,7 +249,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 
 	if githubMaxResultsPerQuery >= totalFiles {
 		return []string{
-			cache.RequestString(RangeWithin{0, githubMaxFileSize}),
+			cache.RequestString(RangeWithin{lowerBound, upperBound}),
 		}, nil
 	}
 
@@ -275,6 +277,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 	// range.
 	filesAccessible := uint64(0)
 	sizes := make([]uint64, 0)
+	sizes = append(sizes, lowerBound)
 	for filesAccessible < totalFiles {
 		target := filesAccessible + githubMaxResultsPerQuery
 		if target >= totalFiles {
@@ -284,22 +287,22 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 		logger.Printf("%d accessible files, next target = %d\n",
 			filesAccessible, target)
 
-		cur, err := lowerBoundFileCount(cache, target)
+		size, err := FindFileSize(cache, target, lowerBound, upperBound)
 		if err != nil {
 			return nil, err
 		}
 
 		// If there are more than 1000 files in the next bucket, we must
 		// advance anyway and lose out on some files :(.
-		if l := len(sizes); l > 0 && sizes[l-1] == cur {
-			cur++
+		if l := len(sizes); l > 0 && sizes[l-1] == size {
+			size++
 		}
 
-		nextAccessible, err := cache.CountResults(cur)
+		nextAccessible, err := cache.CountResults(lowerBound, size)
 		if err != nil {
 			return nil, fmt.Errorf(
 				"cache should be populated at %d already, got %v",
-				cur, err)
+				size, err)
 		}
 		if nextAccessible < filesAccessible {
 			return nil, fmt.Errorf(
@@ -309,31 +312,31 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 
 		filesAccessible = nextAccessible
 		if nextAccessible < totalFiles {
-			sizes = append(sizes, cur)
+			sizes = append(sizes, size)
 		}
 	}
-
+	sizes = append(sizes, upperBound)
 	return formatFilesizeRanges(cache, sizes), nil
 }
 
-// lowerBoundFileCount finds the filesize range from [0, return value] that has
+// FindFileSize finds the filesize range from [lowerBound, return value] that has
 // the largest file count that is smaller than or equal to
 // githubMaxResultsPerQuery. It is important to note that this returned value
 // could already be in a previous range if the next file size has more than 1000
 // results. It is left to the caller to handle this bit of logic and guarantee
 // forward progession in this case.
-func lowerBoundFileCount(
-	cache cachedSearch, targetFileCount uint64) (uint64, error) {
+func FindFileSize(
+	cache cachedSearch, targetFileCount, lowerBound, upperBound uint64) (uint64, error) {
 
 	// Binary search for file sizes that make up the next <=1000 element
 	// chunk.
-	cur := uint64(0)
-	increase := githubMaxFileSize / 2
+	cur := lowerBound
+	increase := (upperBound - lowerBound) / 2
 
 	for increase > 0 {
 		mid := cur + increase
 
-		count, err := cache.CountResults(mid)
+		count, err := cache.CountResults(lowerBound, mid)
 		if err != nil {
 			return count, err
 		}
@@ -353,26 +356,24 @@ func lowerBoundFileCount(
 }
 
 func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string {
-	ranges := make([]string, 0, len(sizes)+1)
-
-	if len(sizes) > 0 {
-		ranges = append(ranges, cache.RequestString(
-			RangeLessThan{sizes[0] + 1},
-		))
+	n := len(sizes)
+	if n < 2 {
+		return []string{}
 	}
 
-	for i := 0; i < len(sizes)-1; i += 1 {
-		ranges = append(ranges, cache.RequestString(
-			RangeWithin{sizes[i] + 1, sizes[i+1]},
-		))
-
-		if i != len(sizes)-2 {
-			continue
-		}
-		ranges = append(ranges, cache.RequestString(
-			RangeGreaterThan{sizes[i+1]},
-		))
+	ranges := make([]string, 0, n-1)
+	ranges = append(ranges, cache.RequestString(RangeWithin{sizes[0], sizes[1]}))
+	for i := 1; i < n-1; i++ {
+		ranges = append(ranges, cache.RequestString(RangeWithin{sizes[i] + 1, sizes[i+1]}))
 	}
-
 	return ranges
 }
+
+func RangeSizes(s string) RangeWithin {
+	start := strings.Index(s, "+size:") + len("+size:")
+	end := strings.Index(s, "&")
+	ranges := strings.Split(s[start:end], "..")
+	lowerBound, _ := strconv.ParseUint(ranges[0], 10, 64)
+	upperBound, _ := strconv.ParseUint(ranges[1], 10, 64)
+	return RangeWithin{lowerBound, upperBound}
+}

api/internal/crawl/crawler/github/split_search_ranges_test.go

@@ -2,6 +2,7 @@ package github
 
 import (
 	"fmt"
+	"log"
 	"reflect"
 	"testing"
 )
@@ -10,8 +11,8 @@ type testCachedSearch struct {
 	cache map[uint64]uint64
 }
 
-func (c testCachedSearch) CountResults(upperBound uint64) (uint64, error) {
-	fmt.Printf("CountResults(%05x)\n", upperBound)
+func (c testCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
+	log.Printf("CountResults(%05x)\n", upperBound)
 	count, ok := c.cache[upperBound]
 	if !ok {
 		return count, fmt.Errorf("cache not set at %x", upperBound)
@@ -72,19 +73,29 @@ func TestRangeSplitting(t *testing.T) {
 		},
 	}
 
-	requests, err := FindRangesForRepoSearch(cache)
+	requests, err := FindRangesForRepoSearch(cache, 0, 524288)
 	if err != nil {
 		t.Errorf("Error while finding ranges: %v", err)
 	}
 	expected := []string{
-		"<107",      // cache.RequestString(RangeLessThan{0x6b}),
-		"107..128",  // cache.RequestString(RangeWithin{0x6b, 0x80}),
-		"129..256",  // cache.RequestString(RangeWithin{0x81, 0x100}),
-		"257..4095", // cache.RequestString(RangeWithin{0x101, 0xfff}),
-		">4095",     // cache.RequestString(RangeGreaterThan{0xfff}),
+		"0..106",       // cache.RequestString(RangeWithin{0x00, 0x6a}),
+		"107..128",     // cache.RequestString(RangeWithin{0x6b, 0x80}),
+		"129..256",     // cache.RequestString(RangeWithin{0x81, 0x100}),
+		"257..4095",    // cache.RequestString(RangeWithin{0x101, 0xfff}),
+		"4096..524288", // cache.RequestString(RangeWithin{0x1000, 0x80000}),
 	}
 
 	if !reflect.DeepEqual(requests, expected) {
 		t.Errorf("Expected requests (%v) to equal (%v)", requests, expected)
 	}
 }
+
+func TestRangeSizes(t *testing.T) {
+	s := "https://api.github.com/search/code?q=filename:kustomization.yaml+filename:kustomization.yml" +
+		"+filename:kustomization+size:2365..10000&order=desc&per_page=100&sort=indexed"
+	returnedResult := RangeSizes(s)
+	expectedResult := RangeWithin{uint64(2365), uint64(10000)}
+	if !reflect.DeepEqual(returnedResult, expectedResult) {
+		t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult)
+	}
+}

api/internal/crawl/doc/doc.go

@@ -2,6 +2,8 @@ package doc
 
 import (
 	"fmt"
+	"log"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -44,21 +46,36 @@ type KustomizationDocument struct {
 
 type set map[string]struct{}
 
+func (doc *KustomizationDocument) Copy() *KustomizationDocument {
+	return &KustomizationDocument{
+		Document:    *(doc.Document.Copy()),
+		Kinds:       doc.Kinds,
+		Identifiers: doc.Identifiers,
+		Values:      doc.Values,
+	}
+}
+
 func (doc *KustomizationDocument) String() string {
 	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
 		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
 		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
 }
 
-// Implements the CrawlerDocument interface.
-func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
-	isResource := true
-	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
-		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
-			isResource = false
+// IsKustomizationFile determines whether a file path is a kustomization file
+func IsKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
 		}
 	}
-	if isResource {
+	return false
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *KustomizationDocument) GetResources(
+	includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
+	if !IsKustomizationFile(doc.FilePath) {
 		return []*Document{}, nil
 	}
 
@@ -76,19 +93,46 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 	}
 	k.FixKustomizationPostUnmarshalling()
 
-	res := make([]*Document, 0, len(k.Resources))
-	for _, r := range k.Resources {
-		next, err := doc.Document.FromRelativePath(r)
-		if err != nil {
-			fmt.Printf("GetResources error: %v\n", err)
-			continue
-		}
-		res = append(res, &next)
+	res := make([]*Document, 0)
+
+	if includeResources {
+		resourceDocs := doc.CollectDocuments(k.Resources, "resource")
+		res = append(res, resourceDocs...)
+	}
+
+	if includeGenerators {
+		generatorDocs := doc.CollectDocuments(k.Generators, "generator")
+		res = append(res, generatorDocs...)
+	}
+
+	if includeTransformers {
+		transformerDocs := doc.CollectDocuments(k.Transformers, "transformer")
+		res = append(res, transformerDocs...)
 	}
 
 	return res, nil
 }
 
+// CollectDocuments construct a Document for each path in paths, and return
+// a slice of Document pointers.
+func (doc *KustomizationDocument) CollectDocuments(
+	paths []string, fileType string) []*Document {
+	docs := make([]*Document, 0, len(paths))
+	for _, r := range paths {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
+		next, err := doc.Document.FromRelativePath(r)
+		if err != nil {
+			log.Printf("CollectDocuments error: %v\n", err)
+			continue
+		}
+		next.FileType = fileType
+		docs = append(docs, &next)
+	}
+	return docs
+}
+
 func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
 	data := []byte(doc.DocumentData)
 

api/internal/crawl/doc/doc_test.go

@@ -189,11 +189,13 @@ metadata:
 	}
 }
 
+type TestStructForGetResources struct {
+	doc       KustomizationDocument
+	resources []*Document
+}
+
 func TestGetResources(t *testing.T) {
-	tests := []struct {
-		doc       KustomizationDocument
-		resources []*Document
-	}{
+	tests := []TestStructForGetResources{
 		{
 			doc: KustomizationDocument{
 				Document: Document{
@@ -213,19 +215,27 @@ resources:
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/base",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/otherbase",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "https://github.com/kubernetes-sigs/kustomize",
 					FilePath:      "examples/helloWorld",
 					DefaultBranch: "v3.1.0",
+					FileType:      "resource",
+					User:          "kubernetes-sigs",
 				},
 			},
 		},
@@ -248,9 +258,12 @@ resources:
 			resources: []*Document{},
 		},
 	}
+	runTest(t, tests, true, false, false)
+}
 
+func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
 	for _, test := range tests {
-		res, err := test.doc.GetResources()
+		res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
 		if err != nil {
 			t.Errorf("Unexpected error: %v\n", err)
 			continue
@@ -284,3 +297,83 @@ resources:
 		}
 	}
 }
+
+func TestGetResourcesAndGenerators(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, false, true)
+}
+
+func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/tr.yaml",
+					FileType:      "transformer",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, true, true)
+}

api/internal/crawl/doc/docname.go

@@ -11,12 +11,18 @@ import (
 )
 
 type Document struct {
-	RepositoryURL string     `json:"repositoryUrl,omitempty"`
+	RepositoryURL string `json:"repositoryUrl,omitempty"`
+	// User makes it easy to aggregate data in the user level instead
+	// of the repository level
+	User          string     `json:"user,omitempty"`
 	FilePath      string     `json:"filePath,omitempty"`
 	DefaultBranch string     `json:"defaultBranch,omitempty"`
 	DocumentData  string     `json:"document,omitempty"`
 	CreationTime  *time.Time `json:"creationTime,omitempty"`
 	IsSame        bool       `json:"-"`
+	// FileType can be one of the following:
+	// "generator", "transformer", "resource", "".
+	FileType string `json:"fileType,omitempty"`
 }
 
 // Implements the CrawlerDocument interface.
@@ -27,14 +33,21 @@ func (doc *Document) GetDocument() *Document {
 func (doc *Document) Copy() *Document {
 	return &Document{
 		RepositoryURL: doc.RepositoryURL,
+		User:          doc.User,
 		FilePath:      doc.FilePath,
 		DefaultBranch: doc.DefaultBranch,
 		DocumentData:  doc.DocumentData,
 		CreationTime:  doc.CreationTime,
 		IsSame:        doc.IsSame,
+		FileType:      doc.FileType,
 	}
 }
 
+func (doc *Document) Path() string {
+	return fmt.Sprintf("repoURL: %s filePath: %s branch: %s",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch)
+}
+
 // Implements the CrawlerDocument interface.
 func (doc *Document) WasCached() bool {
 	return doc.IsSame
@@ -47,6 +60,7 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 			RepositoryURL: repoSpec.Host + path.Clean(repoSpec.OrgRepo),
 			FilePath:      path.Clean(repoSpec.Path),
 			DefaultBranch: repoSpec.Ref,
+			User:          UserName(repoSpec.Host + path.Clean(repoSpec.OrgRepo)),
 		}, nil
 	}
 	// else document is probably relative path.
@@ -54,6 +68,7 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 	ret := Document{
 		RepositoryURL: doc.RepositoryURL,
 		DefaultBranch: doc.DefaultBranch,
+		User:          UserName(doc.RepositoryURL),
 	}
 	ogDir, _ := path.Split(doc.FilePath)
 
@@ -78,13 +93,7 @@ func (doc *Document) ID() string {
 }
 
 func (doc *Document) RepositoryFullName() string {
-	url := strings.TrimRight(doc.RepositoryURL, "/")
-
-	gitPrefix := "git@github.com:"
-	if strings.HasPrefix(url, gitPrefix) {
-		url = url[len(gitPrefix):]
-	}
-
+	url := TrimUrl(doc.RepositoryURL)
 	sections := strings.Split(url, "/")
 	l := len(sections)
 	if l < 2 {
@@ -92,3 +101,24 @@ func (doc *Document) RepositoryFullName() string {
 	}
 	return path.Join(sections[l-2], sections[l-1])
 }
+
+// TrimUrl removes all the trailing slashes and the "git@github.com:" prefix (if exists).
+func TrimUrl(s string) string {
+	url := strings.TrimRight(s, "/")
+
+	gitPrefix := "git@github.com:"
+	if strings.HasPrefix(url, gitPrefix) {
+		url = url[len(gitPrefix):]
+	}
+	return url
+}
+
+func UserName(repositoryURL string) string {
+	url := TrimUrl(repositoryURL)
+	sections := strings.Split(url, "/")
+	l := len(sections)
+	if l < 2 {
+		return url
+	}
+	return sections[l-2]
+}

api/internal/crawl/doc/docname_test.go

@@ -28,6 +28,7 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/other/file/resource.yaml",
 						DefaultBranch: "master",
+						User:          "example.com",
 					},
 				},
 				{
@@ -36,6 +37,7 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/other/file/patch.yaml",
 						DefaultBranch: "master",
+						User:          "example.com",
 					},
 				},
 				{
@@ -44,6 +46,7 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/file/service.yaml",
 						DefaultBranch: "master",
+						User:          "example.com",
 					},
 				},
 			},
@@ -65,7 +68,7 @@ func TestFromRelativePath(t *testing.T) {
 
 func TestDocument_RepositoryFullName(t *testing.T) {
 	testCases := []struct {
-		doc Document
+		doc                        Document
 		expectedRepositoryFullName string
 	}{
 		{
@@ -108,4 +111,40 @@ func TestDocument_RepositoryFullName(t *testing.T) {
 				returnedRepositoryFullName)
 		}
 	}
-}
+}
+
+func TestDocument_UserName(t *testing.T) {
+	testCases := []struct {
+		repositoryURL    string
+		expectedUserName string
+	}{
+		{
+			repositoryURL:    "https://github.com/user/repo",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "https://github.com//user/repo////",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "repo/",
+			expectedUserName: "repo",
+		},
+		{
+			repositoryURL:    "",
+			expectedUserName: "",
+		},
+		{
+			repositoryURL:    "git@github.com:user/repo",
+			expectedUserName: "user",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedUserName := UserName(tc.repositoryURL)
+		if returnedUserName != tc.expectedUserName {
+			t.Errorf("UserName expected %s, got %s",
+				tc.expectedUserName, returnedUserName)
+		}
+	}
+}

api/internal/crawl/doc/unique_doc.go

@@ -0,0 +1,36 @@
+package doc
+
+import (
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+)
+
+// UniqueDocuments make sure a Document with a given ID appears only once
+type UniqueDocuments struct {
+	docs   []*Document
+	docIDs utils.SeenMap
+}
+
+func NewUniqueDocuments() UniqueDocuments {
+	return UniqueDocuments{
+		docs:   []*Document{},
+		docIDs: utils.NewSeenMap(),
+	}
+}
+
+func (uds *UniqueDocuments) Add(d *Document) {
+	if uds.docIDs.Seen(d.ID()) {
+		return
+	}
+	uds.docs = append(uds.docs, d)
+	uds.docIDs.Set(d.ID(), "")
+}
+
+func (uds *UniqueDocuments) AddDocuments(docs []*Document) {
+	for _, d := range docs {
+		uds.Add(d)
+	}
+}
+
+func (uds *UniqueDocuments) Documents() []*Document {
+	return uds.docs
+}

api/internal/crawl/go.mod

@@ -8,6 +8,6 @@ require (
 	github.com/gorilla/mux v1.7.3
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/rs/cors v1.7.0
-	sigs.k8s.io/kustomize/api v0.3.0
+	sigs.k8s.io/kustomize/api v0.3.1
 	sigs.k8s.io/yaml v1.1.0
 )

api/internal/crawl/go.sum

@@ -133,13 +133,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -198,7 +193,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.1/go.mod h1:/7E04hlzRG9Jrp6WILZfYYm/REoJWL2l+MlsCO1eH74=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -240,7 +234,6 @@ github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lz
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -387,7 +380,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
@@ -415,8 +407,8 @@ k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.3.0 h1:riR/YsL75nGb+aIPFdIRiqu21+OZbAXQybDS7+FUYRg=
-sigs.k8s.io/kustomize/api v0.3.0/go.mod h1:DWNMJBV1xvLruMpihGgnIPznMwHpwUSrxz6v3gnw5kw=
+sigs.k8s.io/kustomize/api v0.3.1 h1:oqMIXvS6tFEUVuKIRUKDa05eC4Hh+cb9JYg8Zhp2d24=
+sigs.k8s.io/kustomize/api v0.3.1/go.mod h1:A+ATnlHqzictQfQC1q3KB/T6MSr0UWQsrrLxMWkge2E=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/internal/crawl/index/elasticsearch.go

@@ -20,12 +20,18 @@ const IndexConfig = `
 				"repositoryUrl": {
 					"type": "keyword"
 				},
+				"user": {
+					"type": "keyword"
+				},
 				"filePath": {
 					"type": "keyword"
 				},
 				"defaultBranch": {
 					"type": "keyword"
 				},
+				"fileType": {
+					"type": "keyword"
+				},
 				"document": {
 					"type": "text"
 				},
@@ -87,7 +93,7 @@ func (idx *index) responseErrorOrNil(info string, res *esapi.Response,
 
 	defer res.Body.Close()
 	if res.IsError() {
-		return fmt.Errorf("%s: %s", messageStart, res.String())
+		return fmt.Errorf("%s: %s [%d]", messageStart, res.String(), res.StatusCode)
 	}
 
 	if reader != nil {
@@ -309,9 +315,9 @@ func (idx *index) Exists(id string) (bool, error) {
 		op.WithPretty(),
 	)
 
-	if !res.IsError() {
+	if res != nil && !res.IsError() {
 		return true, nil
-	} else if res.StatusCode == 404 {
+	} else if res != nil && res.StatusCode == 404 {
 		return false, nil
 	} else {
 		return false, idx.responseErrorOrNil(

api/internal/crawl/index/kustomize.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"strings"
 	"time"
 
@@ -17,6 +18,7 @@ const (
 )
 
 type Mode int
+
 const (
 	InsertOrUpdate = iota
 	Delete
@@ -97,14 +99,14 @@ type KustomizeIndex struct {
 }
 
 // Create index reference to the index containing the kustomize documents.
-func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
-	idx, err := newIndex(ctx, "kustomize")
+func NewKustomizeIndex(ctx context.Context, indexName string) (*KustomizeIndex, error) {
+	idx, err := newIndex(ctx, indexName)
 	if err != nil {
 		return nil, err
 	}
 
 	indicesExistsOp := idx.client.Indices.Exists
-	resp, err := indicesExistsOp([]string{"kustomize"},
+	resp, err := indicesExistsOp([]string{indexName},
 		indicesExistsOp.WithContext(idx.ctx),
 		indicesExistsOp.WithPretty())
 	if err != nil {
@@ -112,9 +114,9 @@ func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
 	}
 
 	if resp.StatusCode == 200 {
-		fmt.Printf("The kustomize index already exists\n")
+		log.Printf("The %s index already exists", indexName)
 	} else {
-		fmt.Printf("Creating the kustomize index\n")
+		log.Printf("Creating the %s index\n", indexName)
 		if err := idx.CreateIndex([]byte(IndexConfig)); err != nil {
 			return nil, err
 		}
@@ -252,7 +254,7 @@ func (it *KustomizeIterator) Next() bool {
 	}
 
 	if it.err == nil {
-		fmt.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
+		log.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
 		it.err = it.update(*it.scrollImpl.ScrollID, reader)
 	}
 
@@ -268,7 +270,7 @@ func (it *KustomizeIterator) Value() KustomizeResult {
 	return it.scrollImpl
 }
 
-// Check if any errors have occured.
+// Check if any errors have occurred.
 func (it *KustomizeIterator) Err() error {
 	return it.err
 }
@@ -341,7 +343,7 @@ func (ki *KustomizeIndex) Search(query string,
 	if err != nil {
 		return nil, fmt.Errorf("failed to format query %s", query)
 	}
-	fmt.Printf("formated query: %s\n", data)
+	log.Printf("formated query: %s\n", data)
 
 	var kr ElasticKustomizeResult
 	err = ki.index.Search(data, opts.SearchOptions, func(results io.Reader) error {

api/internal/crawl/search_cmds/creationTime.md

@@ -0,0 +1,411 @@
+Find out the largest value of the `creationTime` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "max_creationTime" : { "max" : { "field" : "creationTime" } }
+    }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "min_creationTime" : { "min" : { "field" : "creationTime" } }
+    }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all the kustomization files:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+        { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize resource files:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }, 
+      "filter": [
+        { "regexp": { "fileType": "resource"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize generator files:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }, 
+      "filter": [
+        { "regexp": { "fileType": "generator"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize transformer files:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }, 
+      "filter": [
+        { "regexp": { "fileType": "transformer"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Query all the documents whose `creationTime` <= `2016-07-29T17:38:26.000Z`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "range": {
+      "creationTime": {
+       "lte": "2016-07-29T17:38:26.000Z"
+      }
+    }
+  }
+}
+'
+```
+
+Query all the documents whose `creationTime` falls within the specific range:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "range": {
+      "creationTime": {
+       "gte": "2016-07-29T17:38:26.000Z",
+       "lte": "2016-08-29T17:38:26.000Z"
+      }
+    }
+  }
+}
+'
+```
+
+Query all the kustomization files whose `creationTime` falls within the specific range:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 20,
+  "query": {
+    "bool": {
+      "filter": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }
+      },
+      "must": {
+        "range": {
+          "creationTime": {
+            "gte": "2017-09-24T15:49:57.000Z",
+            "lte": "2017-09-24T15:49:57.000Z"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```
+
+Aggregate how many new kustomization files were added into Github each month:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize resource files were added into Github each month:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize generator files were added into Github each month:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize transformer files were added into Github each month:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomization files were added into Github each year:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize resource files were added into Github each year:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]            
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize generator files were added into Github each year:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]            
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize transformer files were added into Github each year:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]            
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Find the generator files created within the given time range:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "range": {
+          "creationTime": {
+           "gte": "2019-04-26T16:40:02.000Z",
+           "lte": "2019-04-26T16:40:02.000Z"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```
+
+Find the transformer files created within the given time range:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "range": {
+          "creationTime": {
+           "gte": "2019-04-26T16:40:02.000Z",
+           "lte": "2019-04-26T16:40:02.000Z"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/defaultBranch.md

@@ -0,0 +1,32 @@
+Count distinct values of the `defaultBranch` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "defaultBranch_count" : {
+            "cardinality" : {
+                "field" : "defaultBranch",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the github branches where kustomization files and kustomize resource files live, 
+and how many kustomization files and kustomize resource files live in each branch:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "defaultBranch" : {
+            "terms" : {
+              "field" : "defaultBranch",
+              "size": 41
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/fieldExistence.md

@@ -0,0 +1,55 @@
+Count the documents whose `document` field is empty (The reason why the `document` field
+of a document is empty is because of empty documents):
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "document"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Find all the documents having the `creationTime` field set:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "creationTime"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `creationTime` field is not set:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "creationTime"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+The following fields of a document in the kustomize index are always non-empty:
+`repositoryUrl`, `filePath`, `defaultBranch`.
+
+The following fields of a document in the kustomize index may be empty:
+`kinds`, `identifiers`, `values`.

api/internal/crawl/search_cmds/fileType.md

@@ -0,0 +1,301 @@
+Find all the documents having the `fileType` field set:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "fileType"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `fileType` field is not set:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "fileType"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Search for all the documents whose `fileType` field is `resource`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "resource" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files whose `fileType` field is `resource`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
+       { "regexp": { "fileType": "resource" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "resource" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search all the kustomization files including a `generators` field:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": {
+        "match" : {
+          "identifiers" : {
+            "query" : "generators"
+          }
+        }
+      },
+      "filter": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents whose `fileType` field is `generator`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files whose `fileType` field is `generator`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
+       { "regexp": { "fileType": "generator" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize generator files:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search all the kustomization files including a `transformers` field:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": {
+        "match" : {
+          "identifiers" : {
+            "query" : "transformers"
+          }
+        }
+      },
+      "filter": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents whose `fileType` field is `transformer`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files whose `fileType` field is `transformer`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
+       { "regexp": { "fileType": "transformer" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize transformer files:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Count distinct values of the `fileType` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "fileType_count" : {
+            "cardinality" : {
+                "field" : "fileType",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the values of the `fileType` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "fileType" : {
+            "terms" : {
+              "field" : "fileType"
+            }
+        }
+    }
+}
+'
+```
+
+
+For all the kustomization files in the index, list all the values of the
+`fileType` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+             ]
+        }
+    },
+    "aggs" : {
+        "fileType" : {
+            "terms" : {
+              "field" : "fileType"
+            }
+        }
+    }
+}
+'
+```
+
+For all the non-kustomization files in the index, list all the values of the
+`fileType` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            }
+        }
+    },
+    "aggs" : {
+        "fileType" : {
+            "terms" : {
+              "field" : "fileType"
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/generator.md

@@ -0,0 +1,29 @@
+Find all the generator files whose `kinds` field includes `ChartRenderer`, and
+only output certain fields of each document:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 200,
+  "_source": {
+    "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"]
+  },
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "match" : {
+          "kinds" : {
+            "query" : "ChartRenderer"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/id.md

@@ -0,0 +1,12 @@
+Find the document with the given `_id`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "terms": {
+      "_id": [ "b3a03f3327841617db696e2d6abc30e1a1bd653f1a2bbce05637f7dcae1a43f7" ] 
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/keyword_search.md

@@ -0,0 +1,82 @@
+Count the documents in the index whose `repositoryUrl` field starts with
+`https://github.com/`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Count the documents in the index whose `repositoryUrl` field does not start with
+`https://github.com/`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": [
+       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents matching the given `repositoryUrl` and `filePath`, and return 
+a version for each search hit:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "version": true,
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "repositoryUrl": "git@github.com:talos-systems/talos-controller-manager" }},
+       { "regexp": { "filePath": "hack/config.*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents whose filePath ends with one of these following three filenames:
+`kustomization.yaml`, `kustomization.yml`, `kustomization`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents whose filePath does not end with any of these following
+three filenames: `kustomization.yaml`, `kustomization.yml`, `kustomization`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+      ]
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/misc.md

@@ -0,0 +1,32 @@
+Check the health status of an ElasticSearch cluster:
+```
+curl -X GET "${ElasticSearchURL}:9200/_cat/health?v&pretty"
+```
+
+Check the indices in an ElasticSearch cluster:
+```
+curl "${ElasticSearchURL}:9200/_cat/indices?v"
+```
+
+Get the mapping of the index:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty"
+```
+
+Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**):
+```
+curl -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty"
+```
+
+Add a new field into an existing index.
+```
+curl -X PUT "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping/_doc?pretty" -H 'Content-Type: application/json' -d'
+{
+  "properties": {
+    "fileType": {
+      "type": "keyword"
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/repositoryUrl.md

@@ -0,0 +1,255 @@
+Count distinct values of the `repositoryUrl` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count how many Github repositories include kustomization files:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+             ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize resource files in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize generator files in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize transformer files in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize resource dirs in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "resource" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize generator dirs in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "generator" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize transformer dirs in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "transformer" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the github repositories including kustomization files and kustomize resource files,
+and how many kustomization files and kustomize resource files each github repository includes
+(the github repository including the most kustomization files is listed first):
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 2082
+            }
+        }
+    }
+}
+'
+```
+
+List the top 20 Github repositories including the most amount of kustomization files:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+             ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+List the top 20 Github repositories including the most amount of kustomize resource files:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl" : {
+            "terms" : {
+              "field" : "repositoryUrl",
+              "size": 20
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/snapshot.md

@@ -0,0 +1,29 @@
+Retrieve information about all registered snapshot repositories:
+```
+curl -X GET "${ElasticSearchURL}:9200/_snapshot?pretty"
+```
+
+Retrieve information about a given snapshot repository, `kustomize-backup`:
+```
+curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup?pretty"
+```
+
+Verify a snapshot repository, `kustomize-backup`, manually:
+```
+curl -X POST "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/_verify?pretty"
+```
+
+List all the snapshots in a given snapshot repository:
+```
+curl -X GET "${ElasticSearchURL}:9200/_cat/snapshots/kustomize-backup?v&s=id&pretty"
+```
+
+Retrieve a summary information about a given snapshot:
+```
+curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot?pretty"
+```
+
+Retrieve a detailed information about a given snapshot:
+```
+curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot/_status?pretty"
+```

api/internal/crawl/search_cmds/text_search.md

@@ -0,0 +1,148 @@
+Search for all the kustomize resource files including a Deployment object:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files including a Deployment object, but only
+including the `kinds` field in the result:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "_source": {
+    "includes": ["kinds"]
+  },
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files including both a Deployment object and 
+a Service object:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "kinds" : {
+        "query" : "Deployment Service",
+        "operator" : "and"
+      }
+    }
+  }
+}
+'
+```
+
+Count the number of documents including Deployment and the number of documents 
+including Service:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 0,
+  "aggs" : {
+    "messages" : {
+      "filters" : {
+        "filters" : {
+          "Deployment" : { "match" : { "kinds" : "Deployment"   }},
+          "Service" : { "match" : { "kinds" : "Service" }}
+        }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files involving CRDs:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "crds"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files defining configMapGenerator:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "configMapGenerator"
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents having a `kind` field:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "match" : { "identifiers" : { "query" : "kind" }}}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kuostmization files having a `kind` field:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)" }},
+       { "match" : { "identifiers" : { "query" : "kind" }}}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files defining the `generatorOptions:disableNameSuffixHash` feature:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "match" : {
+      "identifiers" : {
+        "query" : "generatorOptions:disableNameSuffixHash"
+      }
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/transformer.md

@@ -0,0 +1,29 @@
+Find all the trasnformer files whose `kinds` field includes `HelmValues`, and
+only output certain fields of each document:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 200,
+  "_source": {
+    "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"]
+  },
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "match" : {
+          "kinds" : {
+            "query" : "HelmValues"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/user.md

@@ -0,0 +1,380 @@
+Find all the documents having the `user` field set:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "user"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `user` field is not set:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "user"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Search for all the documents whose `user` field is `kubernetes-sigs`:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "user": "kubernetes-sigs" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Count distinct values of the `user` field:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the values of the `user` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size" : 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomization files in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+             ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomization files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+             ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize resource files in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize resource files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize generator files in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize generator files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+              "field" : "user",
+              "size": 20  
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize transformer files in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize transformer files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize generator dirs in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "generator" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize generator dirs in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "generator" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+              "field" : "user",
+              "size": 20  
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize transformer dirs in the index:
+```
+curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "transformer" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize transformer dirs in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "transformer" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+              "field" : "user"
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/utils/utils.go

@@ -0,0 +1,21 @@
+package utils
+
+type SeenMap map[string]string
+
+func (seen SeenMap) Seen(item string) bool {
+	_, ok := seen[item]
+	return ok
+}
+
+func (seen SeenMap) Set(k, v string) {
+	seen[k] = v
+}
+
+// The caller should make sure that key is in the map.
+func (seen SeenMap) Value(k string) string {
+	return seen[k]
+}
+
+func NewSeenMap() SeenMap {
+	return make(map[string]string)
+}

api/internal/git/cloner.go

@@ -27,90 +27,28 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 	if err != nil {
 		return err
 	}
+
+	if repoSpec.Ref == "" {
+		repoSpec.Ref = "master"
+	}
 	cmd := exec.Command(
 		gitProgram,
-		"init",
+		"clone",
+		"--depth=1",
+		repoSpec.CloneSpec(),
+		"-b",
+		repoSpec.Ref,
 		repoSpec.Dir.String())
 	var out bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &out
 	err = cmd.Run()
 	if err != nil {
-		log.Printf("Error initializing empty git repo: %s", out.String())
+		log.Printf("Error cloning git repo: %s", out.String())
 		return errors.Wrapf(
 			err,
-			"trouble initializing empty git repo in %s",
-			repoSpec.Dir.String())
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"remote",
-		"add",
-		"origin",
-		repoSpec.CloneSpec())
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error setting git remote: %s", out.String())
-		return errors.Wrapf(
-			err,
-			"trouble adding remote %s",
-			repoSpec.CloneSpec())
-	}
-	if repoSpec.Ref == "" {
-		repoSpec.Ref = "master"
-	}
-	cmd = exec.Command(
-		gitProgram,
-		"fetch",
-		"--depth=1",
-		"origin",
-		repoSpec.Ref)
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		cmd = exec.Command(
-			gitProgram,
-			"pull",
-			"origin",
-			"master")
-		var out bytes.Buffer
-		cmd.Stdout = &out
-		cmd.Dir = repoSpec.Dir.String()
-		err := cmd.Run()
-		if err != nil {
-			return errors.Wrapf(err, "trouble pulling %s", repoSpec.OrgRepo)
-		}
-		if repoSpec.Ref == "" {
-			repoSpec.Ref = "master"
-		}
-		cmd = exec.Command(gitProgram, "checkout", repoSpec.Ref)
-		cmd.Dir = repoSpec.Dir.String()
-		err = cmd.Run()
-		if err != nil {
-			return errors.Wrapf(
-				err, "trouble checking out href %s", repoSpec.Ref)
-		}
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"reset",
-		"--hard",
-		"FETCH_HEAD")
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error performing git reset: %s", out.String())
-		return errors.Wrapf(
-			err, "trouble hard resetting empty repository to %s", repoSpec.Ref)
+			"trouble cloning git repo %v in %s",
+			repoSpec.CloneSpec(), repoSpec.Dir.String())
 	}
 
 	cmd = exec.Command(
@@ -120,10 +58,11 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 		"--init",
 		"--recursive")
 	cmd.Stdout = &out
+	cmd.Stderr = &out
 	cmd.Dir = repoSpec.Dir.String()
 	err = cmd.Run()
 	if err != nil {
-		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.Ref)
+		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec())
 	}
 
 	return nil

api/internal/tools/tools.go

@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // This file exists to automatically trigger installs
-// of the given tools, and is the offical 'unofficial'
+// of the given tools, and is the official 'unofficial'
 // way to declare a dependence on a Go binary until
 // some better technique comes along.
 

api/krusty/baseandoverlaymedium_test.go

@@ -154,8 +154,9 @@ LEGUME=chickpea
 `)
 	th.WriteF("/app/overlay/configmap/dummy.txt",
 		`Lorem ipsum dolor sit amet, consectetur
-adipiscing elit, sed do eiusmod tempor
-incididunt ut labore et dolore magna aliqua. 
+ 		 
+adipiscing elit, sed do eiusmod tempor		
+incididunt ut labore et dolore magna aliqua.   
 `)
 	th.WriteF("/app/overlay/deployment/deployment.yaml", `
 apiVersion: apps/v1
@@ -292,8 +293,11 @@ metadata:
 ---
 apiVersion: v1
 data:
-  nonsense: "Lorem ipsum dolor sit amet, consectetur\nadipiscing elit, sed do eiusmod
-    tempor\nincididunt ut labore et dolore magna aliqua. \n"
+  nonsense: |
+    Lorem ipsum dolor sit amet, consectetur
+
+    adipiscing elit, sed do eiusmod tempor
+    incididunt ut labore et dolore magna aliqua.
 kind: ConfigMap
 metadata:
   annotations:
@@ -302,6 +306,6 @@ metadata:
     app: mungebot
     org: kubernetes
     repo: test-infra
-  name: test-infra-app-config-f462h769f9
+  name: test-infra-app-config-hh272bg5d4
 `)
 }

api/kv/kv.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"regexp"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -85,11 +86,17 @@ func (kvl *loader) keyValuesFromFileSources(sources []string) ([]types.Pair, err
 		if err != nil {
 			return nil, err
 		}
-		kvs = append(kvs, types.Pair{Key: k, Value: string(content)})
+		kvs = append(kvs, types.Pair{Key: k, Value: trimTrailingSpacesInLines(string(content))})
 	}
 	return kvs, nil
 }
 
+// trimTrailingSpacesInLines takes string with multiple lines and trims the trailing white spaces and tabs from each line.
+func trimTrailingSpacesInLines(str string) string {
+	re := regexp.MustCompile(`[ \t]*\n`)
+	return re.ReplaceAllString(str, "\n")
+}
+
 func (kvl *loader) keyValuesFromEnvFiles(paths []string) ([]types.Pair, error) {
 	var kvs []types.Pair
 	for _, p := range paths {

api/kv/kv_test.go

@@ -95,3 +95,12 @@ func TestKeyValuesFromFileSources(t *testing.T) {
 		}
 	}
 }
+
+func TestTrimTrailingSpacesInLines(t *testing.T) {
+	input := "\"fooKey\": \"fooValue\"   \t\n \t\t  \n\t\"barKey\": \"barValue\""
+	expected := "\"fooKey\": \"fooValue\"\n\n\t\"barKey\": \"barValue\""
+	res := trimTrailingSpacesInLines(input)
+	if !reflect.DeepEqual(res, expected) {
+		t.Errorf("Trim trailing spaces in lines should succeed, got: %s exptected: %s", res, expected)
+	}
+}

cmd/config/.golangci.yml

@@ -23,7 +23,7 @@ linters:
     - gofmt
     - goimports
 #    - golint
-    - gosec
+#    - gosec
     - gosimple
     - govet
     - ineffassign

cmd/config/configcobra/cmds.go

@@ -43,6 +43,29 @@ Advanced Documentation Topics:
 `,
 }
 
+// Export commands publicly for composition
+var (
+	Annotate     = commands.AnnotateCommand
+	Cat          = commands.CatCommand
+	Count        = commands.CountCommand
+	CreateSetter = commands.CreateSetterCommand
+	Fmt          = commands.FmtCommand
+	Grep         = commands.GrepCommand
+	ListSetters  = commands.ListSettersCommand
+	Merge        = commands.MergeCommand
+	Merge3       = commands.Merge3Command
+	RunFn        = commands.RunFnCommand
+	Set          = commands.SetCommand
+	Sink         = commands.SinkCommand
+	Source       = commands.SourceCommand
+	Tree         = commands.TreeCommand
+	Wrap         = commands.WrapCommand
+	XArgs        = commands.XArgsCommand
+
+	StackOnError = &commands.StackOnError
+	ExitOnError  = &commands.ExitOnError
+)
+
 // NewConfigCommand returns a new *cobra.Command for the config command group.  This may
 // be embedded into other go binaries as a way of packaging the "config" command as part
 // of another binary.
@@ -69,6 +92,7 @@ func NewConfigCommand(name string) *cobra.Command {
 
 	name = strings.TrimSpace(name + " config")
 	commands.ExitOnError = true
+	root.AddCommand(commands.AnnotateCommand(name))
 	root.AddCommand(commands.GrepCommand(name))
 	root.AddCommand(commands.TreeCommand(name))
 	root.AddCommand(commands.CatCommand(name))
@@ -77,8 +101,14 @@ func NewConfigCommand(name string) *cobra.Command {
 	root.AddCommand(commands.Merge3Command(name))
 	root.AddCommand(commands.CountCommand(name))
 	root.AddCommand(commands.RunFnCommand(name))
+	root.AddCommand(commands.XArgsCommand())
+	root.AddCommand(commands.WrapCommand())
+
 	root.AddCommand(commands.SetCommand(name))
+	root.AddCommand(commands.ListSettersCommand(name))
 	root.AddCommand(commands.CreateSetterCommand(name))
+	root.AddCommand(commands.SinkCommand(name))
+	root.AddCommand(commands.SourceCommand(name))
 
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-merge",
@@ -92,8 +122,13 @@ func NewConfigCommand(name string) *cobra.Command {
 	})
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-fn",
-		Short: "[Alpha] Documentation for writing containerized functions run by run.",
-		Long:  api.ConfigFnLong,
+		Short: "[Alpha] Documentation for developing and invoking Configuration Functions.",
+		Long:  api.FunctionsImplLong,
+	})
+	root.AddCommand(&cobra.Command{
+		Use:   "docs-fn-spec",
+		Short: "[Alpha] Documentation for Configuration Functions Specification.",
+		Long:  api.FunctionsSpecLong,
 	})
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-io-annotations",

cmd/config/docs/api-conventions/config-fn.md

@@ -1,281 +0,0 @@
-# Configuration Functions API Semantics
-
-  Configuration Functions are functions packaged as executables in containers which enable
-  **shift-left practices**.  They configure applications and infrastructure through
-  Kubernetes style Resource Configuration, but run locally pre-commit.
-
-  Configuration functions enable shift-left practices (client-side) through:
-
-  - Pre-commit / delivery validation and linting of configuration
-    - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
-  - Implementation of abstractions as client actuated APIs (e.g. templating)
-    - e.g. Create a client-side *"CRD"* for generating configuration checked into git
-  - Aspect Orient configuration / Injection of cross-cutting configuration
-    - e.g. T-Shirt size containers by annotating Resources with `small`, `medium`, `large`
-      and inject the cpu and memory resources into containers accordingly.
-    - e.g. Inject `init` and `side-car` containers into Resources based off of Resource
-      Type, annotations, etc.
-
-  Performing these on the client rather than the server enables:
-
-  - Configuration to be reviewed prior to being sent to the API server
-  - Configuration to be validated as part of the CD pipeline
-  - Configuration for Resources to validated holistically rather than individually
-    per-Resource -- e.g. ensure the `Service.selector` and `Deployment.spec.template` labels
-    match.
-    - MutatingWebHooks are scoped to a single Resource instance at a time.
-  - Low-level tweaks to the output of high-level abstractions -- e.g. add an `init container`
-    to a client *"CRD"* Resource after it was generated.
-  - Composition and layering of multiple functions together
-    - Compose generation, injection, validation together
-
-  Configuration Functions are implemented as executable programs published in containers which:
-
-  - Accept as input (stdin):
-    - A list of Resource Configuration
-    - A Function Configuration (to configure the function itself)
-  - Emit as output (stdout + exit):
-    - A list of Resource Configuration
-    - An exit code for success / failure
-
-### Function Specification
-
-  - Functions **SHOULD** be published as container images containing a `CMD` invoking an executable.
-  - Functions **MUST** accept input on STDIN a `ResourceList` containing the Resources and
-    `functionConfig`.
-  - Functions **MUST** emit output on STDOUT a `ResourceList` containing the modified
-    Resources.
-  - Functions **MUST** exit non-0 on failure, and exit 0 on success.
-  - Functions **MAY** emit output on STDERR with error messaging.
-  - Functions performing validation **SHOULD** exit failure and emit error messaging
-    on a validation failure.
-  - Functions generating Resources **SHOULD** retain non-conflicting changes on the
-    generated Resources -- e.g. 1. the function generates a Deployment, but doesn't
-    specify `cpu`, 2. the user sets the `cpu` on the generated Resource, 3. the
-    function should keep the `cpu` when regenerating the Resource a second time.
-  - Functions **SHOULD** be usable outside `kustomize config run` -- e.g. though pipeline
-    mechanisms such as Tekton.
-
-#### Input Format
-
-  Functions must accept on STDIN:
-
-  `ResourceList`:
-  - contains `items` field, same as `List.items`
-  - contains `functionConfig` field -- a single item with the configuration for the function itself
-
-  Example `ResourceList` Input:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    functionConfig:
-      apiVersion: example.com/v1beta1
-      kind: Nginx
-      metadata:
-        name: my-instance
-        annotations:
-          config.kubernetes.io/local-config: "true"
-      spec:
-        replicas: 5
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 3
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Output Format
-
-  Functions must emit on STDOUT:
-
-  `ResourceList`:
-  - contains `items` field, same as `List.items`
-
-  Example `ResourceList` Output:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 5
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Container Environment
-
-  When run by `kustomize config run`, functions are run in containers with the
-  following environment:
-
-  - Network: `none`
-  - User: `nobody`
-  - Security Options: `no-new-privileges`
-  - Volumes: the volume containing the `functionConfig` yaml is mounted under `/local` as `ro`
-
-### Example Function Implementation
-
-  Following is an example for implementing an nginx abstraction using a config
-  function.
-
-#### `nginx-template.sh`
-
-  `nginx-template.sh` is a simple bash script which uses a *heredoc* as a templating solution
-  for generating Resources from the functionConfig input fields.
-
-  The script wraps itself using `config run wrap -- $0` which will:
-
-  1. Parse the `ResourceList.functionConfig` (provided to the container stdin) into env vars
-  2. Merge the stdout into the original list of Resources
-  3. Defaults filenames for newly generated Resources (if they are not set as annotations)
-     to `config/NAME_KIND.yaml`
-  4. Format the output
-
-    #!/bin/bash
-    # script must run wrapped by `kustomize config run wrap`
-    # for parsing input the functionConfig into env vars
-    if [ -z ${WRAPPED} ]; then
-      export WRAPPED=true
-      config run wrap -- $0
-      exit $?
-    fi
-
-    cat <<End-of-message
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: ${NAME}
-    ---
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      replicas: ${REPLICAS}
-      selector:
-        matchLabels:
-          app: nginx
-          instance: ${NAME}
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: ${NAME}
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-    End-of-message
-
-#### `Dockerfile`
-
-  `Dockerfile` installs `kustomize config` and copies the script into the container image.
-
-    FROM golang:1.13-stretch
-    RUN go get sigs.k8s.io/kustomize/cmd/config
-    RUN mv /go/bin/config /usr/bin/config
-    COPY nginx-template.sh /usr/bin/nginx-template.sh
-    CMD ["nginx-template.sh]
-
-### Example Function Usage
-
-Following is an example of running the `kustomize config run` using the preceding API.
-
-#### `nginx.yaml` (Input)
-
-  `dir/nginx.yaml` contains a reference to the Function.  The contents of `nginx.yaml`
-  are passed to the Function through the `ResourceList.functionConfig` field.
-
-    apiVersion: example.com/v1beta1
-    kind: Nginx
-    metadata:
-      name: my-instance
-      annotations:
-        config.kubernetes.io/local-config: "true"
-      configFn:
-        container:
-          image: gcr.io/example-functions/nginx-template:v1.0.0
-    spec:
-      replicas: 5
-
-  - `configFn.container.image`: the image to use for this API
-  - `annotations[config.kubernetes.io/local-config]`: mark this as not a Resource that should
-    be applied
-
-#### `kustomize config run dir/` (Output)
-
-  `dir/my-instance_deployment.yaml` contains the Deployment:
-
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      replicas: 5
-      selector:
-        matchLabels:
-          app: nginx
-          instance: my-instance
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: my-instance
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-
-  `dir/my-instance_service.yaml` contains the Service:
-
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: my-instance

cmd/config/docs/api-conventions/config-io.md

@@ -29,7 +29,7 @@ metadata:
 ### `config.kubernetes.io/index`
 
 Records the index of a Resource in file. In a multi-object YAML file, Resources are separated
-by three dashes (`---`), and the index represents the positon of the Resource starting from zero.
+by three dashes (`---`), and the index represents the position of the Resource starting from zero.
 
 This annotation **SHOULD** be set when reading Resources from files.
 It **SHOULD** be unset when writing Resources to files.

cmd/config/docs/api-conventions/functions-impl.md

@@ -0,0 +1,181 @@
+# Running Configuration Functions using kustomize CLI
+
+Configuration functions can be implemented using any toolchain and invoked using any
+container workflow orchestrator including Tekton, Cloud Build, or run directly using `docker run`.
+
+Run `config help docs-fn-spec` to see the Configuration Functions Specification.
+
+`kustomize config run` is an example orchestrator for invoking Configuration Functions. This
+document describes how to implement and invoke an example function.
+
+## Example Function Implementation
+
+Following is an example for implementing an nginx abstraction using a configuration
+function.
+
+### `nginx-template.sh`
+
+`nginx-template.sh` is a simple bash script which uses a _heredoc_ as a templating solution
+for generating Resources from the functionConfig input fields.
+
+The script wraps itself using `config run wrap -- $0` which will:
+
+1. Parse the `ResourceList.functionConfig` (provided to the container stdin) into env vars
+2. Merge the stdout into the original list of Resources
+3. Defaults filenames for newly generated Resources (if they are not set as annotations)
+   to `config/NAME_KIND.yaml`
+4. Format the output
+
+```bash
+#!/bin/bash
+# script must run wrapped by "kustomize config run wrap"
+# for parsing input the functionConfig into env vars
+if [ -z ${WRAPPED} ]; then
+  export WRAPPED=true
+  config run wrap -- $0
+  exit $?
+fi
+
+cat <<End-of-message
+apiVersion: v1
+kind: Service
+metadata:
+  name: ${NAME}
+  labels:
+    app: nginx
+    instance: ${NAME}
+spec:
+  ports:
+  - port: 80
+    targetPort: 80
+    name: http
+  selector:
+    app: nginx
+    instance: ${NAME}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${NAME}
+  labels:
+    app: nginx
+    instance: ${NAME}
+spec:
+  replicas: ${REPLICAS}
+  selector:
+    matchLabels:
+      app: nginx
+      instance: ${NAME}
+  template:
+    metadata:
+      labels:
+        app: nginx
+        instance: ${NAME}
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+End-of-message
+```
+
+### Dockerfile
+
+`Dockerfile` installs `kustomize config` and copies the script into the container image.
+
+```
+FROM golang:1.13-stretch
+RUN go get sigs.k8s.io/kustomize/cmd/config
+RUN mv /go/bin/config /usr/bin/config
+COPY nginx-template.sh /usr/bin/nginx-template.sh
+CMD ["nginx-template.sh]
+```
+
+## Example Function Usage
+
+Following is an example of running the `kustomize config run` using the preceding API.
+
+When run by `kustomize config run`, functions are run in containers with the
+following environment:
+
+- Network: `none`
+- User: `nobody`
+- Security Options: `no-new-privileges`
+- Volumes: the volume containing the `functionConfig` yaml is mounted under `/local` as `ro`
+
+### Input
+
+`dir/nginx.yaml` contains a reference to the Function. The contents of `nginx.yaml`
+are passed to the Function through the `ResourceList.functionConfig` field.
+
+```yaml
+apiVersion: example.com/v1beta1
+kind: Nginx
+metadata:
+  name: my-instance
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    config.k8s.io/function: |
+      container:
+        image: gcr.io/example-functions/nginx-template:v1.0.0
+spec:
+  replicas: 5
+```
+
+- `annotations[config.k8s.io/function].container.image`: the image to use for this API
+- `annotations[config.kubernetes.io/local-config]`: mark this as not a Resource that should
+  be applied
+
+### Output
+
+The function is invoked using byrunning `kustomize config run dir/`.
+
+`dir/my-instance_deployment.yaml` contains the Deployment:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: my-instance
+  labels:
+    app: nginx
+    instance: my-instance
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: nginx
+      instance: my-instance
+  template:
+    metadata:
+      labels:
+        app: nginx
+        instance: my-instance
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:1.7.9
+          ports:
+            - containerPort: 80
+```
+
+`dir/my-instance_service.yaml` contains the Service:
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: my-instance
+  labels:
+    app: nginx
+    instance: my-instance
+spec:
+  ports:
+    - port: 80
+      targetPort: 80
+      name: http
+  selector:
+    app: nginx
+    instance: my-instance
+```

cmd/config/docs/api-conventions/functions-spec.md

@@ -0,0 +1,186 @@
+# Configuration Functions Specification
+
+This document specifies a standard for client-side functions that operate on
+Kubernetes declarative configurations. This standard enables creating
+small, interoperable, and language-independent executable programs packaged as
+containers that can be chained together as part of a configuration management pipeline.
+The end result of such a pipeline are fully rendered configurations that can then be
+applied to a control plane (e.g. Using ‘kubectl apply’ for Kubernetes control plane).
+As such, although this document references Kubernetes Resource Model and API conventions,
+it is completely decoupled from Kuberentes API machinery and does not depend on any
+in-cluster components.
+
+This document references terms described in [Kubernetes API Conventions][1].
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD",
+"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be
+interpreted as described in [RFC 2119][2].
+
+## Use Cases
+
+_Configuration functions_ enable shift-left practices (client-side) through:
+
+- Pre-commit / delivery validation and linting of configuration
+  - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
+- Implementation of abstractions as client actuated APIs (e.g. templating)
+  - e.g. Create a client-side _"CRD"_ for generating configuration checked into git
+- Aspect Orient configuration / Injection of cross-cutting configuration
+  - e.g. T-Shirt size containers by annotating Resources with `small`, `medium`, `large`
+    and inject the cpu and memory resources into containers accordingly.
+  - e.g. Inject `init` and `side-car` containers into Resources based off of Resource
+    Type, annotations, etc.
+
+Performing these on the client rather than the server enables:
+
+- Configuration to be reviewed prior to being sent to the API server
+- Configuration to be validated as part of the CI?CD pipeline
+- Configuration for Resources to validated holistically rather than individually
+  per-Resource
+  - e.g. ensure the `Service.selector` and `Deployment.spec.template` labels
+    match.
+  - e.g. MutatingWebHooks are scoped to a single Resource instance at a time.
+- Low-level tweaks to the output of high-level abstractions
+  - e.g. add an `init container` to a client _"CRD"_ Resource after it was generated.
+- Composition and layering of multiple functions together
+  - Compose generation, injection, validation together
+
+## Spec
+
+### Input Type
+
+A function MUST accept as input a single [Kubernetes List type][3].
+The `items` field in the input will contain a sequence of [Object types][3].
+A function MAY not support [Simple types][3] and List types.
+
+An example using `v1/ConfigMapList` as input:
+
+```yaml
+apiVersion: v1
+kind: ConfigMapList
+items:
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: config1
+    data:
+      p1: v1
+      p2: v2
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: config2
+```
+
+An example using `v1/List` as input:
+
+```yaml
+apiVersion: v1
+kind: List
+items:
+  spec:
+   - apiVersion: foo-corp.com/v1
+    kind: FulfillmentCenter
+    metadata:
+      name: staging
+    address: "100 Main St."
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: namespace-reader
+    rules:
+      - resources:
+          - namespaces
+        apiGroups:
+          - ""
+        verbs:
+          - get
+          - watch
+          - list
+```
+
+In addition, a function MUST accept as input a List of kind `ResourceList` where the
+`functionConfig` field, if present, will contain the invocation-specific configuration passed to the function
+by the orchestrator.
+Functions MAY consider this field optional so that they can be triggered in an ad-hoc fashion.
+
+An example using `config.kubernetes.io/v1beta1/ResourceList` as input:
+
+```yaml
+apiVersion: config.kubernetes.io/v1beta1
+kind: ResourceList
+functionConfig:
+  apiVersion: foo-corp.com/v1
+  kind: FulfillmentCenter
+  metadata:
+    name: staging
+    metadata:
+      annotations:
+        config.k8s.io/function: |
+          container:
+            image: gcr.io/example/foo:v1.0.0
+  spec:
+    address: "100 Main St."
+items:
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: namespace-reader
+    rules:
+      - resources:
+          - namespaces
+        apiGroups:
+          - ""
+        verbs:
+          - get
+          - watch
+          - list
+```
+
+Here `FulfillmentCenter` kind with name `staging` is passed as the invocation-specific configuration
+to the function.
+
+### Output Type
+
+A function’s output MUST be the same as the input specification above
+-- i.e. `ResourceList` or `List`.
+This is necessary to enable chaining two or more functions together in a pipeline.
+The serialization format of the output SHOULD match that of its input on each invocation
+-- e.g. if the input was a `ResourceList`, the output should also be a `ResourceList`.
+
+### Serialization Format
+
+A function MUST support YAML as a serialization format for the input and output.
+A function MUST use utf8 encoding (as YAML is a superset of JSON, JSON will also be supported
+by any conforming function).
+
+### Operations
+
+A function MAY Create, Update, or Delete any number of items in the `items` field and output the
+resultant list.
+
+A function MAY modify annotations with prefix `config.kubernetes.io`, but must be careful about
+doing so since they’re used for orchestration purposes and will likely impact subsequent functions
+in the pipeline.
+
+A function SHOULD preserve comments when input serialization format is YAML.
+This allows for human authoring of configuration to coexist with changes made by functions.
+
+### Containerization
+
+A function MUST be implemented as a container.
+
+A function container MUST be capable of running as a non-root user if it does not require
+access to host filesystem or makes network calls.
+
+### stdin/stdout/stderr and Exit Codes
+
+A function MUST accept input from stdin and emit output to stdout.
+
+Any error messages MUST be emitted to stderr.
+
+An exit code of zero indicates function execution was successful.
+A non-zero exit code indicates a failure.
+
+[1]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md
+[2]: https://tools.ietf.org/html/rfc2119
+[3]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds

cmd/config/docs/commands/annotate.md

@@ -0,0 +1,18 @@
+## annotate
+
+[Alpha] Set an annotation on Resources.
+
+### Synopsis
+
+[Alpha]  Set an annotation on Resources.
+
+  DIR:
+    Path to local directory.
+
+### Examples
+
+    kustomize config annotate my-dir/ --kv foo=bar
+
+    kustomize config annotate my-dir/ --kv foo=bar --kv a=b
+
+    kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo

cmd/config/docs/commands/list-setters.md

@@ -0,0 +1,23 @@
+## set
+
+[Alpha] List setters for Resources.
+
+### Synopsis
+
+List setters for Resources.
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    Optional.  The name of the setter to display.
+
+### Examples
+
+  Show setters:
+
+    $ config set DIR/
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    name-prefix   ''            PREFIX    string   2

cmd/config/docs/commands/run-fns.md

@@ -22,8 +22,8 @@ order they appear in the file).
 
 #### Config Functions:
 
-  Config functions are specified as Kubernetes types containing a metadata.configFn.container.image
-  field.  This field tells run how to invoke the container.
+  Config functions are specified as Kubernetes types containing a metadata.annotations.[config.kubernetes.io/function]
+  field specifying an image for the container to run.  This image tells run how to invoke the container.
 
   Example config function:
 
@@ -31,17 +31,17 @@ order they appear in the file).
 	apiVersion: fn.example.com/v1beta1
 	kind: ExampleFunctionKind
 	metadata:
-	  configFn:
-	    container:
-	      # function is invoked as a container running this image
-	      image: gcr.io/example/examplefunction:v1.0.1
 	  annotations:
+	    config.kubernetes.io/function: |
+	      container:
+	        # function is invoked as a container running this image
+	        image: gcr.io/example/examplefunction:v1.0.1
 	    config.kubernetes.io/local-config: "true" # tools should ignore this
 	spec:
 	  configField: configValue
 
   In the preceding example, 'kustomize config run example/' would identify the function by
-  the metadata.configFn field.  It would then write all Resources in the directory to
+  the metadata.annotations.[config.kubernetes.io/function] field.  It would then write all Resources in the directory to
   a container stdin (running the gcr.io/example/examplefunction:v1.0.1 image).  It
   would then write the container stdout back to example/, replacing the directory
   file contents.

cmd/config/docs/commands/set.md

@@ -59,19 +59,19 @@ To create a custom setter for a field see: `kustomize help config create-setter`
   List setters: Show the possible setters
 
     $ config set DIR/
-    NAME          DESCRIPTION    VALUE     TYPE    COUNT   OWNER
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
     name-prefix   ''            PREFIX    string   2
 
-  Perform substitution: set a new value, owner and description
+  Perform set: set a new value, owner and description
 
     $ kustomize config set DIR/ name-prefix "test" --description "test environment" --set-by "dev"
-    performed 2 substitutions
+    set 2 values
 
-  Show substitutions: Show the new values
+  List setters: Show the new values
 
-    $ config set dir
-    NAME       DESCRIPTION       VALUE    TYPE    COUNT   SUBSTITUTED   OWNER
-    prefix   'test environment'   test   string   2       true          dev
+    $ config set DIR/
+        NAME      DESCRIPTION         VALUE     TYPE     COUNT     SETBY 
+    name-prefix   'test environment'   test     string   2          dev
 
   New Resource YAML:
 

cmd/config/docs/commands/sink.md

@@ -0,0 +1,18 @@
+## sink
+
+[Alpha] Implement a Sink by writing input to a local directory.
+
+### Synopsis
+
+[Alpha] Implement a Sink by writing input to a local directory.
+
+    kustomize config sink [DIR]
+
+  DIR:
+    Path to local directory.  If unspecified, sink will write to stdout as if it were a single file.
+
+`sink` writes its input to a directory
+
+### Examples
+
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/

cmd/config/docs/commands/source.md

@@ -0,0 +1,22 @@
+## source
+
+[Alpha] Implement a Source by reading a local directory.
+
+### Synopsis
+
+[Alpha] Implement a Source by reading a local directory.
+
+    kustomize config source DIR...
+
+  DIR:
+    One or more paths to local directories.  Contents from directories will be concatenated.
+    If no directories are provided, source will read from stdin as if it were a single file.
+
+`source` emits configuration to act as input to a function
+
+### Examples
+
+    # emity configuration directory as input source to a function
+    kustomize config source DIR/
+
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/

cmd/config/go.mod

@@ -9,7 +9,6 @@ require (
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	k8s.io/apimachinery v0.17.0
 	sigs.k8s.io/kustomize/kyaml v0.0.0
 )

cmd/config/go.sum

@@ -101,6 +101,8 @@ github.com/posener/complete/v2 v2.0.1-alpha.12/go.mod h1://JlL91cS2JV7rOl6LVHrRq
 github.com/posener/script v1.0.4 h1:nSuXW5ZdmFnQIueLB2s0qvs4oNsUloM1Zydzh75v42w=
 github.com/posener/script v1.0.4/go.mod h1:Rg3ijooqulo05aGLyGsHoLmIOUzHUVK19WVgrYBPU/E=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=

cmd/config/internal/commands/annotate.go

@@ -0,0 +1,103 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"strings"
+
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// NewAnnotateRunner returns a command runner.
+func NewAnnotateRunner(parent string) *AnnotateRunner {
+	r := &AnnotateRunner{}
+	c := &cobra.Command{
+		Use:     "annotate [DIR]",
+		Args:    cobra.MaximumNArgs(1),
+		Short:   commands.AnnotateShort,
+		Long:    commands.AnnotateLong,
+		Example: commands.AnnotateExamples,
+		RunE:    r.runE,
+	}
+	fixDocs(parent, c)
+	r.Command = c
+	c.Flags().StringVar(&r.Kind, "kind", "", "Resource kind to annotate")
+	c.Flags().StringVar(&r.ApiVersion, "apiVersion", "", "Resource apiVersion to annotate")
+	c.Flags().StringVar(&r.Name, "name", "", "Resource name to annotate")
+	c.Flags().StringVar(&r.Namespace, "namespace", "", "Resource namespace to annotate")
+	c.Flags().StringSliceVar(&r.Values, "kv", []string{}, "annotation as KEY=VALUE")
+	return r
+}
+
+func AnnotateCommand(parent string) *cobra.Command {
+	return NewAnnotateRunner(parent).Command
+}
+
+type AnnotateRunner struct {
+	Command    *cobra.Command
+	Values     []string
+	Kind       string
+	Name       string
+	ApiVersion string
+	Namespace  string
+	Path       string
+}
+
+func (r *AnnotateRunner) runE(c *cobra.Command, args []string) error {
+	var input []kio.Reader
+	var output []kio.Writer
+	if len(args) == 0 {
+		rw := &kio.ByteReadWriter{Reader: c.InOrStdin(), Writer: c.OutOrStdout()}
+		input = []kio.Reader{rw}
+		output = []kio.Writer{rw}
+	} else {
+		rw := &kio.LocalPackageReadWriter{PackagePath: args[0], NoDeleteFiles: true}
+		input = []kio.Reader{rw}
+		output = []kio.Writer{rw}
+	}
+	return handleError(c, kio.Pipeline{
+		Inputs:  input,
+		Filters: []kio.Filter{r},
+		Outputs: output,
+	}.Execute())
+}
+
+func (r *AnnotateRunner) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	for i := range nodes {
+		n := nodes[i]
+		m, err := n.GetMeta()
+		if err != nil {
+			return nil, err
+		}
+		if r.Kind != "" && r.Kind != m.Kind {
+			continue
+		}
+		if r.ApiVersion != "" && r.ApiVersion != m.APIVersion {
+			continue
+		}
+		if r.Namespace != "" && r.Namespace != m.Namespace {
+			continue
+		}
+		if r.Name != "" && r.Name != m.Name {
+			continue
+		}
+
+		for i := range r.Values {
+			// split key, value pairs
+			kv := strings.SplitN(r.Values[i], "=", 2)
+			if len(kv) != 2 {
+				return nil, errors.Errorf("must specify --kv as KEY=VALUE: %s", r.Values[i])
+			}
+			if err := n.PipeE(yaml.SetAnnotation(kv[0], kv[1])); err != nil {
+				return nil, err
+			}
+		}
+
+	}
+	return nodes, nil
+}

cmd/config/internal/commands/annotate_test.go

@@ -0,0 +1,488 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func TestAnnotateCommand(t *testing.T) {
+	var tests = []struct {
+		name     string
+		args     []string
+		expected string
+	}{
+		{
+			name:     "single value",
+			args:     []string{"--kv", "a=b"},
+			expected: expectedSingleValue,
+		},
+		{
+			name:     "multi value",
+			args:     []string{"--kv", "a=b", "--kv", "c=d"},
+			expected: expectedMultiValue,
+		},
+		{
+			name:     "filter kind",
+			args:     []string{"--kv", "a=b", "--kind", "Service"},
+			expected: expectedFilterKindService,
+		},
+		{
+			name:     "filter apiVersion",
+			args:     []string{"--kv", "a=b", "--apiVersion", "v1"},
+			expected: expectedFilterApiVersionV1,
+		},
+		{
+			name:     "filter name",
+			args:     []string{"--kv", "a=b", "--name", "bar"},
+			expected: expectedFilterNameBar,
+		},
+		{
+			name:     "filter namespace",
+			args:     []string{"--kv", "a=b", "--namespace", "bar"},
+			expected: expectedFilterNamespaceBar,
+		},
+	}
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			d := initTestDir(t)
+			defer os.RemoveAll(d)
+
+			a := NewAnnotateRunner("")
+			a.Command.SetArgs(append([]string{d}, tt.args...))
+			a.Command.SilenceUsage = true
+			a.Command.SilenceErrors = true
+
+			err := a.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			actual := &bytes.Buffer{}
+			err = kio.Pipeline{
+				Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: d}},
+				Outputs: []kio.Writer{kio.ByteWriter{Writer: actual, KeepReaderAnnotations: true}},
+			}.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(tt.expected),
+				strings.TrimSpace(actual.String())) {
+				t.FailNow()
+			}
+		})
+	}
+}
+
+func initTestDir(t *testing.T) string {
+	d, err := ioutil.TempDir("", "kustomize-annotate-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	err = ioutil.WriteFile(filepath.Join(d, "f1.yaml"), []byte(f1Input), 0600)
+	if !assert.NoError(t, err) {
+		defer os.RemoveAll(d)
+		t.FailNow()
+	}
+	err = ioutil.WriteFile(filepath.Join(d, "f2.yaml"), []byte(f2Input), 0600)
+	if !assert.NoError(t, err) {
+		defer os.RemoveAll(d)
+		t.FailNow()
+	}
+	return d
+}
+
+var (
+	f1Input = `
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`
+
+	f2Input = `
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedSingleValue = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedMultiValue = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterKindService = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterApiVersionV1 = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterNameBar = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterNamespaceBar = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+)

cmd/config/internal/commands/cat.go

@@ -121,6 +121,13 @@ func (r *CatRunner) runE(c *cobra.Command, args []string) error {
 		out = o
 	}
 
+	// remove this annotation explicitly, the ByteWriter won't clear it by
+	// default because it doesn't set it
+	clear := []string{"config.kubernetes.io/path"}
+	if r.KeepAnnotations {
+		clear = nil
+	}
+
 	var outputs []kio.Writer
 	outputs = append(outputs, kio.ByteWriter{
 		Writer:                out,
@@ -129,6 +136,7 @@ func (r *CatRunner) runE(c *cobra.Command, args []string) error {
 		WrappingAPIVersion:    r.WrapApiVersion,
 		FunctionConfig:        functionConfig,
 		Style:                 yaml.GetStyle(r.Styles...),
+		ClearAnnotations:      clear,
 	})
 
 	return handleError(c, kio.Pipeline{Inputs: inputs, Filters: fltr, Outputs: outputs}.Execute())

cmd/config/internal/commands/cat_test.go

@@ -90,8 +90,6 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
 ---
@@ -100,8 +98,6 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:
     app: nginx
@@ -114,8 +110,6 @@ metadata:
     app: nginx
   annotations:
     app: nginx
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f2.yaml'
 spec:
   replicas: 3
 `, b.String()) {
@@ -196,8 +190,6 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
 ---
@@ -206,8 +198,6 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:
     app: nginx
@@ -218,8 +208,6 @@ metadata:
   name: foo
   annotations:
     config.kubernetes.io/local-config: "true"
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f2.yaml'
   configFn:
     container:
       image: gcr.io/example/image:version
@@ -233,8 +221,6 @@ metadata:
   name: bar
   annotations:
     app: nginx
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f2.yaml'
 spec:
   replicas: 3
 `, b.String()) {
@@ -314,8 +300,6 @@ metadata:
   name: foo
   annotations:
     config.kubernetes.io/local-config: "true"
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f2.yaml'
   configFn:
     container:
       image: gcr.io/example/reconciler:v1
@@ -414,8 +398,6 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
 ---
@@ -424,8 +406,6 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:
     app: nginx
@@ -438,8 +418,6 @@ metadata:
     app: nginx
   annotations:
     app: nginx
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f2.yaml'
 spec:
   replicas: 3
 `, string(actual)) {
@@ -536,8 +514,6 @@ metadata:
   name: foo
   annotations:
     app: nginx2
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
 ---
@@ -546,8 +522,6 @@ metadata:
   name: foo
   annotations:
     app: nginx
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:
     app: nginx
@@ -560,8 +534,6 @@ metadata:
     app: nginx
   annotations:
     app: nginx
-    config.kubernetes.io/package: '.'
-    config.kubernetes.io/path: 'f2.yaml'
 spec:
   replicas: 3
 `, string(actual)) {

cmd/config/internal/commands/cmdcreatesetter.go

@@ -34,9 +34,10 @@ func NewCreateSetterRunner(parent string) *CreateSetterRunner {
 		"kind of the Resource on which to create the setter.")
 	set.Flags().StringVar(&r.Set.SetPartialField.Type, "type", "",
 		"valid OpenAPI field type -- e.g. integer,boolean,string.")
+	set.Flags().BoolVar(&r.Set.SetPartialField.Partial, "partial", false,
+		"create a partial setter for only part of the field value.")
 	fixDocs(parent, set)
 	set.MarkFlagRequired("type")
-	set.MarkFlagRequired("field")
 	r.Command = set
 	return r
 }

cmd/config/internal/commands/cmdlistsetters.go

@@ -0,0 +1,47 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/setters"
+)
+
+// NewListSettersRunner returns a command runner.
+func NewListSettersRunner(parent string) *ListSettersRunner {
+	r := &ListSettersRunner{}
+	c := &cobra.Command{
+		Use:     "list-setters DIR [NAME]",
+		Args:    cobra.RangeArgs(1, 2),
+		Short:   commands.ListSettersShort,
+		Long:    commands.ListSettersLong,
+		Example: commands.ListSettersExamples,
+		PreRunE: r.preRunE,
+		RunE:    r.runE,
+	}
+	fixDocs(parent, c)
+	r.Command = c
+	return r
+}
+
+func ListSettersCommand(parent string) *cobra.Command {
+	return NewListSettersRunner(parent).Command
+}
+
+type ListSettersRunner struct {
+	Command *cobra.Command
+	Lookup  setters.LookupSetters
+}
+
+func (r *ListSettersRunner) preRunE(c *cobra.Command, args []string) error {
+	if len(args) > 1 {
+		r.Lookup.Name = args[1]
+	}
+	return nil
+}
+
+func (r *ListSettersRunner) runE(c *cobra.Command, args []string) error {
+	return handleError(c, lookup(r.Lookup, c, args))
+}

cmd/config/internal/commands/cmdset.go

@@ -5,6 +5,7 @@ package commands
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
@@ -27,6 +28,10 @@ func NewSetRunner(parent string) *SetRunner {
 	}
 	fixDocs(parent, c)
 	r.Command = c
+	c.Flags().StringVar(&r.Perform.SetBy, "set-by", "",
+		"annotate the field with who set it")
+	c.Flags().StringVar(&r.Perform.Description, "description", "",
+		"annotate the field with a description of its value")
 
 	return r
 }
@@ -59,14 +64,14 @@ func (r *SetRunner) runE(c *cobra.Command, args []string) error {
 		return handleError(c, r.perform(c, args))
 	}
 
-	return handleError(c, r.lookup(c, args))
+	return handleError(c, lookup(r.Lookup, c, args))
 }
 
-func (r *SetRunner) lookup(c *cobra.Command, args []string) error {
+func lookup(l setters.LookupSetters, c *cobra.Command, args []string) error {
 	// lookup the setters
 	err := kio.Pipeline{
 		Inputs:  []kio.Reader{&kio.LocalPackageReader{PackagePath: args[0]}},
-		Filters: []kio.Filter{&r.Lookup},
+		Filters: []kio.Filter{&l},
 	}.Execute()
 	if err != nil {
 		return err
@@ -82,8 +87,8 @@ func (r *SetRunner) lookup(c *cobra.Command, args []string) error {
 	table.SetHeader([]string{
 		"NAME", "DESCRIPTION", "VALUE", "TYPE", "COUNT", "SETBY",
 	})
-	for i := range r.Lookup.SetterCounts {
-		s := r.Lookup.SetterCounts[i]
+	for i := range l.SetterCounts {
+		s := l.SetterCounts[i]
 		v := s.Value
 		if s.Value == "" {
 			v = s.Value
@@ -98,6 +103,11 @@ func (r *SetRunner) lookup(c *cobra.Command, args []string) error {
 		})
 	}
 	table.Render()
+
+	if len(l.SetterCounts) == 0 {
+		// exit non-0 if no matching setters are found
+		os.Exit(1)
+	}
 	return nil
 }
 

cmd/config/internal/commands/fmt.go

@@ -31,6 +31,8 @@ formatting substitution verbs {'%n': 'metadata.name', '%s': 'metadata.namespace'
 		`if true, keep index and filename annotations set on Resources.`)
 	c.Flags().BoolVar(&r.Override, "override", false,
 		`if true, override existing filepath annotations.`)
+	c.Flags().BoolVar(&r.UseSchema, "use-schema", false,
+		`if true, uses openapi resource schema to format resources.`)
 	r.Command = c
 	return r
 }
@@ -46,6 +48,7 @@ type FmtRunner struct {
 	SetFilenames    bool
 	KeepAnnotations bool
 	Override        bool
+	UseSchema       bool
 }
 
 func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
@@ -56,7 +59,9 @@ func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
 }
 
 func (r *FmtRunner) runE(c *cobra.Command, args []string) error {
-	f := []kio.Filter{filters.FormatFilter{}}
+	f := []kio.Filter{filters.FormatFilter{
+		UseSchema: r.UseSchema,
+	}}
 
 	// format with file names
 	if r.SetFilenames {

cmd/config/internal/commands/grep_test.go

@@ -76,7 +76,6 @@ metadata:
   annotations:
     app: nginx2
     config.kubernetes.io/index: '0'
-    config.kubernetes.io/package: '.'
     config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
@@ -87,7 +86,6 @@ metadata:
   annotations:
     app: nginx
     config.kubernetes.io/index: '1'
-    config.kubernetes.io/package: '.'
     config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:

cmd/config/internal/commands/merge3.go

@@ -25,6 +25,8 @@ func GetMerge3Runner(name string) *Merge3Runner {
 		"Path to updated package")
 	c.Flags().StringVar(&r.toDir, "to", "",
 		"Path to destination package")
+	c.Flags().BoolVar(&r.path, "path-merge-key", false,
+		"Use the path as part of the merge key when merging resources")
 
 	r.Command = c
 	return r
@@ -40,6 +42,7 @@ type Merge3Runner struct {
 	ancestor string
 	fromDir  string
 	toDir    string
+	path     bool
 }
 
 func (r *Merge3Runner) runE(c *cobra.Command, args []string) error {
@@ -47,6 +50,7 @@ func (r *Merge3Runner) runE(c *cobra.Command, args []string) error {
 		OriginalPath: r.ancestor,
 		UpdatedPath:  r.fromDir,
 		DestPath:     r.toDir,
+		MergeOnPath:  r.path,
 	}.Merge()
 	if err != nil {
 		return err

cmd/config/internal/commands/run-fns.go

@@ -4,22 +4,27 @@
 package commands
 
 import (
+	"fmt"
+	"io"
+	"strings"
+
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/runfn"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
 // GetCatRunner returns a RunFnRunner.
 func GetRunFnRunner(name string) *RunFnRunner {
 	r := &RunFnRunner{}
 	c := &cobra.Command{
-		Use:     "run DIR",
-		Aliases: []string{"run-fns"},
+		Use:     "run [DIR]",
 		Short:   commands.RunFnsShort,
 		Long:    commands.RunFnsLong,
 		Example: commands.RunFnsExamples,
 		RunE:    r.runE,
-		Args:    cobra.ExactArgs(1),
+		PreRunE: r.preRunE,
 	}
 	fixDocs(name, c)
 	c.Flags().BoolVar(&r.IncludeSubpackages, "include-subpackages", true,
@@ -27,11 +32,14 @@ func GetRunFnRunner(name string) *RunFnRunner {
 	r.Command = c
 	r.Command.Flags().BoolVar(
 		&r.DryRun, "dry-run", false, "print results to stdout")
+	r.Command.Flags().BoolVar(
+		&r.GlobalScope, "global-scope", false, "set global scope for functions.")
 	r.Command.Flags().StringSliceVar(
 		&r.FnPaths, "fn-path", []string{},
-		"directories containing functions without configuration")
-	r.Command.AddCommand(XArgsCommand())
-	r.Command.AddCommand(WrapCommand())
+		"read functions from these directories instead of the configuration directory.")
+	r.Command.Flags().StringVar(
+		&r.Image, "image", "",
+		"run this image as a function instead of discovering them.")
 	return r
 }
 
@@ -44,13 +52,144 @@ type RunFnRunner struct {
 	IncludeSubpackages bool
 	Command            *cobra.Command
 	DryRun             bool
+	GlobalScope        bool
 	FnPaths            []string
+	Image              string
+	RunFns             runfn.RunFns
 }
 
 func (r *RunFnRunner) runE(c *cobra.Command, args []string) error {
-	rec := runfn.RunFns{Path: args[0], FunctionPaths: r.FnPaths}
-	if r.DryRun {
-		rec.Output = c.OutOrStdout()
-	}
-	return handleError(c, rec.Execute())
+	return handleError(c, r.RunFns.Execute())
+}
+
+// getFunctions parses the commandline flags and arguments into explicit
+// Functions to run.
+func (r *RunFnRunner) getFunctions(c *cobra.Command, args, dataItems []string) (
+	[]*yaml.RNode, error) {
+	// if image isn't specified, then Functions is empty
+	if r.Image == "" {
+		return nil, nil
+	}
+
+	// create the function spec to set as an annotation
+	fn, err := yaml.Parse(`container: {}`)
+	if err != nil {
+		return nil, err
+	}
+	// TODO: add support network, volumes, etc based on flag values
+	err = fn.PipeE(
+		yaml.Lookup("container"),
+		yaml.SetField("image", yaml.NewScalarRNode(r.Image)))
+	if err != nil {
+		return nil, err
+	}
+
+	// create the function config
+	rc, err := yaml.Parse(`
+metadata:
+  name: function-input
+data: {}
+`)
+	if err != nil {
+		return nil, err
+	}
+
+	// set the function annotation on the function config so it
+	// is parsed by RunFns
+	value, err := fn.String()
+	if err != nil {
+		return nil, err
+	}
+	err = rc.PipeE(
+		yaml.LookupCreate(yaml.MappingNode, "metadata", "annotations"),
+		yaml.SetField("config.kubernetes.io/function", yaml.NewScalarRNode(value)))
+	if err != nil {
+		return nil, err
+	}
+
+	// default the function config kind to ConfigMap, this may be overridden
+	var kind = "ConfigMap"
+	var version = "v1"
+
+	// populate the function config with data.  this is a convention for functions
+	// to be more commandline friendly
+	if len(dataItems) > 0 {
+		dataField, err := rc.Pipe(yaml.Lookup("data"))
+		if err != nil {
+			return nil, err
+		}
+		for i, s := range dataItems {
+			kv := strings.SplitN(s, "=", 2)
+			if i == 0 && len(kv) == 1 {
+				// first argument may be the kind
+				kind = s
+				continue
+			}
+			if len(kv) != 2 {
+				return nil, fmt.Errorf("args must have keys and values separated by =")
+			}
+			err := dataField.PipeE(yaml.SetField(kv[0], yaml.NewScalarRNode(kv[1])))
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	err = rc.PipeE(yaml.SetField("kind", yaml.NewScalarRNode(kind)))
+	if err != nil {
+		return nil, err
+	}
+	err = rc.PipeE(yaml.SetField("apiVersion", yaml.NewScalarRNode(version)))
+	if err != nil {
+		return nil, err
+	}
+	return []*yaml.RNode{rc}, nil
+}
+
+func (r *RunFnRunner) preRunE(c *cobra.Command, args []string) error {
+	if c.ArgsLenAtDash() >= 0 && r.Image == "" {
+		return errors.Errorf("must specify --image")
+	}
+
+	var dataItems []string
+	if c.ArgsLenAtDash() >= 0 {
+		dataItems = args[c.ArgsLenAtDash():]
+		args = args[:c.ArgsLenAtDash()]
+	}
+	if len(args) > 1 {
+		return errors.Errorf("0 or 1 arguments supported, function arguments go after '--'")
+	}
+
+	fns, err := r.getFunctions(c, args, dataItems)
+	if err != nil {
+		return err
+	}
+
+	// set the output to stdout if in dry-run mode or no arguments are specified
+	var output io.Writer
+	var input io.Reader
+	if len(args) == 0 {
+		output = c.OutOrStdout()
+		input = c.InOrStdin()
+	} else if r.DryRun {
+		output = c.OutOrStdout()
+	}
+
+	// set the path if specified as an argument
+	var path string
+	if len(args) == 1 {
+		// argument is the directory
+		path = args[0]
+	}
+
+	r.RunFns = runfn.RunFns{
+		FunctionPaths: r.FnPaths,
+		GlobalScope:   r.GlobalScope,
+		Functions:     fns,
+		Output:        output,
+		Input:         input,
+		Path:          path,
+	}
+
+	// don't consider args for the function
+	return nil
 }

cmd/config/internal/commands/run_test.go

@@ -0,0 +1,232 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestRunFnCommand_preRunE verifies that preRunE correctly parses the commandline
+// flags and arguments into the RunFns structure to be executed.
+func TestRunFnCommand_preRunE(t *testing.T) {
+	tests := []struct {
+		name          string
+		args          []string
+		expected      string
+		err           string
+		path          string
+		input         io.Reader
+		output        io.Writer
+		functionPaths []string
+	}{
+		{
+			name: "config map",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {a: b, c: d, e: f}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name:   "config map stdin / stdout",
+			args:   []string{"run", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
+			input:  os.Stdin,
+			output: os.Stdout,
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {a: b, c: d, e: f}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name:   "config map dry-run",
+			args:   []string{"run", "dir", "--image", "foo:bar", "--dry-run", "--", "a=b", "c=d", "e=f"},
+			output: os.Stdout,
+			path:   "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {a: b, c: d, e: f}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name: "config map no args",
+			args: []string{"run", "dir", "--image", "foo:bar"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name: "custom kind",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "Foo", "g=h"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {g: h}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name: "custom kind '=' in data",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "Foo", "g=h", "i=j=k"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {g: h, i: j=k}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name:          "function paths",
+			args:          []string{"run", "dir", "--fn-path", "path1", "--fn-path", "path2"},
+			path:          "dir",
+			functionPaths: []string{"path1", "path2"},
+		},
+		{
+			name: "custom kind with function paths",
+			args: []string{
+				"run", "dir", "--fn-path", "path", "--image", "foo:bar", "--", "Foo", "g=h", "i=j=k"},
+			path:          "dir",
+			functionPaths: []string{"path"},
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {g: h, i: j=k}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name: "config map multi args",
+			args: []string{"run", "dir", "dir2", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
+			err:  "0 or 1 arguments supported",
+		},
+		{
+			name: "config map not image",
+			args: []string{"run", "dir", "--", "a=b", "c=d", "e=f"},
+			err:  "must specify --image",
+		},
+		{
+			name: "config map bad data",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "a=b", "c", "e=f"},
+			err:  "must have keys and values separated by",
+		},
+	}
+
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			r := GetRunFnRunner("kustomize")
+			// Don't run the actual command
+			r.Command.Run = nil
+			r.Command.RunE = func(cmd *cobra.Command, args []string) error { return nil }
+			r.Command.SilenceErrors = true
+			r.Command.SilenceUsage = true
+
+			// hack due to https://github.com/spf13/cobra/issues/42
+			root := &cobra.Command{Use: "root"}
+			root.AddCommand(r.Command)
+			root.SetArgs(tt.args)
+
+			// error case
+			err := r.Command.Execute()
+			if tt.err != "" {
+				if !assert.Error(t, err) {
+					t.FailNow()
+				}
+				if !assert.Contains(t, err.Error(), tt.err) {
+					t.FailNow()
+				}
+				// don't check anything else in error case
+				return
+			}
+
+			// non-error case
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			// check if Input was set
+			if !assert.Equal(t, tt.input, r.RunFns.Input) {
+				t.FailNow()
+			}
+
+			// check if Output was set
+			if !assert.Equal(t, tt.output, r.RunFns.Output) {
+				t.FailNow()
+			}
+
+			// check if Path was set
+			if !assert.Equal(t, tt.path, r.RunFns.Path) {
+				t.FailNow()
+			}
+
+			// check if FunctionPaths were set
+			if tt.functionPaths == nil {
+				// make Equal work against flag default
+				tt.functionPaths = []string{}
+			}
+			if !assert.Equal(t, tt.functionPaths, r.RunFns.FunctionPaths) {
+				t.FailNow()
+			}
+
+			// check if Functions were set
+			if tt.expected != "" {
+				if !assert.Len(t, r.RunFns.Functions, 1) {
+					t.FailNow()
+				}
+				actual := strings.TrimSpace(r.RunFns.Functions[0].MustString())
+				if !assert.Equal(t, strings.TrimSpace(tt.expected), actual) {
+					t.FailNow()
+				}
+			}
+
+		})
+	}
+
+}

cmd/config/internal/commands/sink.go

@@ -0,0 +1,53 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+)
+
+// GetSinkRunner returns a command for Sink.
+func GetSinkRunner(name string) *SinkRunner {
+	r := &SinkRunner{}
+	c := &cobra.Command{
+		Use:     "sink DIR",
+		Short:   commands.SinkShort,
+		Long:    commands.SinkLong,
+		Example: commands.SinkExamples,
+		RunE:    r.runE,
+		Args:    cobra.MaximumNArgs(1),
+	}
+	fixDocs(name, c)
+	r.Command = c
+	return r
+}
+
+func SinkCommand(name string) *cobra.Command {
+	return GetSinkRunner(name).Command
+}
+
+// SinkRunner contains the run function
+type SinkRunner struct {
+	Command *cobra.Command
+}
+
+func (r *SinkRunner) runE(c *cobra.Command, args []string) error {
+	var outputs []kio.Writer
+	if len(args) == 1 {
+		outputs = []kio.Writer{&kio.LocalPackageWriter{PackagePath: args[0]}}
+	} else {
+		outputs = []kio.Writer{&kio.ByteWriter{
+			Writer:           c.OutOrStdout(),
+			ClearAnnotations: []string{kioutil.PathAnnotation}},
+		}
+	}
+
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}},
+		Outputs: outputs}.Execute()
+	return handleError(c, err)
+}

cmd/config/internal/commands/sink_test.go

@@ -0,0 +1,251 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+)
+
+func TestSinkCommand(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(d)
+
+	r := commands.GetSinkRunner("")
+	r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    selector:
+      app: nginx
+- apiVersion: v1
+  kind: Abstraction
+  metadata:
+    name: foo
+    annotations:
+      config.kubernetes.io/function: |
+        container:
+          image: gcr.io/example/reconciler:v1
+      config.kubernetes.io/local-config: "true"
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    labels:
+      app: nginx
+    name: bar
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+`))
+	r.Command.SetArgs([]string{d})
+	if !assert.NoError(t, r.Command.Execute()) {
+		t.FailNow()
+	}
+
+	actual, err := ioutil.ReadFile(filepath.Join(d, "f1.yaml"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	expected := `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`
+	if !assert.Equal(t, expected, string(actual)) {
+		t.FailNow()
+	}
+
+	actual, err = ioutil.ReadFile(filepath.Join(d, "f2.yaml"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	expected = `apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: gcr.io/example/reconciler:v1
+    config.kubernetes.io/local-config: "true"
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+spec:
+  replicas: 3
+`
+	if !assert.Equal(t, expected, string(actual)) {
+		t.FailNow()
+	}
+}
+
+func TestSinkCommand_Stdout(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(d)
+
+	// fmt the files
+	out := &bytes.Buffer{}
+	r := commands.GetSinkRunner("")
+	r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    selector:
+      app: nginx
+- apiVersion: v1
+  kind: Abstraction
+  metadata:
+    name: foo
+    annotations:
+      config.kubernetes.io/function: |
+        container:
+          image: gcr.io/example/reconciler:v1
+      config.kubernetes.io/local-config: "true"
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    labels:
+      app: nginx
+    name: bar
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+`))
+
+	r.Command.SetOut(out)
+	r.Command.SetArgs([]string{})
+	if !assert.NoError(t, r.Command.Execute()) {
+		t.FailNow()
+	}
+
+	expected := `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: gcr.io/example/reconciler:v1
+    config.kubernetes.io/local-config: "true"
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+spec:
+  replicas: 3
+`
+	if !assert.Equal(t, expected, out.String()) {
+		t.FailNow()
+	}
+}

cmd/config/internal/commands/source.go

@@ -0,0 +1,82 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// GetSourceRunner returns a command for Source.
+func GetSourceRunner(name string) *SourceRunner {
+	r := &SourceRunner{}
+	c := &cobra.Command{
+		Use:     "source DIR",
+		Short:   commands.SourceShort,
+		Long:    commands.SourceLong,
+		Example: commands.SourceExamples,
+		RunE:    r.runE,
+	}
+	fixDocs(name, c)
+	c.Flags().StringVar(&r.WrapKind, "wrap-kind", kio.ResourceListKind,
+		"output using this format.")
+	c.Flags().StringVar(&r.WrapApiVersion, "wrap-version", kio.ResourceListAPIVersion,
+		"output using this format.")
+	c.Flags().StringVar(&r.FunctionConfig, "function-config", "",
+		"path to function config.")
+	r.Command = c
+	_ = c.MarkFlagFilename("function-config", "yaml", "json", "yml")
+	return r
+}
+
+func SourceCommand(name string) *cobra.Command {
+	return GetSourceRunner(name).Command
+}
+
+// SourceRunner contains the run function
+type SourceRunner struct {
+	WrapKind       string
+	WrapApiVersion string
+	FunctionConfig string
+	Command        *cobra.Command
+}
+
+func (r *SourceRunner) runE(c *cobra.Command, args []string) error {
+	// if there is a function-config specified, emit it
+	var functionConfig *yaml.RNode
+	if r.FunctionConfig != "" {
+		configs, err := kio.LocalPackageReader{PackagePath: r.FunctionConfig}.Read()
+		if err != nil {
+			return err
+		}
+		if len(configs) != 1 {
+			return fmt.Errorf("expected exactly 1 functionConfig, found %d", len(configs))
+		}
+		functionConfig = configs[0]
+	}
+
+	var outputs []kio.Writer
+	outputs = append(outputs, kio.ByteWriter{
+		Writer:                c.OutOrStdout(),
+		KeepReaderAnnotations: true,
+		WrappingKind:          r.WrapKind,
+		WrappingAPIVersion:    r.WrapApiVersion,
+		FunctionConfig:        functionConfig,
+	})
+
+	var inputs []kio.Reader
+	for _, a := range args {
+		inputs = append(inputs, kio.LocalPackageReader{PackagePath: a})
+	}
+	if len(inputs) == 0 {
+		inputs = []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}}
+	}
+
+	err := kio.Pipeline{Inputs: inputs, Outputs: outputs}.Execute()
+	return handleError(c, err)
+}

cmd/config/internal/commands/source_test.go

@@ -0,0 +1,200 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+)
+
+func TestSourceCommand(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer os.RemoveAll(d)
+
+	err = ioutil.WriteFile(filepath.Join(d, "f1.yaml"), []byte(`
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`), 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+	err = ioutil.WriteFile(filepath.Join(d, "f2.yaml"), []byte(`
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: gcr.io/example/reconciler:v1
+    config.kubernetes.io/local-config: "true"
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+spec:
+  replicas: 3
+`), 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// fmt the files
+	b := &bytes.Buffer{}
+	r := commands.GetSourceRunner("")
+	r.Command.SetArgs([]string{d})
+	r.Command.SetOut(b)
+	if !assert.NoError(t, r.Command.Execute()) {
+		return
+	}
+
+	if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    selector:
+      app: nginx
+- apiVersion: v1
+  kind: Abstraction
+  metadata:
+    name: foo
+    annotations:
+      config.kubernetes.io/function: |
+        container:
+          image: gcr.io/example/reconciler:v1
+      config.kubernetes.io/local-config: "true"
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    labels:
+      app: nginx
+    name: bar
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+`, b.String()) {
+		return
+	}
+}
+
+func TestSourceCommand_Stdin(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer os.RemoveAll(d)
+
+	in := bytes.NewBufferString(`
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`)
+
+	out := &bytes.Buffer{}
+	r := commands.GetSourceRunner("")
+	r.Command.SetArgs([]string{})
+	r.Command.SetIn(in)
+	r.Command.SetOut(out)
+	if !assert.NoError(t, r.Command.Execute()) {
+		return
+	}
+
+	if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+  spec:
+    selector:
+      app: nginx
+`, out.String()) {
+		return
+	}
+}

cmd/config/internal/commands/tree_test.go

@@ -104,7 +104,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -118,7 +117,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -132,7 +130,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -146,7 +143,6 @@ metadata:
   namespace: default2
   annotations:
     app: nginx2
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -160,7 +156,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx3
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -171,8 +166,7 @@ metadata:
     app: nginx
   annotations:
     app: nginx
-    config.kubernetes.io/package: bar-package
-    config.kubernetes.io/path: f2.yaml
+    config.kubernetes.io/path: bar-package/f2.yaml
   name: bar
 spec:
   replicas: 3
@@ -183,7 +177,6 @@ metadata:
   namespace: default
   annotations:
     app: nginx
-    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   selector:

cmd/config/internal/generateddocs/api/docs.go

@@ -4,288 +4,6 @@
 // Code generated by "mdtogo"; DO NOT EDIT.
 package api
 
-var ConfigFnLong = `# Configuration Functions API Semantics
-
-  Configuration Functions are functions packaged as executables in containers which enable
-  **shift-left practices**.  They configure applications and infrastructure through
-  Kubernetes style Resource Configuration, but run locally pre-commit.
-
-  Configuration functions enable shift-left practices (client-side) through:
-
-  - Pre-commit / delivery validation and linting of configuration
-    - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
-  - Implementation of abstractions as client actuated APIs (e.g. templating)
-    - e.g. Create a client-side *"CRD"* for generating configuration checked into git
-  - Aspect Orient configuration / Injection of cross-cutting configuration
-    - e.g. T-Shirt size containers by annotating Resources with ` + "`" + `small` + "`" + `, ` + "`" + `medium` + "`" + `, ` + "`" + `large` + "`" + `
-      and inject the cpu and memory resources into containers accordingly.
-    - e.g. Inject ` + "`" + `init` + "`" + ` and ` + "`" + `side-car` + "`" + ` containers into Resources based off of Resource
-      Type, annotations, etc.
-
-  Performing these on the client rather than the server enables:
-
-  - Configuration to be reviewed prior to being sent to the API server
-  - Configuration to be validated as part of the CD pipeline
-  - Configuration for Resources to validated holistically rather than individually
-    per-Resource -- e.g. ensure the ` + "`" + `Service.selector` + "`" + ` and ` + "`" + `Deployment.spec.template` + "`" + ` labels
-    match.
-    - MutatingWebHooks are scoped to a single Resource instance at a time.
-  - Low-level tweaks to the output of high-level abstractions -- e.g. add an ` + "`" + `init container` + "`" + `
-    to a client *"CRD"* Resource after it was generated.
-  - Composition and layering of multiple functions together
-    - Compose generation, injection, validation together
-
-  Configuration Functions are implemented as executable programs published in containers which:
-
-  - Accept as input (stdin):
-    - A list of Resource Configuration
-    - A Function Configuration (to configure the function itself)
-  - Emit as output (stdout + exit):
-    - A list of Resource Configuration
-    - An exit code for success / failure
-
-### Function Specification
-
-  - Functions **SHOULD** be published as container images containing a ` + "`" + `CMD` + "`" + ` invoking an executable.
-  - Functions **MUST** accept input on STDIN a ` + "`" + `ResourceList` + "`" + ` containing the Resources and
-    ` + "`" + `functionConfig` + "`" + `.
-  - Functions **MUST** emit output on STDOUT a ` + "`" + `ResourceList` + "`" + ` containing the modified
-    Resources.
-  - Functions **MUST** exit non-0 on failure, and exit 0 on success.
-  - Functions **MAY** emit output on STDERR with error messaging.
-  - Functions performing validation **SHOULD** exit failure and emit error messaging
-    on a validation failure.
-  - Functions generating Resources **SHOULD** retain non-conflicting changes on the
-    generated Resources -- e.g. 1. the function generates a Deployment, but doesn't
-    specify ` + "`" + `cpu` + "`" + `, 2. the user sets the ` + "`" + `cpu` + "`" + ` on the generated Resource, 3. the
-    function should keep the ` + "`" + `cpu` + "`" + ` when regenerating the Resource a second time.
-  - Functions **SHOULD** be usable outside ` + "`" + `kustomize config run` + "`" + ` -- e.g. though pipeline
-    mechanisms such as Tekton.
-
-#### Input Format
-
-  Functions must accept on STDIN:
-
-  ` + "`" + `ResourceList` + "`" + `:
-  - contains ` + "`" + `items` + "`" + ` field, same as ` + "`" + `List.items` + "`" + `
-  - contains ` + "`" + `functionConfig` + "`" + ` field -- a single item with the configuration for the function itself
-
-  Example ` + "`" + `ResourceList` + "`" + ` Input:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    functionConfig:
-      apiVersion: example.com/v1beta1
-      kind: Nginx
-      metadata:
-        name: my-instance
-        annotations:
-          config.kubernetes.io/local-config: "true"
-      spec:
-        replicas: 5
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 3
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Output Format
-
-  Functions must emit on STDOUT:
-
-  ` + "`" + `ResourceList` + "`" + `:
-  - contains ` + "`" + `items` + "`" + ` field, same as ` + "`" + `List.items` + "`" + `
-
-  Example ` + "`" + `ResourceList` + "`" + ` Output:
-
-    apiVersion: config.kubernetes.io/v1alpha1
-    kind: ResourceList
-    items:
-    -  apiVersion: apps/v1
-       kind: Deployment
-       metadata:
-         name: my-instance
-       spec:
-         replicas: 5
-         ...
-    - apiVersion: v1
-      kind: Service
-      metadata:
-        name: my-instance
-      spec:
-        ...
-
-#### Container Environment
-
-  When run by ` + "`" + `kustomize config run` + "`" + `, functions are run in containers with the
-  following environment:
-
-  - Network: ` + "`" + `none` + "`" + `
-  - User: ` + "`" + `nobody` + "`" + `
-  - Security Options: ` + "`" + `no-new-privileges` + "`" + `
-  - Volumes: the volume containing the ` + "`" + `functionConfig` + "`" + ` yaml is mounted under ` + "`" + `/local` + "`" + ` as ` + "`" + `ro` + "`" + `
-
-### Example Function Implementation
-
-  Following is an example for implementing an nginx abstraction using a config
-  function.
-
-#### ` + "`" + `nginx-template.sh` + "`" + `
-
-  ` + "`" + `nginx-template.sh` + "`" + ` is a simple bash script which uses a *heredoc* as a templating solution
-  for generating Resources from the functionConfig input fields.
-
-  The script wraps itself using ` + "`" + `config run wrap -- $0` + "`" + ` which will:
-
-  1. Parse the ` + "`" + `ResourceList.functionConfig` + "`" + ` (provided to the container stdin) into env vars
-  2. Merge the stdout into the original list of Resources
-  3. Defaults filenames for newly generated Resources (if they are not set as annotations)
-     to ` + "`" + `config/NAME_KIND.yaml` + "`" + `
-  4. Format the output
-
-    #!/bin/bash
-    # script must run wrapped by ` + "`" + `kustomize config run wrap` + "`" + `
-    # for parsing input the functionConfig into env vars
-    if [ -z ${WRAPPED} ]; then
-      export WRAPPED=true
-      config run wrap -- $0
-      exit $?
-    fi
-
-    cat <<End-of-message
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: ${NAME}
-    ---
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: ${NAME}
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      replicas: ${REPLICAS}
-      selector:
-        matchLabels:
-          app: nginx
-          instance: ${NAME}
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: ${NAME}
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-    End-of-message
-
-#### ` + "`" + `Dockerfile` + "`" + `
-
-  ` + "`" + `Dockerfile` + "`" + ` installs ` + "`" + `kustomize config` + "`" + ` and copies the script into the container image.
-
-    FROM golang:1.13-stretch
-    RUN go get sigs.k8s.io/kustomize/cmd/config
-    RUN mv /go/bin/config /usr/bin/config
-    COPY nginx-template.sh /usr/bin/nginx-template.sh
-    CMD ["nginx-template.sh]
-
-### Example Function Usage
-
-Following is an example of running the ` + "`" + `kustomize config run` + "`" + ` using the preceding API.
-
-#### ` + "`" + `nginx.yaml` + "`" + ` (Input)
-
-  ` + "`" + `dir/nginx.yaml` + "`" + ` contains a reference to the Function.  The contents of ` + "`" + `nginx.yaml` + "`" + `
-  are passed to the Function through the ` + "`" + `ResourceList.functionConfig` + "`" + ` field.
-
-    apiVersion: example.com/v1beta1
-    kind: Nginx
-    metadata:
-      name: my-instance
-      annotations:
-        config.kubernetes.io/local-config: "true"
-      configFn:
-        container:
-          image: gcr.io/example-functions/nginx-template:v1.0.0
-    spec:
-      replicas: 5
-
-  - ` + "`" + `configFn.container.image` + "`" + `: the image to use for this API
-  - ` + "`" + `annotations[config.kubernetes.io/local-config]` + "`" + `: mark this as not a Resource that should
-    be applied
-
-#### ` + "`" + `kustomize config run dir/` + "`" + ` (Output)
-
-  ` + "`" + `dir/my-instance_deployment.yaml` + "`" + ` contains the Deployment:
-
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      replicas: 5
-      selector:
-        matchLabels:
-          app: nginx
-          instance: my-instance
-      template:
-        metadata:
-          labels:
-            app: nginx
-            instance: my-instance
-        spec:
-          containers:
-          - name: nginx
-            image: nginx:1.7.9
-            ports:
-            - containerPort: 80
-
-  ` + "`" + `dir/my-instance_service.yaml` + "`" + ` contains the Service:
-
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: my-instance
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      ports:
-      - port: 80
-        targetPort: 80
-        name: http
-      selector:
-        app: nginx
-        instance: my-instance`
-
 var ConfigIoLong = `# Configuration IO API Semantics
 
 Resource Configuration may be read / written from / to sources such as directories,
@@ -315,7 +33,7 @@ Example:
 ### ` + "`" + `config.kubernetes.io/index` + "`" + `
 
 Records the index of a Resource in file. In a multi-object YAML file, Resources are separated
-by three dashes (` + "`" + `---` + "`" + `), and the index represents the positon of the Resource starting from zero.
+by three dashes (` + "`" + `---` + "`" + `), and the index represents the position of the Resource starting from zero.
 
 This annotation **SHOULD** be set when reading Resources from files.
 It **SHOULD** be unset when writing Resources to files.
@@ -346,6 +64,355 @@ Example:
 	  annotations:
 	    config.kubernetes.io/local-config: "true"`
 
+var FunctionsImplShort = `Following is an example for implementing an nginx abstraction using a configuration`
+var FunctionsImplLong = `# Running Configuration Functions using kustomize CLI
+
+Configuration functions can be implemented using any toolchain and invoked using any
+container workflow orchestrator including Tekton, Cloud Build, or run directly using ` + "`" + `docker run` + "`" + `.
+
+Run ` + "`" + `config help docs-fn-spec` + "`" + ` to see the Configuration Functions Specification.
+
+` + "`" + `kustomize config run` + "`" + ` is an example orchestrator for invoking Configuration Functions. This
+document describes how to implement and invoke an example function.
+
+function.
+
+### ` + "`" + `nginx-template.sh` + "`" + `
+
+` + "`" + `nginx-template.sh` + "`" + ` is a simple bash script which uses a _heredoc_ as a templating solution
+for generating Resources from the functionConfig input fields.
+
+The script wraps itself using ` + "`" + `config run wrap -- $0` + "`" + ` which will:
+
+1. Parse the ` + "`" + `ResourceList.functionConfig` + "`" + ` (provided to the container stdin) into env vars
+2. Merge the stdout into the original list of Resources
+3. Defaults filenames for newly generated Resources (if they are not set as annotations)
+   to ` + "`" + `config/NAME_KIND.yaml` + "`" + `
+4. Format the output
+
+	#!/bin/bash
+	# script must run wrapped by "kustomize config run wrap"
+	# for parsing input the functionConfig into env vars
+	if [ -z ${WRAPPED} ]; then
+	  export WRAPPED=true
+	  config run wrap -- $0
+	  exit $?
+	fi
+	
+	cat <<End-of-message
+	apiVersion: v1
+	kind: Service
+	metadata:
+	  name: ${NAME}
+	  labels:
+	    app: nginx
+	    instance: ${NAME}
+	spec:
+	  ports:
+	  - port: 80
+	    targetPort: 80
+	    name: http
+	  selector:
+	    app: nginx
+	    instance: ${NAME}
+	---
+	apiVersion: apps/v1
+	kind: Deployment
+	metadata:
+	  name: ${NAME}
+	  labels:
+	    app: nginx
+	    instance: ${NAME}
+	spec:
+	  replicas: ${REPLICAS}
+	  selector:
+	    matchLabels:
+	      app: nginx
+	      instance: ${NAME}
+	  template:
+	    metadata:
+	      labels:
+	        app: nginx
+	        instance: ${NAME}
+	    spec:
+	      containers:
+	      - name: nginx
+	        image: nginx:1.7.9
+	        ports:
+	        - containerPort: 80
+	End-of-message
+
+### Dockerfile
+
+` + "`" + `Dockerfile` + "`" + ` installs ` + "`" + `kustomize config` + "`" + ` and copies the script into the container image.
+
+	FROM golang:1.13-stretch
+	RUN go get sigs.k8s.io/kustomize/cmd/config
+	RUN mv /go/bin/config /usr/bin/config
+	COPY nginx-template.sh /usr/bin/nginx-template.sh
+	CMD ["nginx-template.sh]
+
+## Example Function Usage
+
+Following is an example of running the ` + "`" + `kustomize config run` + "`" + ` using the preceding API.
+
+When run by ` + "`" + `kustomize config run` + "`" + `, functions are run in containers with the
+following environment:
+
+- Network: ` + "`" + `none` + "`" + `
+- User: ` + "`" + `nobody` + "`" + `
+- Security Options: ` + "`" + `no-new-privileges` + "`" + `
+- Volumes: the volume containing the ` + "`" + `functionConfig` + "`" + ` yaml is mounted under ` + "`" + `/local` + "`" + ` as ` + "`" + `ro` + "`" + `
+
+### Input
+
+` + "`" + `dir/nginx.yaml` + "`" + ` contains a reference to the Function. The contents of ` + "`" + `nginx.yaml` + "`" + `
+are passed to the Function through the ` + "`" + `ResourceList.functionConfig` + "`" + ` field.
+
+	apiVersion: example.com/v1beta1
+	kind: Nginx
+	metadata:
+	  name: my-instance
+	  annotations:
+	    config.kubernetes.io/local-config: "true"
+	    config.k8s.io/function: |
+	      container:
+	        image: gcr.io/example-functions/nginx-template:v1.0.0
+	spec:
+	  replicas: 5
+
+- ` + "`" + `annotations[config.k8s.io/function].container.image` + "`" + `: the image to use for this API
+- ` + "`" + `annotations[config.kubernetes.io/local-config]` + "`" + `: mark this as not a Resource that should
+  be applied
+
+### Output
+
+The function is invoked using byrunning ` + "`" + `kustomize config run dir/` + "`" + `.
+
+` + "`" + `dir/my-instance_deployment.yaml` + "`" + ` contains the Deployment:
+
+	apiVersion: apps/v1
+	kind: Deployment
+	metadata:
+	  name: my-instance
+	  labels:
+	    app: nginx
+	    instance: my-instance
+	spec:
+	  replicas: 5
+	  selector:
+	    matchLabels:
+	      app: nginx
+	      instance: my-instance
+	  template:
+	    metadata:
+	      labels:
+	        app: nginx
+	        instance: my-instance
+	    spec:
+	      containers:
+	        - name: nginx
+	          image: nginx:1.7.9
+	          ports:
+	            - containerPort: 80
+
+` + "`" + `dir/my-instance_service.yaml` + "`" + ` contains the Service:
+
+	apiVersion: v1
+	kind: Service
+	metadata:
+	  name: my-instance
+	  labels:
+	    app: nginx
+	    instance: my-instance
+	spec:
+	  ports:
+	    - port: 80
+	      targetPort: 80
+	      name: http
+	  selector:
+	    app: nginx
+	    instance: my-instance`
+
+var FunctionsSpecShort = `_Configuration functions_ enable shift-left practices (client-side) through:`
+var FunctionsSpecLong = `# Configuration Functions Specification
+
+This document specifies a standard for client-side functions that operate on
+Kubernetes declarative configurations. This standard enables creating
+small, interoperable, and language-independent executable programs packaged as
+containers that can be chained together as part of a configuration management pipeline.
+The end result of such a pipeline are fully rendered configurations that can then be
+applied to a control plane (e.g. Using ‘kubectl apply’ for Kubernetes control plane).
+As such, although this document references Kubernetes Resource Model and API conventions,
+it is completely decoupled from Kuberentes API machinery and does not depend on any
+in-cluster components.
+
+This document references terms described in [Kubernetes API Conventions][1].
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD",
+"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be
+interpreted as described in [RFC 2119][2].
+
+
+- Pre-commit / delivery validation and linting of configuration
+  - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
+- Implementation of abstractions as client actuated APIs (e.g. templating)
+  - e.g. Create a client-side _"CRD"_ for generating configuration checked into git
+- Aspect Orient configuration / Injection of cross-cutting configuration
+  - e.g. T-Shirt size containers by annotating Resources with ` + "`" + `small` + "`" + `, ` + "`" + `medium` + "`" + `, ` + "`" + `large` + "`" + `
+    and inject the cpu and memory resources into containers accordingly.
+  - e.g. Inject ` + "`" + `init` + "`" + ` and ` + "`" + `side-car` + "`" + ` containers into Resources based off of Resource
+    Type, annotations, etc.
+
+Performing these on the client rather than the server enables:
+
+- Configuration to be reviewed prior to being sent to the API server
+- Configuration to be validated as part of the CI?CD pipeline
+- Configuration for Resources to validated holistically rather than individually
+  per-Resource
+  - e.g. ensure the ` + "`" + `Service.selector` + "`" + ` and ` + "`" + `Deployment.spec.template` + "`" + ` labels
+    match.
+  - e.g. MutatingWebHooks are scoped to a single Resource instance at a time.
+- Low-level tweaks to the output of high-level abstractions
+  - e.g. add an ` + "`" + `init container` + "`" + ` to a client _"CRD"_ Resource after it was generated.
+- Composition and layering of multiple functions together
+  - Compose generation, injection, validation together
+
+## Spec
+
+### Input Type
+
+A function MUST accept as input a single [Kubernetes List type][3].
+The ` + "`" + `items` + "`" + ` field in the input will contain a sequence of [Object types][3].
+A function MAY not support [Simple types][3] and List types.
+
+An example using ` + "`" + `v1/ConfigMapList` + "`" + ` as input:
+
+	apiVersion: v1
+	kind: ConfigMapList
+	items:
+	  - apiVersion: v1
+	    kind: ConfigMap
+	    metadata:
+	      name: config1
+	    data:
+	      p1: v1
+	      p2: v2
+	  - apiVersion: v1
+	    kind: ConfigMap
+	    metadata:
+	      name: config2
+
+An example using ` + "`" + `v1/List` + "`" + ` as input:
+
+	apiVersion: v1
+	kind: List
+	items:
+	  spec:
+	   - apiVersion: foo-corp.com/v1
+	    kind: FulfillmentCenter
+	    metadata:
+	      name: staging
+	    address: "100 Main St."
+	  - apiVersion: rbac.authorization.k8s.io/v1
+	    kind: ClusterRole
+	    metadata:
+	      name: namespace-reader
+	    rules:
+	      - resources:
+	          - namespaces
+	        apiGroups:
+	          - ""
+	        verbs:
+	          - get
+	          - watch
+	          - list
+
+In addition, a function MUST accept as input a List of kind ` + "`" + `ResourceList` + "`" + ` where the
+` + "`" + `functionConfig` + "`" + ` field, if present, will contain the invocation-specific configuration passed to the function
+by the orchestrator.
+Functions MAY consider this field optional so that they can be triggered in an ad-hoc fashion.
+
+An example using ` + "`" + `config.kubernetes.io/v1beta1/ResourceList` + "`" + ` as input:
+
+	apiVersion: config.kubernetes.io/v1beta1
+	kind: ResourceList
+	functionConfig:
+	  apiVersion: foo-corp.com/v1
+	  kind: FulfillmentCenter
+	  metadata:
+	    name: staging
+	    metadata:
+	      annotations:
+	        config.k8s.io/function: |
+	          container:
+	            image: gcr.io/example/foo:v1.0.0
+	  spec:
+	    address: "100 Main St."
+	items:
+	  - apiVersion: rbac.authorization.k8s.io/v1
+	    kind: ClusterRole
+	    metadata:
+	      name: namespace-reader
+	    rules:
+	      - resources:
+	          - namespaces
+	        apiGroups:
+	          - ""
+	        verbs:
+	          - get
+	          - watch
+	          - list
+
+Here ` + "`" + `FulfillmentCenter` + "`" + ` kind with name ` + "`" + `staging` + "`" + ` is passed as the invocation-specific configuration
+to the function.
+
+### Output Type
+
+A function’s output MUST be the same as the input specification above
+-- i.e. ` + "`" + `ResourceList` + "`" + ` or ` + "`" + `List` + "`" + `.
+This is necessary to enable chaining two or more functions together in a pipeline.
+The serialization format of the output SHOULD match that of its input on each invocation
+-- e.g. if the input was a ` + "`" + `ResourceList` + "`" + `, the output should also be a ` + "`" + `ResourceList` + "`" + `.
+
+### Serialization Format
+
+A function MUST support YAML as a serialization format for the input and output.
+A function MUST use utf8 encoding (as YAML is a superset of JSON, JSON will also be supported
+by any conforming function).
+
+### Operations
+
+A function MAY Create, Update, or Delete any number of items in the ` + "`" + `items` + "`" + ` field and output the
+resultant list.
+
+A function MAY modify annotations with prefix ` + "`" + `config.kubernetes.io` + "`" + `, but must be careful about
+doing so since they’re used for orchestration purposes and will likely impact subsequent functions
+in the pipeline.
+
+A function SHOULD preserve comments when input serialization format is YAML.
+This allows for human authoring of configuration to coexist with changes made by functions.
+
+### Containerization
+
+A function MUST be implemented as a container.
+
+A function container MUST be capable of running as a non-root user if it does not require
+access to host filesystem or makes network calls.
+
+### stdin/stdout/stderr and Exit Codes
+
+A function MUST accept input from stdin and emit output to stdout.
+
+Any error messages MUST be emitted to stderr.
+
+An exit code of zero indicates function execution was successful.
+A non-zero exit code indicates a failure.
+
+[1]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md
+[2]: https://tools.ietf.org/html/rfc2119
+[3]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds`
+
 var Merge2Long = `# Merge (2-way)
 
   2-way merges fields from a source to a destination, overriding the destination fields

cmd/config/internal/generateddocs/commands/docs.go

@@ -4,6 +4,20 @@
 // Code generated by "mdtogo"; DO NOT EDIT.
 package commands
 
+var AnnotateShort = `[Alpha] Set an annotation on Resources.`
+var AnnotateLong = `
+[Alpha]  Set an annotation on Resources.
+
+  DIR:
+    Path to local directory.
+`
+var AnnotateExamples = `
+    kustomize config annotate my-dir/ --kv foo=bar
+
+    kustomize config annotate my-dir/ --kv foo=bar --kv a=b
+
+    kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo`
+
 var CatShort = `[Alpha] Print Resource Config from a local directory.`
 var CatLong = `
 [Alpha]  Print Resource Config from a local directory.
@@ -142,6 +156,25 @@ var GrepExamples = `
     # look for Resources matching a specific container image
     kustomize config grep "spec.template.spec.containers[name=nginx].image=nginx:1\.7\.9" my-dir/ | kustomize config tree`
 
+var ListSettersShort = `[Alpha] List setters for Resources.`
+var ListSettersLong = `
+List setters for Resources.
+
+  DIR
+
+    A directory containing Resource configuration.
+
+  NAME
+
+    Optional.  The name of the setter to display.
+`
+var ListSettersExamples = `
+  Show setters:
+
+    $ config set DIR/
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    name-prefix   ''            PREFIX    string   2`
+
 var MergeShort = `[Alpha] Merge Resource configuration files`
 var MergeLong = `
 [Alpha] Merge Resource configuration files
@@ -201,8 +234,8 @@ order they appear in the file).
 
 #### Config Functions:
 
-  Config functions are specified as Kubernetes types containing a metadata.configFn.container.image
-  field.  This field tells run how to invoke the container.
+  Config functions are specified as Kubernetes types containing a metadata.annotations.[config.kubernetes.io/function]
+  field specifying an image for the container to run.  This image tells run how to invoke the container.
 
   Example config function:
 
@@ -210,17 +243,17 @@ order they appear in the file).
 	apiVersion: fn.example.com/v1beta1
 	kind: ExampleFunctionKind
 	metadata:
-	  configFn:
-	    container:
-	      # function is invoked as a container running this image
-	      image: gcr.io/example/examplefunction:v1.0.1
 	  annotations:
+	    config.kubernetes.io/function: |
+	      container:
+	        # function is invoked as a container running this image
+	        image: gcr.io/example/examplefunction:v1.0.1
 	    config.kubernetes.io/local-config: "true" # tools should ignore this
 	spec:
 	  configField: configValue
 
   In the preceding example, 'kustomize config run example/' would identify the function by
-  the metadata.configFn field.  It would then write all Resources in the directory to
+  the metadata.annotations.[config.kubernetes.io/function] field.  It would then write all Resources in the directory to
   a container stdin (running the gcr.io/example/examplefunction:v1.0.1 image).  It
   would then write the container stdout back to example/, replacing the directory
   file contents.
@@ -286,19 +319,19 @@ var SetExamples = `
   List setters: Show the possible setters
 
     $ config set DIR/
-    NAME          DESCRIPTION    VALUE     TYPE    COUNT   OWNER
+        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
     name-prefix   ''            PREFIX    string   2
 
-  Perform substitution: set a new value, owner and description
+  Perform set: set a new value, owner and description
 
     $ kustomize config set DIR/ name-prefix "test" --description "test environment" --set-by "dev"
-    performed 2 substitutions
+    set 2 values
 
-  Show substitutions: Show the new values
+  List setters: Show the new values
 
-    $ config set dir
-    NAME       DESCRIPTION       VALUE    TYPE    COUNT   SUBSTITUTED   OWNER
-    prefix   'test environment'   test   string   2       true          dev
+    $ config set DIR/
+        NAME      DESCRIPTION         VALUE     TYPE     COUNT     SETBY 
+    name-prefix   'test environment'   test     string   2          dev
 
   New Resource YAML:
 
@@ -313,6 +346,38 @@ var SetExamples = `
         name: test-app2 # {"description":"test environment","type":"string","x-kustomize":{"setBy":"dev","partialFieldSetters":[{"name":"name-prefix","value":"test"}]}}
     ...`
 
+var SinkShort = `[Alpha] Implement a Sink by writing input to a local directory.`
+var SinkLong = `
+[Alpha] Implement a Sink by writing input to a local directory.
+
+    kustomize config sink [DIR]
+
+  DIR:
+    Path to local directory.  If unspecified, sink will write to stdout as if it were a single file.
+
+` + "`" + `sink` + "`" + ` writes its input to a directory
+`
+var SinkExamples = `
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/`
+
+var SourceShort = `[Alpha] Implement a Source by reading a local directory.`
+var SourceLong = `
+[Alpha] Implement a Source by reading a local directory.
+
+    kustomize config source DIR...
+
+  DIR:
+    One or more paths to local directories.  Contents from directories will be concatenated.
+    If no directories are provided, source will read from stdin as if it were a single file.
+
+` + "`" + `source` + "`" + ` emits configuration to act as input to a function
+`
+var SourceExamples = `
+    # emity configuration directory as input source to a function
+    kustomize config source DIR/
+
+    kustomize config source DIR/ | your-function | kustomize config sink DIR/`
+
 var TreeShort = `[Alpha] Display Resource structure from a directory or stdin.`
 var TreeLong = `
 [Alpha] Display Resource structure from a directory or stdin.

cmd/config/main.go

@@ -9,6 +9,7 @@ package main
 import (
 	"os"
 
+	"sigs.k8s.io/kustomize/cmd/config/complete"
 	"sigs.k8s.io/kustomize/cmd/config/configcobra"
 	"sigs.k8s.io/kustomize/kyaml/commandutil"
 )
@@ -16,7 +17,10 @@ import (
 func main() {
 	// enable the config commands
 	os.Setenv(commandutil.EnableAlphaCommmandsEnvName, "true")
-	if err := configcobra.NewConfigCommand("").Execute(); err != nil {
+	cmd := configcobra.NewConfigCommand("")
+	complete.Complete(cmd).Complete("config")
+
+	if err := cmd.Execute(); err != nil {
 		os.Exit(1)
 	}
 }

cmd/kubectl/go.mod

@@ -3,12 +3,21 @@ module sigs.k8s.io/kustomize/cmd/kubectl
 go 1.13
 
 require (
+	github.com/go-errors/errors v1.0.1
 	github.com/spf13/cobra v0.0.5
+	github.com/stretchr/testify v1.4.0
+	k8s.io/api v0.17.0
+	k8s.io/apimachinery v0.17.0
 	k8s.io/cli-runtime v0.17.0
 	k8s.io/client-go v0.17.0
 	k8s.io/component-base v0.17.0 // indirect
 	k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd
-	sigs.k8s.io/kustomize/kyaml v0.0.0
+	sigs.k8s.io/controller-runtime v0.4.0
+	sigs.k8s.io/kustomize/kstatus v0.0.1
+	sigs.k8s.io/kustomize/kyaml v0.0.2
 )
 
-replace sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml
+replace (
+	sigs.k8s.io/kustomize/kstatus v0.0.1 => ../../kstatus
+	sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml
+)

cmd/kubectl/go.sum

@@ -23,6 +23,7 @@ github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -31,15 +32,23 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -51,8 +60,10 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -60,6 +71,8 @@ github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
+github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
@@ -68,39 +81,76 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54=
+github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
+github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
+github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
+github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
+github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
+github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
 github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
+github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 h1:u4bArs140e9+AfE52mFHOXVFnOSBJBRlzTHrOPLOIhE=
+github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -116,20 +166,30 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
+github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -138,12 +198,15 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
+github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -164,6 +227,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -185,13 +250,18 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -202,12 +272,16 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
@@ -215,6 +289,7 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -235,19 +310,35 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -255,18 +346,24 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMx
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -279,7 +376,9 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -288,14 +387,18 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
+golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -303,17 +406,24 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuA
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
 gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
@@ -325,17 +435,25 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
@@ -345,21 +463,30 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48=
 k8s.io/api v0.0.0-20191214185829-ca1d04f8b0d3/go.mod h1:itOjKREfmUTvcjantxOsyYU5mbFsU7qUnyUuRfF5+5M=
 k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
 k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
+k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783 h1:V6ndwCPoao1yZ52agqOKaUAl7DYWVGiXjV7ePA2i610=
+k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY=
+k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4=
 k8s.io/apimachinery v0.0.0-20191214185652-442f8fb2f03a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY=
 k8s.io/apimachinery v0.0.0-20191216025728-0ee8b4573e3a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY=
 k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
 k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg=
 k8s.io/cli-runtime v0.0.0-20191214191754-e6dc6d5c8724/go.mod h1:wzlq80lvjgHW9if6MlE4OIGC86MDKsy5jtl9nxz/IYY=
 k8s.io/cli-runtime v0.17.0 h1:XEuStbJBHCQlEKFyTQmceDKEWOSYHZkcYWKp3SsQ9Hk=
 k8s.io/cli-runtime v0.17.0/go.mod h1:1E5iQpMODZq2lMWLUJELwRu2MLWIzwvMgDBpn3Y81Qo=
+k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk=
 k8s.io/client-go v0.0.0-20191214190045-a32a6f7a3052/go.mod h1:tAaoc/sYuIL0+njJefSAmE28CIcxyaFV4kbIujBlY2s=
 k8s.io/client-go v0.0.0-20191219150334-0b8da7416048/go.mod h1:ZEe8ZASDUAuqVGJ+UN0ka0PfaR+b6a6E1PGsSNZRui8=
 k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
 k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
+k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE=
 k8s.io/code-generator v0.0.0-20191214185510-0b9b3c99f9f2/go.mod h1:BjGKcoq1MRUmcssvHiSxodCco1T6nVIt4YeCT5CMSao=
+k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA=
 k8s.io/component-base v0.0.0-20191214190519-d868452632e2/go.mod h1:wupxkh1T/oUDqyTtcIjiEfpbmIHGm8By/vqpSKC6z8c=
 k8s.io/component-base v0.17.0 h1:BnDFcmBDq+RPpxXjmuYnZXb59XNN9CaFrX8ba9+3xrA=
 k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc=
@@ -367,13 +494,16 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd h1:nZX5+wEqTu/EBIYjrZlFOA63z4+Zcy96lDkCZPU9a9c=
 k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd/go.mod h1:9ehGcuUGjXVZh0qbYSB0vvofQw2JQe6c6cO0k4wu/Oo=
 k8s.io/metrics v0.0.0-20191214191643-6b1944c9f765/go.mod h1:5V7rewilItwK0cz4nomU0b3XCcees2Ka5EBYWS1HBeM=
+k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
@@ -381,9 +511,16 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+sigs.k8s.io/controller-runtime v0.4.0 h1:wATM6/m+3w8lj8FXNaO6Fs/rq/vqoOjO1Q116Z9NPsg=
+sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns=
 sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
+sigs.k8s.io/kustomize/kyaml v0.0.2 h1:Rl/wMrnpZzZjsVeFIIOAb92Kz/UfLrTUEXjiHW6oS0o=
+sigs.k8s.io/kustomize/kyaml v0.0.2/go.mod h1:rywm/rcR5LmCBghz9956tE45OdUPChFoXVVs+WmhMTI=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
+sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA=
+sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM=
+sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI=

cmd/kubectl/kubectlcobra/applier.go

@@ -0,0 +1,205 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kubectlcobra
+
+import (
+	"context"
+	"time"
+
+	"github.com/go-errors/errors"
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/cli-runtime/pkg/resource"
+	"k8s.io/kubectl/pkg/cmd/apply"
+	"k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kustomize/kstatus/wait"
+)
+
+// newApplier returns a new Applier. It will set up the applyOptions and
+// statusOptions which are responsible for capturing any command line flags.
+// It currently requires IOStreams, but this is a legacy from when
+// the ApplyOptions were responsible for printing progress. This is now
+// handled by a separate printer with the KubectlPrinterAdapter bridging
+// between the two.
+func newApplier(factory util.Factory, ioStreams genericclioptions.IOStreams) *Applier {
+	return &Applier{
+		applyOptions:  apply.NewApplyOptions(ioStreams),
+		statusOptions: NewStatusOptions(),
+		factory:       factory,
+		ioStreams:     ioStreams,
+	}
+}
+
+// resolver defines the interface the applier needs to observe status for resources.
+type resolver interface {
+	WaitForStatusOfObjects(ctx context.Context, objects []wait.KubernetesObject) <-chan wait.Event
+}
+
+// Applier performs the step of applying a set of resources into a cluster,
+// conditionally waits for all of them to be fully reconciled and finally
+// performs prune to clean up any resources that has been deleted.
+type Applier struct {
+	factory   util.Factory
+	ioStreams genericclioptions.IOStreams
+
+	applyOptions  *apply.ApplyOptions
+	statusOptions *StatusOptions
+	resolver      resolver
+}
+
+// Initialize sets up the Applier for actually doing an apply against
+// a cluster. This involves validating command line inputs and configuring
+// clients for communicating with the cluster.
+func (a *Applier) Initialize(cmd *cobra.Command) error {
+	a.applyOptions.PreProcessorFn = PrependGroupingObject(a.applyOptions)
+	err := a.applyOptions.Complete(a.factory, cmd)
+	if err != nil {
+		return errors.WrapPrefix(err, "error setting up ApplyOptions", 1)
+	}
+	// Default PostProcessor is configured in "Complete" function,
+	// so the prune must happen after "Complete".
+	a.applyOptions.PostProcessorFn = prune(a.factory, a.applyOptions)
+
+	resolver, err := a.newResolver(a.statusOptions.period)
+	if err != nil {
+		return errors.WrapPrefix(err, "error creating resolver", 1)
+	}
+	a.resolver = resolver
+	return nil
+}
+
+// SetFlags configures the command line flags needed for apply and
+// status. This is a temporary solution as we should separate the configuration
+// of cobra flags from the Applier.
+func (a *Applier) SetFlags(cmd *cobra.Command) {
+	a.applyOptions.DeleteFlags.AddFlags(cmd)
+	a.applyOptions.RecordFlags.AddFlags(cmd)
+	a.applyOptions.PrintFlags.AddFlags(cmd)
+	a.statusOptions.AddFlags(cmd)
+	a.applyOptions.Overwrite = true
+}
+
+// newResolver sets up a new Resolver for computing status. The configuration
+// needed for the resolver is taken from the Factory.
+func (a *Applier) newResolver(pollInterval time.Duration) (*wait.Resolver, error) {
+	config, err := a.factory.ToRESTConfig()
+	if err != nil {
+		return nil, errors.WrapPrefix(err, "error getting RESTConfig", 1)
+	}
+
+	mapper, err := a.factory.ToRESTMapper()
+	if err != nil {
+		return nil, errors.WrapPrefix(err, "error getting RESTMapper", 1)
+	}
+
+	c, err := client.New(config, client.Options{Scheme: scheme.Scheme, Mapper: mapper})
+	if err != nil {
+		return nil, errors.WrapPrefix(err, "error creating client", 1)
+	}
+
+	return wait.NewResolver(c, mapper, pollInterval), nil
+}
+
+// Run performs the Apply step. This happens asynchronously with updates
+// on progress and any errors are reported back on the event channel.
+// Cancelling the operation or setting timeout on how long to wait
+// for it complete can be done with the passed in context.
+// Note: There sn't currently any way to interrupt the operation
+// before all the given resources have been applied to the cluster. Any
+// cancellation or timeout will only affect how long we wait for the
+// resources to become current.
+func (a *Applier) Run(ctx context.Context) <-chan Event {
+	ch := make(chan Event)
+
+	go func() {
+		defer close(ch)
+		adapter := &KubectlPrinterAdapter{
+			ch: ch,
+		}
+		// The adapter is used to intercept what is meant to be printing
+		// in the ApplyOptions, and instead turn those into events.
+		a.applyOptions.ToPrinter = adapter.toPrinterFunc()
+		// This provides us with a slice of all the objects that will be
+		// applied to the cluster.
+		infos, _ := a.applyOptions.GetObjects()
+		err := a.applyOptions.Run()
+		if err != nil {
+			// If we see an error here we just report it on the channel and then
+			// give up. Eventually we might be able to determine which errors
+			// are fatal and which might allow us to continue.
+			ch <- Event{
+				EventType: ErrorEventType,
+				ErrorEvent: ErrorEvent{
+					Err: errors.WrapPrefix(err, "error applying resources", 1),
+				},
+			}
+			return
+		}
+
+		if a.statusOptions.wait {
+			statusChannel := a.resolver.WaitForStatusOfObjects(ctx, infosToObjects(infos))
+			// As long as the statusChannel remains open, we take every statusEvent,
+			// wrap it in an Event and send it on the channel.
+			for statusEvent := range statusChannel {
+				ch <- Event{
+					EventType:   StatusEventType,
+					StatusEvent: statusEvent,
+				}
+			}
+		}
+
+	}()
+	return ch
+}
+
+func infosToObjects(infos []*resource.Info) []wait.KubernetesObject {
+	var objects []wait.KubernetesObject
+	for _, info := range infos {
+		u := info.Object.(*unstructured.Unstructured)
+		objects = append(objects, u)
+	}
+	return objects
+}
+
+// EventType determines the type of events that are available.
+type EventType string
+
+const (
+	ErrorEventType  EventType = "error"
+	ApplyEventType  EventType = "apply"
+	StatusEventType EventType = "status"
+)
+
+// Event is the type of the objects that will be returned through
+// the channel that is returned from a call to Run. It contains
+// information about progress and errors encountered during
+// the process of doing apply, waiting for status and doing a prune.
+type Event struct {
+	// EventType is the type of event.
+	EventType EventType
+
+	// ErrorEvent contains information about any errors encountered.
+	ErrorEvent ErrorEvent
+
+	// ApplyEvent contains information about progress pertaining to
+	// applying a resource to the cluster.
+	ApplyEvent ApplyEvent
+
+	// StatusEvents contains information about the status of one of
+	// the applied resources.
+	StatusEvent wait.Event
+}
+
+type ErrorEvent struct {
+	Err error
+}
+
+type ApplyEvent struct {
+	Operation string
+	Object    runtime.Object
+}

cmd/kubectl/kubectlcobra/applier_test.go

@@ -0,0 +1,48 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kubectlcobra
+
+import (
+	"context"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	cmdtesting "k8s.io/kubectl/pkg/cmd/testing"
+	"testing"
+)
+
+// The applier is currently hard to test, as the dependencies on the ApplyOptions and
+// the resolver are hard to stub out. As we work to better separate the different
+// responsibilities of the apply functionality, we should also make it easier to test.
+// This provides some basic tests for now.
+
+func TestApplierWithUnknownFile(t *testing.T) {
+	tf := cmdtesting.NewTestFactory()
+	defer tf.Cleanup()
+	iostreams, _, _, _ := genericclioptions.NewTestIOStreams()
+	cmd := NewCmdApply("base", tf, iostreams)
+
+	applier := newApplier(tf, iostreams)
+	filenames := []string{"file.yaml"}
+	applier.applyOptions.DeleteFlags.FileNameFlags.Filenames = &filenames
+
+	err := applier.Initialize(cmd)
+	assert.NoError(t, err)
+
+	ch := applier.Run(context.TODO())
+
+	var events []Event
+	for msg := range ch {
+		events = append(events, msg)
+	}
+
+	if !assert.Equal(t, 1, len(events)) {
+		return
+	}
+
+	event := events[0]
+	if !assert.Equal(t, ErrorEventType, event.EventType) {
+		return
+	}
+	assert.Contains(t, event.ErrorEvent.Err.Error(), "does not exist")
+}

cmd/kubectl/kubectlcobra/basic_printer.go

@@ -0,0 +1,62 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kubectlcobra
+
+import (
+	"fmt"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"sigs.k8s.io/kustomize/kstatus/wait"
+)
+
+// BasicPrinter is a simple implementation that just prints the events
+// from the channel in the default format for kubectl.
+// We need to support different printers for different output formats.
+type BasicPrinter struct {
+	ioStreams genericclioptions.IOStreams
+}
+
+// Print outputs the events from the provided channel in a simple
+// format on StdOut. As we support other printer implementations
+// this should probably be an interface.
+// This function will block until the channel is closed.
+func (b *BasicPrinter) Print(ch <-chan Event) {
+	for event := range ch {
+		switch event.EventType {
+		case ErrorEventType:
+			cmdutil.CheckErr(event.ErrorEvent.Err)
+		case ApplyEventType:
+			obj := event.ApplyEvent.Object
+			gvk := obj.GetObjectKind().GroupVersionKind()
+			name := "<unknown>"
+			if acc, err := meta.Accessor(obj); err == nil {
+				if n := acc.GetName(); len(n) > 0 {
+					name = n
+				}
+			}
+			fmt.Fprintf(b.ioStreams.Out, "%s %s\n", resourceIdToString(gvk.GroupKind(), name), event.ApplyEvent.Operation)
+		case StatusEventType:
+			statusEvent := event.StatusEvent
+			switch statusEvent.Type {
+			case wait.ResourceUpdate:
+				id := statusEvent.EventResource.ResourceIdentifier
+				gk := id.GroupKind
+				fmt.Fprintf(b.ioStreams.Out, "%s is %s: %s\n", resourceIdToString(gk, id.Name), statusEvent.EventResource.Status.String(), statusEvent.EventResource.Message)
+			case wait.Completed:
+				fmt.Fprint(b.ioStreams.Out, "all resources has reached the Current status\n")
+			case wait.Aborted:
+				fmt.Fprintf(b.ioStreams.Out, "resources failed to the reached Current status\n")
+			}
+		}
+	}
+}
+
+// resourceIdToString returns the string representation of a GroupKind and a resource name.
+func resourceIdToString(gk schema.GroupKind, name string) string {
+	return fmt.Sprintf("%s/%s", strings.ToLower(gk.String()), name)
+}

cmd/kubectl/kubectlcobra/commands.go

@@ -5,7 +5,9 @@
 package kubectlcobra
 
 import (
+	"context"
 	"flag"
+	"fmt"
 	"os"
 	"strings"
 
@@ -17,9 +19,6 @@ import (
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
 	"k8s.io/kubectl/pkg/util/i18n"
 	"sigs.k8s.io/kustomize/kyaml/commandutil"
-
-	// initialize auth
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
 )
 
 // GetCommand returns a command from kubectl to install
@@ -81,7 +80,10 @@ func updateHelp(names []string, c *cobra.Command) {
 
 // NewCmdApply creates the `apply` command
 func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command {
-	o := apply.NewApplyOptions(ioStreams)
+	applier := newApplier(f, ioStreams)
+	printer := &BasicPrinter{
+		ioStreams: ioStreams,
+	}
 
 	cmd := &cobra.Command{
 		Use:                   "apply (-f FILENAME | -k DIRECTORY)",
@@ -93,25 +95,67 @@ func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IO
 		Run: func(cmd *cobra.Command, args []string) {
 			if len(args) > 0 {
 				// check is kustomize, if so update
-				o.DeleteFlags.FileNameFlags.Kustomize = &args[0]
+				applier.applyOptions.DeleteFlags.FileNameFlags.Kustomize = &args[0]
 			}
 
-			cmdutil.CheckErr(o.Complete(f, cmd))
-			cmdutil.CheckErr(o.Run())
+			cmdutil.CheckErr(applier.Initialize(cmd))
+
+			// Create a context with the provided timout from the cobra parameter.
+			ctx, cancel := context.WithTimeout(context.Background(), applier.statusOptions.timeout)
+			defer cancel()
+			// Run the applier. It will return a channel where we can receive updates
+			// to keep track of progress and any issues.
+			ch := applier.Run(ctx)
+
+			// The printer will print updates from the channel. It will block
+			// until the channel is closed.
+			printer.Print(ch)
 		},
 	}
 
-	// bind flag structs
-	o.DeleteFlags.AddFlags(cmd)
-	o.RecordFlags.AddFlags(cmd)
-	o.PrintFlags.AddFlags(cmd)
-
-	o.Overwrite = true
+	applier.SetFlags(cmd)
 
 	cmdutil.AddValidateFlags(cmd)
-	cmd.Flags().BoolVar(&o.ServerDryRun, "server-dry-run", o.ServerDryRun, "If true, request will be sent to server with dry-run flag, which means the modifications won't be persisted. This is an alpha feature and flag.")
+	cmd.Flags().BoolVar(&applier.applyOptions.ServerDryRun, "server-dry-run", applier.applyOptions.ServerDryRun, "If true, request will be sent to server with dry-run flag, which means the modifications won't be persisted. This is an alpha feature and flag.")
 	cmd.Flags().Bool("dry-run", false, "If true, only print the object that would be sent, without sending it. Warning: --dry-run cannot accurately output the result of merging the local manifest and the server-side data. Use --server-dry-run to get the merged result instead.")
 	cmdutil.AddServerSideApplyFlags(cmd)
 
 	return cmd
 }
+
+// PrependGroupingObject orders the objects to apply so the "grouping"
+// object stores the inventory, and it is first to be applied.
+func PrependGroupingObject(o *apply.ApplyOptions) func() error {
+	return func() error {
+		if o == nil {
+			return fmt.Errorf("ApplyOptions are nil")
+		}
+		infos, err := o.GetObjects()
+		if err != nil {
+			return err
+		}
+		_, exists := findGroupingObject(infos)
+		if exists {
+			if err := addInventoryToGroupingObj(infos); err != nil {
+				return err
+			}
+			if !sortGroupingObject(infos) {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+// Prune deletes previously applied objects that have been
+// omitted in the current apply. The previously applied objects
+// are reached through ConfigMap grouping objects.
+func prune(f util.Factory, o *apply.ApplyOptions) func() error {
+	return func() error {
+		po, err := NewPruneOptions(f, o)
+		if err != nil {
+			return err
+		}
+		return po.Prune()
+	}
+}

cmd/kubectl/kubectlcobra/commands_test.go

@@ -0,0 +1,58 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"testing"
+
+	"k8s.io/cli-runtime/pkg/resource"
+	"k8s.io/kubectl/pkg/cmd/apply"
+)
+
+func TestPrependGroupingObject(t *testing.T) {
+	tests := []struct {
+		infos []*resource.Info
+	}{
+		{
+			infos: []*resource.Info{copyGroupingInfo()},
+		},
+		{
+			infos: []*resource.Info{pod1Info, pod3Info, copyGroupingInfo()},
+		},
+		{
+			infos: []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
+		},
+	}
+
+	for _, test := range tests {
+		applyOptions := createApplyOptions(test.infos)
+		f := PrependGroupingObject(applyOptions)
+		err := f()
+		if err != nil {
+			t.Errorf("Error running pre-processor callback: %s", err)
+		}
+		infos, _ := applyOptions.GetObjects()
+		if len(test.infos) != len(infos) {
+			t.Fatalf("Wrong number of objects after prepending grouping object")
+		}
+		groupingInfo := infos[0]
+		if !isGroupingObject(groupingInfo.Object) {
+			t.Fatalf("First object is not the grouping object")
+		}
+		inventory, _ := retrieveInventoryFromGroupingObj(infos)
+		if len(inventory) != (len(infos) - 1) {
+			t.Errorf("Wrong number of inventory items stored in grouping object")
+		}
+	}
+
+}
+
+// createApplyOptions is a helper function to assemble the ApplyOptions
+// with the passed objects (infos).
+func createApplyOptions(infos []*resource.Info) *apply.ApplyOptions {
+	applyOptions := &apply.ApplyOptions{}
+	applyOptions.SetObjects(infos)
+	return applyOptions
+}

cmd/kubectl/kubectlcobra/grouping.go

@@ -0,0 +1,268 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strconv"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/cli-runtime/pkg/resource"
+)
+
+const (
+	GroupingLabel = "kustomize.config.k8s.io/inventory-id"
+	GroupingHash  = "kustomize.config.k8s.io/inventory-hash"
+)
+
+// retrieveGroupingLabel returns the string value of the GroupingLabel
+// for the passed object. Returns error if the passed object is nil or
+// is not a grouping object.
+func retrieveGroupingLabel(obj runtime.Object) (string, error) {
+	var groupingLabel string
+	if obj == nil {
+		return "", fmt.Errorf("Grouping object is nil.\n")
+	}
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return "", err
+	}
+	labels := accessor.GetLabels()
+	groupingLabel, exists := labels[GroupingLabel]
+	if !exists {
+		return "", fmt.Errorf("Grouping label does not exist for grouping object: %s\n", GroupingLabel)
+	}
+	return strings.TrimSpace(groupingLabel), nil
+}
+
+// isGroupingObject returns true if the passed object has the
+// grouping label.
+// TODO(seans3): Check type is ConfigMap.
+func isGroupingObject(obj runtime.Object) bool {
+	if obj == nil {
+		return false
+	}
+	groupingLabel, err := retrieveGroupingLabel(obj)
+	if err == nil && len(groupingLabel) > 0 {
+		return true
+	}
+	return false
+}
+
+// findGroupingObject returns the "Grouping" object (ConfigMap with
+// grouping label) if it exists, and a boolean describing if it was found.
+func findGroupingObject(infos []*resource.Info) (*resource.Info, bool) {
+	for _, info := range infos {
+		if info != nil && isGroupingObject(info.Object) {
+			return info, true
+		}
+	}
+	return nil, false
+}
+
+// sortGroupingObject reorders the infos slice to place the grouping
+// object in the first position. Returns true if grouping object found,
+// false otherwise.
+func sortGroupingObject(infos []*resource.Info) bool {
+	for i, info := range infos {
+		if info != nil && isGroupingObject(info.Object) {
+			// If the grouping object is not already in the first position,
+			// swap the grouping object with the first object.
+			if i > 0 {
+				infos[0], infos[i] = infos[i], infos[0]
+			}
+			return true
+		}
+	}
+	return false
+}
+
+// Adds the inventory of all objects (passed as infos) to the
+// grouping object. Returns an error if a grouping object does not
+// exist, or we are unable to successfully add the inventory to
+// the grouping object; nil otherwise. Each object is in
+// unstructured.Unstructured format.
+func addInventoryToGroupingObj(infos []*resource.Info) error {
+
+	// Iterate through the objects (infos), creating an Inventory struct
+	// as metadata for each object, or if it's the grouping object, store it.
+	var groupingInfo *resource.Info
+	var groupingObj *unstructured.Unstructured
+	inventoryMap := map[string]string{}
+	for _, info := range infos {
+		obj := info.Object
+		if isGroupingObject(obj) {
+			// If we have more than one grouping object--error.
+			if groupingObj != nil {
+				return fmt.Errorf("Error--applying more than one grouping object.")
+			}
+			var ok bool
+			groupingObj, ok = obj.(*unstructured.Unstructured)
+			if !ok {
+				return fmt.Errorf("Grouping object is not an Unstructured: %#v", groupingObj)
+			}
+			groupingInfo = info
+		} else {
+			if obj == nil {
+				return fmt.Errorf("Creating inventory; object is nil")
+			}
+			gk := obj.GetObjectKind().GroupVersionKind().GroupKind()
+			inventory, err := createInventory(info.Namespace, info.Name, gk)
+			if err != nil {
+				return err
+			}
+			inventoryMap[inventory.String()] = ""
+		}
+	}
+
+	// If we've found the grouping object, store the object metadata inventory
+	// in the grouping config map.
+	if groupingObj == nil {
+		return fmt.Errorf("Grouping object not found")
+	}
+
+	if len(inventoryMap) > 0 {
+		// Adds the inventory map to the ConfigMap "data" section.
+		err := unstructured.SetNestedStringMap(groupingObj.UnstructuredContent(),
+			inventoryMap, "data")
+		if err != nil {
+			return err
+		}
+		// Adds the hash of the inventory strings as an annotation to the
+		// grouping object. Inventory strings must be sorted to make hash
+		// deterministic.
+		inventoryList := mapKeysToSlice(inventoryMap)
+		sort.Strings(inventoryList)
+		invHash, err := calcInventoryHash(inventoryList)
+		if err != nil {
+			return err
+		}
+		// Add the hash as a suffix to the grouping object's name.
+		invHashStr := strconv.FormatUint(uint64(invHash), 16)
+		if err := addSuffixToName(groupingInfo, invHashStr); err != nil {
+			return err
+		}
+		annotations := groupingObj.GetAnnotations()
+		if annotations == nil {
+			annotations = map[string]string{}
+		}
+		annotations[GroupingHash] = invHashStr
+		groupingObj.SetAnnotations(annotations)
+	}
+	return nil
+}
+
+// retrieveInventoryFromGroupingObj returns a slice of pointers to the
+// inventory metadata. This function finds the grouping object, then
+// parses the stored resource metadata into Inventory structs. Returns
+// an error if there is a problem parsing the data into Inventory
+// structs, or if the grouping object is not in Unstructured format; nil
+// otherwise. If a grouping object does not exist, or it does not have a
+// "data" map, then returns an empty slice and no error.
+func retrieveInventoryFromGroupingObj(infos []*resource.Info) ([]*Inventory, error) {
+	inventory := []*Inventory{}
+	groupingInfo, exists := findGroupingObject(infos)
+	if exists {
+		groupingObj, ok := groupingInfo.Object.(*unstructured.Unstructured)
+		if !ok {
+			err := fmt.Errorf("Grouping object is not an Unstructured: %#v", groupingObj)
+			return inventory, err
+		}
+		invMap, exists, err := unstructured.NestedStringMap(groupingObj.Object, "data")
+		if err != nil {
+			err := fmt.Errorf("Error retrieving inventory from grouping object.")
+			return inventory, err
+		}
+		if exists {
+			for invStr := range invMap {
+				inv, err := parseInventory(invStr)
+				if err != nil {
+					return inventory, err
+				}
+				inventory = append(inventory, inv)
+			}
+		}
+	}
+	return inventory, nil
+}
+
+// calcInventoryHash returns an unsigned int32 representing the hash
+// of the inventory strings. If there is an error writing bytes to
+// the hash, then the error is returned; nil is returned otherwise.
+// Used to quickly identify the set of resources in the grouping object.
+func calcInventoryHash(inv []string) (uint32, error) {
+	h := fnv.New32a()
+	for _, is := range inv {
+		_, err := h.Write([]byte(is))
+		if err != nil {
+			return uint32(0), err
+		}
+	}
+	return h.Sum32(), nil
+}
+
+// retrieveInventoryHash takes a grouping object (encapsulated by
+// a resource.Info), and returns the string representing the hash
+// of the grouping inventory; returns empty string if the grouping
+// object is not in Unstructured format, or if the hash annotation
+// does not exist.
+func retrieveInventoryHash(groupingInfo *resource.Info) string {
+	var invHash = ""
+	groupingObj, ok := groupingInfo.Object.(*unstructured.Unstructured)
+	if ok {
+		annotations := groupingObj.GetAnnotations()
+		if annotations != nil {
+			invHash = annotations[GroupingHash]
+		}
+	}
+	return invHash
+}
+
+// mapKeysToSlice returns the map keys as a slice of strings.
+func mapKeysToSlice(m map[string]string) []string {
+	s := make([]string, len(m))
+	i := 0
+	for k := range m {
+		s[i] = k
+		i++
+	}
+	return s
+}
+
+// addSuffixToName adds the passed suffix (usually a hash) as a suffix
+// to the name of the passed object stored in the Info struct. Returns
+// an error if the object is not "*unstructured.Unstructured" or if the
+// name stored in the object differs from the name in the Info struct.
+func addSuffixToName(info *resource.Info, suffix string) error {
+
+	if info == nil {
+		return fmt.Errorf("Nil resource.Info")
+	}
+	suffix = strings.TrimSpace(suffix)
+	if len(suffix) == 0 {
+		return fmt.Errorf("Passed empty suffix")
+	}
+
+	accessor, _ := meta.Accessor(info.Object)
+	name := accessor.GetName()
+	if name != info.Name {
+		return fmt.Errorf("Grouping object (%s) and resource.Info (%s) have different names\n", name, info.Name)
+	}
+	// Error if name alread has suffix.
+	suffix = "-" + suffix
+	if strings.HasSuffix(name, suffix) {
+		return fmt.Errorf("Name already has suffix: %s\n", name)
+	}
+	name += suffix
+	accessor.SetName(name)
+	info.Name = name
+
+	return nil
+}

cmd/kubectl/kubectlcobra/grouping_test.go

@@ -0,0 +1,600 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"fmt"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/cli-runtime/pkg/resource"
+)
+
+var testNamespace = "test-grouping-namespace"
+var groupingObjName = "test-grouping-obj"
+var pod1Name = "pod-1"
+var pod2Name = "pod-2"
+var pod3Name = "pod-3"
+
+var testGroupingLabel = "test-app-label"
+
+var groupingObj = unstructured.Unstructured{
+	Object: map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "ConfigMap",
+		"metadata": map[string]interface{}{
+			"name":      groupingObjName,
+			"namespace": testNamespace,
+			"labels": map[string]interface{}{
+				GroupingLabel: testGroupingLabel,
+			},
+		},
+	},
+}
+
+var pod1 = unstructured.Unstructured{
+	Object: map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "Pod",
+		"metadata": map[string]interface{}{
+			"name":      pod1Name,
+			"namespace": testNamespace,
+		},
+	},
+}
+
+var pod1Info = &resource.Info{
+	Namespace: testNamespace,
+	Name:      pod1Name,
+	Object:    &pod1,
+}
+
+var pod2 = unstructured.Unstructured{
+	Object: map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "Pod",
+		"metadata": map[string]interface{}{
+			"name":      pod2Name,
+			"namespace": testNamespace,
+		},
+	},
+}
+
+var pod2Info = &resource.Info{
+	Namespace: testNamespace,
+	Name:      pod2Name,
+	Object:    &pod2,
+}
+
+var pod3 = unstructured.Unstructured{
+	Object: map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "Pod",
+		"metadata": map[string]interface{}{
+			"name":      pod3Name,
+			"namespace": testNamespace,
+		},
+	},
+}
+
+var pod3Info = &resource.Info{
+	Namespace: testNamespace,
+	Name:      pod3Name,
+	Object:    &pod3,
+}
+
+var nonUnstructuredGroupingObj = &corev1.ConfigMap{
+	ObjectMeta: metav1.ObjectMeta{
+		Namespace: testNamespace,
+		Name:      groupingObjName,
+		Labels: map[string]string{
+			GroupingLabel: "true",
+		},
+	},
+}
+
+var nonUnstructuredGroupingInfo = &resource.Info{
+	Namespace: testNamespace,
+	Name:      groupingObjName,
+	Object:    nonUnstructuredGroupingObj,
+}
+
+var nilInfo = &resource.Info{
+	Namespace: testNamespace,
+	Name:      groupingObjName,
+	Object:    nil,
+}
+
+var groupingObjLabelWithSpace = unstructured.Unstructured{
+	Object: map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "ConfigMap",
+		"metadata": map[string]interface{}{
+			"name":      groupingObjName,
+			"namespace": testNamespace,
+			"labels": map[string]interface{}{
+				GroupingLabel: "\tgrouping-label ",
+			},
+		},
+	},
+}
+
+func TestRetrieveGroupingLabel(t *testing.T) {
+	tests := []struct {
+		obj           runtime.Object
+		groupingLabel string
+		isError       bool
+	}{
+		// Nil grouping object throws error.
+		{
+			obj:           nil,
+			groupingLabel: "",
+			isError:       true,
+		},
+		// Pod is not a grouping object.
+		{
+			obj:           &pod2,
+			groupingLabel: "",
+			isError:       true,
+		},
+		// Retrieves label without preceding/trailing whitespace.
+		{
+			obj:           &groupingObjLabelWithSpace,
+			groupingLabel: "grouping-label",
+			isError:       false,
+		},
+		{
+			obj:           &groupingObj,
+			groupingLabel: testGroupingLabel,
+			isError:       false,
+		},
+	}
+
+	for _, test := range tests {
+		actual, err := retrieveGroupingLabel(test.obj)
+		if test.isError && err == nil {
+			t.Errorf("Did not receive expected error.\n")
+		}
+		if !test.isError {
+			if err != nil {
+				t.Fatalf("Received unexpected error: %s\n", err)
+			}
+			if test.groupingLabel != actual {
+				t.Errorf("Expected grouping label (%s), got (%s)\n", test.groupingLabel, actual)
+			}
+		}
+	}
+}
+
+func TestIsGroupingObject(t *testing.T) {
+	tests := []struct {
+		obj        runtime.Object
+		isGrouping bool
+	}{
+		{
+			obj:        nil,
+			isGrouping: false,
+		},
+		{
+			obj:        &groupingObj,
+			isGrouping: true,
+		},
+		{
+			obj:        &pod2,
+			isGrouping: false,
+		},
+	}
+
+	for _, test := range tests {
+		grouping := isGroupingObject(test.obj)
+		if test.isGrouping && !grouping {
+			t.Errorf("Grouping object not identified: %#v", test.obj)
+		}
+		if !test.isGrouping && grouping {
+			t.Errorf("Non-grouping object identifed as grouping obj: %#v", test.obj)
+		}
+	}
+}
+
+func TestFindGroupingObject(t *testing.T) {
+	tests := []struct {
+		infos  []*resource.Info
+		exists bool
+		name   string
+	}{
+		{
+			infos:  []*resource.Info{},
+			exists: false,
+			name:   "",
+		},
+		{
+			infos:  []*resource.Info{nil},
+			exists: false,
+			name:   "",
+		},
+		{
+			infos:  []*resource.Info{copyGroupingInfo()},
+			exists: true,
+			name:   groupingObjName,
+		},
+		{
+			infos:  []*resource.Info{pod1Info},
+			exists: false,
+			name:   "",
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info, pod3Info},
+			exists: false,
+			name:   "",
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
+			exists: true,
+			name:   groupingObjName,
+		},
+	}
+
+	for _, test := range tests {
+		groupingObj, found := findGroupingObject(test.infos)
+		if test.exists && !found {
+			t.Errorf("Should have found grouping object")
+		}
+		if !test.exists && found {
+			t.Errorf("Grouping object found, but it does not exist: %#v", groupingObj)
+		}
+		if test.exists && found && test.name != groupingObj.Name {
+			t.Errorf("Grouping object name does not match: %s/%s", test.name, groupingObj.Name)
+		}
+	}
+}
+
+func TestSortGroupingObject(t *testing.T) {
+	tests := []struct {
+		infos  []*resource.Info
+		sorted bool
+	}{
+		{
+			infos:  []*resource.Info{},
+			sorted: false,
+		},
+		{
+			infos:  []*resource.Info{copyGroupingInfo()},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{pod1Info},
+			sorted: false,
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info},
+			sorted: false,
+		},
+		{
+			infos:  []*resource.Info{copyGroupingInfo(), pod1Info},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{pod1Info, copyGroupingInfo()},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{pod1Info, pod2Info, pod3Info, copyGroupingInfo()},
+			sorted: true,
+		},
+		{
+			infos:  []*resource.Info{copyGroupingInfo(), pod1Info, pod2Info, pod3Info},
+			sorted: true,
+		},
+	}
+
+	for _, test := range tests {
+		wasSorted := sortGroupingObject(test.infos)
+		if wasSorted && !test.sorted {
+			t.Errorf("Grouping object was sorted, but it shouldn't have been")
+		}
+		if !wasSorted && test.sorted {
+			t.Errorf("Grouping object was NOT sorted, but it should have been")
+		}
+		if wasSorted {
+			first := test.infos[0]
+			if !isGroupingObject(first.Object) {
+				t.Errorf("Grouping object was not sorted into first position")
+			}
+		}
+	}
+}
+
+func TestAddRetrieveInventoryToFromGroupingObject(t *testing.T) {
+	tests := []struct {
+		infos    []*resource.Info
+		expected []*Inventory
+		isError  bool
+	}{
+		// No grouping object is an error.
+		{
+			infos:   []*resource.Info{},
+			isError: true,
+		},
+		// No grouping object is an error.
+		{
+			infos:   []*resource.Info{pod1Info, pod2Info},
+			isError: true,
+		},
+		// Grouping object without other objects is OK.
+		{
+			infos:   []*resource.Info{copyGroupingInfo(), nilInfo},
+			isError: true,
+		},
+		{
+			infos:   []*resource.Info{nonUnstructuredGroupingInfo},
+			isError: true,
+		},
+		{
+			infos:    []*resource.Info{copyGroupingInfo()},
+			expected: []*Inventory{},
+			isError:  false,
+		},
+		// More than one grouping object is an error.
+		{
+			infos:    []*resource.Info{copyGroupingInfo(), copyGroupingInfo()},
+			expected: []*Inventory{},
+			isError:  true,
+		},
+		// More than one grouping object is an error.
+		{
+			infos:    []*resource.Info{copyGroupingInfo(), pod1Info, copyGroupingInfo()},
+			expected: []*Inventory{},
+			isError:  true,
+		},
+		// Basic test case: one grouping object, one pod.
+		{
+			infos: []*resource.Info{copyGroupingInfo(), pod1Info},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+		{
+			infos: []*resource.Info{pod1Info, copyGroupingInfo()},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+		{
+			infos: []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod2Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod3Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+		{
+			infos: []*resource.Info{pod1Info, pod2Info, pod3Info, copyGroupingInfo()},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod2Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod3Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+		{
+			infos: []*resource.Info{copyGroupingInfo(), pod1Info, pod2Info, pod3Info},
+			expected: []*Inventory{
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod1Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod2Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+				&Inventory{
+					Namespace: testNamespace,
+					Name:      pod3Name,
+					GroupKind: schema.GroupKind{
+						Group: "",
+						Kind:  "Pod",
+					},
+				},
+			},
+			isError: false,
+		},
+	}
+
+	for _, test := range tests {
+		err := addInventoryToGroupingObj(test.infos)
+		if test.isError && err == nil {
+			t.Errorf("Should have produced an error, but returned none.")
+		}
+		if !test.isError {
+			if err != nil {
+				t.Fatalf("Received error when expecting none (%s)\n", err)
+			}
+			retrieved, err := retrieveInventoryFromGroupingObj(test.infos)
+			if err != nil {
+				t.Fatalf("Error retrieving inventory: %s\n", err)
+			}
+			if len(test.expected) != len(retrieved) {
+				t.Errorf("Expected inventory for %d resources, actual %d",
+					len(test.expected), len(retrieved))
+			}
+			for _, expected := range test.expected {
+				found := false
+				for _, actual := range retrieved {
+					if expected.Equals(actual) {
+						found = true
+						continue
+					}
+				}
+				if !found {
+					t.Errorf("Expected inventory (%s) not found", expected)
+				}
+			}
+			// If the grouping object has an inventory, check the
+			// grouping object has an inventory hash.
+			groupingInfo, exists := findGroupingObject(test.infos)
+			if exists && len(test.expected) > 0 {
+				invHash := retrieveInventoryHash(groupingInfo)
+				if len(invHash) == 0 {
+					t.Errorf("Grouping object missing inventory hash")
+				}
+			}
+		}
+	}
+}
+
+func TestAddSuffixToName(t *testing.T) {
+	tests := []struct {
+		info     *resource.Info
+		suffix   string
+		expected string
+		isError  bool
+	}{
+		// Nil info should return error.
+		{
+			info:     nil,
+			suffix:   "",
+			expected: "",
+			isError:  true,
+		},
+		// Empty suffix should return error.
+		{
+			info:     copyGroupingInfo(),
+			suffix:   "",
+			expected: "",
+			isError:  true,
+		},
+		// Empty suffix should return error.
+		{
+			info:     copyGroupingInfo(),
+			suffix:   " \t",
+			expected: "",
+			isError:  true,
+		},
+		{
+			info:     copyGroupingInfo(),
+			suffix:   "hashsuffix",
+			expected: groupingObjName + "-hashsuffix",
+			isError:  false,
+		},
+	}
+
+	for _, test := range tests {
+		//t.Errorf("%#v [%s]", test.info, test.suffix)
+		err := addSuffixToName(test.info, test.suffix)
+		if test.isError {
+			if err == nil {
+				t.Errorf("Should have produced an error, but returned none.")
+			}
+		}
+		if !test.isError {
+			if err != nil {
+				t.Fatalf("Received error when expecting none (%s)\n", err)
+			}
+			actualName, err := getObjectName(test.info.Object)
+			if err != nil {
+				t.Fatalf("Error getting object name: %s", err)
+			}
+			if actualName != test.info.Name {
+				t.Errorf("Object name (%s) does not match info name (%s)\n", actualName, test.info.Name)
+			}
+			if test.expected != actualName {
+				t.Errorf("Expected name (%s), got (%s)\n", test.expected, actualName)
+			}
+		}
+	}
+}
+
+func getObjectName(obj runtime.Object) (string, error) {
+	u, ok := obj.(*unstructured.Unstructured)
+	if !ok {
+		return "", fmt.Errorf("Grouping object is not Unstructured format")
+	}
+	return u.GetName(), nil
+}
+
+func copyGroupingInfo() *resource.Info {
+	groupingObjCopy := groupingObj.DeepCopy()
+	var groupingInfo = &resource.Info{
+		Namespace: testNamespace,
+		Name:      groupingObjName,
+		Object:    groupingObjCopy,
+	}
+	return groupingInfo
+}

cmd/kubectl/kubectlcobra/inventory.go

@@ -0,0 +1,193 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// Separates inventory fields. This string is allowable as a
+// ConfigMap key, but it is not allowed as a character in
+// resource name.
+const fieldSeparator = "_"
+
+// Inventory organizes and stores the indentifying information
+// for an object. This struct (as a string) is stored in a
+// grouping object to keep track of sets of applied objects.
+type Inventory struct {
+	Namespace string
+	Name      string
+	GroupKind schema.GroupKind
+}
+
+// createInventory returns a pointer to an Inventory struct filled
+// with the passed values. This function validates the passed fields
+// and returns an error for bad parameters.
+func createInventory(namespace string,
+	name string, gk schema.GroupKind) (*Inventory, error) {
+
+	// Namespace can be empty, but name cannot.
+	name = strings.TrimSpace(name)
+	if name == "" {
+		return nil, fmt.Errorf("Empty name for inventory object")
+	}
+	if gk.Empty() {
+		return nil, fmt.Errorf("Empty GroupKind for inventory object")
+	}
+
+	return &Inventory{
+		Namespace: strings.TrimSpace(namespace),
+		Name:      name,
+		GroupKind: gk,
+	}, nil
+}
+
+// parseInventory takes a string, splits it into its five fields,
+// and returns a pointer to an Inventory struct storing the
+// five fields. Example inventory string:
+//
+//   test-namespace/test-name/apps/v1/ReplicaSet
+//
+// Returns an error if unable to parse and create the Inventory
+// struct.
+func parseInventory(inv string) (*Inventory, error) {
+	parts := strings.Split(inv, fieldSeparator)
+	if len(parts) == 4 {
+		gk := schema.GroupKind{
+			Group: strings.TrimSpace(parts[2]),
+			Kind:  strings.TrimSpace(parts[3]),
+		}
+		return createInventory(parts[0], parts[1], gk)
+	}
+	return nil, fmt.Errorf("Unable to decode inventory: %s\n", inv)
+}
+
+// Equals returns true if the Inventory structs are identical;
+// false otherwise.
+func (i *Inventory) Equals(other *Inventory) bool {
+	if other == nil {
+		return false
+	}
+	return i.String() == other.String()
+}
+
+// String create a string version of the Inventory struct.
+func (i *Inventory) String() string {
+	return fmt.Sprintf("%s%s%s%s%s%s%s",
+		i.Namespace, fieldSeparator,
+		i.Name, fieldSeparator,
+		i.GroupKind.Group, fieldSeparator,
+		i.GroupKind.Kind)
+}
+
+// InventorySet encapsulates a grouping of unique Inventory
+// structs. Organizes the Inventory structs with a map,
+// which ensures there are no duplicates. Allows set
+// operations such as merging sets and subtracting sets.
+type InventorySet struct {
+	set map[string]*Inventory
+}
+
+// NewInventorySet returns a pointer to an InventorySet
+// struct grouping the passed Inventory items.
+func NewInventorySet(items []*Inventory) *InventorySet {
+	invSet := InventorySet{set: map[string]*Inventory{}}
+	invSet.AddItems(items)
+	return &invSet
+}
+
+// GetItems returns the set of pointers to Inventory
+// structs.
+func (is *InventorySet) GetItems() []*Inventory {
+	items := []*Inventory{}
+	for _, item := range is.set {
+		items = append(items, item)
+	}
+	return items
+}
+
+// AddItems adds Inventory structs to the set which
+// are not already in the set.
+func (is *InventorySet) AddItems(items []*Inventory) {
+	for _, item := range items {
+		if item != nil {
+			is.set[item.String()] = item
+		}
+	}
+}
+
+// DeleteItem removes an Inventory struct from the
+// set if it exists in the set. Returns true if the
+// Inventory item was deleted, false if it did not exist
+// in the set.
+func (is *InventorySet) DeleteItem(item *Inventory) bool {
+	if item == nil {
+		return false
+	}
+	if _, ok := is.set[item.String()]; ok {
+		delete(is.set, item.String())
+		return true
+	}
+	return false
+}
+
+// Merge combines the unique set of Inventory items from the
+// current set with the passed "other" set, returning a new
+// set or error. Returns an error if the passed set to merge
+// is nil.
+func (is *InventorySet) Merge(other *InventorySet) (*InventorySet, error) {
+	if other == nil {
+		return nil, fmt.Errorf("InventorySet to merge is nil.")
+	}
+	// Copy the current InventorySet into result
+	result := NewInventorySet(is.GetItems())
+	result.AddItems(other.GetItems())
+	return result, nil
+}
+
+// Subtract removes the Inventory items in the "other" set from the
+// current set, returning a new set. This does not modify the current
+// set. Returns an error if the passed set to subtract is nil.
+func (is *InventorySet) Subtract(other *InventorySet) (*InventorySet, error) {
+	if other == nil {
+		return nil, fmt.Errorf("InventorySet to subtract is nil.")
+	}
+	// Copy the current InventorySet into result
+	result := NewInventorySet(is.GetItems())
+	// Remove each item in "other" which exists in "result"
+	for _, item := range other.GetItems() {
+		result.DeleteItem(item)
+	}
+	return result, nil
+}
+
+// Equals returns true if the "other" inventory set is the same
+// as this current inventory set. Relies on the fact that the
+// inventory items are sorted for the String() function.
+func (is *InventorySet) Equals(other *InventorySet) bool {
+	if other == nil {
+		return false
+	}
+	return is.String() == other.String()
+}
+
+// String returns a string describing set of Inventory structs.
+func (is *InventorySet) String() string {
+	strs := []string{}
+	for _, item := range is.GetItems() {
+		strs = append(strs, item.String())
+	}
+	sort.Strings(strs)
+	return strings.Join(strs, ", ")
+}
+
+// Size returns the number of Inventory structs in the set.
+func (is *InventorySet) Size() int {
+	return len(is.set)
+}

cmd/kubectl/kubectlcobra/inventory_test.go

@@ -0,0 +1,558 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// package kubectlcobra contains cobra commands from kubectl
+package kubectlcobra
+
+import (
+	"testing"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func TestCreateInventory(t *testing.T) {
+	tests := []struct {
+		namespace string
+		name      string
+		gk        schema.GroupKind
+		expected  string
+		isError   bool
+	}{
+		{
+			namespace: "  \n",
+			name:      " test-name\t",
+			gk: schema.GroupKind{
+				Group: "apps",
+				Kind:  "ReplicaSet",
+			},
+			expected: "_test-name_apps_ReplicaSet",
+			isError:  false,
+		},
+		{
+			namespace: "test-namespace ",
+			name:      " test-name\t",
+			gk: schema.GroupKind{
+				Group: "apps",
+				Kind:  "ReplicaSet",
+			},
+			expected: "test-namespace_test-name_apps_ReplicaSet",
+			isError:  false,
+		},
+		// Error with empty name.
+		{
+			namespace: "test-namespace ",
+			name:      " \t",
+			gk: schema.GroupKind{
+				Group: "apps",
+				Kind:  "ReplicaSet",
+			},
+			expected: "",
+			isError:  true,
+		},
+		// Error with empty GroupKind.
+		{
+			namespace: "test-namespace",
+			name:      "test-name",
+			gk:        schema.GroupKind{},
+			expected:  "",
+			isError:   true,
+		},
+	}
+
+	for _, test := range tests {
+		inv, err := createInventory(test.namespace, test.name, test.gk)
+		if !test.isError {
+			if err != nil {
+				t.Errorf("Error creating inventory when it should have worked.")
+			} else if test.expected != inv.String() {
+				t.Errorf("Expected inventory (%s) != created inventory(%s)\n", test.expected, inv.String())
+			}
+		}
+		if test.isError && err == nil {
+			t.Errorf("Should have returned an error in createInventory()")
+		}
+	}
+}
+
+func TestInventoryEqual(t *testing.T) {
+	tests := []struct {
+		inventory1 *Inventory
+		inventory2 *Inventory
+		isEqual    bool
+	}{
+		// "Other" inventory is nil, then not equal.
+		{
+			inventory1: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			inventory2: nil,
+			isEqual:    false,
+		},
+		// Two equal inventories without a namespace
+		{
+			inventory1: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			inventory2: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			isEqual: true,
+		},
+		// Two equal inventories with a namespace
+		{
+			inventory1: &Inventory{
+				Namespace: "test-namespace",
+				Name:      "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			inventory2: &Inventory{
+				Namespace: "test-namespace",
+				Name:      "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			isEqual: true,
+		},
+		// One inventory with a namespace, one without -- not equal.
+		{
+			inventory1: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			inventory2: &Inventory{
+				Namespace: "test-namespace",
+				Name:      "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			isEqual: false,
+		},
+		// One inventory with a Deployment, one with a ReplicaSet -- not equal.
+		{
+			inventory1: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			inventory2: &Inventory{
+				Name: "test-inv",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "ReplicaSet",
+				},
+			},
+			isEqual: false,
+		},
+	}
+
+	for _, test := range tests {
+		actual := test.inventory1.Equals(test.inventory2)
+		if test.isEqual && !actual {
+			t.Errorf("Expected inventories equal, but actual is not: (%s)/(%s)\n", test.inventory1, test.inventory2)
+		}
+	}
+}
+
+func TestParseInventory(t *testing.T) {
+	tests := []struct {
+		invStr    string
+		inventory *Inventory
+		isError   bool
+	}{
+		{
+			invStr: "_test-name_apps_ReplicaSet\t",
+			inventory: &Inventory{
+				Name: "test-name",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "ReplicaSet",
+				},
+			},
+			isError: false,
+		},
+		{
+			invStr: "test-namespace_test-name_apps_Deployment",
+			inventory: &Inventory{
+				Namespace: "test-namespace",
+				Name:      "test-name",
+				GroupKind: schema.GroupKind{
+					Group: "apps",
+					Kind:  "Deployment",
+				},
+			},
+			isError: false,
+		},
+		// Not enough fields -- error
+		{
+			invStr:    "_test-name_apps",
+			inventory: &Inventory{},
+			isError:   true,
+		},
+	}
+
+	for _, test := range tests {
+		actual, err := parseInventory(test.invStr)
+		if !test.isError {
+			if err != nil {
+				t.Errorf("Error parsing inventory when it should have worked.")
+			} else if !test.inventory.Equals(actual) {
+				t.Errorf("Expected inventory (%s) != parsed inventory (%s)\n", test.inventory, actual)
+			}
+		}
+		if test.isError && err == nil {
+			t.Errorf("Should have returned an error in parseInventory()")
+		}
+	}
+}
+
+var inventory1 = Inventory{
+	Namespace: "test-namespace",
+	Name:      "test-inv-1",
+	GroupKind: schema.GroupKind{
+		Group: "apps",
+		Kind:  "Deployment",
+	},
+}
+
+var inventory2 = Inventory{
+	Namespace: "test-namespace",
+	Name:      "test-inv-2",
+	GroupKind: schema.GroupKind{
+		Group: "",
+		Kind:  "Pod",
+	},
+}
+
+var inventory3 = Inventory{
+	Namespace: "test-namespace",
+	Name:      "test-inv-3",
+	GroupKind: schema.GroupKind{
+		Group: "",
+		Kind:  "Service",
+	},
+}
+
+var inventory4 = Inventory{
+	Namespace: "test-namespace",
+	Name:      "test-inv-4",
+	GroupKind: schema.GroupKind{
+		Group: "apps",
+		Kind:  "DaemonSet",
+	},
+}
+
+func TestNewInventorySet(t *testing.T) {
+	tests := []struct {
+		items        []*Inventory
+		expectedStr  string
+		expectedSize int
+	}{
+		{
+			items:        []*Inventory{},
+			expectedStr:  "",
+			expectedSize: 0,
+		},
+		{
+			items:        []*Inventory{&inventory1},
+			expectedStr:  "test-namespace_test-inv-1_apps_Deployment",
+			expectedSize: 1,
+		},
+		{
+			items:        []*Inventory{&inventory1, &inventory2},
+			expectedStr:  "test-namespace_test-inv-1_apps_Deployment, test-namespace_test-inv-2__Pod",
+			expectedSize: 2,
+		},
+	}
+
+	for _, test := range tests {
+		invSet := NewInventorySet(test.items)
+		actualStr := invSet.String()
+		actualSize := invSet.Size()
+		if test.expectedStr != actualStr {
+			t.Errorf("Expected InventorySet (%s), got (%s)\n", test.expectedStr, actualStr)
+		}
+		if test.expectedSize != actualSize {
+			t.Errorf("Expected InventorySet size (%d), got (%d)\n", test.expectedSize, actualSize)
+		}
+		actualItems := invSet.GetItems()
+		if len(test.items) != len(actualItems) {
+			t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.items), len(actualItems))
+		}
+	}
+}
+
+func TestInventorySetAddItems(t *testing.T) {
+	tests := []struct {
+		initialItems  []*Inventory
+		addItems      []*Inventory
+		expectedItems []*Inventory
+	}{
+		// Adding no items to empty inventory set.
+		{
+			initialItems:  []*Inventory{},
+			addItems:      []*Inventory{},
+			expectedItems: []*Inventory{},
+		},
+		// Adding item to empty inventory set.
+		{
+			initialItems:  []*Inventory{},
+			addItems:      []*Inventory{&inventory1},
+			expectedItems: []*Inventory{&inventory1},
+		},
+		// Adding no items does not change the inventory set
+		{
+			initialItems:  []*Inventory{&inventory1},
+			addItems:      []*Inventory{},
+			expectedItems: []*Inventory{&inventory1},
+		},
+		// Adding an item which alread exists does not increase size.
+		{
+			initialItems:  []*Inventory{&inventory1, &inventory2},
+			addItems:      []*Inventory{&inventory1},
+			expectedItems: []*Inventory{&inventory1, &inventory2},
+		},
+		{
+			initialItems:  []*Inventory{&inventory1, &inventory2},
+			addItems:      []*Inventory{&inventory3, &inventory4},
+			expectedItems: []*Inventory{&inventory1, &inventory2, &inventory3, &inventory4},
+		},
+	}
+
+	for _, test := range tests {
+		invSet := NewInventorySet(test.initialItems)
+		invSet.AddItems(test.addItems)
+		if len(test.expectedItems) != invSet.Size() {
+			t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.expectedItems), invSet.Size())
+		}
+	}
+}
+
+func TestInventorySetDeleteItem(t *testing.T) {
+	tests := []struct {
+		initialItems  []*Inventory
+		deleteItem    *Inventory
+		expected      bool
+		expectedItems []*Inventory
+	}{
+		{
+			initialItems:  []*Inventory{},
+			deleteItem:    nil,
+			expected:      false,
+			expectedItems: []*Inventory{},
+		},
+		{
+			initialItems:  []*Inventory{},
+			deleteItem:    &inventory1,
+			expected:      false,
+			expectedItems: []*Inventory{},
+		},
+		{
+			initialItems:  []*Inventory{&inventory2},
+			deleteItem:    &inventory1,
+			expected:      false,
+			expectedItems: []*Inventory{&inventory2},
+		},
+		{
+			initialItems:  []*Inventory{&inventory1},
+			deleteItem:    &inventory1,
+			expected:      true,
+			expectedItems: []*Inventory{},
+		},
+		{
+			initialItems:  []*Inventory{&inventory1, &inventory2},
+			deleteItem:    &inventory1,
+			expected:      true,
+			expectedItems: []*Inventory{&inventory2},
+		},
+	}
+
+	for _, test := range tests {
+		invSet := NewInventorySet(test.initialItems)
+		actual := invSet.DeleteItem(test.deleteItem)
+		if test.expected != actual {
+			t.Errorf("Expected return value (%t), got (%t)\n", test.expected, actual)
+		}
+		if len(test.expectedItems) != invSet.Size() {
+			t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.expectedItems), invSet.Size())
+		}
+	}
+}
+
+func TestInventorySetMerge(t *testing.T) {
+	tests := []struct {
+		set1   []*Inventory
+		set2   []*Inventory
+		merged []*Inventory
+	}{
+		{
+			set1:   []*Inventory{},
+			set2:   []*Inventory{},
+			merged: []*Inventory{},
+		},
+		{
+			set1:   []*Inventory{},
+			set2:   []*Inventory{&inventory1},
+			merged: []*Inventory{&inventory1},
+		},
+		{
+			set1:   []*Inventory{&inventory1},
+			set2:   []*Inventory{},
+			merged: []*Inventory{&inventory1},
+		},
+		{
+			set1:   []*Inventory{&inventory1, &inventory2},
+			set2:   []*Inventory{&inventory1},
+			merged: []*Inventory{&inventory1, &inventory2},
+		},
+		{
+			set1:   []*Inventory{&inventory1, &inventory2},
+			set2:   []*Inventory{&inventory1, &inventory2},
+			merged: []*Inventory{&inventory1, &inventory2},
+		},
+		{
+			set1:   []*Inventory{&inventory1, &inventory2},
+			set2:   []*Inventory{&inventory3, &inventory4},
+			merged: []*Inventory{&inventory1, &inventory2, &inventory3, &inventory4},
+		},
+	}
+
+	for _, test := range tests {
+		invSet1 := NewInventorySet(test.set1)
+		invSet2 := NewInventorySet(test.set2)
+		expected := NewInventorySet(test.merged)
+		merged, _ := invSet1.Merge(invSet2)
+		if expected.Size() != merged.Size() {
+			t.Errorf("Expected merged inventory set size (%d), got (%d)\n", expected.Size(), merged.Size())
+		}
+	}
+}
+
+func TestInventorySetSubtract(t *testing.T) {
+	tests := []struct {
+		initialItems  []*Inventory
+		subtractItems []*Inventory
+		expected      []*Inventory
+	}{
+		{
+			initialItems:  []*Inventory{},
+			subtractItems: []*Inventory{},
+			expected:      []*Inventory{},
+		},
+		{
+			initialItems:  []*Inventory{},
+			subtractItems: []*Inventory{&inventory1},
+			expected:      []*Inventory{},
+		},
+		{
+			initialItems:  []*Inventory{&inventory1},
+			subtractItems: []*Inventory{},
+			expected:      []*Inventory{&inventory1},
+		},
+		{
+			initialItems:  []*Inventory{&inventory1, &inventory2},
+			subtractItems: []*Inventory{&inventory1},
+			expected:      []*Inventory{&inventory2},
+		},
+		{
+			initialItems:  []*Inventory{&inventory1, &inventory2},
+			subtractItems: []*Inventory{&inventory1, &inventory2},
+			expected:      []*Inventory{},
+		},
+		{
+			initialItems:  []*Inventory{&inventory1, &inventory2},
+			subtractItems: []*Inventory{&inventory3, &inventory4},
+			expected:      []*Inventory{&inventory1, &inventory2},
+		},
+	}
+
+	for _, test := range tests {
+		invInitialItems := NewInventorySet(test.initialItems)
+		invSubtractItems := NewInventorySet(test.subtractItems)
+		expected := NewInventorySet(test.expected)
+		actual, _ := invInitialItems.Subtract(invSubtractItems)
+		if expected.Size() != actual.Size() {
+			t.Errorf("Expected subtracted inventory set size (%d), got (%d)\n", expected.Size(), actual.Size())
+		}
+	}
+}
+
+func TestInventorySetEquals(t *testing.T) {
+	tests := []struct {
+		set1    []*Inventory
+		set2    []*Inventory
+		isEqual bool
+	}{
+		{
+			set1:    []*Inventory{},
+			set2:    []*Inventory{&inventory1},
+			isEqual: false,
+		},
+		{
+			set1:    []*Inventory{&inventory1},
+			set2:    []*Inventory{},
+			isEqual: false,
+		},
+		{
+			set1:    []*Inventory{&inventory1, &inventory2},
+			set2:    []*Inventory{&inventory1},
+			isEqual: false,
+		},
+		{
+			set1:    []*Inventory{&inventory1, &inventory2},
+			set2:    []*Inventory{&inventory3, &inventory4},
+			isEqual: false,
+		},
+		// Empty sets are equal.
+		{
+			set1:    []*Inventory{},
+			set2:    []*Inventory{},
+			isEqual: true,
+		},
+		{
+			set1:    []*Inventory{&inventory1},
+			set2:    []*Inventory{&inventory1},
+			isEqual: true,
+		},
+		// Ordering of the inventory items does not matter for equality.
+		{
+			set1:    []*Inventory{&inventory1, &inventory2},
+			set2:    []*Inventory{&inventory2, &inventory1},
+			isEqual: true,
+		},
+	}
+
+	for _, test := range tests {
+		invSet1 := NewInventorySet(test.set1)
+		invSet2 := NewInventorySet(test.set2)
+		if !invSet1.Equals(invSet2) && test.isEqual {
+			t.Errorf("Expected equal inventory sets; got unequal (%s)/(%s)\n", invSet1, invSet2)
+		}
+		if invSet1.Equals(invSet2) && !test.isEqual {
+			t.Errorf("Expected inequal inventory sets; got equal (%s)/(%s)\n", invSet1, invSet2)
+		}
+	}
+}